diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..cc93c0b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,61 @@ +# .dockerignore does NOT use .gitignore semantics. Docker matches with +# moby/patternmatcher: filepath.Match plus `**`, so `*` does not cross +# `/` and an unprefixed pattern is anchored at the context root. Every +# depth-independent pattern therefore needs `**/`, or `config/.env` and +# `certs/server.key` still ship while this file reads as solved. Only +# genuinely root-anchored entries go unprefixed. Never transplant these +# into .gitignore, where `**/` is wrong. +# +# Matching is case-sensitive, so secrets use character ranges rather +# than an ALL-CAPS twin, which would still miss `Server.Key`. +# +# Extend with this repo's own host-built artifacts, written anchored: +# `/myapp`, never `**/myapp`, which also matches `cmd/myapp/` and +# deletes the package directory from the context. + +# Excluding .git means `git describe` cannot run in any build stage and +# fails quietly there; pass the version in with --build-arg VERSION. +.git + +# Agent scratch: one full checkout of the repo per in-flight agent. +# Anchored because it occurs once where agents run at the repo root. +# KNOWN GAP: a repo running agents in subdirectories still ships +# `services/api/.claude/` and must add its own anchored entry. +.claude + +# Environment files. `*.env` covers bare `.env` and the `prod.env` +# convention. Re-include a committed template with a negation if the +# build needs one: `!docs/example.env`. +**/*.[eE][nN][vV] +**/.[eE][nN][vV].* +**/.[eE][nN][vV][rR][cC] + +# Private keys and the bundles carrying them. Public certificates +# (*.crt, *.cer) are deliberately absent: they are legitimate inputs. +**/*.[pP][eE][mM] +**/*.[kK][eE][yY] +**/*.[pP]12 +**/*.[pP][fF][xX] +**/[iI][dD]_[rR][sS][aA] +**/[iI][dD]_[dD][sS][aA] +**/[iI][dD]_[eE][cC][dD][sS][aA] +**/[iI][dD]_[eE][dD]25519 + +# Dependencies: restored inside the image, never copied in. +**/node_modules + +# The binary `make build` writes on the host; the image builds its own. +/bin + +# OS metadata. +**/.DS_Store +**/Thumbs.db + +# Editor state: never a build input, and it churns COPY. +**/*.swp +**/*.swo +**/*~ +**/*.bak +**/.idea +**/.vscode +**/*.sublime-* diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..92ec261 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +root = true + +[*] +indent_style = space +indent_size = 4 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[Makefile] +indent_style = tab + +[*.go] +indent_style = tab diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml new file mode 100644 index 0000000..ee73864 --- /dev/null +++ b/.gitea/workflows/check.yml @@ -0,0 +1,9 @@ +name: check +on: [push] +jobs: + check: + runs-on: ubuntu-latest + steps: + # actions/checkout v4.2.2, 2026-02-22 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - run: script/cibuild diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f706024 --- /dev/null +++ b/.gitignore @@ -0,0 +1,33 @@ +# OS +.DS_Store +Thumbs.db + +# Editors +*.swp +*.swo +*~ +*.bak +.idea/ +.vscode/ +*.sublime-* + +# Agent scratch (worktrees of this repo, created and destroyed by +# in-flight tooling). Unanchored: .gitignore patterns already match at +# every depth, so no prefix is wanted here. This is not a .dockerignore +# entry and must not be given a `**/` prefix on the way into one. +.claude/ + +# Node +node_modules/ + +# Environment / secrets +.env +.env.* +*.pem +*.key + +# Go: the binary `make build` writes, test binaries, profiles and logs +/bin/ +*.test +*.out +*.log diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..a7a74c2 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,98 @@ +version: "2" + +# Config schema uses the golangci-lint v2 layout (settings live under +# linters.settings, not top-level linters-settings) so that the +# thresholds below are actually applied by golangci-lint >= v2. + +run: + timeout: 5m + modules-download-mode: readonly + +linters: + default: all + enable: + # Successor to the deprecated gomodguard. Named explicitly, rather than + # left to `default: all`, because it carries the module policy below. + - gomodguard_v2 + disable: + # Genuinely incompatible with project patterns + - exhaustruct # Requires all struct fields + - godot # Requires comments to end with periods + - wrapcheck # Too verbose for internal packages + - varnamelen # Short names like db, id are idiomatic Go + # Deprecated: the warning is attached to the old name, so it is + # silenced by disabling that name, not by enabling the successor. + - wsl # Deprecated, replaced by wsl_v5 + - gomodguard # Deprecated, replaced by gomodguard_v2 + settings: + lll: + line-length: 88 + funlen: + lines: 80 + statements: 50 + cyclop: + max-complexity: 15 + dupl: + threshold: 100 + depguard: + # Test-support code must not be compiled into the shipped binary. A + # test-support package exists to hand a test privileges the program + # itself must never have, so a file that is not a test must not import + # one. Test files, and the files inside a package whose directory name + # ends in `test`, are where that code belongs, and are exempt. + # + # The deny list below is the one part of this file a repository is + # expected to extend, and the only part it may. depguard matches an + # import path against a list of prefixes, so it cannot be told "any path + # whose last segment ends in test"; a repository's own test-support + # packages have to be named here one at a time, by full import path, + # under a module path that differs from repository to repository. Add + # them; change nothing else. + rules: + test-support: + list-mode: lax + files: + - "$all" + - "!$test" + - "!**/*test/**" + deny: + - pkg: net/http/httptest + desc: >- + Test-support code belongs in test files and in packages whose + directory name ends in test, not in the shipped binary. + # Only decisions already recorded in the Go package defaults are + # listed here. Every entry matches the module path exactly. + gomodguard_v2: + blocked: + - module: github.com/rs/zerolog + recommendations: + - log/slog + reason: "Structured logging is stdlib log/slog." + # One entry per pre-fork module path, because the later releases + # are separate paths. A prefix match would be shorter but would + # also reach github.com/go-redis/redismock, the test double for + # the successor these entries recommend. + - module: github.com/go-redis/redis + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/go-redis/redis/v7 + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/go-redis/redis/v8 + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/sergi/go-diff + recommendations: + - github.com/aymanbagabas/go-udiff + reason: "No unified diff output; use go-udiff." + - module: github.com/hexops/gotextdiff + recommendations: + - github.com/aymanbagabas/go-udiff + reason: "Unmaintained fork; use go-udiff." + +issues: + max-issues-per-linter: 0 + max-same-issues: 0 diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..23d67fc --- /dev/null +++ b/.prettierignore @@ -0,0 +1,2 @@ +node_modules/ +yarn.lock diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..8af31cd --- /dev/null +++ b/.prettierrc @@ -0,0 +1,4 @@ +{ + "tabWidth": 4, + "proseWrap": "always" +} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..84da0ff --- /dev/null +++ b/Dockerfile @@ -0,0 +1,65 @@ +# Lint phase. The linter is invoked directly rather than through `make +# lint` or `script/lint`, which are themselves a docker build and would +# recurse into a daemon that does not exist in a build step. +# +# golangci/golangci-lint v2.12.2 (built with go1.26.2), 2026-05-06 +FROM golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint + +WORKDIR /src + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +RUN golangci-lint run --config .golangci.yml ./... + +# Test phase, same shape and for the same reason. The go directive in +# go.mod is a minimum, so this Go may be newer than the linter's. The +# Debian image rather than the Alpine one, because the race detector +# needs the C compiler it carries. +# +# golang 1.27.1-trixie, 2026-09-19 +FROM golang@sha256:3b77fc618ec235a1ab412de7737f120dd507c57e8d87de4cbb7994fb94275ed5 AS test + +WORKDIR /src + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +RUN go test -count=1 -timeout 90s -race -cover ./... || \ + { echo "--- Rerunning with -v for details ---"; \ + go test -count=1 -timeout 90s -race -v ./...; exit 1; } + +# Build stage, and the last one: a plain `docker build .` names no +# target and so builds this one. Nothing is wanted from the two phases +# above; the copies are what make BuildKit build them first, so this +# image cannot be produced unless lint and test passed. The image an +# app's Dockerfile builds FROM comes with milestone 2 +# (https://git.eeqj.de/sneak/smallwebwaf/issues/12); until then this +# stage builds the binary and can run it. +# +# golang 1.27.1-trixie, 2026-09-19 +FROM golang@sha256:3b77fc618ec235a1ab412de7737f120dd507c57e8d87de4cbb7994fb94275ed5 + +COPY --from=lint /src/go.sum /dev/null +COPY --from=test /src/go.sum /dev/null + +WORKDIR /src + +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +# The version is computed on the host and passed in, because +# .dockerignore excludes .git. +ARG VERSION=dev +RUN CGO_ENABLED=0 go build -trimpath \ + -ldflags="-s -w -X main.Version=${VERSION}" \ + -o /usr/local/bin/smallwebwaf ./cmd/smallwebwaf + +EXPOSE 8080 +ENTRYPOINT ["/usr/local/bin/smallwebwaf"] diff --git a/EVALUATION.md b/EVALUATION.md index f671f13..4ea3fc9 100644 --- a/EVALUATION.md +++ b/EVALUATION.md @@ -17,8 +17,8 @@ configured only by environment variables, that provides: - R5: temporary blocks for abusers, permanent bans for repeat offenders - R6: one or more RBLs or IP reputation APIs - R7: AS number lookup -- R8: thresholds biased by AS number or country (for example, listed AS - numbers get 50 percent of the normal limit) +- R8: thresholds biased by AS number or country (for example, listed AS numbers + get 50 percent of the normal limit) - R9: WAF-style attack detection and prevention - R10: runs as a plain env-var-configured sidecar between traefik and one app @@ -37,19 +37,19 @@ Closest existing options, and why each still falls short: - BunkerWeb is the closest single product. - Meets: R1 (rates in requests per second, minute, hour or day), R2 - (`LIMIT_IGNORE_IP`, also by AS number and reverse DNS), R3 partly (webhook, - Slack, Discord, Matrix plugins, fired only on denied requests; ntfy only - through the generic webhook, payload format unverified), R5 partly - (`BAD_BEHAVIOR_BAN_TIME`, `0` means permanent; no escalation for repeat - offenders, the ban length is one fixed value), R6 (DNSBL plugin, external - blacklist URLs, optional CrowdSec), R7 partly (AS number used for + (`LIMIT_IGNORE_IP`, also by AS number and reverse DNS), R3 partly + (webhook, Slack, Discord, Matrix plugins, fired only on denied requests; + ntfy only through the generic webhook, payload format unverified), R5 + partly (`BAD_BEHAVIOR_BAN_TIME`, `0` means permanent; no escalation for + repeat offenders, the ban length is one fixed value), R6 (DNSBL plugin, + external blacklist URLs, optional CrowdSec), R7 partly (AS number used for blacklist and whitelist decisions), R9 (ModSecurity with the Core Rule Set, or Coraza plugin), env-var settings. - Fails: R8 (AS number and country can only allow or deny, never scale a limit), R4 (no volume or byte threshold alerts in the free edition; - reporting is a paid feature), R5 escalation, and R10 in spirit: since - 1.6 it needs a `bunkerweb` container plus a `bw-scheduler` container and - a database, or the all-in-one image that bundles nginx, scheduler, UI and + reporting is a paid feature), R5 escalation, and R10 in spirit: since 1.6 + it needs a `bunkerweb` container plus a `bw-scheduler` container and a + database, or the all-in-one image that bundles nginx, scheduler, UI and Redis in one container. It is designed to be the front door for many sites, not a per-app sidecar. AGPL-3.0. - Unverified: whether several rates (minute, hour, day) can be stacked on @@ -61,15 +61,14 @@ Closest existing options, and why each still falls short: (community blocklist, further blocklists, reputation API), R7 (alerts are enriched with AS number and country), R9 (AppSec component with virtual patching and ModSecurity-syntax rules), R2 (allowlists). - - Partly: R1 and R8. Detection is by leaky-bucket scenarios over logs, and - a scenario can filter on AS number or country, so a stricter bucket for + - Partly: R1 and R8. Detection is by leaky-bucket scenarios over logs, and a + scenario can filter on AS number or country, so a stricter bucket for listed AS numbers is possible, but each is a hand-written YAML scenario, it reacts after the fact by banning, and it is not an inline limiter that answers 429. - - Fails: R10 (needs the security engine container with persistent state, - log acquisition from traefik, a bouncer such as the traefik plugin, and - YAML for acquisition, profiles, scenarios and notifications), bytes half - of R4. + - Fails: R10 (needs the security engine container with persistent state, log + acquisition from traefik, a bouncer such as the traefik plugin, and YAML + for acquisition, profiles, scenarios and notifications), bytes half of R4. - CrowdSec plus traefik's own `rateLimit` middleware is the best combination with no new code. It gives inline limiting (one window per middleware, keyed by IP, with `sourceCriterion` exclusions), bans with escalation, alerts and @@ -89,19 +88,19 @@ Closest existing options, and why each still falls short: central API, or AppSec only. Supports captcha remediation. - Covers: R5, R6, R9, R3 and R7 through the engine (see Verdict). - Misses: the plugin itself does no rate limiting; R8; R4 bytes. - - Configuration: traefik static config to load the plugin, dynamic config - or labels for the middleware, CrowdSec YAML for everything else. + - Configuration: traefik static config to load the plugin, dynamic config or + labels for the middleware, CrowdSec YAML for everything else. - Sidecar fit: no. It lives inside traefik, plus a separate engine container. - Maturity: widely used (about 900 stars, listed in the traefik plugin catalog, documented by CrowdSec itself), actively maintained. Traefik - plugins run in an interpreter inside traefik, which costs some - per-request time. + plugins run in an interpreter inside traefik, which costs some per-request + time. - CrowdSec generic bouncers (nginx, Caddy, firewall) - Same engine, different enforcement point. The firewall bouncer blocks at - nftables level on the host, which is cheap and covers every service on - the host at once; worth considering fleet-wide regardless of this - project. Not a sidecar, same misses as above. + nftables level on the host, which is cheap and covers every service on the + host at once; worth considering fleet-wide regardless of this project. Not + a sidecar, same misses as above. - BunkerWeb 1.6.14 (`bunkerity/bunkerweb`) - What it is: nginx with Lua plugins, ModSecurity and the Core Rule Set, configured by settings that are passed as env vars to its scheduler @@ -123,23 +122,22 @@ Closest existing options, and why each still falls short: edition is capped at 10 applications. - Configuration: web UI backed by PostgreSQL. No env-var configuration. - Sidecar fit: no. Seven containers (postgres, management, detector, - tengine, and three helpers); the proxy container uses host networking. - The detection engine is closed source. + tengine, and three helpers); the proxy container uses host networking. The + detection engine is closed source. - Maturity: very active, vendor-driven. - Coraza (`corazawaf/coraza`) and Coraza-based proxies - - What it is: a Go library that implements the ModSecurity rule language - and runs the OWASP Core Rule Set. OWASP project, actively maintained, the + - What it is: a Go library that implements the ModSecurity rule language and + runs the OWASP Core Rule Set. OWASP project, actively maintained, the successor path now that ModSecurity is in maintenance only. - Packagings: `coraza-caddy` (Caddy module), `coraza-spoa` (HAProxy), `coraza-proxy-wasm` (Envoy), a traefik WASM plugin, and - `coreruleset/coraza-crs-docker` (Caddy plus Coraza plus the Core Rule - Set, with env vars for backend address, engine mode and rule-set - tuning). + `coreruleset/coraza-crs-docker` (Caddy plus Coraza plus the Core Rule Set, + with env vars for backend address, engine mode and rule-set tuning). - Covers: R9 only. `coraza-crs-docker` fits R10 well: one container, env vars, backend address. - - Misses: R1 to R8. ModSecurity-language rules can count requests per IP - in a persistent collection, but Coraza's support for persistent - collections is limited and this is not a practical rate limiter. + - Misses: R1 to R8. ModSecurity-language rules can count requests per IP in + a persistent collection, but Coraza's support for persistent collections + is limited and this is not a practical rate limiter. - Value here: the right library to embed for R9 in a purpose-built sidecar. - ModSecurity Core Rule Set containers (`owasp/modsecurity-crs`) - What it is: official images of Apache or nginx with ModSecurity and the @@ -149,21 +147,21 @@ Closest existing options, and why each still falls short: - Covers: R9, and R10 (single container, env vars, one backend). - Misses: R1 to R8. - Maturity: rule set is very actively maintained; the ModSecurity engine - itself is in maintenance under OWASP after Trustwave ended support in - 2024. + itself is in maintenance under OWASP after Trustwave ended support + in 2024. - Anubis (`TecharoHQ/anubis`), 1.27 current - What it is: a single-binary reverse proxy that makes browsers solve a proof-of-work challenge before passing them to `TARGET`. Aimed at scrapers, which is likely a large share of unwanted traffic on a public gitea. - Covers: R10 well (one container, env vars for listener, target, - difficulty, cookies). Policy rules can match on path, user agent, - headers, IP ranges, and with the vendor's hosted data service also AS - number and country, and can weigh a request toward a harder challenge. + difficulty, cookies). Policy rules can match on path, user agent, headers, + IP ranges, and with the vendor's hosted data service also AS number and + country, and can weigh a request toward a harder challenge. - Misses: R1, R3, R4, R5, R6, R9. Bot policy needs a YAML file, not env vars. AS number and country matching depend on the vendor's hosted - service. Breaks non-browser clients unless paths are exempted; for - gitea, git-over-HTTP and API paths must be allowed through by rule. + service. Breaks non-browser clients unless paths are exempted; for gitea, + git-over-HTTP and API paths must be allowed through by rule. - Maturity: very active, widely deployed on code forges since 2025. - Value here: complementary. It can be chained (traefik, then the sidecar, then Anubis, then the app) if challenge pages are wanted. @@ -187,29 +185,28 @@ Closest existing options, and why each still falls short: - Sidecar fit: no; they live inside traefik. - Traefik built-in middlewares - `rateLimit` (one average-and-burst window per middleware, optional Redis - in traefik 3.x), `inFlightReq`, `ipAllowList`. No bans, alerts, - reputation or AS number awareness. + in traefik 3.x), `inFlightReq`, `ipAllowList`. No bans, alerts, reputation + or AS number awareness. - caddy-waf (`fabriziosalmi/caddy-waf`), 0.4.x - What it is: a Caddy module with regex rules and anomaly scoring, per-IP and per-path rate limiting with one configurable window, IP and DNS - blacklists, Tor exit list fetch, country and AS number allow or deny - from MaxMind databases. + blacklists, Tor exit list fetch, country and AS number allow or deny from + MaxMind databases. - Misses: R8 (allow or deny only), R3, R4, R5 (no documented ban state or alerting), R1's three windows. Caddyfile configuration. One maintainer, pre-1.0, AGPL-3.0. - open-appsec (Check Point) - - Machine-learning WAF agent attached to nginx, Kong, Envoy or similar, - with a declarative policy file or the vendor's cloud console. Covers R9 - only; rate limiting and richer features are in paid tiers. Not a sidecar - in the required sense. + - Machine-learning WAF agent attached to nginx, Kong, Envoy or similar, with + a declarative policy file or the vendor's cloud console. Covers R9 only; + rate limiting and richer features are in paid tiers. Not a sidecar in the + required sense. - iocaine - - Serves generated garbage pages to clients the fronting proxy classifies - as scrapers. Not a limiter, WAF or ban tool; out of scope except as a + - Serves generated garbage pages to clients the fronting proxy classifies as + scrapers. Not a limiter, WAF or ban tool; out of scope except as a curiosity for scraper traffic. - Pangolin - - A tunnelled access platform that bundles traefik and optionally - CrowdSec. Replaces the ingress rather than adding a sidecar; out of - scope. + - A tunnelled access platform that bundles traefik and optionally CrowdSec. + Replaces the ingress rather than adding a sidecar; out of scope. ## Requirement by requirement, across the field diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..e8da879 --- /dev/null +++ b/Makefile @@ -0,0 +1,38 @@ +.PHONY: bootstrap setup test lint fmt fmt-check check docker hooks build run + +# Makefile targets are thin shims; the implementations live in script/ +# per the scripts-to-rule-them-all pattern (see the Entrypoints section +# of README.md). build and run are for working on the code by hand. + +bootstrap: + @script/bootstrap + +setup: + @script/setup + +test: + @script/test + +lint: + @script/lint + +fmt: + @script/fmt + +fmt-check: + @script/fmt-check + +check: + @script/check + +docker: + @script/docker + +hooks: + @script/install-precommit + +build: + @script/build + +run: + @script/run diff --git a/README.md b/README.md index 60f7bc6..20b760c 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,134 @@ # smallwebwaf -`smallwebwaf` is a simple, fast, logging web application firewall for people who -host their own services. It runs inside the container of the one application it -protects, between your reverse proxy (traefik) and the app: the app's Dockerfile -builds `FROM` the `smallwebwaf` image, traefik sends the app's requests to -`smallwebwaf` on port 8080, and `smallwebwaf` passes them on to the app on -`127.0.0.1:8081`. It needs no setting, and protects the app from the first -request with defaults chosen for a service on the open internet. It keeps its -state in memory and in JSON files you can read and edit, and writes a detailed -JSON log line for every request. +`smallwebwaf` is a simple, fast, logging web application firewall, written in Go +by [@sneak](https://sneak.berlin), for people who host their own services. It +runs inside the container of the one application it protects, between your +reverse proxy (traefik) and the app: the app's Dockerfile builds `FROM` the +`smallwebwaf` image, traefik sends the app's requests to `smallwebwaf` on port +8080, and `smallwebwaf` passes them on to the app on `127.0.0.1:8081`. It needs +no setting, and protects the app from the first request with defaults chosen for +a service on the open internet. It keeps its state in memory and in JSON files +you can read and edit, and writes a detailed JSON log line for every request. -Status: design stage. This repository currently holds the documents only; no -code has been written. The design is in [`SPEC.md`](SPEC.md), and the survey of -existing tools that led to it is in [`EVALUATION.md`](EVALUATION.md). +Status: the first milestone is built +(https://git.eeqj.de/sneak/smallwebwaf/issues/13). `smallwebwaf` passes each +request to the app and the app's answer back, unchanged, within its timeouts and +size limits, works out each client's address, and writes a JSON log line for +every request. Rate limits per client, the country lists and the image an app +builds on come with milestone 2 +(https://git.eeqj.de/sneak/smallwebwaf/issues/14), and the rest of the design +after that, in the order of the build order in [`SPEC.md`](SPEC.md). The survey +of existing tools that led to the design is in [`EVALUATION.md`](EVALUATION.md). + +## Getting started + +`smallwebwaf` is one Go binary. Until milestone 2 brings its image, build and +run it from a clone, with Go installed: + +```sh +git clone https://git.eeqj.de/sneak/smallwebwaf.git +cd smallwebwaf +make build +SWWAF_UPSTREAM_URL=http://127.0.0.1:3000 ./bin/smallwebwaf +``` + +It then listens on port 8080 and passes every request to the app at +`SWWAF_UPSTREAM_URL`, here an app on port 3000; with no setting at all, to an +app on `127.0.0.1:8081`. On `SIGTERM` or `SIGINT` it stops taking requests and +gives those in progress five seconds to finish. + +## What milestone 1 does + +- Passes each request to the app and the app's answer back unchanged: method, + path, query, headers, body and status. Bodies stream through in both + directions and are never held whole in memory. A WebSocket, or any other + upgraded connection, passes through, and the timeouts do not cut it. +- Works out the client's address. A TCP peer outside `SWWAF_TRUSTED_PROXIES` is + the client, and the forwarded headers it sends are replaced, not passed on. + For a peer inside it, `X-Forwarded-For` is read from the right, and the first + address outside `SWWAF_TRUSTED_PROXIES` is the client; if every address in it + is inside, the leftmost is, and with no header the peer is. The app sees what + it would see from traefik directly: the same `Host`, the same + `X-Forwarded-Proto`, and `X-Forwarded-For` with the peer added at the end. +- Enforces the four timeouts and the two size limits below. A limit passed + before the response has started gets `smallwebwaf`'s own answer: `408` for a + client too slow to send its request, `413` for a request body that is too + large, `504` for an app too slow to answer, and `502` for a response that is + too large or an app that cannot be reached. A request that announces a body + over the limit is refused before anything reaches the app. While a request + body is still on its way, a request timeout that runs out answers `408` if + `smallwebwaf` was waiting for the client to send more, and `504` if it was + waiting for the app to take what it had. Once the response has started, a + limit can only cut the connection. +- Writes a line in the request log for each request (see "Request log" below). + +## Settings + +Each setting is an environment variable, and each has a default, so none has to +be set. A setting that is set but invalid stops the start with a message naming +it, and the effective settings are logged at start. + +- `SWWAF_LISTEN_ADDR` (default `:8080`): where `smallwebwaf` listens. +- `SWWAF_UPSTREAM_URL` (default `http://127.0.0.1:8081`): the app, as `http` or + `https`, a host and an optional port, and nothing more. +- `SWWAF_TRUSTED_PROXIES` (default `10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`, + the private address ranges): the netblocks whose `X-Forwarded-For` is + believed. A list given replaces the default; set but empty, it trusts nothing. +- `SWWAF_CLIENT_REQUEST_TIMEOUT` (default `60s`): how long a client may take to + send its request line and headers, and then, from the end of the headers, its + body. +- `SWWAF_CLIENT_RESPONSE_TIMEOUT` (default `30m`): how long the response may + take to reach the client, from the end of the request to the last byte. +- `SWWAF_UPSTREAM_REQUEST_TIMEOUT` (default `60s`): how long connecting to the + app and sending it the whole request may take. +- `SWWAF_UPSTREAM_RESPONSE_TIMEOUT` (default `30m`): how long the app may take + to send its whole answer, from the end of the request to the last byte. +- `SWWAF_REQUEST_MAX_BYTES` (default `100M`): the largest request body. +- `SWWAF_RESPONSE_MAX_BYTES` (default `5G`): the largest response body. + +Durations are in Go's syntax, with `d` for days (`90s`, `15m`, `7d`). Sizes are +bytes, with an optional `K`, `M` or `G`, which are powers of 1024 (`1K` is 1024 +bytes). Netblocks are in CIDR form, and a bare address stands for itself alone. +`off` switches a timeout or a size limit off. + +Two limits are fixed rather than settings: the request line and headers may take +up to 32 KiB, above which the answer is `431` and nothing reaches the app, and a +kept-open connection that sends nothing for 120 seconds is closed. That is +longer than the 90 seconds after which traefik closes a connection it is not +using, so traefik never sends a request on a connection `smallwebwaf` is +closing. + +## Request log + +`smallwebwaf` writes one JSON object per line on stdout for every request, +refused ones included: + +``` +{"type":"request","time":"2026-10-03T12:00:00.123Z","client_ip":"203.0.113.9","peer_ip":"172.18.0.2","method":"GET","host":"app.example","path":"/","query":"","protocol":"HTTP/1.1","status":200,"upstream_status":200,"request_bytes":0,"response_bytes":5120,"referer":"","user_agent":"curl/8.9.1","action":"forward","duration_total":3.217,"duration_upstream_total":3.104} +``` + +- `time` is when the request arrived, in UTC. `peer_ip` is the TCP peer, + normally traefik. `path` and `query` are as the client sent them. +- `status` is what the client was sent, `0` if nothing was; `upstream_status` is + what the app answered, and is left out when the app did not answer. +- `request_bytes` and `response_bytes` count body bytes. +- `action` is `forward` for a request passed to the app, `too_large` for a + request or response over its size limit, `timed_out` for one that ran out of + time, and `upstream_error` when the app could not be reached or its answer + broke off. +- `aborted` is there, and true, when the client went away early. +- `duration_total` and `duration_upstream_total` are in milliseconds. + +No body and no other header is logged. `smallwebwaf`'s own messages (start, the +settings, stop, errors) share the stream as JSON lines marked +`"type":"process"`. + +Go's HTTP server, on which `smallwebwaf` is built, reads a request's line and +headers before `smallwebwaf` sees the request, and some requests end there, +without a line in the log: headers over 32 KiB, which it answers `431`, headers +slower than `SWWAF_CLIENT_REQUEST_TIMEOUT`, whose connection it closes without +an answer, and requests it cannot read at all, which it answers itself, mostly +with `400`. ## Why @@ -249,8 +365,67 @@ visitor on your local network, another container or your monitoring, has no country: `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` refuses it unless you list it in `SWWAF_ALLOW_NETS`. Such addresses are never sent to GeoJS. +## How the code is laid out + +- `cmd/smallwebwaf`: the binary, which only calls `internal/smallwebwaf`. +- `internal/smallwebwaf`: the process: it reads the settings, listens, serves + requests until `SIGTERM` or `SIGINT`, and stops. +- `internal/config`: reads the settings, the one place they are read. +- `internal/proxy`: what happens to each request: it works out the client, runs + the checks, passes the request to the app and the answer back with the + standard library's `httputil.ReverseProxy` within the timeouts and size + limits, and writes the request's log line. Its `check` method is where + milestone 2's rate limits and country lists refuse a request. +- `internal/requestlog`: the lines on stdout: the request log line and the + process's own messages. + +Only the Go standard library is used. + +## Entrypoints + +This repository adheres to the +[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) +standard: the scripts in `script/` are the entrypoints for working on it, and +the `Makefile` targets are thin shims that call them. The scripts are POSIX sh, +so that they run in minimal containers. + +- `script/bootstrap`: installs what the other scripts need on the host: `make`, + `git`, Go for `gofmt`, node and yarn, and prettier. +- `script/setup`: readies a fresh clone: runs `script/bootstrap`, then + `script/install-precommit`. +- `script/projectname`: prints the project's name, `smallwebwaf`, which + `script/docker` and the others tag their images with. +- `script/test`: runs the tests, as the `test` phase of the `Dockerfile`. +- `script/lint`: runs golangci-lint, as the `lint` phase of the `Dockerfile`. +- `script/fmt`: formats the Go code with `gofmt` and the Markdown with prettier. +- `script/fmt-check`: checks the formatting, and changes nothing. +- `script/check`: runs `script/test`, `script/lint` and `script/fmt-check`. +- `script/docker`: builds the image, whose build runs the tests and the linter + first. +- `script/cibuild`: what CI runs: `script/bootstrap`, `script/check`, then the + image build. +- `script/precommit`: run by the git pre-commit hook; runs `script/check`. +- `script/install-precommit`: installs that hook; `make hooks` runs it. +- `script/build`: builds `bin/smallwebwaf` on the host, with Go installed, for + working on the code by hand; `make build` runs it. +- `script/run`: builds `bin/smallwebwaf` with `script/build` and runs it; + `make run` runs it. + +## TODO + +- Milestone 2: rate limits per client, the country lists and the image an app + builds on (https://git.eeqj.de/sneak/smallwebwaf/issues/14). +- The licence (https://git.eeqj.de/sneak/smallwebwaf/issues/15). +- The rest of the design, in the order of the build order in + [`SPEC.md`](SPEC.md). + ## Documents - [`SPEC.md`](SPEC.md): the design. - [`EVALUATION.md`](EVALUATION.md): what already exists, what each tool covers and misses, and why none was adopted. +- [`REPO_POLICIES.md`](REPO_POLICIES.md): the policies this repository follows. + +## Author + +[@sneak](https://sneak.berlin) diff --git a/REPO_POLICIES.md b/REPO_POLICIES.md new file mode 100644 index 0000000..2256291 --- /dev/null +++ b/REPO_POLICIES.md @@ -0,0 +1,603 @@ +--- +title: Repository Policies +last_modified: 2026-09-08 +--- + +This document covers repository structure, tooling, and workflow standards. Code +style conventions are in separate documents: + +- [Code Styleguide](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE.md) + (general, bash, Docker) +- [Go](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_GO.md) +- [JavaScript](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_JS.md) +- [Python](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_PYTHON.md) +- [Go HTTP Server Conventions](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/GO_HTTP_SERVER_CONVENTIONS.md) + +--- + +- Cross-project documentation (such as this file) must include + `last_modified: YYYY-MM-DD` in the YAML front matter so it can be kept in sync + with the authoritative source as policies evolve. + +- **ALL external references must be pinned by cryptographic hash.** This + includes Docker base images, Go modules, npm packages, GitHub Actions, and + anything else fetched from a remote source. Version tags (`@v4`, `@latest`, + `:3.21`, etc.) are server-mutable and therefore remote code execution + vulnerabilities. The ONLY acceptable way to reference an external dependency + is by its content hash (Docker `@sha256:...`, Go module hash in `go.sum`, npm + integrity hash in lockfile, GitHub Actions `@`). No exceptions. + This also means never `curl | bash` to install tools like pyenv, nvm, rustup, + etc. Instead, download a specific release archive from GitHub, verify its hash + (hardcoded in the Dockerfile or script), and only then install. Unverified + install scripts are arbitrary remote code execution. This is the single most + important rule in this document. Double-check every external reference in + every file before committing. There are zero exceptions to this rule. + +- Every repo with software must have a root `Makefile` with these targets: + `make bootstrap`, `make setup`, `make test`, `make lint`, `make fmt` (writes), + `make fmt-check` (read-only), `make check` (runs `test`, `lint`, `fmt-check`), + `make docker`, and `make hooks` (installs pre-commit hook). A model Makefile + is at `https://git.eeqj.de/sneak/prompts/raw/branch/main/Makefile`. + +- Repos follow the + [Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) + pattern: the implementation of each Makefile target lives in an executable + script in `script/` (`script/bootstrap`, `script/setup`, `script/test`, + `script/lint`, `script/fmt`, `script/fmt-check`, `script/check`, + `script/docker`), and the Makefile targets are thin shims that call them. The + scripts must be POSIX sh (`#!/bin/sh`, `set -eu`, no bashisms) so they run in + minimal containers (e.g. alpine images have no bash); locate the repo root + with `$(cd "$(dirname "$0")/.." && pwd -P)` and `cd` there before acting. From + the standard's canonical set we use `bootstrap`, `setup` (make the repo ready + for development after a fresh clone: runs `bootstrap`, then + `install-precommit`, plus any repo-specific initialization), `test`, and + `cibuild`. `script/bootstrap` installs all dependencies idempotently and + assumes nothing is present: base tools come from nix, apt, brew, or apk + (detected in that order; apt runs noninteractive). For node it uses the + installed node if present; otherwise it installs a PINNED node version via + nvm, first installing nvm itself if missing — from a hash-verified GitHub + release archive (never `curl | sh`), with bash installed as an explicit + prerequisite since nvm requires bash. yarn is then pinned via + `corepack prepare yarn@ --activate`. Never install "latest" or "lts"; + always exact versions. `script/cibuild` runs the CI build: it changes to the + repo root, runs `script/bootstrap`, runs `script/check`, and builds the image + with the version; the Gitea workflow calls it. **`script/cibuild` runs + `script/bootstrap` first**, because the workflow checks out the repo and runs + nothing else, while `script/fmt-check` runs the formatter on the host: on a + pristine checkout with nothing installed the run dies there, after the + containerised gates have passed. **The bootstrap alone is not enough**: + `script/bootstrap` installs node and yarn under nvm and leaves neither on the + `PATH` of the shell that called it, so a bare `yarn` still exits 127. The host + entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore + source nvm for the pinned node version before invoking it, exactly as + `script/bootstrap`'s own install step does. A runner carrying nothing but + docker and git then gets through `script/check`. Four further scripts are our + own extensions to the standard: `script/check` runs `script/test`, + `script/lint` and `script/fmt-check`; `script/precommit` is what the git + pre-commit hook runs, and it calls `script/check`; `script/install-precommit` + installs the git pre-commit hook (the `make hooks` target shims to it); and + `script/projectname` (literally that filename) simply outputs the project's + name. Scripts that need the name call `script/projectname` — e.g. + `script/docker` assembles its image tag from it — so those scripts stay + byte-identical across all repos. Repo-type-specific pre-commit extras (e.g. + `go mod tidy` verification in Go repos) belong in `script/precommit`, not in + the hook itself. Model scripts are at + `https://git.eeqj.de/sneak/prompts/raw/branch/main/script/`. The README + must document the provided scripts in an **Entrypoints** section (see the + README requirements below). + +- Always use Makefile targets (`make fmt`, `make test`, `make lint`, etc.) + instead of invoking the underlying tools directly. The Makefile is the single + source of truth for how these operations are run. + +- The Makefile is authoritative documentation for how the repo is used. Beyond + the required targets above, it should have targets for every common operation: + running a local development server (`make run`, `make dev`), re-initializing + or migrating the database (`make db-reset`, `make migrate`), building + artifacts (`make build`), generating code, seeding data, or anything else a + developer would do regularly. If someone checks out the repo and types + `make`, they should see every meaningful operation available. A new + contributor should be able to understand the entire development workflow by + reading the Makefile. + +- Every repo should have a `Dockerfile`, and it carries the repo's gates: a + `lint` phase and a `test` phase, with the final stage depending on both so the + image cannot be built unless they pass. For non-server repos the final stage + brings up a development environment; for server repos it is the runtime image. + Dockerfiles install development prerequisites by running `script/bootstrap` + rather than duplicating installs inline; COPY `script/` and the dependency + manifests (`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before + running it. + +- **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is + no separate lint file. `script/lint` and `script/test` each build one phase + and nothing else: + + ```sh + docker build --no-cache --target lint -t "$(script/projectname)-lint" . + docker build --no-cache --target test -t "$(script/projectname)-test" . + ``` + + **A stage that is not the last one in the file is built only when the final + stage's chain depends on it, or when `--target` names it.** That is why the + two gates are always invoked by name here, and why the final stage carries a + `COPY --from=` of a harmless file from each of them: without that edge a + plain `docker build .` builds the last stage alone and exits 0 having linted + and tested nothing. + + **Every `docker build` in `script/` is tagged**, here and in + `script/cibuild` and `script/docker`. An untagged build leaves a dangling + image behind on every invocation, on every developer host and every CI + runner; a tagged one replaces the previous image. + + Inside a phase the tool is invoked directly — `golangci-lint`, `go test`, + `eslint`, `prettier` — never through `make lint` or `script/test`, which are + themselves a `docker build` and would recurse into a daemon that does not + exist in a build step. Formatting is the exception and stays on the host: + `script/fmt` writes the working tree, and `script/fmt-check` is its + read-only twin. + + **No lint verdict may come from a host invocation of the linter.** On a + shared host golangci-lint reads a result cache keyed on file content rather + than location, so a second checkout of the same content is served the first + one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs + exit non-zero with `parallel golangci-lint is running` — a status a caller + cannot tell from real findings. Both have produced wrong verdicts in this + org, in both directions. A container has its own cache, its own `TMPDIR` and + a digest-pinned binary, so neither is reachable. + +- **Any build that runs checks is built with `--no-cache`.** Docker invalidates + a `COPY` layer only when the copied content changes, so on an unchanged tree + the check `RUN` is served from cache, nothing executes, and the build still + exits 0. Every `docker build` in `script/` therefore passes `--no-cache`: + `script/lint`, `script/test`, `script/cibuild` and `script/docker` are the + four, and there is no fifth — `script/check` runs the two gate phases and + `script/fmt-check`, and builds no image of its own. A bare `docker build .` is + not evidence that anything ran: a sub-second build reporting success is a + cache hit, not a result. Never invalidate by pruning — `docker builder prune` + and friends destroy a build cache shared with every other build on the host. + +- **The gate phases are separate stages, and the build stage depends on both.** + The lint phase is based on the `golangci/golangci-lint` image (pinned by + hash), so lint failures surface in seconds rather than after a full compile, + and the test phase is based on the Go image. The canonical Go repo + `Dockerfile`: + + ```dockerfile + # Lint phase + # golangci/golangci-lint:v2.x.x, YYYY-MM-DD + FROM golangci/golangci-lint@sha256:... AS lint + WORKDIR /src + COPY go.mod go.sum ./ + RUN go mod download + COPY . . + RUN golangci-lint run --config .golangci.yml ./... + + # Test phase + # golang:1.x-alpine, YYYY-MM-DD + FROM golang@sha256:... AS test + WORKDIR /src + COPY go.mod go.sum ./ + RUN go mod download + COPY . . + RUN go test -timeout 90s -race -cover ./... || \ + { echo "--- Rerunning with -v for details ---"; \ + go test -timeout 90s -race -v ./...; exit 1; } + + # Build stage. Nothing is wanted from either phase above; the copies + # are what make BuildKit build them first, so this stage cannot run + # unless lint and test passed. + # golang:1.x-alpine, YYYY-MM-DD + FROM golang@sha256:... AS builder + COPY --from=lint /src/go.sum /dev/null + COPY --from=test /src/go.sum /dev/null + WORKDIR /src + COPY go.mod go.sum ./ + RUN go mod download + COPY . . + + ARG VERSION=dev + RUN CGO_ENABLED=0 go build -trimpath \ + -ldflags="-s -w -X main.Version=${VERSION}" \ + -o /app ./cmd/app/ + + # Runtime stage, and the last one + FROM alpine@sha256:... + COPY --from=builder /app /usr/local/bin/app + ENTRYPOINT ["app"] + ``` + + Key points: + - The lint phase uses the `golangci/golangci-lint` image directly (it has + both Go and the linter), so nothing needs installing. + - `COPY --from= /src/go.sum /dev/null` is a no-op copy whose only + purpose is the ordering edge. BuildKit runs stages in parallel by default, + and a stage nothing depends on is not built at all, so without these two + lines a red gate would not fail the build. + - Keep the runtime stage last, and if you add a stage after it, give it the + same two copies. A plain `docker build .` builds the last stage's chain + and nothing else. + - If the project uses `//go:embed` directives that reference build artifacts + (e.g. a web frontend compiled in a separate stage), the lint phase must + create placeholder files so the embed directives resolve. Example: + `RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`. + - If the project requires CGO or system libraries for linting (e.g. + `vips-dev`), install them in the lint phase with `apk add`. + - `ARG VERSION=dev` is declared in the stage that compiles and supplied by + `script/docker` and `script/cibuild`; no stage may call `git describe`. + +- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that + runs `script/cibuild` on push, and checks out the repo as its only other step. + That script bootstraps, runs the gate phases, and then builds the image, so a + successful run means every check passed; a bare `docker build .` does not + carry the same guarantee, because its gate phases may come from the cache. The + image build is uncached and so runs the gate phases a second time. That is the + price of the rule above, and it is worth paying: the image that ships is built + from a run of its own gates rather than from a cache entry. + +- Use platform-standard formatters: `black` for Python, `prettier` for + JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with + two exceptions: four-space indents (except Go), and `proseWrap: always` for + Markdown (hard-wrap at 80 columns). Documentation and writing repos (Markdown, + HTML, CSS) should also have `.prettierrc` and `.prettierignore`. + +- Pre-commit hook: runs `script/precommit`, which calls `script/check`. If local + testing is not possible in the repo, `script/precommit` may skip `script/test` + and run only `script/lint` and `script/fmt-check`. The hook is installed by + `script/install-precommit`; the Makefile must provide a `make hooks` target + that shims to it. + +- All repos with software must have tests that run via the platform-standard + test framework (`go test`, `pytest`, `jest`/`vitest`, etc.). If no meaningful + tests exist yet, add the most minimal test possible — e.g. importing the + module under test to verify it compiles/parses. There is no excuse for + `make test` to be a no-op. + +- `make test` must complete in under 60 seconds. That is the hard cap, and a + suite that exceeds it fails. Under 20 seconds is the target. A suite between + 20 and 60 seconds is still green, but the overage must be filed as an + improvement bug against that repo. Add a 90-second timeout to the test + invocation (`go test -timeout 90s`). The backstop deliberately sits above the + hard cap so that it catches a genuinely hung test rather than a merely slow + one. + +- **The test command should use the conditional verbose rerun pattern.** Run + tests without `-v` (verbose) first. If tests fail, automatically rerun with + `-v` to show full output. This keeps CI logs and `docker build` output clean + on success (just package/suite summaries) while providing full diagnostic + detail on failure (every test case, every assertion). The command lives in the + `test` phase of the `Dockerfile`, since `script/test` builds that phase; the + Makefile form below is the same pattern for any repo-local invocation: + + ```makefile + test: + @ || \ + { echo "--- Rerunning with -v for details ---"; \ + ; exit 1; } + ``` + + Go example: + + ```makefile + test: + @go test -count=1 -timeout 90s -race -cover ./... || \ + { echo "--- Rerunning with -v for details ---"; \ + go test -count=1 -timeout 90s -race -v ./...; exit 1; } + ``` + + `-count=1` is required on both invocations: it defeats Go's test _result_ + cache, so the target cannot report a pass it did not earn, and the rerun + reproduces a failure instead of replaying it. It leaves the build cache + alone, so it costs the runtime of the suite and no recompilation. + + Note that this is a second, independent cache, stacked below the Docker + layer cache that [issue #26](https://git.eeqj.de/sneak/prompts/issues/26) + addresses. `CHECK_EPOCH` guarantees the `RUN make test` _step_ re-executes; + it does not guarantee `go test` inside that step does any work, because the + `GOCACHE` baked into earlier image layers survives into the re-executed + step. They are two separate defects requiring two separate fixes, and a fix + for one must not be recorded as covering the other. + + Python example: + + ```makefile + test: + @python -m pytest || \ + { echo "--- Rerunning with -v for details ---"; \ + python -m pytest -v; exit 1; } + ``` + + The `exit 1` ensures the target always fails after a rerun — the first run + already proved the tests are broken, so the build must not pass even if a + flaky test happens to succeed on the second attempt. The rerun exists solely + for diagnostic output. + +- Docker builds must complete in under 5 minutes. + +- `make check` must not modify any files in the repo. Tests may use temporary + directories. + +- `main` must always pass `make check`, no exceptions. + +- Never commit secrets. `.env` files, credentials, API keys, and private keys + must be in `.gitignore`. No exceptions. + +- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`), + editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`), + language build artifacts, and `node_modules/`. Fetch the standard `.gitignore` + from `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when + setting up a new repo. These patterns are written to `.gitignore`'s own + semantics, in which an unanchored pattern already matches at every depth; they + are not a `.dockerignore` and must not be transplanted into one unmodified. + +- **`.dockerignore` does not use `.gitignore` semantics, and copying patterns + across unmodified leaves secrets in the build context.** Docker matches with + `moby/patternmatcher`: `filepath.Match` semantics plus a `**` extension, so + `*` does not cross `/` and a pattern without a leading `**/` is anchored at + the build-context root. A `.dockerignore` listing `.env`, `*.pem` and `*.key` + therefore excludes only the copies at the repository root, while `config/.env` + and `certs/server.key` still reach the context and can land in an image layer + — which is more dangerous than a short file with no secret patterns at all, + because it reads as solved and stops anyone looking. Give every + depth-independent pattern the `**/` prefix and leave only genuinely + root-anchored entries unprefixed: `.git`, and the repo's own host-built + binary, written `/myapp` and never `**/myapp`, which would also match + `cmd/myapp/` and delete the package directory from the context. Matching is + case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so + secret names use character ranges — `**/*.[kK][eE][yY]`, `**/*.[pP][eE][mM]`, + and likewise for `.envrc` and the extensionless SSH keys. Where such a pattern + also catches something the build needs, re-include it with a negation + (`!docs/example.env`); deleting the pattern reopens the exposure for every + other file it covers. Fetch the standard `.dockerignore` from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.dockerignore` and extend + it with the repo's own artifacts. + +- **In-repo agent scratch belongs in both files, written to each file's own + semantics.** `.claude/` holds one worktree per in-flight agent — an entire + additional checkout of the repo — so under `COPY . .` the build context + inflates by a multiple of the repo and another session's unreviewed work can + be copied into an image layer. In `.gitignore` the entry is `.claude/`, + unanchored. In `.dockerignore` it is `.claude`, anchored and with **no** `**/` + prefix, because the prefixed form would also delete any nested directory of + that name from the build. Anchoring carries a known gap that the canonical + `.dockerignore` states in its own comment, since consuming repos receive the + file and not the tracker: the directory is created in the agent's working + directory, so a repo running agents in subdirectories still ships + `services/api/.claude/` and must add its own anchored entry there. + +- **Excluding `.git` means `git describe` cannot run inside any build stage, and + it fails quietly there.** In a build stage there is no repository, so + `git describe` writes nothing to stdout, `-X main.Version=` comes out empty, + the binary reports no version at all, and the build still exits 0. Compute the + version on the host and thread it in as a build arg. `script/docker` and + `script/cibuild` do this, byte-identically across repos: + + ```sh + # Own line: a failing command substitution inside an argument does not + # trip `set -e`, so the inline form degrades to an empty constant. + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + docker build --no-cache \ + --build-arg VERSION="$version" \ + -t "$(script/projectname)" . + ``` + + `--always` makes an untagged repo yield an abbreviated commit hash rather + than failing, and the `[ -n "$version" ]` line is the single place the + fallback is applied — a live check that fires on a build from an export with + no `.git` and on a repository with no commits yet. Do not fold it into the + substitution as `|| echo unknown`, which makes the guard unreachable. The + Dockerfile's side is `ARG VERSION=dev` in the stage that compiles, declared + there because `ARG` is stage-scoped; passing `VERSION` to a repo whose + Dockerfile declares no such `ARG` is ignored and costs nothing, which is why + the scripts stay byte-identical. One consequence for CI: the standard + checkout action clones shallow and fetches no tags, so a repo that embeds a + tag-derived version must set `fetch-depth: 0` on its checkout step. + +- **Verify `.dockerignore` by enumerating the image, not by reading the + patterns.** Plant files at the root _and_ at least two directories deep, build + a probe image that does `COPY . .`, and list what actually landed + (`docker run --rm --entrypoint find IMAGE /app`). The `transferring context` + size is not a substitute: a nested secret is a few bytes, and BuildKit + transfers only the delta from the previous build. + +- **No build artifacts in version control.** Code-derived data (compiled + bundles, minified output, generated assets) must never be committed to the + repository if it can be avoided. The build process (e.g. Dockerfile, Makefile) + should generate these at build time. Notable exception: Go protobuf generated + files (`.pb.go`) ARE committed because repos need to work with `go get`, which + downloads code but does not execute code generation. + +- Never use `git add -A` or `git add .`. Always stage files explicitly by name. + +- Never force-push to `main`. + +- Make all changes on a feature branch. You can do whatever you want on a + feature branch. + +- `.golangci.yml` is standardized. The vendored copy in a consuming repo must + _NEVER_ be modified by an agent: fetch it from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml` and keep it + byte-identical, so that no repo can quietly loosen its own linting. Linter + configuration changes are made to the canonical copy in the `prompts` repo and + reach consuming repos by re-vendoring; an agent may open a PR against + canonical, which only the user merges. One list is exempt from byte-identity, + because it cannot be written once for every repo: the `deny` list of the + `test-support` depguard rule, where a repo names its own test-support packages + by full import path. A repo adds entries there and changes nothing else, and a + re-vendor carries its entries forward. The canonical golangci-lint version is + v2.12.2 (released 2026-05-06), pinned as the digest of the lint phase's base + image + (`golangci/golangci-lint@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240`, + which reports `2.12.2 built with go1.26.2 from c0d3ddc9`). That digest is the + only pin, since no repo installs golangci-lint on the host: bumping the + version means changing it and nothing else. + +- **`script/bootstrap` installs a pinned tool by comparing versions, never by + testing presence.** An `if ! command -v ; then install; fi` guard tests + `PATH` only, so on an already-provisioned machine the pin is inert and a + version bump is a silent no-op — while the Dockerfile, installing into a clean + image, gets the pinned version, so a local `make check` and `make docker` can + disagree about what the tool even is. The canonical form: + - compares the installed version against the pin over the **whole** version + token; a parser that stops at the first `-` reports `2.12.2` for a host + running `2.12.2-rc1` and skips the install; + - treats absent, non-zero, empty or unrecognised `--version` output as a + mismatch, so the failure direction is a redundant install and never a + skipped one; + - after installing, re-resolves the binary the way callers do — `hash -r`, + then through `PATH`, not through the directory the installer wrote to — + and fails naming the resolved path, since an install that a shadowing + binary hides succeeds while changing nothing any caller sees; + - is actually called, and prints the version on both success paths: a + function defined and never invoked has the same exit status and the same + empty output as one that worked. + + Keep it POSIX sh: no arrays, no `[[`, no `grep -P`. + +- When pinning images or packages by hash, add a comment above the reference + with the version and date (YYYY-MM-DD). + +- Use `yarn`, not `npm`. + +- Write all dates as YYYY-MM-DD (ISO 8601). + +- Simple projects should be configured with environment variables. + +- Dockerized web services listen on port 8080 by default, overridable with + `PORT`. + +- **HTTP/web services must be hardened for production internet exposure before + tagging 1.0.** This means full compliance with security best practices + including, without limitation, all of the following: + - **Security headers** on every response: + - `Strict-Transport-Security` (HSTS) with `max-age` of at least one year + and `includeSubDomains`. + - `Content-Security-Policy` (CSP) with a restrictive default policy + (`default-src 'self'` as a baseline, tightened per-resource as + needed). Never use `unsafe-inline` or `unsafe-eval` unless + unavoidable, and document the reason. + - `X-Frame-Options: DENY` (or `SAMEORIGIN` if framing is required). + Prefer the `frame-ancestors` CSP directive as the primary control. + - `X-Content-Type-Options: nosniff`. + - `Referrer-Policy: strict-origin-when-cross-origin` (or stricter). + - `Permissions-Policy` restricting access to browser features the + application does not use (camera, microphone, geolocation, etc.). + - **Request and response limits:** + - Maximum request body size enforced on all endpoints (e.g. Go + `http.MaxBytesReader`). Choose a sane default per-route; never accept + unbounded input. + - Maximum response body size where applicable (e.g. paginated APIs). + - `ReadTimeout` and `ReadHeaderTimeout` on the `http.Server` to defend + against slowloris attacks. + - `WriteTimeout` on the `http.Server`. + - `IdleTimeout` on the `http.Server`. + - Per-handler execution time limits via `context.WithTimeout` or + chi/stdlib `middleware.Timeout`. + - **Authentication and session security:** + - Rate limiting on password-based authentication endpoints. API keys are + high-entropy and not susceptible to brute force, so they are exempt. + - CSRF tokens on all state-mutating HTML forms. API endpoints + authenticated via `Authorization` header (Bearer token, API key) are + exempt because the browser does not attach these automatically. + - Passwords stored using bcrypt, scrypt, or argon2 — never plain-text, + MD5, or SHA. + - Session cookies set with `HttpOnly`, `Secure`, and `SameSite=Lax` (or + `Strict`) attributes. + - **Reverse proxy awareness:** + - True client IP detection when behind a reverse proxy + (`X-Forwarded-For`, `X-Real-IP`). The application must accept + forwarded headers only from a configured set of trusted proxy + addresses — never trust `X-Forwarded-For` unconditionally. + - **CORS:** + - Authenticated endpoints must restrict `Access-Control-Allow-Origin` to + an explicit allowlist of known origins. Wildcard (`*`) is acceptable + only for public, unauthenticated read-only APIs. + - **Error handling:** + - Internal errors must never leak stack traces, SQL queries, file paths, + or other implementation details to the client. Return generic error + messages in production; detailed errors only when `DEBUG` is enabled. + - **TLS:** + - Services never terminate TLS directly. They are always deployed behind + a TLS-terminating reverse proxy. The service itself listens on plain + HTTP. However, HSTS headers and `Secure` cookie flags must still be + set by the application so that the browser enforces HTTPS end-to-end. + + This list is non-exhaustive. Apply defense-in-depth: if a standard security + hardening measure exists for HTTP services and is not listed here, it is + still expected. When in doubt, harden. + +- `README.md` is the primary documentation. Required sections: + - **Description**: First line must include the project name, purpose, + category (web server, SPA, CLI tool, etc.), license, and author. Example: + "µPaaS is an MIT-licensed Go web application by @sneak that receives + git-frontend webhooks and deploys applications via Docker in realtime." + - **Getting Started**: Copy-pasteable install/usage code block. + - **Entrypoints**: Opens by stating that the repo adheres to the + [Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) + standard (with that link), then documents each provided `script/` + entrypoint and its purpose. + - **Rationale**: Why does this exist? + - **Design**: How is the program structured? + - **TODO**: Update meticulously, even between commits. When planning, put + the todo list in the README so a new agent can pick up where the last one + left off. + - **License**: MIT, GPL, or WTFPL. Ask the user for new projects. Include a + `LICENSE` file in the repo root and a License section in the README. + - **Author**: [@sneak](https://sneak.berlin). + +- First commit of a new repo should contain only `README.md`. + +- Go module root: `sneak.berlin/go/`. Always run `go mod tidy` before + committing. + +- Use SemVer. + +- Database migrations live in `internal/db/migrations/` and must be embedded in + the binary. + - `000_migration.sql` — contains ONLY the creation of the migrations + tracking table itself. Nothing else. + - `001_schema.sql` — the full application schema. + - **Pre-1.0.0:** never add additional migration files (002, 003, etc.). + There is no installed base to migrate. Edit `001_schema.sql` directly. + - **Post-1.0.0:** add new numbered migration files for each schema change. + Never edit existing migrations after release. + +- All repos should have an `.editorconfig` enforcing the project's indentation + settings. + +- Avoid putting files in the repo root unless necessary. Root should contain + only project-level config files (`README.md`, `Makefile`, `Dockerfile`, + `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and + language-specific config). Everything else goes in a subdirectory. Canonical + subdirectory names: + - `bin/` — executable scripts and tools + - `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose + body is a single call into `internal/` or `pkg/`, no project logic in + `cmd/` + - `configs/` — configuration templates and examples + - `deploy/` — deployment manifests (k8s, compose, terraform) + - `docs/` — documentation and markdown (README.md stays in root) + - `internal/` — Go internal packages + - `internal/db/migrations/` — database migrations + - `pkg/` — Go library packages + - `share/` — systemd units, data files + - `static/` — static assets (images, fonts, etc.) + - `web/` — web frontend source + +- When setting up a new repo, files from the `prompts` repo may be used as + templates. Fetch them from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/`. + +- New repos must contain at minimum: + - `README.md`, `.git`, `.gitignore`, `.editorconfig` + - `LICENSE`, `REPO_POLICIES.md` (copy from the `prompts` repo) + - `Makefile` + - `script/` entrypoints (`bootstrap`, `setup`, `projectname`, `test`, + `lint`, `fmt`, `fmt-check`, `check`, `docker`, `cibuild`, `precommit`, + `install-precommit`) + - `Dockerfile`, `.dockerignore` + - `.gitea/workflows/check.yml` + - Go: `go.mod`, `go.sum`, `.golangci.yml` + - JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore` + - Python: `pyproject.toml` diff --git a/SPEC.md b/SPEC.md index 08a00de..918ed49 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1,8 +1,8 @@ # smallwebwaf SPEC (draft): protective reverse proxy for one app -Status: fourth draft, with the owner's rulings to date applied. Nothing has been -built yet. `EVALUATION.md` beside this file explains why no existing tool was -chosen. +Status: fourth draft, with the owner's rulings to date applied. Milestone 1 of +the build order is built. `EVALUATION.md` beside this file explains why no +existing tool was chosen. ## Purpose @@ -409,7 +409,8 @@ The settings, by group: Bodies stream straight through, so a request body reaches the app while the client is still sending it. - `SWWAF_CLIENT_REQUEST_TIMEOUT` (default `60s`): how long a client may take - to send its whole request, headers and body. + to send its request line and headers, and then, from the end of the + headers, its body. - `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES` (default `32K`): the largest request line and headers a client may send. Over it, `smallwebwaf` answers `431` and closes the connection, and nothing reaches the app. @@ -436,6 +437,12 @@ The settings, by group: an app that is too slow. A request that announces a body larger than its limit is refused before anything reaches the app. Once the response has started it can only be cut off, and the connection is closed. + - Go's HTTP server, on which `smallwebwaf` is built, reads a request's line + and headers before `smallwebwaf` sees the request. A client that takes + longer than `SWWAF_CLIENT_REQUEST_TIMEOUT` to send them gets no answer: + the server closes its connection. Headers over + `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES` are answered `431` by the server + itself. Neither request gets a line in the request log. - A WebSocket connection leaves these limits behind once it is upgraded: it stays open until either side closes it. - Lookup of AS number and country (R7). On by default through GeoJS, which needs @@ -1012,10 +1019,12 @@ and the running `smallwebwaf` takes the edit in. ## Request log -One JSON object per line on stdout for every request, including refused ones. -stdout is always on. When `SWWAF_LOG_REMOTE_URL` is set the same lines are also -sent to the remote endpoint, so a deployment can stop depending on docker's log -handling while `docker logs` keeps working. +One JSON object per line on stdout for every request, including refused ones, +apart from those Go's HTTP server ends before `smallwebwaf` sees them (see +"Configuration surface", size and time limits). stdout is always on. When +`SWWAF_LOG_REMOTE_URL` is set the same lines are also sent to the remote +endpoint, so a deployment can stop depending on docker's log handling while +`docker logs` keeps working. - Standard web log fields: `time` (RFC 3339 with milliseconds), `instance`, `client_ip`, `method`, `scheme`, `host`, `path`, `query`, `protocol`, diff --git a/cmd/smallwebwaf/main.go b/cmd/smallwebwaf/main.go new file mode 100644 index 0000000..108247c --- /dev/null +++ b/cmd/smallwebwaf/main.go @@ -0,0 +1,18 @@ +// Command smallwebwaf is a web application firewall for one app: it runs +// between traefik and the app, and passes requests through within its +// limits. Everything it does is in internal/smallwebwaf. +package main + +import ( + "os" + + "sneak.berlin/go/smallwebwaf/internal/smallwebwaf" +) + +// Version is the version of the binary, set when it is built with +// -ldflags "-X main.Version=...". +var Version = "dev" //nolint:gochecknoglobals // the linker sets it + +func main() { + os.Exit(smallwebwaf.Main(Version)) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..c60a421 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module sneak.berlin/go/smallwebwaf + +go 1.26.0 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..e69de29 diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..a0236c2 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,352 @@ +// Package config reads smallwebwaf's settings. Every setting is an +// environment variable whose name starts with SWWAF_, every setting has a +// default, and this package is the one place they are read. +package config + +import ( + "errors" + "fmt" + "log/slog" + "math" + "net" + "net/netip" + "net/url" + "strconv" + "strings" + "time" +) + +// Config is smallwebwaf's settings. A timeout or size of zero is off. +type Config struct { + // ListenAddr is where smallwebwaf listens (SWWAF_LISTEN_ADDR). + ListenAddr string + // UpstreamURL is the app (SWWAF_UPSTREAM_URL). + UpstreamURL *url.URL + // TrustedProxies are the netblocks whose X-Forwarded-For is + // believed (SWWAF_TRUSTED_PROXIES). + TrustedProxies []netip.Prefix + // ClientRequestTimeout bounds reading the whole request from the + // client (SWWAF_CLIENT_REQUEST_TIMEOUT). + ClientRequestTimeout time.Duration + // ClientResponseTimeout bounds writing the whole response to the + // client (SWWAF_CLIENT_RESPONSE_TIMEOUT). + ClientResponseTimeout time.Duration + // UpstreamRequestTimeout bounds connecting to the app and writing + // the whole request to it (SWWAF_UPSTREAM_REQUEST_TIMEOUT). + UpstreamRequestTimeout time.Duration + // UpstreamResponseTimeout bounds reading the whole response from + // the app (SWWAF_UPSTREAM_RESPONSE_TIMEOUT). + UpstreamResponseTimeout time.Duration + // RequestMaxBytes is the largest request body + // (SWWAF_REQUEST_MAX_BYTES). + RequestMaxBytes int64 + // ResponseMaxBytes is the largest response body + // (SWWAF_RESPONSE_MAX_BYTES). + ResponseMaxBytes int64 + + // settings are the values read, as given or by default, for the + // log line at start. + settings []slog.Attr +} + +// off is the value that switches a timeout or a size limit off. +const off = "off" + +const ( + day = 24 * time.Hour + kibibyte = 1 << 10 + mebibyte = 1 << 20 + gibibyte = 1 << 30 +) + +var ( + errNotDuration = errors.New( + "is not a duration such as 90s, 15m or 7d, or off") + errNotSize = errors.New( + "is not a size such as 512K, 100M or 5G, or off") + errNotPositive = errors.New("must be more than zero, or off") + errEmptyItem = errors.New("has an empty item in its list") + errNotNetblock = errors.New( + "is not a netblock such as 10.0.0.0/8, or an address") + errNotListenAddr = errors.New( + "is not an address to listen on, such as :8080") + errNotUpstreamURL = errors.New( + "is not a URL with only a scheme, a host and an optional port, " + + "such as http://127.0.0.1:8081") +) + +// FromEnvironment reads the settings with lookupEnv, normally +// os.LookupEnv. A setting that is not set takes its default. A setting +// that is set but invalid is an error that names it. +func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) { + env := &environment{lookupEnv: lookupEnv} + cfg := &Config{ + ListenAddr: env.address("SWWAF_LISTEN_ADDR", ":8080"), + UpstreamURL: env.appURL("SWWAF_UPSTREAM_URL", "http://127.0.0.1:8081"), + TrustedProxies: env.netblocks("SWWAF_TRUSTED_PROXIES", privateRanges), + ClientRequestTimeout: env.duration("SWWAF_CLIENT_REQUEST_TIMEOUT", "60s"), + ClientResponseTimeout: env.duration("SWWAF_CLIENT_RESPONSE_TIMEOUT", "30m"), + UpstreamRequestTimeout: env.duration("SWWAF_UPSTREAM_REQUEST_TIMEOUT", "60s"), + UpstreamResponseTimeout: env.duration("SWWAF_UPSTREAM_RESPONSE_TIMEOUT", "30m"), + RequestMaxBytes: env.size("SWWAF_REQUEST_MAX_BYTES", "100M"), + ResponseMaxBytes: env.size("SWWAF_RESPONSE_MAX_BYTES", "5G"), + } + + if env.err != nil { + return nil, env.err + } + + cfg.settings = env.settings + + return cfg, nil +} + +// privateRanges are the private address ranges, the default trusted +// proxies. +const privateRanges = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16" + +// LogValue makes a Config log as each setting's name with the value it was +// given, or its default. +func (c *Config) LogValue() slog.Value { + return slog.GroupValue(c.settings...) +} + +// environment is where FromEnvironment reads the settings: it notes each +// value for the log, and keeps the first error. +type environment struct { + lookupEnv func(string) (string, bool) + settings []slog.Attr + err error +} + +// value returns a setting's value, or its default when it is not set, +// and notes it for the log. +func (e *environment) value(name, defaultValue string) string { + value, ok := e.lookupEnv(name) + if !ok { + value = defaultValue + } + + e.settings = append(e.settings, slog.String(name, value)) + + return value +} + +// check keeps the first error, naming the setting it is about. +func (e *environment) check(name string, err error) { + if err != nil && e.err == nil { + e.err = fmt.Errorf("%s: %w", name, err) + } +} + +// address reads a setting that is an address to listen on. +func (e *environment) address(name, defaultValue string) string { + address, err := parseListenAddr(e.value(name, defaultValue)) + e.check(name, err) + + return address +} + +// appURL reads a setting that is the app's URL. +func (e *environment) appURL(name, defaultValue string) *url.URL { + upstream, err := parseUpstreamURL(e.value(name, defaultValue)) + e.check(name, err) + + return upstream +} + +// netblocks reads a setting that is a list of netblocks. +func (e *environment) netblocks(name, defaultValue string) []netip.Prefix { + netblocks, err := parseNetblocks(e.value(name, defaultValue)) + e.check(name, err) + + return netblocks +} + +// duration reads a setting that is a duration. +func (e *environment) duration(name, defaultValue string) time.Duration { + duration, err := parseDuration(e.value(name, defaultValue)) + e.check(name, err) + + return duration +} + +// size reads a setting that is a number of bytes. +func (e *environment) size(name, defaultValue string) int64 { + size, err := parseSize(e.value(name, defaultValue)) + e.check(name, err) + + return size +} + +// parseDuration reads a duration in Go's syntax, such as 90s or 15m, a +// whole number of days such as 7d, or off. +func parseDuration(value string) (time.Duration, error) { + if value == off { + return 0, nil + } + + duration, err := durationOrDays(value) + if err != nil { + return 0, fmt.Errorf("%q %w", value, errNotDuration) + } + + if duration <= 0 { + return 0, fmt.Errorf("%q %w", value, errNotPositive) + } + + return duration, nil +} + +// durationOrDays reads Go's duration syntax, or a whole number of days. +func durationOrDays(value string) (time.Duration, error) { + days, isDays := strings.CutSuffix(value, "d") + if !isDays { + return time.ParseDuration(value) + } + + n, err := strconv.ParseInt(days, 10, 64) + if err != nil || n < 0 || n > math.MaxInt64/int64(day) { + return 0, errNotDuration + } + + return time.Duration(n) * day, nil +} + +// parseSize reads a number of bytes with an optional K, M or G suffix, in +// powers of 1024 (1K is 1024 bytes), or off. +func parseSize(value string) (int64, error) { + if value == off { + return 0, nil + } + + number, unit := splitUnit(value) + + n, err := strconv.ParseInt(number, 10, 64) + if err != nil || n > math.MaxInt64/unit { + return 0, fmt.Errorf("%q %w", value, errNotSize) + } + + if n <= 0 { + return 0, fmt.Errorf("%q %w", value, errNotPositive) + } + + return n * unit, nil +} + +// splitUnit splits a size into its number and the bytes its suffix +// stands for. +func splitUnit(value string) (string, int64) { + switch { + case strings.HasSuffix(value, "K"): + return strings.TrimSuffix(value, "K"), kibibyte + case strings.HasSuffix(value, "M"): + return strings.TrimSuffix(value, "M"), mebibyte + case strings.HasSuffix(value, "G"): + return strings.TrimSuffix(value, "G"), gibibyte + default: + return value, 1 + } +} + +// parseList splits a comma-separated list and trims the spaces around +// each item. An empty value is an empty list. +func parseList(value string) ([]string, error) { + if strings.TrimSpace(value) == "" { + return []string{}, nil + } + + items := strings.Split(value, ",") + for i, item := range items { + items[i] = strings.TrimSpace(item) + if items[i] == "" { + return nil, fmt.Errorf("%q %w", value, errEmptyItem) + } + } + + return items, nil +} + +// parseNetblocks reads a comma-separated list of netblocks. +func parseNetblocks(value string) ([]netip.Prefix, error) { + items, err := parseList(value) + if err != nil { + return nil, err + } + + netblocks := make([]netip.Prefix, 0, len(items)) + + for _, item := range items { + netblock, err := parseNetblock(item) + if err != nil { + return nil, err + } + + netblocks = append(netblocks, netblock) + } + + return netblocks, nil +} + +// parseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare +// address is a netblock of that address alone, a /32 or a /128. +func parseNetblock(value string) (netip.Prefix, error) { + if strings.Contains(value, "/") { + netblock, err := netip.ParsePrefix(value) + if err != nil { + return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock) + } + + return netblock.Masked(), nil + } + + addr, err := netip.ParseAddr(value) + if err != nil || addr.Zone() != "" { + return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock) + } + + return netip.PrefixFrom(addr, addr.BitLen()), nil +} + +// parseListenAddr checks an address to listen on: an optional host and a +// port number. +func parseListenAddr(value string) (string, error) { + _, port, err := net.SplitHostPort(value) + if err != nil { + return "", fmt.Errorf("%q %w", value, errNotListenAddr) + } + + _, err = strconv.ParseUint(port, 10, 16) + if err != nil { + return "", fmt.Errorf("%q %w", value, errNotListenAddr) + } + + return value, nil +} + +// parseUpstreamURL reads the app's URL: http or https, a host and an +// optional port from 1 to 65535, and nothing else, since the request's +// own path and query go to the app unchanged. +func parseUpstreamURL(value string) (*url.URL, error) { + upstream, err := url.Parse(value) + if err != nil { + return nil, fmt.Errorf("%q %w", value, errNotUpstreamURL) + } + + onlySchemeAndHost := (upstream.Scheme == "http" || upstream.Scheme == "https") && + upstream.Hostname() != "" && upstream.User == nil && upstream.Opaque == "" && + (upstream.Path == "" || upstream.Path == "/") && + upstream.RawQuery == "" && upstream.Fragment == "" + if !onlySchemeAndHost { + return nil, fmt.Errorf("%q %w", value, errNotUpstreamURL) + } + + if upstream.Port() != "" { + port, err := strconv.ParseUint(upstream.Port(), 10, 16) + if err != nil || port == 0 { + return nil, fmt.Errorf("%q %w", value, errNotUpstreamURL) + } + } + + return upstream, nil +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..964ffa1 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,244 @@ +package config_test + +import ( + "bytes" + "encoding/json" + "log/slog" + "maps" + "net/netip" + "slices" + "strings" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/config" +) + +// The settings, by name. +const ( + listenAddr = "SWWAF_LISTEN_ADDR" + upstreamURL = "SWWAF_UPSTREAM_URL" + trustedProxies = "SWWAF_TRUSTED_PROXIES" + clientRequestTimeout = "SWWAF_CLIENT_REQUEST_TIMEOUT" + clientResponseTimeout = "SWWAF_CLIENT_RESPONSE_TIMEOUT" + upstreamRequestTimeout = "SWWAF_UPSTREAM_REQUEST_TIMEOUT" + upstreamResponseTimeout = "SWWAF_UPSTREAM_RESPONSE_TIMEOUT" + requestMaxBytes = "SWWAF_REQUEST_MAX_BYTES" + responseMaxBytes = "SWWAF_RESPONSE_MAX_BYTES" +) + +// off switches a timeout or a size limit off. +const off = "off" + +// environment is a set of environment variables, for FromEnvironment. +type environment map[string]string + +// lookupEnv reads one of the variables, as os.LookupEnv does. +func (e environment) lookupEnv(name string) (string, bool) { + value, ok := e[name] + + return value, ok +} + +// fromEnvironment reads the settings from env, which must be valid. +func fromEnvironment(t *testing.T, env environment) *config.Config { + t.Helper() + + cfg, err := config.FromEnvironment(env.lookupEnv) + if err != nil { + t.Fatalf("settings %v: %v", env, err) + } + + return cfg +} + +func TestDefaults(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{}) + + wantSettings(t, cfg, config.Config{ + ListenAddr: ":8080", + ClientRequestTimeout: time.Minute, + ClientResponseTimeout: 30 * time.Minute, + UpstreamRequestTimeout: time.Minute, + UpstreamResponseTimeout: 30 * time.Minute, + RequestMaxBytes: 100 << 20, + ResponseMaxBytes: 5 << 30, + }) + + if cfg.UpstreamURL.String() != "http://127.0.0.1:8081" { + t.Errorf("%s is %s", upstreamURL, cfg.UpstreamURL) + } + + wantNetblocks(t, cfg.TrustedProxies, + "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16") +} + +func TestValuesAsSet(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{ + listenAddr: "127.0.0.1:9000", + upstreamURL: "https://app.internal:8443/", + trustedProxies: " 192.0.2.1, 10.1.2.3/8 ,2001:db8::/32", + clientRequestTimeout: "90s", + clientResponseTimeout: "7d", + upstreamRequestTimeout: "1h30m", + upstreamResponseTimeout: off, + requestMaxBytes: "512K", + responseMaxBytes: "1234", + }) + + wantSettings(t, cfg, config.Config{ + ListenAddr: "127.0.0.1:9000", + ClientRequestTimeout: 90 * time.Second, + ClientResponseTimeout: 7 * 24 * time.Hour, + UpstreamRequestTimeout: 90 * time.Minute, + UpstreamResponseTimeout: 0, + RequestMaxBytes: 512 << 10, + ResponseMaxBytes: 1234, + }) + + if cfg.UpstreamURL.String() != "https://app.internal:8443/" { + t.Errorf("%s is %s", upstreamURL, cfg.UpstreamURL) + } + + wantNetblocks(t, cfg.TrustedProxies, "192.0.2.1/32", "10.0.0.0/8", "2001:db8::/32") +} + +func TestSizesAndOff(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{ + requestMaxBytes: "3G", + responseMaxBytes: off, + clientRequestTimeout: off, + }) + + if cfg.RequestMaxBytes != 3<<30 || cfg.ResponseMaxBytes != 0 || + cfg.ClientRequestTimeout != 0 { + t.Errorf("3G, off and off read as %d, %d and %s", + cfg.RequestMaxBytes, cfg.ResponseMaxBytes, cfg.ClientRequestTimeout) + } +} + +func TestTrustedProxiesSetButEmptyTrustNothing(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{trustedProxies: ""}) + if len(cfg.TrustedProxies) != 0 { + t.Errorf("trusted proxies %v, want none", cfg.TrustedProxies) + } +} + +func TestInvalidValueStopsTheStart(t *testing.T) { + t.Parallel() + + for _, tc := range []struct{ name, value string }{ + {listenAddr, "8080"}, + {listenAddr, ":http"}, + {listenAddr, ":65536"}, + {upstreamURL, "127.0.0.1:8081"}, + {upstreamURL, "ftp://127.0.0.1:8081"}, + {upstreamURL, "http://"}, + {upstreamURL, "http://:8081"}, + {upstreamURL, "http://127.0.0.1:0"}, + {upstreamURL, "http://127.0.0.1:99999"}, + {upstreamURL, "http://127.0.0.1:8081/app"}, + {upstreamURL, "http://127.0.0.1:8081/?a=1"}, + {upstreamURL, "http://user:secret@127.0.0.1:8081"}, + {trustedProxies, "10.0.0.0/33"}, + {trustedProxies, "traefik"}, + {trustedProxies, "10.0.0.0/8,,192.168.0.0/16"}, + {trustedProxies, "fe80::1%eth0"}, + {clientRequestTimeout, "60"}, + {clientRequestTimeout, ""}, + {clientResponseTimeout, "1y"}, + {upstreamRequestTimeout, "-1s"}, + {upstreamResponseTimeout, "0s"}, + {upstreamResponseTimeout, "1.5d"}, + {requestMaxBytes, "100MB"}, + {requestMaxBytes, "100m"}, + {requestMaxBytes, "1.5M"}, + {responseMaxBytes, "0"}, + {responseMaxBytes, "-5"}, + {responseMaxBytes, "99999999999G"}, + } { + t.Run(tc.name+"="+tc.value, func(t *testing.T) { + t.Parallel() + + _, err := config.FromEnvironment(environment{tc.name: tc.value}.lookupEnv) + if err == nil { + t.Fatalf("%s=%q was accepted", tc.name, tc.value) + } + + if !strings.HasPrefix(err.Error(), tc.name+": ") { + t.Errorf("error %q does not name %s", err, tc.name) + } + }) + } +} + +func TestLogsEachSettingWithItsValue(t *testing.T) { + t.Parallel() + + cfg := fromEnvironment(t, environment{clientRequestTimeout: "45s"}) + + var out bytes.Buffer + + slog.New(slog.NewJSONHandler(&out, nil)).Info("starting", "settings", cfg) + + var line struct { + Settings map[string]string `json:"settings"` + } + + err := json.Unmarshal(out.Bytes(), &line) + if err != nil { + t.Fatalf("decode %s: %v", out.Bytes(), err) + } + + want := map[string]string{ + listenAddr: ":8080", + upstreamURL: "http://127.0.0.1:8081", + trustedProxies: "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16", + clientRequestTimeout: "45s", + clientResponseTimeout: "30m", + upstreamRequestTimeout: "60s", + upstreamResponseTimeout: "30m", + requestMaxBytes: "100M", + responseMaxBytes: "5G", + } + if !maps.Equal(line.Settings, want) { + t.Errorf("logged settings\n%v\nwant\n%v", line.Settings, want) + } +} + +// wantSettings checks the settings that are plain values. +func wantSettings(t *testing.T, got *config.Config, want config.Config) { + t.Helper() + + if got.ListenAddr != want.ListenAddr || + got.ClientRequestTimeout != want.ClientRequestTimeout || + got.ClientResponseTimeout != want.ClientResponseTimeout || + got.UpstreamRequestTimeout != want.UpstreamRequestTimeout || + got.UpstreamResponseTimeout != want.UpstreamResponseTimeout || + got.RequestMaxBytes != want.RequestMaxBytes || + got.ResponseMaxBytes != want.ResponseMaxBytes { + t.Errorf("settings\n%+v\nwant\n%+v", got, want) + } +} + +// wantNetblocks checks a list of netblocks. +func wantNetblocks(t *testing.T, got []netip.Prefix, want ...string) { + t.Helper() + + gotText := make([]string, 0, len(got)) + for _, netblock := range got { + gotText = append(gotText, netblock.String()) + } + + if !slices.Equal(gotText, want) { + t.Errorf("netblocks %v, want %v", gotText, want) + } +} diff --git a/internal/proxy/bodies.go b/internal/proxy/bodies.go new file mode 100644 index 0000000..c9b223f --- /dev/null +++ b/internal/proxy/bodies.go @@ -0,0 +1,171 @@ +package proxy + +import ( + "errors" + "io" + "net/http" + "sync/atomic" + + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// errResponseTooLarge ends an app's response body that is longer than +// SWWAF_RESPONSE_MAX_BYTES. +var errResponseTooLarge = errors.New( + "the response body is over SWWAF_RESPONSE_MAX_BYTES") + +// requestBody is the client's request body on its way to the app. The +// transport reads it on a goroutine of its own. +type requestBody struct { + // body is the client's body, ending in an *http.MaxBytesError past + // SWWAF_REQUEST_MAX_BYTES. + body io.ReadCloser + rq *request + // waiting is true while a Read waits for the client to send more. + waiting atomic.Bool + // received is true once the client has sent the whole body. + received atomic.Bool + // bytes is how much of the body has been read. + bytes atomic.Int64 +} + +// Read reads from the client's body. +func (b *requestBody) Read(p []byte) (int, error) { + b.waiting.Store(true) + n, err := b.body.Read(p) + b.waiting.Store(false) + b.bytes.Add(int64(n)) + + var tooLarge *http.MaxBytesError + + switch { + case errors.Is(err, io.EOF): + b.received.Store(true) + b.rq.bodyReceived() + case errors.As(err, &tooLarge): + b.rq.refuse(refusal{ + status: http.StatusRequestEntityTooLarge, + action: requestlog.ActionTooLarge, + }) + } + + return n, err +} + +// Close closes the client's body. +func (b *requestBody) Close() error { + return b.body.Close() +} + +// responseBody is the app's response body on its way to the client. +type responseBody struct { + // body is the app's body, ending in an *http.MaxBytesError past + // SWWAF_RESPONSE_MAX_BYTES. + body io.ReadCloser + rq *request +} + +// Read reads from the app's body. +func (b *responseBody) Read(p []byte) (int, error) { + n, err := b.body.Read(p) + if err == nil { + return n, nil + } + + var tooLarge *http.MaxBytesError + + switch { + case errors.Is(err, io.EOF): + b.rq.responseReceived() + case errors.As(err, &tooLarge): + b.rq.refuse(refusal{ + status: http.StatusBadGateway, + action: requestlog.ActionTooLarge, + }) + + return n, errResponseTooLarge + case b.rq.in.Context().Err() == nil: + // The answer broke off, not because the client went away. If a + // timeout cut it, that refusal came first and is the one kept. + b.rq.refuse(refusal{ + status: http.StatusBadGateway, + action: requestlog.ActionUpstreamError, + }) + } + + return n, err +} + +// Close closes the app's body. +func (b *responseBody) Close() error { + return b.body.Close() +} + +// limitBody returns body, cut off with an *http.MaxBytesError after +// maxBytes, or unchanged if maxBytes is zero, which is off. +func limitBody(body io.ReadCloser, maxBytes int64) io.ReadCloser { + if maxBytes == 0 { + return body + } + + // Without a ResponseWriter, MaxBytesReader only counts and cuts off. + return http.MaxBytesReader(nil, body, maxBytes) +} + +// responseWriter is the response to the client. It notes the status and +// size for the log line, and the first error writing to the client. +type responseWriter struct { + http.ResponseWriter + + // status is the final status sent, or zero before one is. + status int + bytes int64 + err error +} + +// WriteHeader sends the status and headers. An informational 1xx status +// is passed on and the final status still comes later. +func (w *responseWriter) WriteHeader(status int) { + if status >= http.StatusOK && w.status == 0 { + w.status = status + } + + w.ResponseWriter.WriteHeader(status) +} + +// Write sends part of the body. +func (w *responseWriter) Write(p []byte) (int, error) { + if w.status == 0 { + w.status = http.StatusOK + } + + n, err := w.ResponseWriter.Write(p) + w.bytes += int64(n) + w.noteError(err) + + return n, err +} + +// FlushError sends what has been written so far. +// http.ResponseController calls it, as ReverseProxy does after each +// write. +func (w *responseWriter) FlushError() error { + err := http.NewResponseController(w.ResponseWriter).Flush() + w.noteError(err) + + return err +} + +// Unwrap lets http.ResponseController reach net/http's own +// ResponseWriter, which is how ReverseProxy takes over the connection of +// an upgraded request. +func (w *responseWriter) Unwrap() http.ResponseWriter { + return w.ResponseWriter +} + +// noteError keeps the first error writing to the client. +func (w *responseWriter) noteError(err error) { + if w.err == nil { + w.err = err + } +} diff --git a/internal/proxy/client.go b/internal/proxy/client.go new file mode 100644 index 0000000..1f9f1e9 --- /dev/null +++ b/internal/proxy/client.go @@ -0,0 +1,101 @@ +package proxy + +import ( + "net/http" + "net/netip" + "slices" + "strings" +) + +// peerAddress is the address of the request's TCP peer, normally traefik. +func peerAddress(r *http.Request) netip.Addr { + addrPort, err := netip.ParseAddrPort(r.RemoteAddr) + if err != nil { + return netip.Addr{} + } + + return addrPort.Addr().Unmap() +} + +// clientAddress works out who the client is. A peer outside the trusted +// proxies is the client, and what it says in X-Forwarded-For is ignored. +// For a peer inside them, X-Forwarded-For is read from the right, and the +// first address outside them is the client; if every address in it is +// inside, the leftmost is, and with no header, the peer. An entry that is +// not an address ends the reading, since nothing to its left can be +// believed. +func clientAddress( + peer netip.Addr, forwardedFor []string, trusted []netip.Prefix, +) netip.Addr { + client := peer + if !isInside(peer, trusted) { + return client + } + + entries := strings.Split(strings.Join(forwardedFor, ","), ",") + for _, entry := range slices.Backward(entries) { + addr, err := netip.ParseAddr(strings.TrimSpace(entry)) + if err != nil { + break + } + + client = addr.Unmap() + if !isInside(client, trusted) { + break + } + } + + return client +} + +// isInside reports whether addr is in one of the netblocks. +func isInside(addr netip.Addr, netblocks []netip.Prefix) bool { + return slices.ContainsFunc(netblocks, func(netblock netip.Prefix) bool { + return netblock.Contains(addr) + }) +} + +// setForwardedHeaders sets the headers in which the app learns about the +// client, so that it sees what it would see from traefik directly. A +// trusted proxy's forwarded headers pass on, with the proxy's own address +// added to X-Forwarded-For. Those of any other peer are its own claims and +// are replaced: X-Forwarded-For names the peer, X-Forwarded-Host the host +// it asked for, and X-Forwarded-Proto plain http, which is how it reached +// smallwebwaf. +func setForwardedHeaders(in, out *http.Request, peer netip.Addr, trusted bool) { + forwardedFor := peer.String() + + if trusted { + // ReverseProxy removes these from out before Rewrite. + for _, name := range []string{"Forwarded", "X-Forwarded-Host", "X-Forwarded-Proto"} { + values, ok := in.Header[name] + if ok { + out.Header[name] = values + } + } + + prior := in.Header.Values("X-Forwarded-For") + if len(prior) > 0 { + forwardedFor = strings.Join(prior, ", ") + ", " + forwardedFor + } + + out.Header.Set("X-Forwarded-For", forwardedFor) + + return + } + + // ReverseProxy has removed Forwarded and the three set below; these + // are the other headers in which traefik tells the app about the + // client and its request. + for _, name := range []string{ + "X-Forwarded-Port", "X-Forwarded-Server", "X-Forwarded-Uri", + "X-Forwarded-Method", "X-Forwarded-Prefix", "X-Forwarded-Tls-Client-Cert", + "X-Forwarded-Tls-Client-Cert-Info", "X-Real-Ip", + } { + out.Header.Del(name) + } + + out.Header.Set("X-Forwarded-For", forwardedFor) + out.Header.Set("X-Forwarded-Host", in.Host) + out.Header.Set("X-Forwarded-Proto", "http") +} diff --git a/internal/proxy/client_test.go b/internal/proxy/client_test.go new file mode 100644 index 0000000..16c48e1 --- /dev/null +++ b/internal/proxy/client_test.go @@ -0,0 +1,161 @@ +package proxy_test + +import ( + "encoding/json" + "net/http" + "testing" +) + +const ( + // trustLocalhost trusts the address every test connects from, and a + // network for proxies in front of it. + trustLocalhost = localhost + "/32,10.0.0.0/8" + // appHost is the host every test asks for. + appHost = "app.example" + // client is the client's address, as a proxy names it. + client = "203.0.113.9" + // forwardedFor is the header that lists the client and its proxies. + forwardedFor = "X-Forwarded-For" + // secure is the scheme a client reached traefik with. + secure = "https" +) + +// appHeaders is what the app tells about the headers it received. +type appHeaders struct { + Host string `json:"host"` + ForwardedFor string `json:"forwardedFor"` + ForwardedHost string `json:"forwardedHost"` + ForwardedProto string `json:"forwardedProto"` + RealIP string `json:"realIp"` +} + +// clientAddressCase is a request and what smallwebwaf makes of it. +type clientAddressCase struct { + name string + env map[string]string + header http.Header + wantClient string + wantApp appHeaders +} + +func TestClientAddressAndForwardedHeaders(t *testing.T) { + t.Parallel() + + for _, tc := range clientAddressCases() { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got, line := requestWithHeaders(t, tc.env, tc.header) + + tc.wantApp.Host = appHost + if got != tc.wantApp { + t.Errorf("app received %+v, want %+v", got, tc.wantApp) + } + + if line.ClientIP != tc.wantClient || line.PeerIP != localhost { + t.Errorf("log line has client_ip %q and peer_ip %q, want %q and %q", + line.ClientIP, line.PeerIP, tc.wantClient, localhost) + } + }) + } +} + +// clientAddressCases are the requests TestClientAddressAndForwardedHeaders +// sends, from 127.0.0.1, which the default trusted proxies leave out. +func clientAddressCases() []clientAddressCase { + trusted := map[string]string{trustedProxies: trustLocalhost} + forged := http.Header{ + forwardedFor: {client}, + "X-Forwarded-Host": {"forged.example"}, + "X-Forwarded-Proto": {secure}, + "X-Real-Ip": {client}, + } + replaced := appHeaders{ + ForwardedFor: localhost, ForwardedHost: appHost, ForwardedProto: "http", + } + + return []clientAddressCase{{ + name: "a peer outside the trusted proxies is the client, " + + "and its forwarded headers are replaced", + header: forged, wantClient: localhost, wantApp: replaced, + }, { + name: "set but empty, the trusted proxies trust nothing", + env: map[string]string{trustedProxies: ""}, + header: forged, wantClient: localhost, wantApp: replaced, + }, { + name: "behind a trusted peer, the client is the first address " + + "outside the trusted proxies from the right", + env: trusted, + header: http.Header{ + forwardedFor: {"198.51.100.7, " + client + ", 10.0.0.2"}, + "X-Forwarded-Host": {appHost}, + "X-Forwarded-Proto": {secure}, + "X-Real-Ip": {client}, + }, + wantClient: client, + wantApp: appHeaders{ + ForwardedFor: "198.51.100.7, " + client + ", 10.0.0.2, " + localhost, + ForwardedHost: appHost, ForwardedProto: secure, RealIP: client, + }, + }, { + name: "when every address is a trusted proxy, the leftmost is the client", + env: trusted, + header: http.Header{forwardedFor: {"10.0.0.5, 10.0.0.2"}}, + wantClient: "10.0.0.5", + wantApp: appHeaders{ForwardedFor: "10.0.0.5, 10.0.0.2, " + localhost}, + }, { + name: "with no header, a trusted peer is the client", + env: trusted, + wantClient: localhost, + wantApp: appHeaders{ForwardedFor: localhost}, + }, { + name: "an entry that is not an address ends the reading", + env: trusted, + header: http.Header{forwardedFor: {client + ", unknown, 10.0.0.2"}}, + wantClient: "10.0.0.2", + wantApp: appHeaders{ + ForwardedFor: client + ", unknown, 10.0.0.2, " + localhost, + }, + }, { + name: "several header lines are read as one list", + env: trusted, + header: http.Header{forwardedFor: {"2001:db8::7", "10.0.0.2"}}, + wantClient: "2001:db8::7", + wantApp: appHeaders{ForwardedFor: "2001:db8::7, 10.0.0.2, " + localhost}, + }} +} + +// requestWithHeaders sends a request for appHost with header through +// smallwebwaf, with the settings in env, and returns the headers the app +// received and the request's log line. +func requestWithHeaders( + t *testing.T, env map[string]string, header http.Header, +) (appHeaders, logLine) { + t.Helper() + + app := startApp(t, func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(appHeaders{ + Host: r.Host, + ForwardedFor: r.Header.Get(forwardedFor), + ForwardedHost: r.Header.Get("X-Forwarded-Host"), + ForwardedProto: r.Header.Get("X-Forwarded-Proto"), + RealIP: r.Header.Get("X-Real-Ip"), + }) + }) + addr, out := startProxy(t, app.URL, env) + + req := newRequest(t, http.MethodGet, addr, "/", http.NoBody) + req.Host = appHost + req.Header = header.Clone() + + answered := do(t, req) + + var got appHeaders + + err := json.Unmarshal(answered.body, &got) + if err != nil { + t.Fatalf("decode the app's answer %q: %v", answered.body, err) + } + + return got, out.requestLine(t) +} diff --git a/internal/proxy/limits_test.go b/internal/proxy/limits_test.go new file mode 100644 index 0000000..b86c4b9 --- /dev/null +++ b/internal/proxy/limits_test.go @@ -0,0 +1,146 @@ +package proxy_test + +import ( + "bytes" + "errors" + "io" + "net/http" + "strconv" + "sync/atomic" + "testing" + + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// sizeLimit is the size limit the tests set, 1K as a setting. +const ( + sizeLimit = 1 << 10 + sizeLimitSetting = "1K" +) + +func TestRequestBodyLimit(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + size int + // announced sends the size in Content-Length; otherwise the body + // is sent in chunks with no length given. + announced bool + want int + action string + // refusedBeforeApp is a refusal before anything reaches the app. + // A body over the limit with no length given has already partly + // reached the app when it is refused. + refusedBeforeApp bool + }{ + {"announced, over the limit", 2 * sizeLimit, true, + http.StatusRequestEntityTooLarge, requestlog.ActionTooLarge, true}, + {"announced, at the limit", sizeLimit, true, + http.StatusOK, requestlog.ActionForward, false}, + {"not announced, over the limit", 4 * sizeLimit, false, + http.StatusRequestEntityTooLarge, requestlog.ActionTooLarge, false}, + {"not announced, at the limit", sizeLimit, false, + http.StatusOK, requestlog.ActionForward, false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + var calls atomic.Int32 + + app := startApp(t, func(_ http.ResponseWriter, r *http.Request) { + calls.Add(1) + + _, _ = io.Copy(io.Discard, r.Body) + }) + addr, out := startProxy(t, app.URL, map[string]string{ + requestMaxBytes: sizeLimitSetting, + }) + + var body io.Reader = bytes.NewReader(make([]byte, tc.size)) + if !tc.announced { + body = io.MultiReader(body) // hides the length + } + + wantStatus(t, do(t, newRequest(t, http.MethodPost, addr, "/upload", body)), + tc.want) + wantLine(t, out.requestLine(t), tc.want, tc.action) + + if tc.refusedBeforeApp && calls.Load() != 0 { + t.Errorf("the app was called %d times, want never", calls.Load()) + } + }) + } +} + +func TestResponseBodyLimit(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + size int + // announced sends the size in Content-Length; otherwise the body + // is sent in chunks with no length given. + announced bool + want int + action string + // received is how much of a body the client gets, and cutOff + // whether the connection is then cut. + received int + cutOff bool + }{ + {"announced, over the limit", 2 * sizeLimit, true, http.StatusBadGateway, + requestlog.ActionTooLarge, len("Bad Gateway\n"), false}, + {"announced, at the limit", sizeLimit, true, http.StatusOK, + requestlog.ActionForward, sizeLimit, false}, + {"not announced, over the limit", 4 * sizeLimit, false, http.StatusOK, + requestlog.ActionTooLarge, sizeLimit, true}, + {"not announced, at the limit", sizeLimit, false, http.StatusOK, + requestlog.ActionForward, sizeLimit, false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + app := startApp(t, func(w http.ResponseWriter, _ *http.Request) { + answerWithSize(w, tc.size, tc.announced) + }) + addr, out := startProxy(t, app.URL, map[string]string{ + responseMaxBytes: sizeLimitSetting, + }) + + got := get(t, addr, "/download") + wantStatus(t, got, tc.want) + + if len(got.body) != tc.received || + errors.Is(got.err, io.ErrUnexpectedEOF) != tc.cutOff { + t.Errorf("client got %d bytes (%v), want %d", + len(got.body), got.err, tc.received) + } + + line := out.requestLine(t) + wantLine(t, line, tc.want, tc.action) + + if line.UpstreamStatus != http.StatusOK { + t.Errorf("log line has upstream_status %d", line.UpstreamStatus) + } + }) + } +} + +// answerWithSize answers with a body of size bytes, announced in +// Content-Length or sent in chunks with no length given. +func answerWithSize(w http.ResponseWriter, size int, announced bool) { + body := make([]byte, size) + if announced { + w.Header().Set("Content-Length", strconv.Itoa(size)) + _, _ = w.Write(body) + + return + } + + // Sending part of it before the end keeps Go's server from working + // out the length. + _, _ = w.Write(body[:size/2]) + _ = http.NewResponseController(w).Flush() + _, _ = w.Write(body[size/2:]) +} diff --git a/internal/proxy/passthrough_test.go b/internal/proxy/passthrough_test.go new file mode 100644 index 0000000..66caee7 --- /dev/null +++ b/internal/proxy/passthrough_test.go @@ -0,0 +1,426 @@ +package proxy_test + +import ( + "bufio" + "bytes" + "errors" + "io" + "net" + "net/http" + "slices" + "strings" + "sync/atomic" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/proxy" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// A request target with an escaped slash and space in its path, and a +// query with a parameter ReverseProxy cannot parse. +const ( + rawPath = "/some%2Fpath/with%20space" + rawQuery = "b=2&a=1&bad=%zz;x" +) + +// chunkSize is the size of each part of a body a test sends in parts. +const chunkSize = 1 << 10 + +var errNotStreamed = errors.New("the first part never reached the app") + +// appSaw is what the app received. +type appSaw struct { + method string + target string + header http.Header + body []byte +} + +func TestPassesRequestAndAnswerUnchanged(t *testing.T) { + t.Parallel() + + requestBody := bytes.Repeat([]byte("request body "), 8000) + answerBody := bytes.Repeat([]byte("answer body "), 8000) + saw := make(chan appSaw, 1) + + app := startApp(t, func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + saw <- appSaw{r.Method, r.RequestURI, r.Header.Clone(), body} + + w.Header().Set("X-App", "yes") + w.Header().Add("Set-Cookie", "a=1") + w.Header().Add("Set-Cookie", "b=2") + w.WriteHeader(http.StatusTeapot) + _, _ = w.Write(answerBody) + }) + addr, out := startProxy(t, app.URL, nil) + + req := newRequest(t, http.MethodPatch, addr, rawPath+"?"+rawQuery, + bytes.NewReader(requestBody)) + req.Header.Add("X-Test", "one") + req.Header.Add("X-Test", "two") + req.Header.Set("User-Agent", "test-agent") + + got := do(t, req) + + wantAppSaw(t, <-saw, requestBody) + wantAnswer(t, got, answerBody) + + line := out.requestLine(t) + wantLine(t, line, http.StatusTeapot, requestlog.ActionForward) + wantRequestFields(t, line, addr, len(requestBody), len(answerBody)) +} + +// wantAppSaw checks that the app received the test's request unchanged. +func wantAppSaw(t *testing.T, saw appSaw, body []byte) { + t.Helper() + + if saw.method != http.MethodPatch || saw.target != rawPath+"?"+rawQuery { + t.Errorf("app saw %s %s, want %s %s", saw.method, saw.target, + http.MethodPatch, rawPath+"?"+rawQuery) + } + + if !slices.Equal(saw.header.Values("X-Test"), []string{"one", "two"}) { + t.Errorf("app saw X-Test %q", saw.header.Values("X-Test")) + } + + if saw.header.Get("User-Agent") != "test-agent" { + t.Errorf("app saw User-Agent %q", saw.header.Get("User-Agent")) + } + + if !bytes.Equal(saw.body, body) { + t.Errorf("app saw a body of %d bytes, want the %d sent", + len(saw.body), len(body)) + } +} + +// wantAnswer checks that the client received the app's answer unchanged. +func wantAnswer(t *testing.T, got answer, body []byte) { + t.Helper() + + wantStatus(t, got, http.StatusTeapot) + + if got.header.Get("X-App") != "yes" { + t.Errorf("client got X-App %q", got.header.Get("X-App")) + } + + if !slices.Equal(got.header.Values("Set-Cookie"), []string{"a=1", "b=2"}) { + t.Errorf("client got Set-Cookie %q", got.header.Values("Set-Cookie")) + } + + if got.err != nil || !bytes.Equal(got.body, body) { + t.Errorf("client got %d bytes (%v), want the %d the app sent", + len(got.body), got.err, len(body)) + } +} + +// wantRequestFields checks the log line's fields about the request. +func wantRequestFields(t *testing.T, line logLine, host string, sent, received int) { + t.Helper() + + want := requestlog.Line{ + Type: "request", Time: line.Time, ClientIP: localhost, PeerIP: localhost, + Method: http.MethodPatch, Host: host, Path: rawPath, Query: rawQuery, + Protocol: "HTTP/1.1", Status: http.StatusTeapot, + UpstreamStatus: http.StatusTeapot, RequestBytes: int64(sent), + ResponseBytes: int64(received), UserAgent: "test-agent", + Action: requestlog.ActionForward, DurationTotal: line.DurationTotal, + DurationUpstreamTotal: line.DurationUpstreamTotal, + } + if line.Line != want { + t.Errorf("log line\n%+v\nwant\n%+v", line.Line, want) + } + + _, err := time.Parse(time.RFC3339, line.Time) + if err != nil || line.DurationTotal <= 0 || line.DurationUpstreamTotal <= 0 { + t.Errorf("log line has time %q and durations %v and %v", + line.Time, line.DurationTotal, line.DurationUpstreamTotal) + } +} + +func TestStreamsTheRequestBody(t *testing.T) { + t.Parallel() + + chunk := bytes.Repeat([]byte("x"), chunkSize) + firstArrived := make(chan struct{}) + + app := startApp(t, func(w http.ResponseWriter, r *http.Request) { + first := make([]byte, len(chunk)) + + _, err := io.ReadFull(r.Body, first) + if err != nil { + return + } + + close(firstArrived) + + rest, _ := io.ReadAll(r.Body) + _, _ = w.Write(rest) + }) + addr, _ := startProxy(t, app.URL, nil) + + body, writer := io.Pipe() + + go func() { + _, _ = writer.Write(chunk) + + select { + case <-firstArrived: + _, _ = writer.Write(chunk) + _ = writer.Close() + case <-time.After(waitLimit): + _ = writer.CloseWithError(errNotStreamed) + } + }() + + got := do(t, newRequest(t, http.MethodPost, addr, "/upload", body)) + if got.err != nil || !bytes.Equal(got.body, chunk) { + t.Errorf("app read %d bytes after the first part (%v), want %d", + len(got.body), got.err, len(chunk)) + } +} + +func TestStreamsTheAnswerBody(t *testing.T) { + t.Parallel() + + chunk := bytes.Repeat([]byte("y"), chunkSize) + firstArrived := make(chan struct{}) + + app := startApp(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(chunk) + _ = http.NewResponseController(w).Flush() + + select { + case <-firstArrived: + _, _ = w.Write(chunk) + case <-time.After(waitLimit): + } + }) + addr, _ := startProxy(t, app.URL, nil) + + req := newRequest(t, http.MethodGet, addr, "/download", http.NoBody) + + res, err := newClient(t).Do(req) + if err != nil { + t.Fatalf("request: %v", err) + } + + first := make([]byte, len(chunk)) + _, err = io.ReadFull(res.Body, first) + + close(firstArrived) + + got := readAnswer(res) + if err != nil || got.err != nil || !bytes.Equal(got.body, chunk) { + t.Errorf("client read %d bytes after the first part (%v, %v), want %d", + len(got.body), err, got.err, len(chunk)) + } +} + +func TestUpgradedConnectionOutlastsTheTimeouts(t *testing.T) { + t.Parallel() + + app := startApp(t, echoAfterUpgrade) + addr, out := startProxy(t, app.URL, map[string]string{ + clientRequestTimeout: shortTimeoutSetting, + clientResponseTimeout: shortTimeoutSetting, + upstreamRequestTimeout: shortTimeoutSetting, + upstreamResponseTimeout: shortTimeoutSetting, + }) + + conn := dial(t, addr) + send(t, conn, "GET /socket HTTP/1.1\r\nHost: app\r\n"+ + "Connection: Upgrade\r\nUpgrade: websocket\r\n\r\n") + + reader := bufio.NewReader(conn) + + res, err := http.ReadResponse(reader, nil) + if err != nil { + t.Fatalf("read the answer to the upgrade: %v", err) + } + + _ = res.Body.Close() + + if res.StatusCode != http.StatusSwitchingProtocols { + t.Fatalf("status %d, want %d", res.StatusCode, http.StatusSwitchingProtocols) + } + + // Wait past every timeout, then use the connection. + time.Sleep(3 * shortTimeout) + send(t, conn, "still here\n") + + echoed, err := reader.ReadString('\n') + if err != nil || echoed != "still here\n" { + t.Errorf("echo %q (%v), want %q", echoed, err, "still here\n") + } + + _ = conn.Close() + + wantLine(t, out.requestLine(t), http.StatusSwitchingProtocols, + requestlog.ActionForward) +} + +// echoAfterUpgrade is an app that switches protocols on request, and then +// sends back each line it receives. +func echoAfterUpgrade(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Upgrade") != "websocket" { + http.Error(w, "not an upgrade", http.StatusBadRequest) + + return + } + + conn, buffered, err := http.NewResponseController(w).Hijack() + if err != nil { + return + } + + defer func() { + _ = conn.Close() + }() + + _, _ = buffered.WriteString("HTTP/1.1 101 Switching Protocols\r\n" + + "Connection: Upgrade\r\nUpgrade: websocket\r\n\r\n") + _ = buffered.Flush() + + for { + line, err := buffered.ReadString('\n') + if err != nil { + return + } + + _, _ = buffered.WriteString(line) + _ = buffered.Flush() + } +} + +func TestServerHasTheFixedLimits(t *testing.T) { + t.Parallel() + + cfg, err := config.FromEnvironment(func(string) (string, bool) { return "", false }) + if err != nil { + t.Fatalf("default settings: %v", err) + } + + server := proxy.New(proxy.Params{ + Config: cfg, + RequestLog: io.Discard, + ProcessLog: requestlog.NewProcessLogger(io.Discard), + }) + + if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 || + server.IdleTimeout != 2*time.Minute || server.ReadHeaderTimeout != time.Minute { + t.Errorf("server listens on %q with header limit %d, idle time %s and "+ + "header timeout %s", server.Addr, server.MaxHeaderBytes, + server.IdleTimeout, server.ReadHeaderTimeout) + } +} + +func TestRefusesHeadersOver32KiB(t *testing.T) { + t.Parallel() + + var calls atomic.Int32 + + app := startApp(t, func(http.ResponseWriter, *http.Request) { + calls.Add(1) + }) + addr, _ := startProxy(t, app.URL, nil) + + // size counts every byte of the request: the request line, the + // headers and the blank line that ends them. + const ( + start = "GET / HTTP/1.1\r\nHost: app\r\nX-Large: " + end = "\r\n\r\n" + ) + + for _, tc := range []struct { + size int + want int + }{ + {size: 32 << 10, want: http.StatusOK}, + {size: 32<<10 + 1, want: http.StatusRequestHeaderFieldsTooLarge}, + } { + conn := dial(t, addr) + send(t, conn, start+strings.Repeat("a", tc.size-len(start)-len(end))+end) + wantStatus(t, readResponse(t, conn), tc.want) + } + + if calls.Load() != 1 { + t.Errorf("the app was called %d times, want once", calls.Load()) + } +} + +func TestAnswers502WhenTheAppCannotBeReached(t *testing.T) { + t.Parallel() + + listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0") + if err != nil { + t.Fatalf("listen: %v", err) + } + + closedAddr := listener.Addr().String() + _ = listener.Close() + + addr, out := startProxy(t, "http://"+closedAddr, nil) + + wantStatus(t, get(t, addr, "/"), http.StatusBadGateway) + wantLine(t, out.requestLine(t), http.StatusBadGateway, + requestlog.ActionUpstreamError) + + logged := slices.ContainsFunc(out.lines(t), func(line map[string]any) bool { + return line["type"] == "process" && line["msg"] == "request to the app failed" + }) + if !logged { + t.Errorf("no process line says the request to the app failed") + } +} + +func TestLogsAnAnswerThatBrokeOff(t *testing.T) { + t.Parallel() + + app := startApp(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, "the first part") + _ = http.NewResponseController(w).Flush() + + panic(http.ErrAbortHandler) // drops the connection mid-answer + }) + addr, out := startProxy(t, app.URL, nil) + + got := get(t, addr, "/") + if string(got.body) != "the first part" || !errors.Is(got.err, io.ErrUnexpectedEOF) { + t.Errorf("client read %q (%v), want the first part cut off", got.body, got.err) + } + + wantLine(t, out.requestLine(t), http.StatusOK, requestlog.ActionUpstreamError) +} + +func TestLogsAClientThatWentAway(t *testing.T) { + t.Parallel() + + arrived := make(chan struct{}) + + app := startApp(t, func(_ http.ResponseWriter, r *http.Request) { + close(arrived) + <-r.Context().Done() + }) + addr, out := startProxy(t, app.URL, nil) + + conn := dial(t, addr) + send(t, conn, "GET /slow HTTP/1.1\r\nHost: app\r\n\r\n") + + select { + case <-arrived: + case <-time.After(waitLimit): + t.Fatal("the request never reached the app") + } + + _ = conn.Close() + + line := out.requestLine(t) + if !line.Aborted || line.Status != 0 || line.Action != requestlog.ActionForward { + t.Errorf("log line has aborted %v, status %d and action %q, "+ + "want true, 0 and %q", + line.Aborted, line.Status, line.Action, requestlog.ActionForward) + } +} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go new file mode 100644 index 0000000..327ba81 --- /dev/null +++ b/internal/proxy/proxy.go @@ -0,0 +1,105 @@ +// Package proxy passes each request to the app and the app's answer back, +// unchanged, within the size and time limits, and writes one request log +// line for each request. +package proxy + +import ( + "io" + "log" + "log/slog" + "net/http" + "time" + + "sneak.berlin/go/smallwebwaf/internal/config" +) + +// The request line and headers a client may send, and how long a +// kept-open client connection may wait for its next request, are fixed +// rather than settings. The limit on the request line and headers is +// 32 KiB, but Go's server reads 4 KiB past its MaxHeaderBytes before it +// refuses, so MaxHeaderBytes is set 4 KiB lower. The idle time is longer +// than the 90 seconds after which traefik closes a connection it is not +// using, so traefik never sends a request on a connection smallwebwaf is +// closing. +const ( + requestHeaderMaxBytes = 32<<10 - 4<<10 + clientIdleTimeout = 120 * time.Second +) + +// How smallwebwaf keeps connections to the app open between requests. +const ( + appIdleConns = 100 + appIdleConnTimeout = 90 * time.Second +) + +// Params are what New needs. +type Params struct { + Config *config.Config + // RequestLog receives one JSON line per request. + RequestLog io.Writer + // ProcessLog receives the process's own messages. + ProcessLog *slog.Logger +} + +// New returns the server smallwebwaf runs: each request it reads passes +// through the proxy. Go's server itself refuses headers over 32 KiB, with +// 431, closes a connection idle for 120 seconds, and applies +// SWWAF_CLIENT_REQUEST_TIMEOUT while the headers arrive; the proxy +// applies the timeouts and size limits from then on. +func New(params Params) *http.Server { + errorLog := slog.NewLogLogger(params.ProcessLog.Handler(), slog.LevelWarn) + + return &http.Server{ + Addr: params.Config.ListenAddr, + Handler: &handler{ + config: params.Config, + requestLog: params.RequestLog, + processLog: params.ProcessLog, + errorLog: errorLog, + transport: newTransport(), + }, + ReadHeaderTimeout: params.Config.ClientRequestTimeout, + IdleTimeout: clientIdleTimeout, + MaxHeaderBytes: requestHeaderMaxBytes, + ErrorLog: errorLog, + } +} + +// handler is the proxy. It holds what every request shares; what belongs +// to one request is in a request. +type handler struct { + config *config.Config + requestLog io.Writer + processLog *slog.Logger + errorLog *log.Logger + transport http.RoundTripper +} + +// newTransport returns what carries requests to the app. It never goes +// through a proxy named in the environment, and leaves the app's answers +// compressed or not as the app sent them. +func newTransport() *http.Transport { + return &http.Transport{ + MaxIdleConns: appIdleConns, + MaxIdleConnsPerHost: appIdleConns, + IdleConnTimeout: appIdleConnTimeout, + DisableCompression: true, + } +} + +// ServeHTTP handles one request: it works out the client, runs the +// checks, passes the request to the app and the answer back within the +// limits, and writes the request's log line. +func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + rq := h.newRequest(w, r) + defer rq.finish() + + refused := rq.check() + if refused != nil { + rq.answer(*refused) + + return + } + + rq.forward(r.Context()) +} diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go new file mode 100644 index 0000000..9a37a40 --- /dev/null +++ b/internal/proxy/proxy_test.go @@ -0,0 +1,328 @@ +package proxy_test + +import ( + "bufio" + "bytes" + "encoding/json" + "io" + "maps" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/proxy" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +const ( + // shortTimeout is what a test sets a timeout to, to see it run out. + shortTimeout = 300 * time.Millisecond + // shortTimeoutSetting is shortTimeout as a setting's value. + shortTimeoutSetting = "300ms" + // longTimeoutSetting is a timeout that does not run out in a test. + longTimeoutSetting = "10s" + // waitLimit bounds how long a test waits for what should happen. + waitLimit = 10 * time.Second + // pollInterval is how often a test looks for a log line. + pollInterval = 10 * time.Millisecond + // localhost is where every test server listens, and so the address + // smallwebwaf sees each test's requests come from. + localhost = "127.0.0.1" +) + +// The settings the tests set. +const ( + clientRequestTimeout = "SWWAF_CLIENT_REQUEST_TIMEOUT" + clientResponseTimeout = "SWWAF_CLIENT_RESPONSE_TIMEOUT" + upstreamRequestTimeout = "SWWAF_UPSTREAM_REQUEST_TIMEOUT" + upstreamResponseTimeout = "SWWAF_UPSTREAM_RESPONSE_TIMEOUT" + requestMaxBytes = "SWWAF_REQUEST_MAX_BYTES" + responseMaxBytes = "SWWAF_RESPONSE_MAX_BYTES" + trustedProxies = "SWWAF_TRUSTED_PROXIES" +) + +// output collects what smallwebwaf writes on stdout. +type output struct { + mu sync.Mutex + buf bytes.Buffer +} + +// Write adds lines smallwebwaf writes. +func (o *output) Write(p []byte) (int, error) { + o.mu.Lock() + defer o.mu.Unlock() + + return o.buf.Write(p) +} + +// lines returns every line written so far, decoded. +func (o *output) lines(t *testing.T) []map[string]any { + t.Helper() + o.mu.Lock() + defer o.mu.Unlock() + + var lines []map[string]any + + for text := range strings.Lines(o.buf.String()) { + var line map[string]any + + err := json.Unmarshal([]byte(text), &line) + if err != nil { + t.Fatalf("output line %q is not JSON: %v", text, err) + } + + lines = append(lines, line) + } + + return lines +} + +// logLine is a request log line, as typed fields and as the JSON object +// it was written as. +type logLine struct { + requestlog.Line + + fields map[string]any +} + +// requestLines waits for count request log lines and returns them. +func (o *output) requestLines(t *testing.T, count int) []logLine { + t.Helper() + + deadline := time.Now().Add(waitLimit) + for time.Now().Before(deadline) { + var found []logLine + + for _, fields := range o.lines(t) { + if fields["type"] == "request" { + found = append(found, decodeLine(t, fields)) + } + } + + if len(found) >= count { + return found + } + + time.Sleep(pollInterval) + } + + t.Fatalf("fewer than %d request log lines after %s", count, waitLimit) + + return nil +} + +// requestLine waits for the request log line of a test's one request. +func (o *output) requestLine(t *testing.T) logLine { + t.Helper() + + return o.requestLines(t, 1)[0] +} + +// decodeLine reads a request log line's fields into a logLine. +func decodeLine(t *testing.T, fields map[string]any) logLine { + t.Helper() + + encoded, err := json.Marshal(fields) + if err != nil { + t.Fatalf("encode %v: %v", fields, err) + } + + line := logLine{fields: fields} + + err = json.Unmarshal(encoded, &line.Line) + if err != nil { + t.Fatalf("decode %s: %v", encoded, err) + } + + return line +} + +// startApp starts app as the app smallwebwaf passes requests to. +func startApp(t *testing.T, app http.HandlerFunc) *httptest.Server { + t.Helper() + + server := httptest.NewServer(app) + t.Cleanup(server.Close) + + return server +} + +// startProxy starts smallwebwaf in front of the app at appURL, with the +// settings in env on top of the defaults, and returns where it listens and +// what it writes. +func startProxy(t *testing.T, appURL string, env map[string]string) (string, *output) { + t.Helper() + + settings := map[string]string{"SWWAF_UPSTREAM_URL": appURL} + maps.Copy(settings, env) + + cfg, err := config.FromEnvironment(func(name string) (string, bool) { + value, ok := settings[name] + + return value, ok + }) + if err != nil { + t.Fatalf("settings %v: %v", settings, err) + } + + out := &output{} + server := proxy.New(proxy.Params{ + Config: cfg, + RequestLog: out, + ProcessLog: requestlog.NewProcessLogger(out), + }) + + listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0") + if err != nil { + t.Fatalf("listen: %v", err) + } + + go func() { + _ = server.Serve(listener) + }() + + t.Cleanup(func() { + _ = server.Close() + }) + + return listener.Addr().String(), out +} + +// newClient returns an HTTP client that sends requests as they are made, +// with no compression of its own. +func newClient(t *testing.T) *http.Client { + t.Helper() + + transport := &http.Transport{DisableCompression: true} + t.Cleanup(transport.CloseIdleConnections) + + return &http.Client{Transport: transport} +} + +// answer is a response as a test reads it: the status, the headers, as +// much of the body as arrived, and the error that ended the reading, nil +// when the whole body arrived. +type answer struct { + status int + header http.Header + body []byte + err error +} + +// readAnswer reads all of res, and closes its body. +func readAnswer(res *http.Response) answer { + body, err := io.ReadAll(res.Body) + _ = res.Body.Close() + + return answer{status: res.StatusCode, header: res.Header, body: body, err: err} +} + +// newRequest makes a request for path to smallwebwaf at addr. +func newRequest(t *testing.T, method, addr, path string, body io.Reader) *http.Request { + t.Helper() + + req, err := http.NewRequestWithContext(t.Context(), method, "http://"+addr+path, body) + if err != nil { + t.Fatalf("new request: %v", err) + } + + return req +} + +// do sends req and reads the answer. +func do(t *testing.T, req *http.Request) answer { + t.Helper() + + res, err := newClient(t).Do(req) + if err != nil { + t.Fatalf("%s %s: %v", req.Method, req.URL.Path, err) + } + + return readAnswer(res) +} + +// get sends a GET request for path to smallwebwaf at addr. +func get(t *testing.T, addr, path string) answer { + t.Helper() + + return do(t, newRequest(t, http.MethodGet, addr, path, http.NoBody)) +} + +// dial opens a connection to smallwebwaf at addr, for requests the HTTP +// client cannot make, such as one that stops sending halfway. +func dial(t *testing.T, addr string) net.Conn { + t.Helper() + + conn, err := (&net.Dialer{}).DialContext(t.Context(), "tcp", addr) + if err != nil { + t.Fatalf("dial %s: %v", addr, err) + } + + t.Cleanup(func() { + _ = conn.Close() + }) + + return conn +} + +// send writes text to conn. +func send(t *testing.T, conn net.Conn, text string) { + t.Helper() + + _, err := io.WriteString(conn, text) + if err != nil { + t.Fatalf("send: %v", err) + } +} + +// readResponse reads the answer to a request sent on conn. +func readResponse(t *testing.T, conn net.Conn) answer { + t.Helper() + + err := conn.SetReadDeadline(time.Now().Add(waitLimit)) + if err != nil { + t.Fatalf("set read deadline: %v", err) + } + + res, err := http.ReadResponse(bufio.NewReader(conn), nil) + if err != nil { + t.Fatalf("read response: %v", err) + } + + return readAnswer(res) +} + +// wantLine checks the request log line's status and action. +func wantLine(t *testing.T, line logLine, status int, action string) { + t.Helper() + + if line.Status != status || line.Action != action { + t.Errorf("log line has status %d and action %q, want %d and %q", + line.Status, line.Action, status, action) + } +} + +// wantStatus checks an answer's status. +func wantStatus(t *testing.T, got answer, status int) { + t.Helper() + + if got.status != status { + t.Errorf("status %d, want %d", got.status, status) + } +} + +// wantTimedOut checks that what began at start ended once shortTimeout +// had run out, and not much later. +func wantTimedOut(t *testing.T, start time.Time) { + t.Helper() + + took := time.Since(start) + if took < shortTimeout || took > shortTimeout+waitLimit/2 { + t.Errorf("took %s, want %s", took, shortTimeout) + } +} diff --git a/internal/proxy/request.go b/internal/proxy/request.go new file mode 100644 index 0000000..bf73130 --- /dev/null +++ b/internal/proxy/request.go @@ -0,0 +1,452 @@ +package proxy + +import ( + "context" + "errors" + "net/http" + "net/http/httptrace" + "net/http/httputil" + "net/netip" + "os" + "sync" + "sync/atomic" + "time" + + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// flushAfterEachWrite has ReverseProxy pass on each part of the app's +// answer as soon as it arrives. +const flushAfterEachWrite time.Duration = -1 + +// refusal is smallwebwaf refusing a request, or refusing to go on with it: +// the status the client is answered if the response has not started yet, +// and the action the log line names. +type refusal struct { + status int + action string +} + +// request is one request on its way through smallwebwaf, from the moment +// its headers have been read to its log line. +type request struct { + h *handler + in *http.Request + // rc sets the deadlines of the connection to the client. + rc *http.ResponseController + out *responseWriter + body *requestBody // nil for a request without a body + line requestlog.Line + + peer netip.Addr + peerTrusted bool + start time.Time + // upstreamStart is when the request was handed to the app. + upstreamStart time.Time + // cancel ends the request to the app. + cancel context.CancelFunc + // refused is the first refusal, from whichever goroutine meets it. + refused atomic.Pointer[refusal] + // complete is true once the app's whole answer has been passed on. + complete bool + + // mu guards what follows. The timeouts run on goroutines of their + // own, and the transport starts and stops them from its own; once + // timersStopped is set, none of them acts any more. + mu sync.Mutex + timersStopped bool + clientRequestTimer *time.Timer + upstreamRequestTimer *time.Timer + upstreamResponseTimer *time.Timer + // requestSent is when the app had been sent the whole request. + requestSent time.Time +} + +// newRequest starts handling r: it notes the time and works out the +// client. +func (h *handler) newRequest(w http.ResponseWriter, r *http.Request) *request { + start := time.Now() + peer := peerAddress(r) + trusted := h.config.TrustedProxies + client := clientAddress(peer, r.Header.Values("X-Forwarded-For"), trusted) + + rq := &request{ + h: h, + in: r, + rc: http.NewResponseController(w), + out: &responseWriter{ResponseWriter: w}, + peer: peer, + peerTrusted: isInside(peer, trusted), + start: start, + line: requestlog.Line{ + Time: requestlog.FormatTime(start), + ClientIP: client.String(), + PeerIP: peer.String(), + Method: r.Method, + Host: r.Host, + Path: r.URL.EscapedPath(), + Query: r.URL.RawQuery, + Protocol: r.Proto, + Referer: r.Referer(), + UserAgent: r.UserAgent(), + Action: requestlog.ActionForward, + }, + } + if r.Body != http.NoBody { + rq.body = &requestBody{body: limitBody(r.Body, h.config.RequestMaxBytes), rq: rq} + } + + return rq +} + +// check is the one place where a request can be refused once its client +// is known, before its body is read or anything reaches the app; the rate +// limits and country lists of milestone 2 go here. It returns nil to let +// the request through. +func (rq *request) check() *refusal { + maxBytes := rq.h.config.RequestMaxBytes + if maxBytes > 0 && rq.in.ContentLength > maxBytes { + return &refusal{ + status: http.StatusRequestEntityTooLarge, + action: requestlog.ActionTooLarge, + } + } + + return nil +} + +// forward passes the request to the app and the app's answer back. ctx +// is the request's own context. +func (rq *request) forward(ctx context.Context) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + rq.cancel = cancel + ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{ + WroteRequest: rq.wroteRequest, + }) + + out := rq.in.WithContext(ctx) + if rq.body != nil { + out.Body = rq.body + } + + reverseProxy := &httputil.ReverseProxy{ + Rewrite: rq.rewrite, + Transport: rq.h.transport, + FlushInterval: flushAfterEachWrite, + ErrorLog: rq.h.errorLog, + ModifyResponse: rq.modifyResponse, + ErrorHandler: rq.answerError, + } + + rq.startRequestTimers() + rq.upstreamStart = time.Now() + reverseProxy.ServeHTTP(rq.out, out) +} + +// rewrite makes the request the app receives: the client's request, +// unchanged, sent to SWWAF_UPSTREAM_URL, with the forwarded headers set. +func (rq *request) rewrite(pr *httputil.ProxyRequest) { + upstream := rq.h.config.UpstreamURL + pr.Out.URL.Scheme = upstream.Scheme + pr.Out.URL.Host = upstream.Host + // ReverseProxy drops query parameters it cannot parse; the app gets + // the query as the client sent it. + pr.Out.URL.RawQuery = pr.In.URL.RawQuery + setForwardedHeaders(pr.In, pr.Out, rq.peer, rq.peerTrusted) +} + +// modifyResponse looks at the app's answer before ReverseProxy passes it +// on. +func (rq *request) modifyResponse(res *http.Response) error { + rq.line.UpstreamStatus = res.StatusCode + + if res.StatusCode == http.StatusSwitchingProtocols { + // An upgraded connection, such as a WebSocket, is not cut by the + // timeouts. ReverseProxy writes this answer straight to the + // connection it takes over, not through rq.out. + rq.stopTimers() + rq.out.status = res.StatusCode + + return nil + } + + maxBytes := rq.h.config.ResponseMaxBytes + if maxBytes > 0 && res.Body != http.NoBody && res.ContentLength > maxBytes { + rq.refuse(refusal{status: http.StatusBadGateway, action: requestlog.ActionTooLarge}) + + return errResponseTooLarge + } + + res.Body = &responseBody{body: limitBody(res.Body, maxBytes), rq: rq} + rq.startClientResponseTimeout() + + return nil +} + +// answerError is ReverseProxy's ErrorHandler: the request could not be +// passed to the app, or the app's answer cannot be passed on. +func (rq *request) answerError(_ http.ResponseWriter, _ *http.Request, err error) { + refused := rq.refused.Load() + if refused == nil { + if rq.in.Context().Err() != nil { + return // the client has gone, and there is no one to answer + } + + rq.h.processLog.Warn("request to the app failed", "error", err.Error()) + + refused = &refusal{ + status: http.StatusBadGateway, + action: requestlog.ActionUpstreamError, + } + } + + rq.answer(*refused) +} + +// answer sends smallwebwaf's own answer, unless the response has already +// started, and records the refusal for the log line. +func (rq *request) answer(r refusal) { + rq.refused.CompareAndSwap(nil, &r) + + if rq.out.status != 0 { + return // too late to answer: the connection can only be cut + } + + // A client found too slow is read no more; any other may go on + // sending until its time is up, so that Go's server can read the + // rest of the body and end the request cleanly. + deadline := rq.clientRequestDeadline() + if r.status == http.StatusRequestTimeout { + deadline = time.Now() + } + + rq.stopReadingBody(deadline) + + timeout := rq.h.config.ClientResponseTimeout + if timeout > 0 { + _ = rq.rc.SetWriteDeadline(time.Now().Add(timeout)) + } + + http.Error(rq.out, http.StatusText(r.status), r.status) +} + +// refuse records r, unless an earlier refusal was, and ends the request +// to the app. +func (rq *request) refuse(r refusal) { + rq.refused.CompareAndSwap(nil, &r) + rq.cancel() +} + +// finish ends the request's timeouts and writes its log line. +func (rq *request) finish() { + rq.stopTimers() + + refused := rq.refused.Load() + if refused == nil { + rq.stopReadingBody(rq.clientRequestDeadline()) + } + + line := &rq.line + line.Status = rq.out.status + line.ResponseBytes = rq.out.bytes + + if rq.body != nil { + line.RequestBytes = rq.body.bytes.Load() + } + + switch { + case refused != nil: + line.Action = refused.action + case errors.Is(rq.out.err, os.ErrDeadlineExceeded): + // The client took longer than SWWAF_CLIENT_RESPONSE_TIMEOUT to + // take the response. + line.Action = requestlog.ActionTimedOut + case !rq.complete && (rq.out.err != nil || rq.in.Context().Err() != nil): + line.Aborted = true + } + + now := time.Now() + line.DurationTotal = requestlog.Milliseconds(now.Sub(rq.start)) + + if !rq.upstreamStart.IsZero() { + line.DurationUpstreamTotal = requestlog.Milliseconds(now.Sub(rq.upstreamStart)) + } + + err := requestlog.Write(rq.h.requestLog, line) + if err != nil { + rq.h.processLog.Error("writing the request log failed", "error", err.Error()) + } +} + +// clientRequestDeadline is when the client must have sent its whole +// request, or zero when SWWAF_CLIENT_REQUEST_TIMEOUT is off. +func (rq *request) clientRequestDeadline() time.Time { + timeout := rq.h.config.ClientRequestTimeout + if timeout == 0 { + return time.Time{} + } + + return rq.start.Add(timeout) +} + +// stopReadingBody ends, at deadline, the reading of a client body that has +// not arrived whole: Go's server then reads no more of it, and closes the +// connection after the answer. +func (rq *request) stopReadingBody(deadline time.Time) { + if rq.body == nil || rq.body.received.Load() { + return + } + + _ = rq.rc.SetReadDeadline(deadline) +} + +// startRequestTimers starts the timeouts that run while the request goes +// to the app: SWWAF_CLIENT_REQUEST_TIMEOUT until the client has sent its +// whole body, and SWWAF_UPSTREAM_REQUEST_TIMEOUT until the app has been +// sent the whole request. +func (rq *request) startRequestTimers() { + rq.mu.Lock() + defer rq.mu.Unlock() + + if rq.body != nil && rq.h.config.ClientRequestTimeout > 0 { + rq.clientRequestTimer = time.AfterFunc( + time.Until(rq.clientRequestDeadline()), rq.requestTimedOut) + } + + timeout := rq.h.config.UpstreamRequestTimeout + if timeout > 0 { + rq.upstreamRequestTimer = time.AfterFunc(timeout, rq.requestTimedOut) + } +} + +// requestTimedOut is called when a request timeout runs out while the +// request is still on its way to the app. The answer names the side +// smallwebwaf was waiting on at that moment: 408 when it was waiting for +// the client to send more of its body, 504 when it was waiting for the +// app to be reached or to take what it had. +func (rq *request) requestTimedOut() { + rq.mu.Lock() + defer rq.mu.Unlock() + + if rq.timersStopped { + return + } + + if rq.body == nil || !rq.body.waiting.Load() { + rq.refuse(refusal{ + status: http.StatusGatewayTimeout, + action: requestlog.ActionTimedOut, + }) + + return + } + + rq.refuse(refusal{ + status: http.StatusRequestTimeout, + action: requestlog.ActionTimedOut, + }) + // The transport gives up on the app only once its Read of the + // client's body returns, so that Read is ended now. The lock keeps + // this from reaching the connection after the request is handled. + _ = rq.rc.SetReadDeadline(time.Now()) +} + +// bodyReceived is called once the client has sent its whole body. +func (rq *request) bodyReceived() { + rq.mu.Lock() + defer rq.mu.Unlock() + + stopTimer(rq.clientRequestTimer) +} + +// wroteRequest is called once the app has been sent the whole request: +// the request timeouts end and SWWAF_UPSTREAM_RESPONSE_TIMEOUT starts. +func (rq *request) wroteRequest(info httptrace.WroteRequestInfo) { + if info.Err != nil { + return // the transport gives up, or tries again + } + + rq.mu.Lock() + defer rq.mu.Unlock() + + if rq.timersStopped { + return + } + + stopTimer(rq.clientRequestTimer) + stopTimer(rq.upstreamRequestTimer) + rq.requestSent = time.Now() + + timeout := rq.h.config.UpstreamResponseTimeout + if timeout > 0 { + rq.upstreamResponseTimer = time.AfterFunc(timeout, rq.responseTimedOut) + } +} + +// responseTimedOut is called when SWWAF_UPSTREAM_RESPONSE_TIMEOUT runs out +// before the app has sent its whole answer. +func (rq *request) responseTimedOut() { + rq.mu.Lock() + defer rq.mu.Unlock() + + if !rq.timersStopped { + rq.refuse(refusal{ + status: http.StatusGatewayTimeout, + action: requestlog.ActionTimedOut, + }) + } +} + +// responseReceived is called once the app has sent its whole answer. +func (rq *request) responseReceived() { + rq.complete = true + rq.stopTimers() +} + +// startClientResponseTimeout sets SWWAF_CLIENT_RESPONSE_TIMEOUT on the +// connection to the client: the response must reach the client within it +// of the end of the request, or of now if the app answers before it has +// the whole request. +func (rq *request) startClientResponseTimeout() { + timeout := rq.h.config.ClientResponseTimeout + if timeout == 0 { + return + } + + from := rq.sentAt() + if from.IsZero() { + from = time.Now() + } + + _ = rq.rc.SetWriteDeadline(from.Add(timeout)) +} + +// sentAt is when the app had been sent the whole request, or zero. +func (rq *request) sentAt() time.Time { + rq.mu.Lock() + defer rq.mu.Unlock() + + return rq.requestSent +} + +// stopTimers stops the request's timeouts and keeps any from starting +// later: the app's answer is complete, the connection upgraded, or the +// request handled. +func (rq *request) stopTimers() { + rq.mu.Lock() + defer rq.mu.Unlock() + + rq.timersStopped = true + stopTimer(rq.clientRequestTimer) + stopTimer(rq.upstreamRequestTimer) + stopTimer(rq.upstreamResponseTimer) +} + +// stopTimer stops t, which is nil when its timeout is off. +func stopTimer(t *time.Timer) { + if t != nil { + t.Stop() + } +} diff --git a/internal/proxy/timeouts_test.go b/internal/proxy/timeouts_test.go new file mode 100644 index 0000000..e964d57 --- /dev/null +++ b/internal/proxy/timeouts_test.go @@ -0,0 +1,258 @@ +package proxy_test + +import ( + "errors" + "io" + "net" + "net/http" + "strconv" + "sync" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// largeBodySize is more than the connections between the client, +// smallwebwaf and the app can hold while nobody reads, so that a sender +// soon waits. +const largeBodySize = 64 << 20 + +// writeSize is how much a test sender writes at a time. +const writeSize = 32 << 10 + +func TestRequestTimeouts(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + env map[string]string + // appTakesNothing has the app never read, while the client sends + // as fast as it can; otherwise the app reads, and the client + // stops sending halfway. + appTakesNothing bool + want int + }{ + { + name: "client request timeout, waiting on the client", + env: map[string]string{clientRequestTimeout: shortTimeoutSetting}, + want: http.StatusRequestTimeout, + }, + { + name: "upstream request timeout, waiting on the client", + env: map[string]string{ + upstreamRequestTimeout: shortTimeoutSetting, + clientRequestTimeout: longTimeoutSetting, + }, + want: http.StatusRequestTimeout, + }, + { + name: "upstream request timeout, waiting on the app", + env: map[string]string{upstreamRequestTimeout: shortTimeoutSetting}, + appTakesNothing: true, + want: http.StatusGatewayTimeout, + }, + { + name: "client request timeout, waiting on the app", + env: map[string]string{ + clientRequestTimeout: shortTimeoutSetting, + upstreamRequestTimeout: longTimeoutSetting, + }, + appTakesNothing: true, + want: http.StatusGatewayTimeout, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + var ( + appURL string + sendRequest func(*testing.T, string) net.Conn + ) + + if tc.appTakesNothing { + appURL, sendRequest = startAppThatTakesNothing(t), sendLargeBody + } else { + appURL, sendRequest = startApp(t, readBody).URL, sendPartOfBody + } + + addr, out := startProxy(t, appURL, tc.env) + start := time.Now() + conn := sendRequest(t, addr) + + wantStatus(t, readResponse(t, conn), tc.want) + wantTimedOut(t, start) + wantLine(t, out.requestLine(t), tc.want, requestlog.ActionTimedOut) + }) + } +} + +// readBody is an app that reads the request body, then answers. +func readBody(_ http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) +} + +// startAppThatTakesNothing starts an app that accepts connections and +// never reads from them, and returns its URL. +func startAppThatTakesNothing(t *testing.T) string { + t.Helper() + + listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0") + if err != nil { + t.Fatalf("listen: %v", err) + } + + var ( + mu sync.Mutex + held []net.Conn + ) + + hold := func(conn net.Conn) { + mu.Lock() + defer mu.Unlock() + + held = append(held, conn) + } + + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + + hold(conn) + } + }() + + t.Cleanup(func() { + _ = listener.Close() + + mu.Lock() + defer mu.Unlock() + + for _, conn := range held { + _ = conn.Close() + } + }) + + return "http://" + listener.Addr().String() +} + +// sendPartOfBody sends a request that announces a large body, and only +// the first bytes of it. +func sendPartOfBody(t *testing.T, addr string) net.Conn { + t.Helper() + + conn := dial(t, addr) + send(t, conn, "POST /upload HTTP/1.1\r\nHost: app\r\nContent-Length: "+ + strconv.Itoa(largeBodySize)+"\r\n\r\nthe first bytes") + + return conn +} + +// sendLargeBody sends a request with a large body, as fast as smallwebwaf +// takes it, from a goroutine of its own. +func sendLargeBody(t *testing.T, addr string) net.Conn { + t.Helper() + + conn := dial(t, addr) + send(t, conn, "POST /upload HTTP/1.1\r\nHost: app\r\nContent-Length: "+ + strconv.Itoa(largeBodySize)+"\r\n\r\n") + + go func() { + chunk := make([]byte, writeSize) + for range largeBodySize / writeSize { + _, err := conn.Write(chunk) + if err != nil { + return + } + } + }() + + return conn +} + +func TestAppTooSlowToAnswer(t *testing.T) { + t.Parallel() + + app := startApp(t, func(_ http.ResponseWriter, r *http.Request) { + <-r.Context().Done() + }) + addr, out := startProxy(t, app.URL, map[string]string{ + upstreamResponseTimeout: shortTimeoutSetting, + }) + + start := time.Now() + + wantStatus(t, get(t, addr, "/slow"), http.StatusGatewayTimeout) + wantTimedOut(t, start) + + line := out.requestLine(t) + wantLine(t, line, http.StatusGatewayTimeout, requestlog.ActionTimedOut) + + _, answered := line.fields["upstream_status"] + if answered { + t.Errorf("log line has upstream_status %v for an app that never answered", + line.fields["upstream_status"]) + } +} + +func TestAppTooSlowToFinishItsAnswer(t *testing.T) { + t.Parallel() + + app := startApp(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "the first part") + _ = http.NewResponseController(w).Flush() + + <-r.Context().Done() + }) + addr, out := startProxy(t, app.URL, map[string]string{ + upstreamResponseTimeout: shortTimeoutSetting, + }) + + start := time.Now() + got := get(t, addr, "/slow") + wantStatus(t, got, http.StatusOK) + + if string(got.body) != "the first part" || !errors.Is(got.err, io.ErrUnexpectedEOF) { + t.Errorf("client read %q (%v), want the first part cut off", got.body, got.err) + } + + wantTimedOut(t, start) + + line := out.requestLine(t) + wantLine(t, line, http.StatusOK, requestlog.ActionTimedOut) + + if line.UpstreamStatus != http.StatusOK { + t.Errorf("log line has upstream_status %d, want %d", + line.UpstreamStatus, http.StatusOK) + } +} + +func TestClientTooSlowToTakeTheAnswer(t *testing.T) { + t.Parallel() + + app := startApp(t, func(w http.ResponseWriter, _ *http.Request) { + chunk := make([]byte, writeSize) + for range largeBodySize / writeSize { + _, err := w.Write(chunk) + if err != nil { + return + } + } + }) + addr, out := startProxy(t, app.URL, map[string]string{ + clientResponseTimeout: shortTimeoutSetting, + }) + + start := time.Now() + + // The client asks, and never reads the answer. + conn := dial(t, addr) + send(t, conn, "GET /large HTTP/1.1\r\nHost: app\r\n\r\n") + + line := out.requestLine(t) + wantTimedOut(t, start) + wantLine(t, line, http.StatusOK, requestlog.ActionTimedOut) +} diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go new file mode 100644 index 0000000..07f074b --- /dev/null +++ b/internal/requestlog/requestlog.go @@ -0,0 +1,102 @@ +// Package requestlog writes the lines smallwebwaf prints on stdout: one +// JSON object per request, marked "type":"request", and the process's own +// messages as JSON lines marked "type":"process". +package requestlog + +import ( + "encoding/json" + "fmt" + "io" + "log/slog" + "time" +) + +// The action a request line names: what smallwebwaf did with the +// request. +const ( + // ActionForward is a request passed to the app. + ActionForward = "forward" + // ActionTooLarge is a request or response over its size limit. + ActionTooLarge = "too_large" + // ActionTimedOut is a request or response that ran out of time. + ActionTimedOut = "timed_out" + // ActionUpstreamError is a request the app could not be reached + // for, or whose answer could not be passed on. + ActionUpstreamError = "upstream_error" +) + +// timeLayout is RFC 3339 with milliseconds. +const timeLayout = "2006-01-02T15:04:05.000Z07:00" + +// Line is one request's line in the request log. The field names are +// those of the "Request log" section of SPEC.md. +// +//nolint:tagliatelle // SPEC.md's request log names its fields in snake_case +type Line struct { + Type string `json:"type"` + Time string `json:"time"` + ClientIP string `json:"client_ip"` + PeerIP string `json:"peer_ip"` + Method string `json:"method"` + Host string `json:"host"` + Path string `json:"path"` + Query string `json:"query"` + Protocol string `json:"protocol"` + Status int `json:"status"` + UpstreamStatus int `json:"upstream_status,omitempty"` + RequestBytes int64 `json:"request_bytes"` + ResponseBytes int64 `json:"response_bytes"` + Referer string `json:"referer"` + UserAgent string `json:"user_agent"` + Action string `json:"action"` + // Aborted is true when the client went away early. + Aborted bool `json:"aborted,omitempty"` + // DurationTotal and DurationUpstreamTotal are in milliseconds. + DurationTotal float64 `json:"duration_total"` + DurationUpstreamTotal float64 `json:"duration_upstream_total,omitempty"` +} + +// Write writes line to w as one JSON line marked "type":"request". +func Write(w io.Writer, line *Line) error { + line.Type = "request" + + encoded, err := json.Marshal(line) + if err != nil { + return fmt.Errorf("encode the request log line: %w", err) + } + + _, err = w.Write(append(encoded, '\n')) + if err != nil { + return fmt.Errorf("write the request log line: %w", err) + } + + return nil +} + +// FormatTime formats t for a line's time field: RFC 3339 in UTC, with +// milliseconds. +func FormatTime(t time.Time) string { + return t.UTC().Format(timeLayout) +} + +// Milliseconds is d in milliseconds, to the microsecond. +func Milliseconds(d time.Duration) float64 { + return float64(d.Microseconds()) / float64(time.Millisecond/time.Microsecond) +} + +// NewProcessLogger returns the logger for the process's own messages: +// JSON lines on w, marked "type":"process", with the time in the same form +// as a request line's. +func NewProcessLogger(w io.Writer) *slog.Logger { + handler := slog.NewJSONHandler(w, &slog.HandlerOptions{ + ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr { + if attr.Key == slog.TimeKey && len(groups) == 0 { + return slog.String(slog.TimeKey, FormatTime(attr.Value.Time())) + } + + return attr + }, + }) + + return slog.New(handler).With("type", "process") +} diff --git a/internal/requestlog/requestlog_test.go b/internal/requestlog/requestlog_test.go new file mode 100644 index 0000000..84fba56 --- /dev/null +++ b/internal/requestlog/requestlog_test.go @@ -0,0 +1,88 @@ +package requestlog_test + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +func TestWriteWritesOneJSONLineMarkedRequest(t *testing.T) { + t.Parallel() + + var out bytes.Buffer + + err := requestlog.Write(&out, &requestlog.Line{ + Time: requestlog.FormatTime(time.Date(2026, 10, 3, 12, 0, 0, 0, time.UTC)), + ClientIP: "203.0.113.9", + Status: 200, + Action: requestlog.ActionForward, + DurationTotal: requestlog.Milliseconds(1500 * time.Microsecond), + }) + if err != nil { + t.Fatalf("write: %v", err) + } + + text := out.String() + if strings.Count(text, "\n") != 1 || !strings.HasSuffix(text, "\n") { + t.Fatalf("wrote %q, want one line", text) + } + + var fields map[string]any + + err = json.Unmarshal(out.Bytes(), &fields) + if err != nil { + t.Fatalf("decode %q: %v", text, err) + } + + want := map[string]any{ + "type": "request", "time": "2026-10-03T12:00:00.000Z", + "client_ip": "203.0.113.9", "status": 200.0, "action": "forward", + "duration_total": 1.5, + } + for name, value := range want { + if fields[name] != value { + t.Errorf("%s is %v, want %v", name, fields[name], value) + } + } + + unset := []string{"upstream_status", "aborted", "duration_upstream_total"} + for _, name := range unset { + _, present := fields[name] + if present { + t.Errorf("%s is there with no value to give", name) + } + } +} + +func TestProcessLinesAreMarkedProcess(t *testing.T) { + t.Parallel() + + var out bytes.Buffer + + requestlog.NewProcessLogger(&out).Info("starting", "version", "v1") + + var fields map[string]any + + err := json.Unmarshal(out.Bytes(), &fields) + if err != nil { + t.Fatalf("decode %q: %v", out.String(), err) + } + + if fields["type"] != "process" || fields["msg"] != "starting" || + fields["level"] != "INFO" || fields["version"] != "v1" { + t.Errorf("process line %v", fields) + } + + timeText, _ := fields["time"].(string) + + logged, err := time.Parse(time.RFC3339, timeText) + if err != nil || !strings.HasSuffix(timeText, "Z") || + len(timeText) != len("2006-01-02T15:04:05.000Z") || + time.Since(logged) > time.Minute { + t.Errorf("process line time %q, want now in UTC with milliseconds", timeText) + } +} diff --git a/internal/smallwebwaf/smallwebwaf.go b/internal/smallwebwaf/smallwebwaf.go new file mode 100644 index 0000000..7141b63 --- /dev/null +++ b/internal/smallwebwaf/smallwebwaf.go @@ -0,0 +1,130 @@ +// Package smallwebwaf runs the smallwebwaf process: it reads the settings, +// serves requests until it is told to stop, and then stops in an orderly +// way. +package smallwebwaf + +import ( + "context" + "errors" + "io" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/proxy" + "sneak.berlin/go/smallwebwaf/internal/requestlog" +) + +// shutdownTimeout is how long requests in progress may take to finish +// once smallwebwaf is told to stop, before their connections are closed. +// runit and docker wait a little longer before they kill the process. +const shutdownTimeout = 5 * time.Second + +// Params are what Run needs from the process. +type Params struct { + // Version is the version of the binary, set when it is built. + Version string + // LookupEnv reads an environment variable, normally os.LookupEnv. + LookupEnv func(string) (string, bool) + // Stdout receives the request log and the process's own messages. + Stdout io.Writer +} + +// Main runs smallwebwaf until SIGTERM or SIGINT, and returns the +// process's exit status. +func Main(version string) int { + ctx, stop := signal.NotifyContext(context.Background(), + syscall.SIGTERM, os.Interrupt) + defer stop() + + return Run(ctx, Params{ + Version: version, + LookupEnv: os.LookupEnv, + Stdout: os.Stdout, + }) +} + +// Run reads the settings, then serves requests until ctx is done. It +// returns the process's exit status, 1 when smallwebwaf cannot start. +func Run(ctx context.Context, params Params) int { + processLog := requestlog.NewProcessLogger(params.Stdout) + + cfg, err := config.FromEnvironment(params.LookupEnv) + if err != nil { + processLog.Error("invalid setting", "error", err.Error()) + + return 1 + } + + listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", cfg.ListenAddr) + if err != nil { + processLog.Error("cannot listen on SWWAF_LISTEN_ADDR", + "error", err.Error()) + + return 1 + } + + server := proxy.New(proxy.Params{ + Config: cfg, + RequestLog: params.Stdout, + ProcessLog: processLog, + }) + + processLog.Info("starting", + "version", params.Version, + "address", listener.Addr().String(), + "settings", cfg) + + return serve(ctx, server, listener, processLog) +} + +// serve serves requests on listener until ctx is done, then gives the +// requests in progress shutdownTimeout to finish. +func serve( + ctx context.Context, server *http.Server, listener net.Listener, + processLog *slog.Logger, +) int { + served := make(chan error, 1) + + go func() { + served <- server.Serve(listener) + }() + + select { + case err := <-served: + processLog.Error("serving failed", "error", err.Error()) + + return 1 + case <-ctx.Done(): + } + + processLog.Info("stopping") + + shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), + shutdownTimeout) + defer cancel() + + err := server.Shutdown(shutdownCtx) + if err != nil { + processLog.Warn("requests still in progress were cut off", + "error", err.Error()) + + _ = server.Close() + } + + err = <-served + if !errors.Is(err, http.ErrServerClosed) { + processLog.Error("serving failed", "error", err.Error()) + + return 1 + } + + processLog.Info("stopped") + + return 0 +} diff --git a/internal/smallwebwaf/smallwebwaf_test.go b/internal/smallwebwaf/smallwebwaf_test.go new file mode 100644 index 0000000..2013071 --- /dev/null +++ b/internal/smallwebwaf/smallwebwaf_test.go @@ -0,0 +1,225 @@ +package smallwebwaf_test + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "sneak.berlin/go/smallwebwaf/internal/smallwebwaf" +) + +const ( + // waitLimit bounds how long a test waits for what should happen. + waitLimit = 10 * time.Second + // pollInterval is how often a test looks for a line. + pollInterval = 10 * time.Millisecond + // testVersion is the version the tests give smallwebwaf. + testVersion = "test" + // localhost is where the tests listen. + localhost = "127.0.0.1" + listenAddr = "SWWAF_LISTEN_ADDR" +) + +// output collects what smallwebwaf writes on stdout. +type output struct { + mu sync.Mutex + buf bytes.Buffer +} + +// Write adds lines smallwebwaf writes. +func (o *output) Write(p []byte) (int, error) { + o.mu.Lock() + defer o.mu.Unlock() + + return o.buf.Write(p) +} + +// line returns the first line whose field key is value, waiting for it. +func (o *output) line(t *testing.T, key, value string) map[string]any { + t.Helper() + + deadline := time.Now().Add(waitLimit) + for time.Now().Before(deadline) { + o.mu.Lock() + text := o.buf.String() + o.mu.Unlock() + + for line := range strings.Lines(text) { + var fields map[string]any + + err := json.Unmarshal([]byte(line), &fields) + if err != nil { + t.Fatalf("output line %q is not JSON: %v", line, err) + } + + if fields[key] == value { + return fields + } + } + + time.Sleep(pollInterval) + } + + t.Fatalf("no line with %s %q in the output:\n%s", key, value, o.buf.String()) + + return nil +} + +// run runs smallwebwaf with the settings in env until ctx is done, and +// returns its exit status. +func run(ctx context.Context, env map[string]string, out *output) int { + return smallwebwaf.Run(ctx, smallwebwaf.Params{ + Version: testVersion, + LookupEnv: func(name string) (string, bool) { + value, ok := env[name] + + return value, ok + }, + Stdout: out, + }) +} + +func TestInvalidSettingStopsTheStart(t *testing.T) { + t.Parallel() + + out := &output{} + + status := run(t.Context(), map[string]string{"SWWAF_REQUEST_MAX_BYTES": "lots"}, out) + if status != 1 { + t.Errorf("exit status %d, want 1", status) + } + + line := out.line(t, "msg", "invalid setting") + message, _ := line["error"].(string) + + if line["type"] != "process" || line["level"] != "ERROR" || + !strings.HasPrefix(message, "SWWAF_REQUEST_MAX_BYTES: ") { + t.Errorf("start refused with %v", line) + } +} + +func TestAddressInUseStopsTheStart(t *testing.T) { + t.Parallel() + + taken, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0") + if err != nil { + t.Fatalf("listen: %v", err) + } + + defer func() { + _ = taken.Close() + }() + + out := &output{} + + status := run(t.Context(), map[string]string{listenAddr: taken.Addr().String()}, out) + if status != 1 { + t.Errorf("exit status %d, want 1", status) + } + + out.line(t, "msg", "cannot listen on SWWAF_LISTEN_ADDR") +} + +func TestServesUntilToldToStop(t *testing.T) { + t.Parallel() + + app := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, "hello from the app") + })) + defer app.Close() + + ctx, stop := context.WithCancel(t.Context()) + out := &output{} + exited := make(chan int, 1) + + go func() { + exited <- run(ctx, map[string]string{ + listenAddr: localhost + ":0", + "SWWAF_UPSTREAM_URL": app.URL, + }, out) + }() + + starting := out.line(t, "msg", "starting") + wantStartingLine(t, starting, app.URL) + + addr, _ := starting["address"].(string) + wantGreeting(t, "http://"+addr+"/") + out.line(t, "type", "request") + + stop() + + select { + case status := <-exited: + if status != 0 { + t.Errorf("exit status %d, want 0", status) + } + case <-time.After(waitLimit): + t.Fatal("still running after being told to stop") + } + + out.line(t, "msg", "stopped") +} + +// wantStartingLine checks that the line at start gives the version and +// every setting's value. +func wantStartingLine(t *testing.T, line map[string]any, appURL string) { + t.Helper() + + settings, _ := line["settings"].(map[string]any) + want := map[string]any{ + listenAddr: localhost + ":0", + "SWWAF_UPSTREAM_URL": appURL, + "SWWAF_TRUSTED_PROXIES": "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16", + "SWWAF_CLIENT_REQUEST_TIMEOUT": "60s", + "SWWAF_CLIENT_RESPONSE_TIMEOUT": "30m", + "SWWAF_UPSTREAM_REQUEST_TIMEOUT": "60s", + "SWWAF_UPSTREAM_RESPONSE_TIMEOUT": "30m", + "SWWAF_REQUEST_MAX_BYTES": "100M", + "SWWAF_RESPONSE_MAX_BYTES": "5G", + } + + for name, value := range want { + if settings[name] != value { + t.Errorf("starting line gives %s=%v, want %v", name, settings[name], value) + } + } + + if line["version"] != testVersion || line["type"] != "process" { + t.Errorf("starting line %v", line) + } +} + +// wantGreeting checks that a request to url gets the app's answer. +func wantGreeting(t *testing.T, url string) { + t.Helper() + + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, url, + http.NoBody) + if err != nil { + t.Fatalf("new request: %v", err) + } + + transport := &http.Transport{} + defer transport.CloseIdleConnections() + + res, err := (&http.Client{Transport: transport}).Do(req) + if err != nil { + t.Fatalf("request: %v", err) + } + + body, err := io.ReadAll(res.Body) + _ = res.Body.Close() + + if err != nil || string(body) != "hello from the app" { + t.Errorf("got %q (%v), want the app's answer", body, err) + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..dc05cde --- /dev/null +++ b/package.json @@ -0,0 +1,5 @@ +{ + "devDependencies": { + "prettier": "3.8.1" + } +} diff --git a/script/bootstrap b/script/bootstrap new file mode 100755 index 0000000..d0e97bc --- /dev/null +++ b/script/bootstrap @@ -0,0 +1,143 @@ +#!/bin/sh +# script/bootstrap: install all dependencies needed to build and develop +# this repo. Idempotent: every install is guarded by a check so already +# installed tools are skipped. Base tooling comes from nix, apt, brew, +# or apk (detected in that order); assumes nothing is present. Node is +# used directly if installed; otherwise it is installed at a pinned +# version via nvm (installing nvm itself first, from a hash-verified +# release archive, never curl | sh). Go comes from the package manager, +# for gofmt in script/fmt and script/fmt-check: the tests and the linter +# run in docker and need no Go on the host. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +# Pinned versions, 2026-07-06 +NODE_VERSION="22.17.0" +NVM_VERSION="0.40.3" +# sha256 of https://github.com/nvm-sh/nvm/archive/refs/tags/v0.40.3.tar.gz +NVM_SHA256="5f4d6aaa04a177dc93c985e31dbc411ab6b8c6e1e21d8015dbc1372625fcd1d0" +YARN_VERSION="1.22.22" + +PKGMGR="" +SUDO="" + +detect_pkgmgr() { + [ -n "$PKGMGR" ] && return 0 + if command -v nix-env >/dev/null 2>&1; then + PKGMGR="nix" + elif command -v apt-get >/dev/null 2>&1; then + PKGMGR="apt" + elif command -v brew >/dev/null 2>&1; then + PKGMGR="brew" + elif command -v apk >/dev/null 2>&1; then + PKGMGR="apk" + else + echo "bootstrap: no supported package manager (nix, apt, brew, apk)" >&2 + exit 1 + fi + if [ "$PKGMGR" = "apt" ]; then + export DEBIAN_FRONTEND=noninteractive + if [ "$(id -u)" != "0" ]; then + SUDO="sudo" + fi + fi +} + +# pkg_install +pkg_install() { + detect_pkgmgr + case "$PKGMGR" in + nix) nix-env -iA "nixpkgs.$1" ;; + apt) + # A fresh CI runner may carry no package lists at all. + $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update -q + $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y "$2" + ;; + brew) brew install "$3" ;; + apk) apk add --no-cache "$4" ;; + esac +} + +missing() { + ! command -v "$1" >/dev/null 2>&1 +} + +# verify_sha256 +verify_sha256() { + if command -v sha256sum >/dev/null 2>&1; then + actual="$(sha256sum "$1" | cut -d' ' -f1)" + else + actual="$(shasum -a 256 "$1" | cut -d' ' -f1)" + fi + if [ "$actual" != "$2" ]; then + echo "bootstrap: sha256 mismatch for $1" >&2 + echo " expected: $2" >&2 + echo " actual: $actual" >&2 + exit 1 + fi +} + +# nvm is a bash script; run a command in a bash with nvm loaded +nvm_sh() { + bash -c ". \"\$HOME/.nvm/nvm.sh\" && $*" +} + +ensure_nvm() { + [ -s "$HOME/.nvm/nvm.sh" ] && return 0 + # nvm prerequisites; nvm itself requires bash + if missing bash; then pkg_install bash bash bash bash; fi + if missing curl; then pkg_install curl curl curl curl; fi + if missing git; then pkg_install git git git git; fi + tmp="$(mktemp -d)" + curl -fsSL -o "$tmp/nvm.tar.gz" \ + "https://github.com/nvm-sh/nvm/archive/refs/tags/v${NVM_VERSION}.tar.gz" + verify_sha256 "$tmp/nvm.tar.gz" "$NVM_SHA256" + mkdir -p "$HOME/.nvm" + tar -xzf "$tmp/nvm.tar.gz" -C "$HOME/.nvm" --strip-components=1 + rm -rf "$tmp" +} + +ensure_node() { + if ! missing node; then return 0; fi + ensure_nvm + nvm_sh "nvm install $NODE_VERSION" +} + +ensure_yarn() { + if ! missing yarn; then return 0; fi + if ! missing corepack; then + corepack enable + corepack prepare "yarn@$YARN_VERSION" --activate + elif [ -s "$HOME/.nvm/nvm.sh" ]; then + nvm_sh "nvm use $NODE_VERSION >/dev/null && corepack enable && \ + corepack prepare yarn@$YARN_VERSION --activate" + else + npm install -g "yarn@$YARN_VERSION" + fi +} + +install_js_deps() { + if missing yarn && [ -s "$HOME/.nvm/nvm.sh" ]; then + nvm_sh "nvm use $NODE_VERSION >/dev/null && cd \"$ROOT\" && \ + yarn install --frozen-lockfile" + else + yarn install --frozen-lockfile + fi +} + +main() { + cd "$ROOT" + + if missing make; then pkg_install gnumake make make make; fi + if missing git; then pkg_install git git git git; fi + if missing gofmt; then pkg_install go golang go go; fi + + ensure_node + ensure_yarn + install_js_deps + + echo "bootstrap complete" +} + +main "$@" diff --git a/script/build b/script/build new file mode 100755 index 0000000..94bd69c --- /dev/null +++ b/script/build @@ -0,0 +1,18 @@ +#!/bin/sh +# script/build: build bin/smallwebwaf on the host, with Go installed, for +# working on the code by hand. The version it reports comes from git, as +# in script/docker. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" + +main() { + cd "$ROOT" + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + go build -trimpath -ldflags "-X main.Version=$version" \ + -o bin/smallwebwaf ./cmd/smallwebwaf +} + +main "$@" diff --git a/script/check b/script/check new file mode 100755 index 0000000..92875f7 --- /dev/null +++ b/script/check @@ -0,0 +1,16 @@ +#!/bin/sh +# script/check: run all checks (test, lint, fmt-check). Our own +# extension to scripts-to-rule-them-all. test and lint are Docker +# phases; fmt-check is native, because a formatter writes the working +# tree. Must not modify any files. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" + +main() { + "$SCRIPT_DIR/test" + "$SCRIPT_DIR/lint" + "$SCRIPT_DIR/fmt-check" +} + +main "$@" diff --git a/script/cibuild b/script/cibuild new file mode 100755 index 0000000..688299f --- /dev/null +++ b/script/cibuild @@ -0,0 +1,29 @@ +#!/bin/sh +# script/cibuild: run the CI build. It bootstraps first: a CI runner +# checks out and runs this and nothing else, and script/fmt-check runs +# the formatter on the host, which a pristine checkout cannot do. +# --no-cache for the same reason as script/docker: the gate phases the +# final stage depends on are RUN steps, and a cached one is a check that +# did not run. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" + +main() { + cd "$ROOT" + "$SCRIPT_DIR/bootstrap" + "$SCRIPT_DIR/check" + # Own line: a failing command substitution inside an argument does + # not trip `set -e`, so the inline form degrades silently to an + # empty constant. VERSION is computed here because .dockerignore + # excludes .git, so `git describe` in a build stage yields an empty + # version without failing. + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + docker build --no-cache \ + --build-arg VERSION="$version" \ + -t "$("$SCRIPT_DIR/projectname")" . +} + +main "$@" diff --git a/script/docker b/script/docker new file mode 100755 index 0000000..c4688e8 --- /dev/null +++ b/script/docker @@ -0,0 +1,25 @@ +#!/bin/sh +# script/docker: build the Docker image tagged with the project name. +# Identical in all repos; the tag comes from script/projectname. +# --no-cache because the gate phases the final stage depends on are RUN +# steps, and a cached one is a check that did not run. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" + +main() { + cd "$ROOT" + # Own line: a failing command substitution inside an argument does + # not trip `set -e`, so the inline form degrades silently to an + # empty constant. VERSION is computed here because .dockerignore + # excludes .git, so `git describe` in a build stage yields an empty + # version without failing. + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + docker build --no-cache \ + --build-arg VERSION="$version" \ + -t "$("$SCRIPT_DIR/projectname")" . +} + +main "$@" diff --git a/script/fmt b/script/fmt new file mode 100755 index 0000000..9febb30 --- /dev/null +++ b/script/fmt @@ -0,0 +1,33 @@ +#!/bin/sh +# script/fmt: format all files (writes): the Go code with gofmt, the +# Markdown with prettier. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +# Must match the pin in script/bootstrap. +NODE_VERSION="22.17.0" + +# script/bootstrap installs node and yarn under nvm and leaves neither +# on the PATH of the shell that called it, so resolve the pinned +# toolchain here the way bootstrap's own install step does. nvm is a +# bash script, hence the subshell. +run_yarn() { + if command -v yarn >/dev/null 2>&1; then + exec yarn "$@" + fi + if [ ! -s "$HOME/.nvm/nvm.sh" ]; then + echo "fmt: no yarn; run script/bootstrap first" >&2 + exit 1 + fi + exec bash -c '. "$HOME/.nvm/nvm.sh" && nvm use "$1" >/dev/null && + shift && exec yarn "$@"' bash "$NODE_VERSION" "$@" +} + +main() { + cd "$ROOT" + gofmt -w cmd internal + run_yarn run prettier --write '**/*.md' --tab-width 4 --prose-wrap always +} + +main "$@" diff --git a/script/fmt-check b/script/fmt-check new file mode 100755 index 0000000..52ec452 --- /dev/null +++ b/script/fmt-check @@ -0,0 +1,38 @@ +#!/bin/sh +# script/fmt-check: check formatting (read-only): the Go code with +# gofmt, the Markdown with prettier. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +# Must match the pin in script/bootstrap. +NODE_VERSION="22.17.0" + +# script/bootstrap installs node and yarn under nvm and leaves neither +# on the PATH of the shell that called it, so resolve the pinned +# toolchain here the way bootstrap's own install step does. nvm is a +# bash script, hence the subshell. +run_yarn() { + if command -v yarn >/dev/null 2>&1; then + exec yarn "$@" + fi + if [ ! -s "$HOME/.nvm/nvm.sh" ]; then + echo "fmt-check: no yarn; run script/bootstrap first" >&2 + exit 1 + fi + exec bash -c '. "$HOME/.nvm/nvm.sh" && nvm use "$1" >/dev/null && + shift && exec yarn "$@"' bash "$NODE_VERSION" "$@" +} + +main() { + cd "$ROOT" + unformatted="$(gofmt -l cmd internal)" + if [ -n "$unformatted" ]; then + echo "fmt-check: gofmt would change:" >&2 + echo "$unformatted" >&2 + exit 1 + fi + run_yarn run prettier --check '**/*.md' --tab-width 4 --prose-wrap always +} + +main "$@" diff --git a/script/install-precommit b/script/install-precommit new file mode 100755 index 0000000..bef6406 --- /dev/null +++ b/script/install-precommit @@ -0,0 +1,16 @@ +#!/bin/sh +# script/install-precommit: install the git pre-commit hook that runs +# script/precommit. Our own extension to scripts-to-rule-them-all. +set -eu + +ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" + +main() { + cd "$ROOT" + hook=".git/hooks/pre-commit" + printf '#!/bin/sh\nset -e\nscript/precommit\n' > .git/hooks/pre-commit + chmod +x .git/hooks/pre-commit + echo "pre-commit hook installed: runs script/precommit" +} + +main "$@" diff --git a/script/lint b/script/lint new file mode 100755 index 0000000..2d8b075 --- /dev/null +++ b/script/lint @@ -0,0 +1,23 @@ +#!/bin/sh +# script/lint: run the linter. Linting is a phase of the Dockerfile and +# this builds that phase alone; the linter is never installed or run on +# a developer host, where a shared result cache and a host-global lock +# make its answer untrustworthy. +# +# The phase is not the last stage in the file, so it is built only when +# --target names it. --no-cache because a cached lint layer is a lint +# that did not run. The tag makes each build replace the previous image +# instead of leaving a dangling one behind. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" + +main() { + cd "$ROOT" + docker build --no-cache \ + --target lint \ + -t "$("$SCRIPT_DIR/projectname")-lint" . +} + +main "$@" diff --git a/script/precommit b/script/precommit new file mode 100755 index 0000000..c0a7867 --- /dev/null +++ b/script/precommit @@ -0,0 +1,12 @@ +#!/bin/sh +# script/precommit: run by the git pre-commit hook; fails the commit if +# checks fail. Our own extension to scripts-to-rule-them-all. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" + +main() { + "$SCRIPT_DIR/check" +} + +main "$@" diff --git a/script/projectname b/script/projectname new file mode 100755 index 0000000..a822b81 --- /dev/null +++ b/script/projectname @@ -0,0 +1,12 @@ +#!/bin/sh +# script/projectname: output the name of this project. Our own +# extension to scripts-to-rule-them-all. Other scripts that need the +# name (e.g. script/docker) call this, so they can stay identical +# across all repos. +set -eu + +main() { + echo "smallwebwaf" +} + +main "$@" diff --git a/script/run b/script/run new file mode 100755 index 0000000..a64ade2 --- /dev/null +++ b/script/run @@ -0,0 +1,14 @@ +#!/bin/sh +# script/run: build bin/smallwebwaf with script/build and run it, with +# the settings in the environment. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" + +main() { + "$SCRIPT_DIR/build" + exec "$ROOT/bin/smallwebwaf" +} + +main "$@" diff --git a/script/setup b/script/setup new file mode 100755 index 0000000..4cc5b6b --- /dev/null +++ b/script/setup @@ -0,0 +1,13 @@ +#!/bin/sh +# script/setup: set up the repo for development after a fresh clone: +# installs dependencies and the git pre-commit hook. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" + +main() { + "$SCRIPT_DIR/bootstrap" + "$SCRIPT_DIR/install-precommit" +} + +main "$@" diff --git a/script/test b/script/test new file mode 100755 index 0000000..cd239f2 --- /dev/null +++ b/script/test @@ -0,0 +1,19 @@ +#!/bin/sh +# script/test: run the test suite. Testing is a phase of the Dockerfile +# and this builds that phase alone, on the same terms as script/lint: +# --target because a phase that is not the last stage is built only when +# named, --no-cache because a cached test layer is a test that did not +# run, and a tag so each build replaces the previous image. +set -eu + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" + +main() { + cd "$ROOT" + docker build --no-cache \ + --target test \ + -t "$("$SCRIPT_DIR/projectname")-test" . +} + +main "$@" diff --git a/yarn.lock b/yarn.lock new file mode 100644 index 0000000..d846639 --- /dev/null +++ b/yarn.lock @@ -0,0 +1,8 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +prettier@3.8.1: + version "3.8.1" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.8.1.tgz#edf48977cf991558f4fcbd8a3ba6015ba2a3a173" + integrity sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==