Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0fb0675588 |
+6
-12
@@ -117,9 +117,8 @@ RUN git checkout --quiet --detach b4b2c785308b1ce785b6155c7fe5f16879080193 \
|
|||||||
&& CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" \
|
&& CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" \
|
||||||
-o /usr/local/bin/runsvinit .
|
-o /usr/local/bin/runsvinit .
|
||||||
|
|
||||||
# The image, described under "Deployment" in SPEC.md. It runs rproxy unless
|
# The image an app's Dockerfile builds FROM, described under "Deployment"
|
||||||
# SMALLWEBWAF_COMMAND says servewaf, and an app's Dockerfile can build FROM it.
|
# in SPEC.md. It is the last stage, so a plain `docker build .` builds it.
|
||||||
# It is the last stage, so a plain `docker build .` builds it.
|
|
||||||
#
|
#
|
||||||
# ubuntu 26.04, 2026-09-27
|
# ubuntu 26.04, 2026-09-27
|
||||||
FROM ubuntu@sha256:f144425ff09be612d6d9ad965196e9cdc23dae1f42110a8a11a3e9a8198759f7
|
FROM ubuntu@sha256:f144425ff09be612d6d9ad965196e9cdc23dae1f42110a8a11a3e9a8198759f7
|
||||||
@@ -141,14 +140,10 @@ RUN mkdir /etc/sysusers.d \
|
|||||||
# apt also fetches the live archive's InRelease files, which change daily
|
# apt also fetches the live archive's InRelease files, which change daily
|
||||||
# and which the install does not use. The snapshot service is HTTPS only
|
# and which the install does not use. The snapshot service is HTTPS only
|
||||||
# and this image has no CA certificates yet, so this step uses the Go
|
# and this image has no CA certificates yet, so this step uses the Go
|
||||||
# image's. The snapshot service answers 503 at times; apt retries a
|
# image's.
|
||||||
# failed download, waiting twice as long before each retry, and this
|
|
||||||
# step gives it five retries, about half a minute in all, where its
|
|
||||||
# default of three gives up after seven seconds.
|
|
||||||
RUN --mount=type=bind,from=builder,source=/etc/ssl/certs/ca-certificates.crt,target=/tmp/go-image-ca.crt \
|
RUN --mount=type=bind,from=builder,source=/etc/ssl/certs/ca-certificates.crt,target=/tmp/go-image-ca.crt \
|
||||||
apt-get update --snapshot 20261001T000000Z \
|
apt-get update --snapshot 20261001T000000Z \
|
||||||
-o Acquire::https::CaInfo=/tmp/go-image-ca.crt \
|
-o Acquire::https::CaInfo=/tmp/go-image-ca.crt \
|
||||||
-o Acquire::Retries=5 \
|
|
||||||
&& printf '%s\n' \
|
&& printf '%s\n' \
|
||||||
'45f95ce276cdba3e41870516a130e03c58b8b7a79e9546b0efe9e526d255740c snapshot.ubuntu.com_ubuntu_20261001T000000Z_dists_resolute_InRelease' \
|
'45f95ce276cdba3e41870516a130e03c58b8b7a79e9546b0efe9e526d255740c snapshot.ubuntu.com_ubuntu_20261001T000000Z_dists_resolute_InRelease' \
|
||||||
'802e675dd9de4c7f3916434a95e7c1d8eec0e82886622d7805ab19a2c6fe0365 snapshot.ubuntu.com_ubuntu_20261001T000000Z_dists_resolute-updates_InRelease' \
|
'802e675dd9de4c7f3916434a95e7c1d8eec0e82886622d7805ab19a2c6fe0365 snapshot.ubuntu.com_ubuntu_20261001T000000Z_dists_resolute-updates_InRelease' \
|
||||||
@@ -158,7 +153,6 @@ RUN --mount=type=bind,from=builder,source=/etc/ssl/certs/ca-certificates.crt,tar
|
|||||||
&& DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \
|
&& DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \
|
||||||
--snapshot 20261001T000000Z \
|
--snapshot 20261001T000000Z \
|
||||||
-o Acquire::https::CaInfo=/tmp/go-image-ca.crt \
|
-o Acquire::https::CaInfo=/tmp/go-image-ca.crt \
|
||||||
-o Acquire::Retries=5 \
|
|
||||||
ca-certificates nix-bin runit \
|
ca-certificates nix-bin runit \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
@@ -198,11 +192,11 @@ RUN groupadd --system --gid 65532 smallwebwaf \
|
|||||||
|
|
||||||
# The state files' directory, SWWAF_STATE_DIR by default, where a volume
|
# The state files' directory, SWWAF_STATE_DIR by default, where a volume
|
||||||
# is mounted to keep them across deploys. The run script gives it to the
|
# is mounted to keep them across deploys. The run script gives it to the
|
||||||
# smallwebwaf user at each start of servewaf.
|
# smallwebwaf user at each start.
|
||||||
RUN mkdir /var/lib/smallwebwaf
|
RUN mkdir /var/lib/smallwebwaf
|
||||||
|
|
||||||
# The default rule file, in SWWAF_RULES_DIR by default, where a Dockerfile
|
# The default rule file, in SWWAF_RULES_DIR by default, where an app's
|
||||||
# built FROM this image can copy rule files of its own beside it.
|
# Dockerfile can copy rule files of its own beside it.
|
||||||
COPY share/rules.d/00-default.rules /etc/smallwebwaf/rules.d/00-default.rules
|
COPY share/rules.d/00-default.rules /etc/smallwebwaf/rules.d/00-default.rules
|
||||||
|
|
||||||
# runsvinit starts runit's runsvdir on /etc/service, where Ubuntu's sv
|
# runsvinit starts runit's runsvdir on /etc/service, where Ubuntu's sv
|
||||||
|
|||||||
@@ -1,12 +1,11 @@
|
|||||||
.PHONY: bootstrap setup test lint fmt fmt-check tidy check docker hooks build \
|
.PHONY: bootstrap setup test lint fmt fmt-check tidy check docker hooks build \
|
||||||
run example-app gitea-requests
|
run example-app
|
||||||
|
|
||||||
# Makefile targets are thin shims; the implementations live in script/
|
# Makefile targets are thin shims; the implementations live in script/
|
||||||
# per the scripts-to-rule-them-all pattern (see the Entrypoints section
|
# per the scripts-to-rule-them-all pattern (see the Entrypoints section
|
||||||
# of README.md). tidy writes go.mod and go.sum as `go mod tidy` does,
|
# of README.md). tidy writes go.mod and go.sum as `go mod tidy` does,
|
||||||
# which test checks. build and run are for working on the code by hand;
|
# which test checks. build and run are for working on the code by hand;
|
||||||
# example-app checks the image with an app built on it; gitea-requests
|
# example-app checks the image with an app built on it.
|
||||||
# reports the gitea requests the Core Rule Set refuses at the defaults.
|
|
||||||
|
|
||||||
bootstrap:
|
bootstrap:
|
||||||
@script/bootstrap
|
@script/bootstrap
|
||||||
@@ -46,6 +45,3 @@ run:
|
|||||||
|
|
||||||
example-app:
|
example-app:
|
||||||
@script/example-app
|
@script/example-app
|
||||||
|
|
||||||
gitea-requests:
|
|
||||||
@script/gitea-requests
|
|
||||||
|
|||||||
@@ -1,8 +1,6 @@
|
|||||||
// Command smallwebwaf is a web application firewall. `smallwebwaf rproxy` is
|
// Command smallwebwaf is a web application firewall for one app: it runs
|
||||||
// the reverse proxy in front of one app, which asks servewaf about each request.
|
// between traefik and the app, and passes requests through within its
|
||||||
// `smallwebwaf servewaf` decides, for many sites, and keeps what lasts.
|
// limits. Everything it does is in internal/smallwebwaf.
|
||||||
// `smallwebwaf healthcheck` is the container's health check. Everything it does
|
|
||||||
// is in internal/smallwebwaf.
|
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|||||||
@@ -1,60 +0,0 @@
|
|||||||
# Admin endpoints
|
|
||||||
|
|
||||||
The endpoints under `/_smallwebwaf/`, what each takes and what each answers.
|
|
||||||
`smallwebwaf servewaf` answers them at its own address, on the private network.
|
|
||||||
"Admin endpoints" in [`SPEC.md`](../SPEC.md) says how their tokens work.
|
|
||||||
|
|
||||||
- `GET /_smallwebwaf/healthz`: `200` and `ok`, to anyone.
|
|
||||||
- `GET /_smallwebwaf/metrics`: the metrics, for `SWWAF_METRICS_TOKEN` (see
|
|
||||||
[`metrics.md`](metrics.md)).
|
|
||||||
- `GET /_smallwebwaf/bans`: every ban held, under `bans`, each as `bans.json`
|
|
||||||
lists it. Needs `SWWAF_ADMIN_TOKEN`, as do the three below.
|
|
||||||
- `POST /_smallwebwaf/bans`: bans a netblock. The body is a JSON object of
|
|
||||||
`netblock`, `duration` and, if you like, `reason`. `netblock` is a netblock
|
|
||||||
such as `203.0.113.0/24`, or a client's address, which bans its IPv4 netblock
|
|
||||||
(`SWWAF_BAN_SCOPE_V4_PREFIX`) or its IPv6 group. `duration` is a duration such
|
|
||||||
as `1h` or `7d`, or `permanent`. A bad body is answered `400`, saying what is
|
|
||||||
wrong:
|
|
||||||
- one that is not such an object, or has another field;
|
|
||||||
- one longer than 4 KiB;
|
|
||||||
- one that names an IPv4-mapped netblock, or an address with a zone.
|
|
||||||
- `DELETE /_smallwebwaf/bans/<client>`: lifts every active ban covering the
|
|
||||||
address `<client>`. It answers `404` when none is active.
|
|
||||||
- `GET /_smallwebwaf/clients/<ip>`: what `servewaf` knows of the client. Under
|
|
||||||
`client`, its counters and history as `clients.json` holds them, or `null`
|
|
||||||
when `clients.json` does not hold it. Under `bans`, every ban covering the
|
|
||||||
address, with its notes.
|
|
||||||
- `POST /_smallwebwaf/report`: takes `rproxy`'s report, without a token. It
|
|
||||||
answers `204`, or `400` to a body that is no report (see "The report" in
|
|
||||||
[`SPEC.md`](../SPEC.md)).
|
|
||||||
|
|
||||||
No check runs on these requests. A missing or wrong token is answered `401`. An
|
|
||||||
endpoint whose token is unset answers `404`, as does any other path.
|
|
||||||
|
|
||||||
`smallwebwaf rproxy` answers only the health check of these, at its own address.
|
|
||||||
It asks `servewaf` about any other request under `/_smallwebwaf/`. One that
|
|
||||||
passes the checks is answered `404`, so that the app never sees that prefix (see
|
|
||||||
"`rproxy`" in [`SPEC.md`](../SPEC.md)).
|
|
||||||
|
|
||||||
## Examples
|
|
||||||
|
|
||||||
With `SWWAF_ADMIN_TOKEN` set, the token in `$TOKEN`, and `servewaf` reached as
|
|
||||||
`servewaf:8080` on its network:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
# Every ban.
|
|
||||||
curl -H "Authorization: Bearer $TOKEN" http://servewaf:8080/_smallwebwaf/bans
|
|
||||||
|
|
||||||
# Ban 203.0.113.0/24 for seven days.
|
|
||||||
curl -H "Authorization: Bearer $TOKEN" \
|
|
||||||
--json '{"netblock": "203.0.113.0/24", "duration": "7d", "reason": "probes for logins"}' \
|
|
||||||
http://servewaf:8080/_smallwebwaf/bans
|
|
||||||
|
|
||||||
# Lift the bans on 203.0.113.9.
|
|
||||||
curl -H "Authorization: Bearer $TOKEN" -X DELETE \
|
|
||||||
http://servewaf:8080/_smallwebwaf/bans/203.0.113.9
|
|
||||||
|
|
||||||
# What servewaf knows of 203.0.113.9.
|
|
||||||
curl -H "Authorization: Bearer $TOKEN" \
|
|
||||||
http://servewaf:8080/_smallwebwaf/clients/203.0.113.9
|
|
||||||
```
|
|
||||||
-137
@@ -1,137 +0,0 @@
|
|||||||
# Alerts
|
|
||||||
|
|
||||||
The alert events, the JSON object the webhook gets, and the messages Slack and
|
|
||||||
ntfy get. "Alerting" in [`SPEC.md`](../SPEC.md) says when an alert is sent, held
|
|
||||||
back or dropped.
|
|
||||||
|
|
||||||
## Events
|
|
||||||
|
|
||||||
- `ban`: a ban `servewaf` made.
|
|
||||||
- `permanent_ban`: a permanent ban it made, or an existing ban made permanent by
|
|
||||||
a request.
|
|
||||||
- `waf_block`: the Core Rule Set scored a request at or over the threshold. The
|
|
||||||
request was refused unless the alert's `detail` gives `mode`: `detect` or
|
|
||||||
`observe`.
|
|
||||||
- `anomaly`: a count over an anomaly threshold.
|
|
||||||
- `reputation_hit`: a request from a client on a blocklist, the CrowdSec
|
|
||||||
decision list or a DNS blocklist, or with an AbuseIPDB score that is a hit.
|
|
||||||
One alert per source.
|
|
||||||
- `source_failure`: one of:
|
|
||||||
- GeoJS failing;
|
|
||||||
- a list that cannot be fetched;
|
|
||||||
- a DNS blocklist or AbuseIPDB failing or refusing a query;
|
|
||||||
- the day's AbuseIPDB checks used up.
|
|
||||||
- `file_error`: one of:
|
|
||||||
- a rule file or state file edit with an error;
|
|
||||||
- a state file that cannot be written;
|
|
||||||
- a replacement lookup database that cannot be read.
|
|
||||||
- `summary`: the hourly summary of alerts held back.
|
|
||||||
|
|
||||||
## The JSON object
|
|
||||||
|
|
||||||
This is the alert for a ban for a broken rate limit, shown indented; it is sent
|
|
||||||
on one line:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"instance": "fsn1/servewaf",
|
|
||||||
"time": "2026-10-06T12:00:50.123461Z",
|
|
||||||
"event": "ban",
|
|
||||||
"host": "app.example",
|
|
||||||
"client": "203.0.113.9",
|
|
||||||
"netblock": "203.0.113.9/32",
|
|
||||||
"asn": "",
|
|
||||||
"as_name": "",
|
|
||||||
"country": "",
|
|
||||||
"reason": "requests per minute over the limit of 500",
|
|
||||||
"detail": {
|
|
||||||
"ban_expires": "2026-10-06T13:00:50.123Z",
|
|
||||||
"cause": "limit",
|
|
||||||
"notes": {
|
|
||||||
"asn": "",
|
|
||||||
"as_name": "",
|
|
||||||
"country": "",
|
|
||||||
"kind": "requests",
|
|
||||||
"limit": 500,
|
|
||||||
"window": "minute",
|
|
||||||
"count": 501,
|
|
||||||
"request": {
|
|
||||||
"time": "2026-10-06T12:00:50.123456789Z",
|
|
||||||
"method": "GET",
|
|
||||||
"host": "app.example",
|
|
||||||
"path": "/owner/repo/commits/branch/main?page=501",
|
|
||||||
"status": 403,
|
|
||||||
"user_agent": "scraper/1.0"
|
|
||||||
},
|
|
||||||
"requests": 501,
|
|
||||||
"refused": 0,
|
|
||||||
"earlier_bans": {
|
|
||||||
"limit": 0,
|
|
||||||
"attack": 0,
|
|
||||||
"admin": 0,
|
|
||||||
"crowdsec": 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"suppressed_repeats": 0
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
- `instance`: `SWWAF_INSTANCE_NAME`. `time`: when the alert was raised. `event`:
|
|
||||||
one of the events above.
|
|
||||||
- `host`: the site of the request that raised the alert, since one `servewaf`
|
|
||||||
serves many sites. It is left out of `source_failure`, `file_error` and
|
|
||||||
`summary`.
|
|
||||||
- `client`: the address whose request raised the alert. Empty for
|
|
||||||
`source_failure`, `file_error` and `summary`.
|
|
||||||
- `netblock`: empty for `source_failure`, `file_error` and `summary`; otherwise
|
|
||||||
one of:
|
|
||||||
- the ban's netblock;
|
|
||||||
- the netblock an anomaly counted, empty in the `asn` and `total` scopes;
|
|
||||||
- for the other events, the client as rate limiting counts it: an IPv4
|
|
||||||
address as a /32, or an IPv6 group.
|
|
||||||
- `asn`, `as_name` and `country`: the client's, as known when the alert was
|
|
||||||
raised. They are empty when GeoJS had not answered yet.
|
|
||||||
- `reason`: a short sentence, such as
|
|
||||||
`requests per minute over the limit of 500` or `listed by a blocklist`.
|
|
||||||
- `detail`: what is particular to the event:
|
|
||||||
- for a ban, its `cause`, `ban_expires` and `notes` (the notes are listed
|
|
||||||
under `bans.json` in [`state-files.md`](state-files.md));
|
|
||||||
- for an anomaly, the `scope`, `window`, `kind`, `count` and `threshold`;
|
|
||||||
- for a reputation hit, the `source` and, for AbuseIPDB, the `score`;
|
|
||||||
- for a Core Rule Set match, the `rule_ids`, `score`, `method` and `path`;
|
|
||||||
- for a source failure, the `source` and `error`;
|
|
||||||
- for a file error, the `file` and `error`;
|
|
||||||
- for a summary, the `hour` it covers, the `count` of alerts held back, and
|
|
||||||
that count by event in `events`;
|
|
||||||
- `mode`, when the request was not refused: `detect` or `observe`.
|
|
||||||
- `suppressed_repeats`: how many repeats the cooldown held back before this
|
|
||||||
alert.
|
|
||||||
|
|
||||||
## Slack and ntfy
|
|
||||||
|
|
||||||
Slack and ntfy get a message with a title and a text. The title is the instance
|
|
||||||
and the event, such as `fsn1/servewaf: ban`. The text is the `reason`, then one
|
|
||||||
line for each of these that the alert has:
|
|
||||||
|
|
||||||
- the host, the client, the netblock and the country;
|
|
||||||
- the file, the source, the error and the mode from `detail`;
|
|
||||||
- the suppressed repeats, if any.
|
|
||||||
|
|
||||||
Slack gets the title in bold and the text below it, with `&`, `<` and `>`
|
|
||||||
escaped. ntfy gets the text as the message, with these headers:
|
|
||||||
|
|
||||||
- `Title`: the title;
|
|
||||||
- `Authorization`: the token, as `Bearer <token>`, while
|
|
||||||
`SWWAF_ALERT_NTFY_TOKEN` is set;
|
|
||||||
- `Priority` and `Tags`: by event, as listed below.
|
|
||||||
|
|
||||||
The priority and tag by event:
|
|
||||||
|
|
||||||
- `ban`: priority `default`, tag `no_entry`.
|
|
||||||
- `permanent_ban`: priority `high`, tag `no_entry`.
|
|
||||||
- `waf_block`: priority `default`, tag `shield`.
|
|
||||||
- `anomaly`: priority `high`, tag `chart_with_upwards_trend`.
|
|
||||||
- `reputation_hit`: priority `low`, tag `label`.
|
|
||||||
- `source_failure` and `file_error`: priority `high`, tag `warning`.
|
|
||||||
- `summary`: priority `default`, tag `bar_chart`.
|
|
||||||
-220
@@ -1,220 +0,0 @@
|
|||||||
# smallwebwaf in front of gitea
|
|
||||||
|
|
||||||
These notes record what `smallwebwaf` does in front of gitea at the default
|
|
||||||
settings. They are observations, not requirements; the requirements are in
|
|
||||||
[`SPEC.md`](../SPEC.md). `script/gitea-requests` runs gitea behind `rproxy` and
|
|
||||||
`servewaf`, and sends through them a sample of gitea's ordinary use and the
|
|
||||||
requests named below. It reports each request the Core Rule Set refuses, so that
|
|
||||||
these notes can be checked again when gitea or the Core Rule Set changes
|
|
||||||
version.
|
|
||||||
|
|
||||||
## What the defaults cover
|
|
||||||
|
|
||||||
- SSH access to gitea does not pass through `smallwebwaf` and is not protected
|
|
||||||
by it.
|
|
||||||
- A clone is a response. It fits the defaults of 30 minutes and 1 GiB unless the
|
|
||||||
repository is large or the link slow. A repository whose clone is larger than
|
|
||||||
1 GiB needs `SWRP_MAX_RESPONSE_SIZE` raised, and the byte limits with it
|
|
||||||
("Byte limits, per client" in `SPEC.md`).
|
|
||||||
- A push is a request, and so is every other upload:
|
|
||||||
- a git push;
|
|
||||||
- an LFS object;
|
|
||||||
- a container image layer;
|
|
||||||
- a package file;
|
|
||||||
- a release attachment.
|
|
||||||
|
|
||||||
At the defaults, an upload larger than 100 MiB or taking more than 60
|
|
||||||
seconds is cut off ("`rproxy`'s limits" in `SPEC.md`). A gitea that takes
|
|
||||||
large uploads needs `SWRP_MAX_REQUEST_SIZE` and `SWRP_MAX_REQUEST_TIME_MS`
|
|
||||||
raised to fit.
|
|
||||||
|
|
||||||
- Archive download and blame or history pages are what scrapers hammer. The
|
|
||||||
request limits do most of the work there.
|
|
||||||
- An Actions runner that reaches gitea through `smallwebwaf` sends requests all
|
|
||||||
day. One older than version 0.4 (April 2026) asks for work every 2 seconds,
|
|
||||||
43,200 requests a day. While it runs a job, it reports the job's log and state
|
|
||||||
every second instead, two requests a second. So one runner busy all day sends
|
|
||||||
120 requests a minute and 172,800 a day, under the request limits. Two such
|
|
||||||
runners on one address, each busy all day, make 345,600 and pass the day limit
|
|
||||||
of 300,000. Three busy at once make 21,600 an hour and pass the hour limit of
|
|
||||||
20,000. Five busy at once make 600 a minute and pass the minute limit of 500.
|
|
||||||
Their address is then banned, their running jobs fail, and each repeat within
|
|
||||||
a day makes the ban longer. Newer runners ask for work less often. Put the
|
|
||||||
runners' addresses in `SWWAF_RATE_LIMIT_EXEMPT_NETS`.
|
|
||||||
|
|
||||||
## What the Core Rule Set lets through
|
|
||||||
|
|
||||||
At the defaults, the Core Rule Set lets gitea's ordinary use through, apart from
|
|
||||||
the refusals in the next section:
|
|
||||||
|
|
||||||
- browsing and views of files in a repository, with their history, blame and the
|
|
||||||
file tree;
|
|
||||||
- diffs, including their hidden lines and large files, and pull request review;
|
|
||||||
- git's clone, fetch and push over HTTP;
|
|
||||||
- signing in, including the return to the page a visitor came from, and sign-in
|
|
||||||
with Git Credential Manager, git-credential-oauth or tea;
|
|
||||||
- the API's calls for a file and its commits;
|
|
||||||
- pushing and pulling container images and packages;
|
|
||||||
- Actions runners, and artifacts uploaded with `actions/upload-artifact@v4`;
|
|
||||||
- posting issues, pull requests, comments, wiki pages and files saved in the web
|
|
||||||
editor, code included, since no body is read;
|
|
||||||
- the page gitea shows after a change, with its message naming what was done.
|
|
||||||
|
|
||||||
## What the Core Rule Set still refuses
|
|
||||||
|
|
||||||
Each refusal answers only that request ("Bans" in `SPEC.md` says what it counts
|
|
||||||
toward). The request log names the rule in `waf_rule_ids`, which
|
|
||||||
`SWWAF_WAF_DISABLED_RULES` can switch off.
|
|
||||||
|
|
||||||
### A query string that reads as an attack
|
|
||||||
|
|
||||||
Most often this is a search. A search term is a query parameter, which the Core
|
|
||||||
Rule Set checks against its lists of system files and commands (see the fourth
|
|
||||||
change under "Attack detection" in `SPEC.md`). A release attachment uploaded
|
|
||||||
through the API as `docker-compose.yml` is refused the same way. Examples:
|
|
||||||
|
|
||||||
- a name on those lists, such as `package.json`, `.gitignore` or
|
|
||||||
`docker-compose.yml`;
|
|
||||||
- text that starts with a command name, such as `python3` or `ssh key`, or has
|
|
||||||
one right after a `/` in its first word, such as `feat/docker-support`;
|
|
||||||
- a shell command with its options or a system path (`ls -la`, `sed -i`,
|
|
||||||
`/bin/sh`), or a command in backticks;
|
|
||||||
- script code (`fetch(`, `${VAR}`, `process.env`), HTML (`<img src=`) or an SQL
|
|
||||||
statement (`SELECT * FROM users WHERE`);
|
|
||||||
- a URL naming an IP address or `localhost`.
|
|
||||||
|
|
||||||
### A branch, tag or file name that starts with a command name
|
|
||||||
|
|
||||||
The Core Rule Set's list of commands (932260) holds names such as `docker-`,
|
|
||||||
`python3`, `ansible`, `base64` and `whoami`. A query parameter holding a branch,
|
|
||||||
tag or file name that starts with one is refused. So is one that has such a name
|
|
||||||
right after a `/`, such as `feat/docker-support`, `fix/docker-build` or
|
|
||||||
`renovate/docker-build-push-action-6.x`. gitea's own pages send the name in a
|
|
||||||
query parameter that 930120, 932160 and 932260 still check:
|
|
||||||
|
|
||||||
- `refSubUrl`, the branch or tag of a directory listing. A listing sends it when
|
|
||||||
it asks for its entries' last commits in a second request. It does so whenever
|
|
||||||
gitea takes more than a second to work them out;
|
|
||||||
- `name`, when a branch is deleted or restored on the branches page;
|
|
||||||
- `tag`, when a release is started from a tag;
|
|
||||||
- `template`, the file of an issue template, when an issue is opened from it;
|
|
||||||
- `rule_name`, when a branch protection rule is opened for editing in the
|
|
||||||
repository's settings or has just been saved.
|
|
||||||
|
|
||||||
What a visitor sees, for a branch named `docker-build`:
|
|
||||||
|
|
||||||
- A directory listing that makes that second request leaves those last commits
|
|
||||||
out and shows an error.
|
|
||||||
- The branches page shows an error instead of deleting or restoring the branch.
|
|
||||||
- A release started from such a tag gets the `403` answer of `smallwebwaf` in
|
|
||||||
place of its form. So does an issue opened from such a template, and such a
|
|
||||||
rule opened for editing.
|
|
||||||
- Saving such a rule stores it. gitea then returns to the branch settings page,
|
|
||||||
which gets the `403` answer, so the save looks as if it failed.
|
|
||||||
|
|
||||||
### A file name in a query parameter that trips the code rules
|
|
||||||
|
|
||||||
The Core Rule Set's rules for Java, script, PowerShell and SQL find what they
|
|
||||||
look for in some file names. This holds too in the ten query parameters that
|
|
||||||
930120, 932160 and 932260 do not check (see the fourth change under "Attack
|
|
||||||
detection" in `SPEC.md`). Examples:
|
|
||||||
|
|
||||||
- `runtime` in OpenJDK's `src/java.base/share/classes/java/lang/Runtime.java`
|
|
||||||
(944110);
|
|
||||||
- a Java class, as in `java.lang.String.md` (944130);
|
|
||||||
- `document.write.js` (941180);
|
|
||||||
- `process.env.d.ts` (934100);
|
|
||||||
- `__proto__` (934130);
|
|
||||||
- the PowerShell command `Install-Script`, as in `install-script.sh` (932120);
|
|
||||||
- a database's name, as in PostgreSQL's `information_schema.sql` (942140).
|
|
||||||
|
|
||||||
For such a file, these requests are refused:
|
|
||||||
|
|
||||||
- the API's list of its commits (`path`);
|
|
||||||
- the API's call for the contents of files that names it in its query (`body`);
|
|
||||||
- the commit graph of that file alone (`file`);
|
|
||||||
- the hidden lines of its diff (`path`), which stay hidden;
|
|
||||||
- the sign-in page, for a visitor who opens a page of the file that needs
|
|
||||||
signing in, such as its edit page (`redirect_to`). The visitor can still sign
|
|
||||||
in by opening the sign-in page itself.
|
|
||||||
|
|
||||||
### A path
|
|
||||||
|
|
||||||
- A file name ending in `~`, or an `.xhtml` file whose path holds a space.
|
|
||||||
- A path that holds what the Core Rule Set takes for a Java class (944130), as
|
|
||||||
`java.lang.String.md` and
|
|
||||||
`META-INF/services/org.apache.commons.logging.LogFactory` do.
|
|
||||||
- A path that holds what it takes for Node.js code (934100), as
|
|
||||||
`process.env.d.ts` does.
|
|
||||||
- A path with a directory named `...` (930100, 930110), such as `.../notes.txt`.
|
|
||||||
|
|
||||||
For such a file, these are refused:
|
|
||||||
|
|
||||||
- the page of the file;
|
|
||||||
- its raw file;
|
|
||||||
- its history and its blame;
|
|
||||||
- its edit page.
|
|
||||||
|
|
||||||
The directory that holds it lists it, and a clone carries it.
|
|
||||||
|
|
||||||
### A container image push that reuses a layer
|
|
||||||
|
|
||||||
A push can reuse a layer of an image docker has pushed to that gitea before.
|
|
||||||
docker then asks gitea to take the layer from that image, naming it in the query
|
|
||||||
parameter `from`, which 932260 checks. The request is refused when the image's
|
|
||||||
owner starts with an entry on the list of commands. It is refused too when the
|
|
||||||
image's name after the owner starts with one. Examples are `owner/python3-app`,
|
|
||||||
`owner/docker-base`, and any image of an owner named `ansible` or `docker-team`.
|
|
||||||
|
|
||||||
What happens next depends on docker's image store. A docker that keeps its
|
|
||||||
images in containerd's image store fails the push, and fails each new try the
|
|
||||||
same way. A new install of Docker Engine 29 does this. A docker that keeps them
|
|
||||||
in its own older image store uploads the layer instead. The push then succeeds
|
|
||||||
after one refusal for each layer it reuses, and each of those refusals counts
|
|
||||||
toward the error burst ("Attack detection" in `SPEC.md`).
|
|
||||||
|
|
||||||
### Artifact uploads from `actions/upload-artifact@v3`
|
|
||||||
|
|
||||||
This action sends the header `Content-Range`, which the Core Rule Set refuses
|
|
||||||
(920450). The action sends two files at a time, does not retry a `403`, and
|
|
||||||
sends no further file once one is refused. So an upload is refused once or
|
|
||||||
twice, however many files it holds, and the step fails. One upload does not ban
|
|
||||||
the runner. The error burst ("Attack detection" in `SPEC.md`) bans it after more
|
|
||||||
than 30 such refusals from its address in a minute, as when many jobs upload at
|
|
||||||
once. `actions/upload-artifact@v4` does not send the header.
|
|
||||||
|
|
||||||
## What body inspection refuses
|
|
||||||
|
|
||||||
With `SWWAF_WAF_BODY_LIMIT` set, the Core Rule Set reads request bodies too. In
|
|
||||||
front of gitea it then refuses these, until their rule ids, named in the request
|
|
||||||
log, are added to `SWWAF_WAF_DISABLED_RULES`:
|
|
||||||
|
|
||||||
- issue and comment text, wiki pages and files saved in the web editor that hold
|
|
||||||
shell commands or code (932125, 932235, 932250 and others);
|
|
||||||
- package descriptions that show code;
|
|
||||||
- PyPI uploads (922130);
|
|
||||||
- attachments named like `debug.log` or `config.yml` (932180).
|
|
||||||
|
|
||||||
## What `script/gitea-requests` sends
|
|
||||||
|
|
||||||
Of the requests above, the script sends:
|
|
||||||
|
|
||||||
- for each file named under "A file name in a query parameter that trips the
|
|
||||||
code rules", the API's list of its commits;
|
|
||||||
- for `Runtime.java` alone, the commit graph of that file, the hidden lines of
|
|
||||||
its diff, and the API's call for the contents of files that names it;
|
|
||||||
- the sign-in page for a visitor who opens the edit page of `document.write.js`;
|
|
||||||
- the pages of `java.lang.String.md`,
|
|
||||||
`META-INF/services/org.apache.commons.logging.LogFactory`, `process.env.d.ts`
|
|
||||||
and `.../notes.txt`;
|
|
||||||
- one container image push that reuses a layer, which it does not try again.
|
|
||||||
|
|
||||||
It does not send the raw file, history, blame or edit page of those four files,
|
|
||||||
or the pages of the directories `META-INF/services` and `...`. It also sends a
|
|
||||||
short sample of gitea's ordinary use:
|
|
||||||
|
|
||||||
- signing in;
|
|
||||||
- pages of files and commits, and a diff;
|
|
||||||
- the API;
|
|
||||||
- a push, and clone and fetch over HTTP;
|
|
||||||
- two container image pushes.
|
|
||||||
@@ -1,39 +0,0 @@
|
|||||||
# What the image holds
|
|
||||||
|
|
||||||
The contents of the `smallwebwaf` image, and how each part is pinned. The image
|
|
||||||
runs `rproxy` by default, or `servewaf` with `SMALLWEBWAF_COMMAND=servewaf`.
|
|
||||||
"Deployment" in [`SPEC.md`](../SPEC.md) says why each part is there and why it
|
|
||||||
is pinned that way.
|
|
||||||
|
|
||||||
## Contents
|
|
||||||
|
|
||||||
- Ubuntu 26.04 LTS, pinned by digest.
|
|
||||||
- `ca-certificates`, `nix-bin` and `runit` from Ubuntu, from the snapshot of its
|
|
||||||
archive at `20261001T000000Z`.
|
|
||||||
- Nix, with nixpkgs `nixos-26.05.11045.774debe7a0d1` set up for root as
|
|
||||||
`nixpkgs`. No Nix daemon; `build-users-group =` in `/etc/nix/nix.conf`. Root's
|
|
||||||
Nix profile, `/nix/var/nix/profiles/default/bin`, last on the `PATH`.
|
|
||||||
Unpacked, nixpkgs takes about 500 MiB.
|
|
||||||
- `runsvinit` as the entrypoint, built from `github.com/peterbourgon/runsvinit`
|
|
||||||
at a fixed commit hash.
|
|
||||||
- The `smallwebwaf` binary, and the user `smallwebwaf` (uid and gid 65532).
|
|
||||||
- The service directory `/etc/service/smallwebwaf`, with the `run` script
|
|
||||||
`share/smallwebwaf.run`.
|
|
||||||
- `/var/lib/smallwebwaf` for `servewaf`'s state files, and
|
|
||||||
`/etc/smallwebwaf/rules.d` with the default rule file
|
|
||||||
`share/rules.d/00-default.rules`.
|
|
||||||
- `EXPOSE 8080`, and the `HEALTHCHECK` that runs `smallwebwaf healthcheck`.
|
|
||||||
|
|
||||||
## Pins
|
|
||||||
|
|
||||||
- Ubuntu's packages: the snapshot service, `--snapshot 20261001T000000Z` on both
|
|
||||||
`apt-get update` and `apt-get install`. The SHA-256 hash of each of the
|
|
||||||
snapshot's `InRelease` files is named in the `Dockerfile` and checked after
|
|
||||||
`apt-get update`. The hashes are the amd64 archive's.
|
|
||||||
- nixpkgs: the release file `nixexprs.tar.xz` of
|
|
||||||
`nixos-26.05.11045.774debe7a0d1` on `releases.nixos.org`, named with its
|
|
||||||
SHA-256 hash.
|
|
||||||
- `runsvinit`: the archived repository, unchanged since 2015, at its last commit
|
|
||||||
(`b4b2c78`, eight commits after `v2.0.0`), by commit hash, with a `go.mod` the
|
|
||||||
build writes.
|
|
||||||
- The Go image that builds `smallwebwaf`: by digest.
|
|
||||||
@@ -1,93 +0,0 @@
|
|||||||
# The metrics
|
|
||||||
|
|
||||||
What `GET /_smallwebwaf/metrics` answers with, at `servewaf`'s own address.
|
|
||||||
"Metrics endpoint" in [`SPEC.md`](../SPEC.md) describes the endpoint. Prometheus
|
|
||||||
keeps the `instance` label as `exported_instance` unless the scrape sets
|
|
||||||
`honor_labels: true`. `rproxy` has no metrics.
|
|
||||||
|
|
||||||
## Traffic
|
|
||||||
|
|
||||||
- `smallwebwaf_requests_total`, `smallwebwaf_request_bytes_total` and
|
|
||||||
`smallwebwaf_response_bytes_total`, by `status_class` such as `2xx` and by
|
|
||||||
`action`;
|
|
||||||
- `smallwebwaf_request_duration_seconds` and
|
|
||||||
`smallwebwaf_upstream_duration_seconds`, as histograms;
|
|
||||||
- `smallwebwaf_requests_in_flight`: the questions and requests to its own
|
|
||||||
endpoints that `servewaf` is handling.
|
|
||||||
|
|
||||||
## Limits and bans
|
|
||||||
|
|
||||||
- `smallwebwaf_rate_limit_hits_total`, by `window` and by `kind` (`requests`,
|
|
||||||
`bytes` or `refusals`);
|
|
||||||
- `smallwebwaf_offences_total`, by `kind`;
|
|
||||||
- `smallwebwaf_bans_made_total`, by `cause`;
|
|
||||||
- `smallwebwaf_active_bans` and `smallwebwaf_permanent_bans`.
|
|
||||||
|
|
||||||
## Attack detection
|
|
||||||
|
|
||||||
- `smallwebwaf_rule_matches_total`, by `rule_id` and `action`, and
|
|
||||||
`smallwebwaf_rules_loaded`;
|
|
||||||
- `smallwebwaf_waf_matches_total`, by `mode` and `rule_id`, for the rules that
|
|
||||||
fired.
|
|
||||||
|
|
||||||
## Countries and AS numbers
|
|
||||||
|
|
||||||
For the `SWWAF_METRICS_TOP_N` busiest of each, the rest as `other`:
|
|
||||||
|
|
||||||
- `smallwebwaf_country_requests_total`,
|
|
||||||
`smallwebwaf_country_request_bytes_total`,
|
|
||||||
`smallwebwaf_country_response_bytes_total` and
|
|
||||||
`smallwebwaf_country_list_refusals_total`, by `country`;
|
|
||||||
- `smallwebwaf_asn_requests_total`, `smallwebwaf_asn_request_bytes_total` and
|
|
||||||
`smallwebwaf_asn_response_bytes_total`, by `asn`.
|
|
||||||
|
|
||||||
## Lookups
|
|
||||||
|
|
||||||
- `smallwebwaf_geojs_requests_total` and `smallwebwaf_geojs_failures_total`;
|
|
||||||
- `smallwebwaf_geojs_unanswered_total`: the requests that needed an answer and
|
|
||||||
went on without it;
|
|
||||||
- with the file source,
|
|
||||||
`smallwebwaf_lookup_database_last_read_timestamp_seconds` and
|
|
||||||
`smallwebwaf_lookup_database_read_failures_total`.
|
|
||||||
|
|
||||||
## Reputation sources
|
|
||||||
|
|
||||||
By `source`: each list's URL, each zone, and `abuseipdb`:
|
|
||||||
|
|
||||||
- `smallwebwaf_reputation_hits_total`, `smallwebwaf_reputation_queries_total`
|
|
||||||
and `smallwebwaf_reputation_failures_total`;
|
|
||||||
- `smallwebwaf_reputation_last_fetch_timestamp_seconds`, for a list;
|
|
||||||
- `smallwebwaf_reputation_daily_budget_remaining`, for AbuseIPDB.
|
|
||||||
|
|
||||||
## State files
|
|
||||||
|
|
||||||
By `file`:
|
|
||||||
|
|
||||||
- `smallwebwaf_state_file_writes_total`,
|
|
||||||
`smallwebwaf_state_file_write_failures_total`,
|
|
||||||
`smallwebwaf_state_file_last_write_timestamp_seconds` and
|
|
||||||
`smallwebwaf_state_file_size_bytes`;
|
|
||||||
- `smallwebwaf_state_file_edits_taken_in_total` and
|
|
||||||
`smallwebwaf_state_file_edits_set_aside_total`;
|
|
||||||
- `smallwebwaf_tracked_clients`: the clients in the table.
|
|
||||||
|
|
||||||
## Remote log, alerts and the process
|
|
||||||
|
|
||||||
- while a syslog server is set, `smallwebwaf_remote_log_lines_sent_total`,
|
|
||||||
`smallwebwaf_remote_log_lines_dropped_total` and
|
|
||||||
`smallwebwaf_remote_log_buffer_depth`;
|
|
||||||
- by `destination`, `smallwebwaf_alerts_sent_total`,
|
|
||||||
`smallwebwaf_alerts_failed_total`, `smallwebwaf_alerts_suppressed_total` and
|
|
||||||
`smallwebwaf_alerts_dropped_total`;
|
|
||||||
- Go's own `go_` metrics and the process's `process_` metrics.
|
|
||||||
|
|
||||||
## When a request is counted
|
|
||||||
|
|
||||||
`servewaf` counts a denied request at its decision, and a request to its own
|
|
||||||
endpoints when it is answered. It counts an allowed request when its report
|
|
||||||
comes, with the status, bytes and times the report gives. These are not counted:
|
|
||||||
|
|
||||||
- a report itself;
|
|
||||||
- an allowed request whose report never comes;
|
|
||||||
- a question refused with `400`;
|
|
||||||
- a question whose body stopped arriving.
|
|
||||||
@@ -1,206 +0,0 @@
|
|||||||
# The request log's fields
|
|
||||||
|
|
||||||
The fields of the JSON objects `rproxy` and `servewaf` write about each request.
|
|
||||||
"Request log" and "The log" in [`SPEC.md`](../SPEC.md) say when a line is
|
|
||||||
written and how the log is sent to a syslog server. A field that does not apply
|
|
||||||
to a request is left out.
|
|
||||||
|
|
||||||
## `servewaf`'s lines
|
|
||||||
|
|
||||||
`smallwebwaf servewaf` writes two lines for each question it answers, joined by
|
|
||||||
`request_id`. The field `servewaf_line` says which of the two a line is:
|
|
||||||
|
|
||||||
- `decision`: the line it writes once it has answered the question;
|
|
||||||
- `report`: the line it writes when `rproxy`'s report comes, with everything
|
|
||||||
known. The repetition is intended.
|
|
||||||
|
|
||||||
A request to its own endpoints writes one line, without `servewaf_line`. A
|
|
||||||
report writes no line but the `report` line of the request it names.
|
|
||||||
|
|
||||||
These questions write no line:
|
|
||||||
|
|
||||||
- a question refused with `400` (see "The question" under "`servewaf`" in
|
|
||||||
[`SPEC.md`](../SPEC.md));
|
|
||||||
- a question whose body stopped arriving, which gets no answer.
|
|
||||||
|
|
||||||
A `report` line:
|
|
||||||
|
|
||||||
```
|
|
||||||
{"type":"request","servewaf_line":"report","time":"2026-10-03T12:00:00.123Z","instance":"fsn1/servewaf","client_ip":"203.0.113.9","method":"GET","scheme":"https","host":"app.example","path":"/","query":"","protocol":"HTTP/1.1","status":200,"request_bytes":0,"response_bytes":5120,"referer":"","user_agent":"curl/8.9.1","request_id":"7Q2NHZ4KJ3VXW5YB6R3MEFTD2A","peer_ip":"172.18.0.2","forwarded_for":"203.0.113.9","client_group":"203.0.113.9/32","asn":"AS64496","as_name":"Example Net","country":"DE","request_headers":{"accept":"*/*"},"action":"forward","counts":{"minute":1,"hour":12,"day":40,"minute_bytes":5120,"hour_bytes":61440,"day_bytes":204800},"waf_score":0,"duration_total":3.217,"duration_checks":0.041,"duration_waf":0.029,"duration_upstream_total":3.104}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Every line
|
|
||||||
|
|
||||||
Every one of these lines has:
|
|
||||||
|
|
||||||
- `type`: `request` (the process's own messages have `process`);
|
|
||||||
- the standard web log fields;
|
|
||||||
- `request_id`, `peer_ip` and `client_group`;
|
|
||||||
- `asn`, `as_name` and `country`;
|
|
||||||
- `action` and `duration_total`.
|
|
||||||
|
|
||||||
### Standard web log fields
|
|
||||||
|
|
||||||
- `time`: when `servewaf` had read the question's body, RFC 3339 with
|
|
||||||
milliseconds, in UTC. The `report` line gives the same time as the `decision`
|
|
||||||
line.
|
|
||||||
- `instance`: `SWWAF_INSTANCE_NAME`.
|
|
||||||
- `client_ip`: the visitor's address, as `rproxy` worked it out.
|
|
||||||
- `method`, `path` and `query`: the visitor's request.
|
|
||||||
- `scheme`: how the visitor reached `rproxy` or the trusted proxy in front of
|
|
||||||
it.
|
|
||||||
- `host`: the site the visitor asked for.
|
|
||||||
- `protocol`: the question's own, `HTTP/1.1`.
|
|
||||||
- `status`:
|
|
||||||
- on the `decision` line, what `servewaf` answered: `200` for `allow`, the
|
|
||||||
status of `deny`, or `0` for `close`, as the visitor is then sent nothing;
|
|
||||||
- on the `report` line, what the visitor got, `0` if nothing.
|
|
||||||
- `request_bytes`:
|
|
||||||
- on the `decision` line, how much of the question's body the Core Rule Set
|
|
||||||
read;
|
|
||||||
- on the `report` line, the bytes of the visitor's body that `rproxy` read,
|
|
||||||
with what a WebSocket carried from the visitor.
|
|
||||||
- `response_bytes`:
|
|
||||||
- on the `decision` line, the body of `servewaf`'s answer;
|
|
||||||
- on the `report` line, the bytes of the answer's body, with what a
|
|
||||||
WebSocket carried from the app.
|
|
||||||
- `referer` and `user_agent`.
|
|
||||||
|
|
||||||
### Request detail
|
|
||||||
|
|
||||||
- `request_id`: the id `rproxy` made for the request when it arrived.
|
|
||||||
- `peer_ip`: the address of `rproxy`'s TCP peer, normally traefik.
|
|
||||||
- `forwarded_for`: the `X-Forwarded-For` header the visitor's request came with.
|
|
||||||
- `client_group`: the client as rate limiting counts it, an IPv4 address as a
|
|
||||||
/32 or an IPv6 group.
|
|
||||||
- `asn`, `as_name` and `country`: the client's, as looked up. Each is empty
|
|
||||||
when:
|
|
||||||
- lookups are off;
|
|
||||||
- the client is not looked up;
|
|
||||||
- the source cannot place the client;
|
|
||||||
- GeoJS had not answered by the time `servewaf` answered.
|
|
||||||
- `content_type`: the visitor's `Content-Type`.
|
|
||||||
- `content_length`: the body length the visitor announced.
|
|
||||||
- `request_headers`: the headers listed in `SWWAF_LOG_REQUEST_HEADERS`, by name
|
|
||||||
in lower case.
|
|
||||||
- `has_authorization` and `has_cookie`: true when the request had those headers.
|
|
||||||
Their values are never logged.
|
|
||||||
|
|
||||||
### Response detail
|
|
||||||
|
|
||||||
- `response_content_type`: the type of `servewaf`'s own answer.
|
|
||||||
- `aborted`: true when `rproxy` went away before the answer.
|
|
||||||
|
|
||||||
### Decision
|
|
||||||
|
|
||||||
- `action`: what was done:
|
|
||||||
- `forward`: answered `allow`, so that `rproxy` passes it to the app;
|
|
||||||
- `denied`: refused for `SWWAF_DENY_NETS`, or for a reputation source with
|
|
||||||
the action `deny`;
|
|
||||||
- `banned`: refused under a ban, or for a `ban` rule, a trap path or a
|
|
||||||
CrowdSec decision, each of which bans the client;
|
|
||||||
- `country_denied`, `rate_limited`, `rule_blocked` and `waf_blocked`:
|
|
||||||
refused by that check;
|
|
||||||
- `too_large`: over a header limit;
|
|
||||||
- `admin`: answered at one of `servewaf`'s own endpoints, or `404` for a
|
|
||||||
path under `/_smallwebwaf/`.
|
|
||||||
- `would_action`: in `observe` mode, what `enforce` mode would have done with a
|
|
||||||
request it would have refused. `action` then names what was done, normally
|
|
||||||
`forward`.
|
|
||||||
- `limit_percent` and `limit_percent_setting`: the client's percentage of the
|
|
||||||
rate limits and the setting that lowered it, when one did. `bytes_percent` and
|
|
||||||
`bytes_percent_setting` do the same for the byte limits.
|
|
||||||
- `counts`: the client's totals as rate limiting counts them, this request
|
|
||||||
included. It is left out when rate limiting did not count the request. It
|
|
||||||
holds:
|
|
||||||
- `minute`, `hour` and `day`: requests;
|
|
||||||
- `minute_bytes`, `hour_bytes` and `day_bytes`: bytes.
|
|
||||||
|
|
||||||
A count can have a fraction (see "Counting method" in
|
|
||||||
[`SPEC.md`](../SPEC.md)). The bytes of an allowed request are counted when
|
|
||||||
its report comes. So on the `decision` line, the bytes are without this
|
|
||||||
request's.
|
|
||||||
|
|
||||||
- `limit_hit`: the limit broken:
|
|
||||||
- `minute`, `hour` or `day` for a rate limit;
|
|
||||||
- `minute_bytes`, `hour_bytes` or `day_bytes` for a byte limit;
|
|
||||||
- `error_burst`.
|
|
||||||
|
|
||||||
A request whose bytes broke a limit was passed on whole, so its `action`
|
|
||||||
stays `forward`. Only its `report` line gives that limit.
|
|
||||||
|
|
||||||
- `rule_ids`: the rule file rules that matched, in order, up to the one that
|
|
||||||
refused.
|
|
||||||
- `waf_score` and `waf_rule_ids`: the Core Rule Set's score and the rules that
|
|
||||||
matched, for a request it inspected.
|
|
||||||
- `reputation`: the sources the client is on, in this order:
|
|
||||||
- blocklist URLs;
|
|
||||||
- the CrowdSec decision list's URL;
|
|
||||||
- DNS blocklist zones;
|
|
||||||
- `abuseipdb`.
|
|
||||||
- `offence`: `limit`, for a request that broke a limit.
|
|
||||||
- `ban_expires`: when the client's ban ends, for a request that made one or was
|
|
||||||
refused under one: a time, or `permanent`.
|
|
||||||
|
|
||||||
### Timings
|
|
||||||
|
|
||||||
All in milliseconds:
|
|
||||||
|
|
||||||
- `duration_total`:
|
|
||||||
- on the `decision` line, from `servewaf` having read the question's body to
|
|
||||||
the line;
|
|
||||||
- on the `report` line, from `rproxy` having read the request's headers to
|
|
||||||
the end of the answer.
|
|
||||||
- `duration_checks`: to the end of the checks;
|
|
||||||
- `duration_waf`: the part of that the Core Rule Set took;
|
|
||||||
- `duration_upstream_total`: on the `report` line of an allowed request, from
|
|
||||||
the hand-over to the app to the end of the answer.
|
|
||||||
|
|
||||||
### Requests to its own endpoints
|
|
||||||
|
|
||||||
The line of a request to `servewaf`'s own endpoints has these fields, with these
|
|
||||||
meanings where they differ:
|
|
||||||
|
|
||||||
- `client_ip` and `peer_ip`: the request's TCP peer.
|
|
||||||
- `request_id`: a new random one of 26 letters and digits.
|
|
||||||
- `scheme`: `http`.
|
|
||||||
- `time`: when the request's headers had been read.
|
|
||||||
- `request_bytes`: the bytes read of a ban to add, and otherwise `0`.
|
|
||||||
- `action`: `admin`.
|
|
||||||
- `duration_total`: from the request's headers being read to the line.
|
|
||||||
|
|
||||||
## `rproxy`'s line
|
|
||||||
|
|
||||||
`smallwebwaf rproxy` writes one short line for each request. `request_id` joins
|
|
||||||
it to `servewaf`'s two. It has these fields and no others:
|
|
||||||
|
|
||||||
- `type`: `request`.
|
|
||||||
- `time`: when the request arrived, RFC 3339 with milliseconds, in UTC.
|
|
||||||
- `request_id`: the id `rproxy` made for the request when it arrived.
|
|
||||||
- `client_ip`: the visitor's address.
|
|
||||||
- `host` and `method`.
|
|
||||||
- `path` and `query`: as the visitor sent them.
|
|
||||||
- `status`: what the visitor got, `0` if nothing.
|
|
||||||
- `request_bytes`: the bytes of the visitor's body that `rproxy` read, with what
|
|
||||||
a WebSocket carried from the visitor.
|
|
||||||
- `response_bytes`: the bytes of the answer's body, with what a WebSocket
|
|
||||||
carried from the app.
|
|
||||||
- `duration_request`: milliseconds from the request's headers being read until
|
|
||||||
the app had been sent the whole request, or the answer started if that was
|
|
||||||
sooner. It includes the wait for `servewaf`.
|
|
||||||
- `duration_response`: milliseconds from then until the end of the answer.
|
|
||||||
- `disposition`: what was done with the request: `allow`, `deny` or `close`. It
|
|
||||||
is `servewaf`'s answer, or the value of `SWRP_DEFAULT_DISPOSITION`. It is left
|
|
||||||
out when:
|
|
||||||
- `servewaf` was not asked, as for the health check;
|
|
||||||
- the visitor went away while `servewaf` was being asked;
|
|
||||||
- `servewaf`'s answer had no valid `X-Smallwebwaf-Disposition`;
|
|
||||||
- `rproxy` could not pass on `servewaf`'s answer, and answered `502`.
|
|
||||||
- `default_disposition`: `true` when `SWRP_DEFAULT_DISPOSITION` gave
|
|
||||||
`disposition`, and otherwise left out.
|
|
||||||
|
|
||||||
For example:
|
|
||||||
|
|
||||||
```
|
|
||||||
{"type":"request","time":"2026-10-03T12:00:00.123Z","request_id":"7Q2NHZ4KJ3VXW5YB6R3MEFTD2A","client_ip":"203.0.113.9","host":"app.example","method":"GET","path":"/","query":"","status":200,"request_bytes":0,"response_bytes":5120,"duration_request":1.204,"duration_response":3.104,"disposition":"allow"}
|
|
||||||
```
|
|
||||||
@@ -1,72 +0,0 @@
|
|||||||
# Rule files
|
|
||||||
|
|
||||||
The format of a rule file, and the default file the image ships. "Rule files" in
|
|
||||||
[`SPEC.md`](../SPEC.md) says how the files are read, watched and mounted, and
|
|
||||||
what each action leads to.
|
|
||||||
|
|
||||||
## The format
|
|
||||||
|
|
||||||
A rule is a line of four fields separated by spaces or tabs: an id, a target, an
|
|
||||||
action and a regex. The regex runs to the end of the line. Blank lines and lines
|
|
||||||
starting with `#` are ignored.
|
|
||||||
|
|
||||||
```
|
|
||||||
# id target action regex
|
|
||||||
env-file path ban (?i)^/\.env(\.[a-z]+)?$
|
|
||||||
scanner-agent user_agent ban (?i)\b(sqlmap|nikto|nuclei|wpscan)\b
|
|
||||||
```
|
|
||||||
|
|
||||||
- The id is letters, digits, `-` and `_`, unique across all files. The log line,
|
|
||||||
the metrics, the alerts and a ban's notes name the rule by it.
|
|
||||||
- The target is what the regex is matched against:
|
|
||||||
- `path`: the path without the query;
|
|
||||||
- `uri`: the path and the query together;
|
|
||||||
- `query`: the query as sent;
|
|
||||||
- `method`, `host`, `user_agent` and `referer`;
|
|
||||||
- `header:<Name>`: any one request header but `Host` and
|
|
||||||
`Transfer-Encoding`.
|
|
||||||
- The action is `log`, `block` or `ban`.
|
|
||||||
- The regex is in Go's syntax, RE2. It matches anywhere in the target unless
|
|
||||||
anchored with `^` and `$`. `(?i)` at its front makes it ignore case.
|
|
||||||
|
|
||||||
## The default file
|
|
||||||
|
|
||||||
The image ships one file, `share/rules.d/00-default.rules`, installed as
|
|
||||||
`/etc/smallwebwaf/rules.d/00-default.rules`:
|
|
||||||
|
|
||||||
```
|
|
||||||
# 00-default.rules: probes no real visitor sends, anchored at the site root
|
|
||||||
|
|
||||||
# id target action regex
|
|
||||||
env-file path ban (?i)^/\.env(\.[a-z]+)?$
|
|
||||||
vcs-dir path ban (?i)^/\.(git|svn|hg|bzr)(/|$)
|
|
||||||
secrets-dir path ban (?i)^/\.(aws|ssh|docker|kube)/
|
|
||||||
secret-file path ban (?i)^/\.(htpasswd|htaccess|npmrc|netrc|pgpass|git-credentials|bash_history|DS_Store)$
|
|
||||||
editor-dir path ban (?i)^/\.(vscode|idea)/
|
|
||||||
backup-file path ban (?i)^/[^/]+\.(php(\.[a-z0-9]+|~)|sql(\.[a-z0-9]+)?)$
|
|
||||||
log-file path ban (?i)^/(debug|error|access)\.log$
|
|
||||||
compose-file path ban (?i)^/(docker-)?compose\.ya?ml$
|
|
||||||
php-shell path ban (?i)^/(shell|c99|r57|wso|alfa)\.php$
|
|
||||||
scanner-agent user_agent ban (?i)\b(sqlmap|nikto|nuclei|masscan|zgrab|wpscan)\b
|
|
||||||
path-traversal uri block (\.\./){2,}
|
|
||||||
empty-agent user_agent log ^$
|
|
||||||
```
|
|
||||||
|
|
||||||
## A file of your own
|
|
||||||
|
|
||||||
`servewaf` reads the rule files in its own container. Rules of your own go in a
|
|
||||||
file named to sort after the default, such as `50-gitea.rules`:
|
|
||||||
|
|
||||||
```
|
|
||||||
# 50-gitea.rules: WordPress probes, which gitea never serves
|
|
||||||
wp-probe path ban (?i)^/(wp-login\.php|xmlrpc\.php|wp-admin/)
|
|
||||||
```
|
|
||||||
|
|
||||||
A `Dockerfile` built on the `smallwebwaf` image, for `servewaf` to run from,
|
|
||||||
adds it:
|
|
||||||
|
|
||||||
```dockerfile
|
|
||||||
COPY 50-gitea.rules /etc/smallwebwaf/rules.d/50-gitea.rules
|
|
||||||
```
|
|
||||||
|
|
||||||
Or mount it there. A `path` rule applies on every site behind that `servewaf`.
|
|
||||||
@@ -1,402 +0,0 @@
|
|||||||
# Settings
|
|
||||||
|
|
||||||
Every setting, its default and the values it takes, in the groups of "Settings"
|
|
||||||
in [`SPEC.md`](../SPEC.md). That section gives the conventions, the settings
|
|
||||||
that take `off`, the `_FILE` form, and what each group does and why.
|
|
||||||
|
|
||||||
The settings that start with `SWWAF_` are `servewaf`'s. Those that start with
|
|
||||||
`SWRP_` are `rproxy`'s, and are under "`rproxy`" below. `SMALLWEBWAF_COMMAND`
|
|
||||||
chooses between the two (see "The image's command" below).
|
|
||||||
|
|
||||||
## Forms of values
|
|
||||||
|
|
||||||
- A duration is Go's syntax, with `d` for days: `90s`, `15m`, `24h`, `7d`. Zero
|
|
||||||
and negative durations are refused.
|
|
||||||
- A size is a number of bytes with an optional `K`, `M` or `G`, which are powers
|
|
||||||
of 1024, so `1K` is 1024 bytes. Examples: `512K`, `100M`, `5G`.
|
|
||||||
- A count is a whole number of requests, such as `1000`.
|
|
||||||
- A list is comma separated, with spaces around items ignored. An empty item is
|
|
||||||
refused.
|
|
||||||
- A netblock is an address and a prefix length, such as `10.0.0.0/8`. A bare
|
|
||||||
address stands for itself alone.
|
|
||||||
- A country is a two-letter code of ISO 3166-1, in either case: `de` and `DE`
|
|
||||||
are the same. `xk` stands for Kosovo. A code that is not a country is refused,
|
|
||||||
such as `nk` (North Korea is `kp`) or the withdrawn `su`.
|
|
||||||
- An AS number is `AS` and the number, in either case, such as `AS64496`.
|
|
||||||
- A percentage is a whole number from 0 to 100.
|
|
||||||
- A path prefix starts with `/`, such as `/assets/`.
|
|
||||||
|
|
||||||
## Core
|
|
||||||
|
|
||||||
- `SWWAF_LISTEN_ADDR` (default `:8080`): where `servewaf` listens: `:` and a
|
|
||||||
port, such as `:9000`. Leave the host out (see "Ports" in
|
|
||||||
[`SPEC.md`](../SPEC.md)).
|
|
||||||
- `SWWAF_INSTANCE_NAME` (default: the container's host name): the `instance`
|
|
||||||
name in each line `servewaf` logs, and in each alert and metric, such as
|
|
||||||
`fsn1/servewaf`. Valid UTF-8. No control character while
|
|
||||||
`SWWAF_ALERT_NTFY_URL` is set. 1 to 48 printable ASCII characters without a
|
|
||||||
space while `SWWAF_LOG_REMOTE_URL` is set and `SWWAF_LOG_REMOTE_APP_NAME` is
|
|
||||||
not.
|
|
||||||
- `SWWAF_MODE` (default `enforce`): `enforce` or `observe`.
|
|
||||||
- `SWWAF_IPV6_GROUP_PREFIX` (default `64`): the prefix length of an IPv6
|
|
||||||
client's group, from 32 to 128.
|
|
||||||
- `SWWAF_MAX_TRACKED_CLIENTS` (default `20000`): the most clients kept, a whole
|
|
||||||
number above zero.
|
|
||||||
- `SWWAF_MAX_BANS` (default `5000`): how many of its own bans `servewaf` keeps,
|
|
||||||
a whole number above zero.
|
|
||||||
|
|
||||||
## Tokens
|
|
||||||
|
|
||||||
- `SWWAF_ADMIN_TOKEN` (default unset): the token for the ban endpoints and
|
|
||||||
`/_smallwebwaf/clients/<ip>`. At least 32 characters.
|
|
||||||
- `SWWAF_METRICS_TOKEN` (default unset): the token for `/_smallwebwaf/metrics`.
|
|
||||||
At least 32 characters.
|
|
||||||
|
|
||||||
## State files
|
|
||||||
|
|
||||||
- `SWWAF_STATE_DIR` (default `/var/lib/smallwebwaf`): the directory of the state
|
|
||||||
files, an absolute path that exists and can be written.
|
|
||||||
- `SWWAF_STATE_WRITE_DELAY` (default `10s`): how long `bans.json` waits to be
|
|
||||||
written after a ban is made, lifted or made permanent. A duration.
|
|
||||||
- `SWWAF_STATE_COUNTER_INTERVAL` (default `15m`): how often every state file is
|
|
||||||
written. A duration.
|
|
||||||
|
|
||||||
## Logging
|
|
||||||
|
|
||||||
- `SWWAF_LOG_LEVEL` (default `info`): the least severe of `servewaf`'s own
|
|
||||||
messages that are written: `debug`, `info`, `warn` or `error`.
|
|
||||||
- `SWWAF_LOG_REQUEST_HEADERS` (default
|
|
||||||
`accept,accept-language,accept-encoding,content-type,origin,range`): the
|
|
||||||
request headers recorded in the log line, in either case. `Host` and
|
|
||||||
`Transfer-Encoding` are refused; the host is the field `host`.
|
|
||||||
- `SWWAF_LOG_REMOTE_URL` (default unset): a syslog server that also gets every
|
|
||||||
line written to stdout: `syslog+udp://`, `syslog+tcp://` or `syslog+tls://`,
|
|
||||||
each with a host and a port and nothing more.
|
|
||||||
- `SWWAF_LOG_REMOTE_TLS_CA_FILE` (default unset): a file of PEM certificates
|
|
||||||
that a `syslog+tls` server's certificate must chain to, instead of the host's
|
|
||||||
own. A file that cannot be read, or holds no certificate, is refused.
|
|
||||||
- `SWWAF_LOG_REMOTE_BUFFER` (default `10000`): the most lines held while they
|
|
||||||
wait to be sent, a whole number above zero.
|
|
||||||
- `SWWAF_LOG_REMOTE_FACILITY` (default `local0`): the syslog facility: `kern`,
|
|
||||||
`user`, `mail`, `daemon`, `auth`, `syslog`, `lpr`, `news`, `uucp`, `cron`,
|
|
||||||
`authpriv`, `ftp`, or `local0` to `local7`.
|
|
||||||
- `SWWAF_LOG_REMOTE_APP_NAME` (default: `SWWAF_INSTANCE_NAME`): the app name in
|
|
||||||
each syslog record, 1 to 48 printable ASCII characters without a space.
|
|
||||||
|
|
||||||
## Metrics
|
|
||||||
|
|
||||||
- `SWWAF_METRICS_TOP_N` (default `50`): how many AS numbers and how many
|
|
||||||
countries get series of their own. A whole number above zero.
|
|
||||||
|
|
||||||
## Static lists
|
|
||||||
|
|
||||||
- `SWWAF_ALLOW_NETS` (default empty): netblocks you trust, such as your
|
|
||||||
monitoring (see step 3 in [`SPEC.md`](../SPEC.md)).
|
|
||||||
- `SWWAF_RATE_LIMIT_EXEMPT_NETS` (default empty): netblocks whose clients are
|
|
||||||
exempt from rate limiting and the byte limits.
|
|
||||||
- `SWWAF_DENY_NETS` (default empty): netblocks whose clients are refused (see
|
|
||||||
step 3 in [`SPEC.md`](../SPEC.md)).
|
|
||||||
|
|
||||||
## Request rate limits, per client
|
|
||||||
|
|
||||||
- `SWWAF_RATE_LIMIT_PER_MINUTE` (default `500`, about 8 a second): the most
|
|
||||||
requests a client may make in a minute. A count or `off`. The default catches,
|
|
||||||
within seconds, a client firing requests as fast as it can. It leaves room for
|
|
||||||
a fleet of about 20 agents at its busiest, about 120 requests a minute. It
|
|
||||||
also leaves room for a few quick first visits to gitea with an empty cache,
|
|
||||||
about 40 files each.
|
|
||||||
- `SWWAF_RATE_LIMIT_PER_HOUR` (default `20000`, about 5.5 a second): the most
|
|
||||||
requests a client may make in an hour. A count or `off`. The default leaves
|
|
||||||
room for an automation fleet at work. It catches a client that keeps up more
|
|
||||||
than about 5.5 a second for an hour.
|
|
||||||
- `SWWAF_RATE_LIMIT_PER_DAY` (default `300000`, about 3.5 a second): the most
|
|
||||||
requests a client may make in a day. A count or `off`. The default catches a
|
|
||||||
scraper that keeps up 4 a second around the clock.
|
|
||||||
- Counting cannot catch a client that stays under all three limits (see "Request
|
|
||||||
rate limits, per client" in [`SPEC.md`](../SPEC.md)).
|
|
||||||
- `SWWAF_RATE_LIMIT_EXEMPT_PATHS` (default empty): path prefixes exempt from
|
|
||||||
rate limiting and the byte limits, such as `/assets/`.
|
|
||||||
|
|
||||||
## Byte limits, per client
|
|
||||||
|
|
||||||
- `SWWAF_BYTES_LIMIT_PER_MINUTE` (default `4G`), `SWWAF_BYTES_LIMIT_PER_HOUR`
|
|
||||||
(default `20G`) and `SWWAF_BYTES_LIMIT_PER_DAY` (default `200G`): the most
|
|
||||||
bytes counted for a client in each window. A size or `off`.
|
|
||||||
- `SWWAF_BYTES_COUNT` (default `both`): which body bytes the byte limits count:
|
|
||||||
`response`, `request` or `both`.
|
|
||||||
|
|
||||||
## Lookup of AS number and country
|
|
||||||
|
|
||||||
- `SWWAF_LOOKUP_SOURCE` (default `geojs`): `geojs`, `file` or `off`. `off` stops
|
|
||||||
the start while a setting that needs a lookup is set:
|
|
||||||
- the two country lists;
|
|
||||||
- `SWWAF_ADD_LOOKUP_HEADERS`;
|
|
||||||
- the four biased threshold lists and `SWWAF_ASN_LIMIT_PERCENT_URL`;
|
|
||||||
- `SWWAF_UNKNOWN_LIMIT_PERCENT` below `100`;
|
|
||||||
- an anomaly threshold per AS number.
|
|
||||||
- `SWWAF_LOOKUP_DB_PATH` (default empty): the IPinfo Lite database file, in its
|
|
||||||
`.mmdb` form, for `SWWAF_LOOKUP_SOURCE=file`.
|
|
||||||
- `SWWAF_LOOKUP_TIMEOUT` (default `800ms`): how long a request waits for its
|
|
||||||
client's first answer from GeoJS, and how long a request to GeoJS may take. A
|
|
||||||
duration; never `off`.
|
|
||||||
- `SWWAF_ADD_LOOKUP_HEADERS` (default `false`): `true` has `rproxy` pass the app
|
|
||||||
the client's AS number in `X-Client-ASN` and its country in
|
|
||||||
`X-Client-Country`.
|
|
||||||
|
|
||||||
## Country lists
|
|
||||||
|
|
||||||
- `SWWAF_DENIED_COUNTRIES` (default empty): countries whose clients are refused,
|
|
||||||
such as `cn,ru,kp`.
|
|
||||||
- `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` (default empty): when set, the only
|
|
||||||
countries whose clients get through, such as `us,de`. A country on both lists
|
|
||||||
stops the start.
|
|
||||||
|
|
||||||
## Biased thresholds
|
|
||||||
|
|
||||||
- `SWWAF_ASN_LIMIT_PERCENT` (default empty): a list of AS numbers, each with `:`
|
|
||||||
and a percentage, such as `AS14061:50,AS16276:50,AS45102:25`. An AS number
|
|
||||||
listed twice is refused.
|
|
||||||
- `SWWAF_COUNTRY_LIMIT_PERCENT` (default empty): the same by country, such as
|
|
||||||
`cn:25,ru:50`.
|
|
||||||
- `SWWAF_ASN_BYTES_PERCENT` and `SWWAF_COUNTRY_BYTES_PERCENT` (default empty):
|
|
||||||
the same, for the byte limits alone.
|
|
||||||
- `SWWAF_UNKNOWN_LIMIT_PERCENT` (default `100`): the percentage for a client the
|
|
||||||
lookup cannot place.
|
|
||||||
- `SWWAF_ASN_LIMIT_PERCENT_URL` (default unset): the `http` or `https` URL of a
|
|
||||||
text file with one AS number and percentage on each line, such as
|
|
||||||
`AS64496:50`. A URL that is also in `SWWAF_BLOCKLIST_URLS` is refused.
|
|
||||||
|
|
||||||
## Attack detection
|
|
||||||
|
|
||||||
- `SWWAF_RULES_DIR` (default `/etc/smallwebwaf/rules.d`): the directory of the
|
|
||||||
rule files.
|
|
||||||
- `SWWAF_RULES_ENABLED` (default `true`): `false` reads no rule file.
|
|
||||||
- `SWWAF_TRAP_PATHS` (default empty): paths only scanners ask for, such as
|
|
||||||
`/wp-login.php,/xmlrpc.php`. Each starts with `/` and holds no `?`.
|
|
||||||
- `SWWAF_WAF_MODE` (default `block`): `off`, `detect` or `block`.
|
|
||||||
- `SWWAF_WAF_PARANOIA_LEVEL` (default `1`): the Core Rule Set's paranoia level,
|
|
||||||
from 1 to 4.
|
|
||||||
- `SWWAF_WAF_ANOMALY_THRESHOLD` (default `5`): the score at which a request is a
|
|
||||||
match. A whole number above zero, or `off`.
|
|
||||||
- `SWWAF_WAF_DISABLED_RULES` (default
|
|
||||||
`920340,920420,920440,920640,930130,930140`): the ids of Core Rule Set rules
|
|
||||||
to switch off. Each a whole number above zero. An id from 900000 to 900999 is
|
|
||||||
refused: those are the rules that set the Core Rule Set up, and
|
|
||||||
`smallwebwaf`'s own.
|
|
||||||
- `SWWAF_WAF_EXEMPT_PATHS` (default empty): path prefixes the Core Rule Set does
|
|
||||||
not inspect.
|
|
||||||
- `SWWAF_WAF_BODY_LIMIT` (default `off`): `off`, or how much of a request body
|
|
||||||
the Core Rule Set reads, a size of at most `64K`. A question carries no more
|
|
||||||
of a body than 64 KiB, and the Core Rule Set reads at most 64 KiB less one
|
|
||||||
byte of it, whatever the value (see "The answer" in [`SPEC.md`](../SPEC.md)).
|
|
||||||
- `SWWAF_ERROR_BURST_THRESHOLD` (default `30`): the most refusals of one client
|
|
||||||
in a minute (see "Attack detection" in `SPEC.md`). A count or `off`.
|
|
||||||
|
|
||||||
## Ban lengths, response and scope
|
|
||||||
|
|
||||||
- `SWWAF_BAN_RESPONSE` (default `403`): `403`, `429` or `close`.
|
|
||||||
- `SWWAF_LIMIT_BAN_DURATION` (default `1h`): the ban for a first broken limit. A
|
|
||||||
duration.
|
|
||||||
- `SWWAF_LIMIT_BAN_REPEAT_WINDOW` (default `24h`): the time after a ban ends
|
|
||||||
within which breaking a limit again lengthens the next ban. A duration.
|
|
||||||
- `SWWAF_MAX_BAN_DURATION` (default `7d`): the longest ban for a broken limit;
|
|
||||||
one that would be longer is permanent. A duration.
|
|
||||||
- `SWWAF_ATTACK_BAN_DURATION` (default `7d`): the ban for a first clear sign of
|
|
||||||
attack. A duration.
|
|
||||||
- `SWWAF_BAN_SCOPE_V4_PREFIX` (default `32`): how much a ban on an IPv4 client
|
|
||||||
covers: the prefix length of its netblock, from 0 to 32.
|
|
||||||
|
|
||||||
## Reputation
|
|
||||||
|
|
||||||
- `SWWAF_BLOCKLIST_URLS` (default empty): blocklists to fetch, such as
|
|
||||||
`https://www.spamhaus.org/drop/drop.txt`. Each an `http` or `https` URL
|
|
||||||
without a user or a fragment. A URL listed twice is refused.
|
|
||||||
- `SWWAF_BLOCKLIST_REFRESH` (default `24h`): how long a list waits after its
|
|
||||||
last try before it is fetched again. A duration of `1h` or more.
|
|
||||||
- `SWWAF_BLOCKLIST_ACTION` (default `deny`): `deny`, `limit:<percent>` or `log`.
|
|
||||||
- `SWWAF_DNSBL_ZONES` (default empty): the DNS blocklists to ask, such as
|
|
||||||
`dnsbl.dronebl.org`. Each a DNS name of at most 189 characters. A Spamhaus
|
|
||||||
zone carries its key, as in `<key>.xbl.dq.spamhaus.net`. A zone listed twice
|
|
||||||
is refused, whatever its case or key.
|
|
||||||
- `SWWAF_DNSBL_RESOLVER` (default unset): the resolver to ask the zones through,
|
|
||||||
such as `192.0.2.53` or `[2001:db8::53]:5353`. An IP address with an optional
|
|
||||||
port, 53 when none is given. Unset, the host's resolver is used.
|
|
||||||
- `SWWAF_ABUSEIPDB_KEY` (default unset): the key of your AbuseIPDB account. A
|
|
||||||
control character in it is refused.
|
|
||||||
- `SWWAF_ABUSEIPDB_MIN_SCORE` (default `75`): the least abuse confidence score,
|
|
||||||
from 0 to 100, that is a hit.
|
|
||||||
- `SWWAF_ABUSEIPDB_DAILY_BUDGET` (default `900`): the most checks made in a day,
|
|
||||||
a whole number above zero. AbuseIPDB's free accounts may make 1,000.
|
|
||||||
- `SWWAF_CROWDSEC_LAPI_URL` (default unset): the local API of a CrowdSec engine
|
|
||||||
you run, such as `http://172.17.0.1:8080`. An `http` or `https` URL without a
|
|
||||||
user or a fragment; its decision list is at `v1/decisions` under it. Set
|
|
||||||
without the key, it is refused. So is a decision list URL that is also in
|
|
||||||
`SWWAF_BLOCKLIST_URLS`, or that is `SWWAF_ASN_LIMIT_PERCENT_URL`.
|
|
||||||
- `SWWAF_CROWDSEC_LAPI_KEY` (default unset): the key the engine gave for
|
|
||||||
`smallwebwaf`, as `cscli bouncers add smallwebwaf` prints it. Set without the
|
|
||||||
URL, it is refused.
|
|
||||||
- `SWWAF_REPUTATION_ACTION` (default `limit:25`): `deny`, `limit:<percent>` or
|
|
||||||
`log`, for DNS blocklist and AbuseIPDB hits.
|
|
||||||
- `SWWAF_REPUTATION_CACHE_TTL` (default `24h`): how long a zone's verdict or
|
|
||||||
AbuseIPDB's score is used. A duration.
|
|
||||||
- `SWWAF_REPUTATION_TIMEOUT` (default `2s`): how long a query to a zone or a
|
|
||||||
check with AbuseIPDB may take. A duration.
|
|
||||||
|
|
||||||
## Alerting
|
|
||||||
|
|
||||||
- `SWWAF_ALERT_WEBHOOK_URL` (default unset): the webhook that receives each
|
|
||||||
alert as a JSON object. An `http` or `https` URL without a user or a fragment.
|
|
||||||
- `SWWAF_ALERT_WEBHOOK_HEADERS` (default empty): headers sent with each post,
|
|
||||||
such as `Authorization:Bearer 0123456789abcdef`. A list of name, `:` and
|
|
||||||
value. A value cannot hold a comma.
|
|
||||||
- `SWWAF_ALERT_SLACK_WEBHOOK_URL` (default unset): the Slack incoming webhook
|
|
||||||
that receives each alert as a message, checked as the webhook URL is.
|
|
||||||
- `SWWAF_ALERT_NTFY_URL` (default unset): the ntfy topic that receives each
|
|
||||||
alert, as the topic's full URL, such as `https://ntfy.sh/my-alerts`, checked
|
|
||||||
as the webhook URL is.
|
|
||||||
- `SWWAF_ALERT_NTFY_TOKEN` (default unset): an ntfy access token. A control
|
|
||||||
character in it is refused.
|
|
||||||
- `SWWAF_ALERT_EVENTS` (default
|
|
||||||
`ban,permanent_ban,waf_block,anomaly,reputation_hit,source_failure,file_error`):
|
|
||||||
the event types alerts are sent for, from those in [`alerts.md`](alerts.md)
|
|
||||||
but `summary`. Any other name is refused.
|
|
||||||
- `SWWAF_ALERT_COOLDOWN` (default `15m`): how long after an alert is sent its
|
|
||||||
repeats are held back. A duration or `off`.
|
|
||||||
- `SWWAF_ALERT_MAX_PER_HOUR` (default `60`): the most alerts sent in an hour. A
|
|
||||||
whole number above zero, or `off`.
|
|
||||||
|
|
||||||
## Anomaly thresholds
|
|
||||||
|
|
||||||
Each scope has four thresholds: the most requests and the most bytes counted in
|
|
||||||
a minute and in an hour. Requests are counts, bytes are sizes. Each is `off` by
|
|
||||||
default. The four thresholds per client are:
|
|
||||||
|
|
||||||
- `SWWAF_ANOMALY_CLIENT_REQUESTS_PER_MINUTE`;
|
|
||||||
- `SWWAF_ANOMALY_CLIENT_REQUESTS_PER_HOUR`;
|
|
||||||
- `SWWAF_ANOMALY_CLIENT_BYTES_PER_MINUTE`;
|
|
||||||
- `SWWAF_ANOMALY_CLIENT_BYTES_PER_HOUR`.
|
|
||||||
|
|
||||||
The other scopes' names have another prefix before the same four endings, such
|
|
||||||
as `REQUESTS_PER_MINUTE`:
|
|
||||||
|
|
||||||
- `SWWAF_ANOMALY_NET_...`: per netblock around a client. Two settings give the
|
|
||||||
netblock's length:
|
|
||||||
- `SWWAF_ANOMALY_NET_V4_PREFIX` (default `24`, from 0 to 32) for an IPv4
|
|
||||||
client;
|
|
||||||
- `SWWAF_ANOMALY_NET_V6_PREFIX` (default `48`, from 0 to 128) for an IPv6
|
|
||||||
client.
|
|
||||||
- `SWWAF_ANOMALY_ASN_...`: per AS number.
|
|
||||||
- `SWWAF_ANOMALY_TOTAL_...`: the whole service.
|
|
||||||
- `SWWAF_WATCH_...`: each netblock in `SWWAF_WATCH_NETS`, as a whole.
|
|
||||||
`SWWAF_WATCH_NETS` (default empty) is a list of name, `=` and netblock, such
|
|
||||||
as `office=203.0.113.0/24,scraper-x=198.51.100.0/22`. An item without `=` or a
|
|
||||||
name, or with a bad netblock, is refused. So is a name used twice.
|
|
||||||
|
|
||||||
## Settings given as files
|
|
||||||
|
|
||||||
To give a token as a file, as "Tokens" under "Deployment" in
|
|
||||||
[`SPEC.md`](../SPEC.md) describes, make the file on the host:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
mkdir -p /srv/app/tokens
|
|
||||||
openssl rand -hex 32 > /srv/app/tokens/metrics
|
|
||||||
chown 65532:65532 /srv/app/tokens/metrics
|
|
||||||
chmod 0400 /srv/app/tokens/metrics
|
|
||||||
```
|
|
||||||
|
|
||||||
Then mount its directory into the container read-only, and name the file:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
docker run \
|
|
||||||
-v /srv/app/tokens:/etc/smallwebwaf/tokens:ro \
|
|
||||||
-e SMALLWEBWAF_COMMAND=servewaf \
|
|
||||||
-e SWWAF_METRICS_TOKEN_FILE=/etc/smallwebwaf/tokens/metrics \
|
|
||||||
...
|
|
||||||
```
|
|
||||||
|
|
||||||
## Fixed limits
|
|
||||||
|
|
||||||
A few limits are fixed rather than settings:
|
|
||||||
|
|
||||||
- at most 100,000 GeoJS answers are kept;
|
|
||||||
- at most 100,000 DNS blocklist verdicts and 100,000 AbuseIPDB scores are kept;
|
|
||||||
- at most 1,000 DNS blocklist queries are under way at once;
|
|
||||||
- at most 20,000 anomaly counters are kept;
|
|
||||||
- each alert destination's queue holds at most 1,000 alerts;
|
|
||||||
- `servewaf` holds at most 10,000 answered requests awaiting their reports;
|
|
||||||
- `servewaf` refuses a request with more than 50 cookies or more than 100 header
|
|
||||||
fields (see "Answering in time" under "`servewaf`" in
|
|
||||||
[`SPEC.md`](../SPEC.md));
|
|
||||||
- `rproxy` takes a request line and headers of at most 32 KiB, and closes a
|
|
||||||
kept-open connection idle for 120 seconds;
|
|
||||||
- `rproxy` reads at most 1 MiB of the body of `servewaf`'s answer, and answers
|
|
||||||
`502` to a longer one.
|
|
||||||
|
|
||||||
## `servewaf`'s answer and `rproxy`'s wait
|
|
||||||
|
|
||||||
`SWWAF_LOOKUP_TIMEOUT` is the longest a question waits on an outside source.
|
|
||||||
Keep it well under `rproxy`'s `SWRP_TIMEOUT_MS`, as the default is. No question
|
|
||||||
waits for `SWWAF_REPUTATION_TIMEOUT`, since the DNS blocklists and AbuseIPDB are
|
|
||||||
asked in the background (see "Answering in time" under "`servewaf`" in
|
|
||||||
[`SPEC.md`](../SPEC.md)).
|
|
||||||
|
|
||||||
## `rproxy`
|
|
||||||
|
|
||||||
`smallwebwaf rproxy` reads the settings below and no others. They are read once
|
|
||||||
at start, and an invalid value stops the start. None can be given as a file,
|
|
||||||
since `rproxy` never touches disk. "`rproxy`" in [`SPEC.md`](../SPEC.md) says
|
|
||||||
what they do.
|
|
||||||
|
|
||||||
A time here is a whole number of milliseconds above zero, such as `2000`.
|
|
||||||
|
|
||||||
- `SWRP_WAF_BACKEND` (required, no default): `servewaf`'s URL. An `http` or
|
|
||||||
`https` URL with a host and an optional port, and nothing more. `rproxy`
|
|
||||||
speaks HTTP/1.1 to it.
|
|
||||||
- `SWRP_TIMEOUT_MS` (default `2000`): how long `rproxy` waits for `servewaf`'s
|
|
||||||
answer, to a question and to a report. A time; never `off`.
|
|
||||||
- `SWRP_DEFAULT_DISPOSITION` (default `allow`): what is done with a request when
|
|
||||||
`servewaf` cannot be connected to, does not answer at all, or does not answer
|
|
||||||
within `SWRP_TIMEOUT_MS`. `allow` passes it to the app, and `deny` answers
|
|
||||||
`503`. It does not apply to an answer `servewaf` gives, such as an error.
|
|
||||||
- `SWRP_MAX_REQUEST_SIZE` (default `100M`): the largest request body. A size or
|
|
||||||
`off`.
|
|
||||||
- `SWRP_MAX_RESPONSE_SIZE` (default `1G`): the largest response body. A size or
|
|
||||||
`off`.
|
|
||||||
- `SWRP_MAX_REQUEST_TIME_MS` (default `60000`): how long the visitor may take to
|
|
||||||
send its request line and headers. It then also bounds, from the end of the
|
|
||||||
headers, the time until the app has been sent the whole request. A time or
|
|
||||||
`off`.
|
|
||||||
- `SWRP_MAX_RESPONSE_TIME_MS` (default `1800000`): how long the answer may take,
|
|
||||||
from the end of the request to its last byte reaching the visitor. A time or
|
|
||||||
`off`.
|
|
||||||
- `SWRP_LISTEN_ADDR` (default `:8080`): where `rproxy` listens, such as `:9000`.
|
|
||||||
- `SWRP_UPSTREAM_URL` (default `http://127.0.0.1:8081`): the app. An `http` or
|
|
||||||
`https` URL with a host and an optional port, and nothing more.
|
|
||||||
- `SWRP_TRUSTED_PROXIES` (default `10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`):
|
|
||||||
the netblocks whose `X-Forwarded-For` is believed. Set but empty, it trusts
|
|
||||||
nothing. A client outside these netblocks cannot claim another address. Where
|
|
||||||
other containers reach `rproxy` directly, or visitors on a private network
|
|
||||||
reach traefik, they can name another address in the header. Narrowing the
|
|
||||||
setting to traefik's own network closes that gap.
|
|
||||||
|
|
||||||
## The image's command
|
|
||||||
|
|
||||||
`SMALLWEBWAF_COMMAND` (default `rproxy`) chooses what the image runs and what
|
|
||||||
its health check checks: `rproxy` or `servewaf` (see "Deployment" in
|
|
||||||
[`SPEC.md`](../SPEC.md)). It cannot be given as a file.
|
|
||||||
|
|
||||||
## Where earlier settings went
|
|
||||||
|
|
||||||
Nothing reads these names any more:
|
|
||||||
|
|
||||||
- `SWWAF_UPSTREAM_URL` is `SWRP_UPSTREAM_URL`.
|
|
||||||
- `SWWAF_TRUSTED_PROXIES` is `SWRP_TRUSTED_PROXIES`.
|
|
||||||
- `SWWAF_REQUEST_MAX_BYTES` is `SWRP_MAX_REQUEST_SIZE`.
|
|
||||||
- `SWWAF_RESPONSE_MAX_BYTES` is `SWRP_MAX_RESPONSE_SIZE`.
|
|
||||||
- `SWWAF_CLIENT_REQUEST_TIMEOUT` and `SWWAF_UPSTREAM_REQUEST_TIMEOUT` are
|
|
||||||
`SWRP_MAX_REQUEST_TIME_MS`, in milliseconds.
|
|
||||||
- `SWWAF_CLIENT_RESPONSE_TIMEOUT` and `SWWAF_UPSTREAM_RESPONSE_TIMEOUT` are
|
|
||||||
`SWRP_MAX_RESPONSE_TIME_MS`, in milliseconds.
|
|
||||||
- `SWWAF_CLIENT_IDLE_TIMEOUT` and `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES` have
|
|
||||||
no setting. `rproxy` closes a kept-open connection idle for 120 seconds, and
|
|
||||||
takes a request line and headers of at most 32 KiB.
|
|
||||||
@@ -1,104 +0,0 @@
|
|||||||
# State files
|
|
||||||
|
|
||||||
The five JSON files in `SWWAF_STATE_DIR`, and what each holds. "Persistent
|
|
||||||
state" in [`SPEC.md`](../SPEC.md) says how they are written, read and edited.
|
|
||||||
Each has a top-level `version` of 1.
|
|
||||||
|
|
||||||
## `bans.json`
|
|
||||||
|
|
||||||
Active and past bans: every ban an admin made, and up to `SWWAF_MAX_BANS` of
|
|
||||||
those `servewaf` made. Per entry:
|
|
||||||
|
|
||||||
- `netblock`, `start` and `expires` (`null` for a permanent ban);
|
|
||||||
- `cause`: `attack`, `limit`, `admin` or `crowdsec`;
|
|
||||||
- `reason`: a short reason;
|
|
||||||
- `lifted`: when an admin lifted it, if one did;
|
|
||||||
- `notes`: the notes below.
|
|
||||||
|
|
||||||
An entry an admin adds needs `netblock`, `start` and `expires`; `reason`,
|
|
||||||
`notes` and `cause` may be left out, and the cause is then `admin`:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"version": 1,
|
|
||||||
"bans": [
|
|
||||||
{
|
|
||||||
"netblock": "203.0.113.0/24",
|
|
||||||
"start": "2026-10-06T12:00:00Z",
|
|
||||||
"expires": null,
|
|
||||||
"reason": "probes for logins"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
"Bans" in [`SPEC.md`](../SPEC.md) says how an edit lifts, keeps or forgets a
|
|
||||||
ban. The notes:
|
|
||||||
|
|
||||||
- `asn`, `as_name` and `country`: the client's, added when the lookup answers
|
|
||||||
(empty when lookups are off);
|
|
||||||
- for a broken limit, `kind` (`requests`, `bytes` or `refusals`), `limit`,
|
|
||||||
`window` and `count`: what the limit was on, the limit, its window and the
|
|
||||||
count reached;
|
|
||||||
- `limit_percent` and `limit_percent_setting`: the client's percentage of that
|
|
||||||
limit and the setting that gave it, when a setting lowered it;
|
|
||||||
- for a clear sign of attack, `rule_id` and `target`, or `trap_path`;
|
|
||||||
- `reputation`: the sources the client was on, each with its `source` and, for
|
|
||||||
AbuseIPDB, its `score`;
|
|
||||||
- `request`: the request that caused the ban, each text cut to 256 bytes:
|
|
||||||
- `time`, `method` and `host`;
|
|
||||||
- `path`, with its query string;
|
|
||||||
- `status` and `user_agent`;
|
|
||||||
- `requests` and `refused`: the netblock's requests since it was first seen, and
|
|
||||||
how many this ban has refused so far, kept up to date while the ban lasts;
|
|
||||||
- `earlier_bans`: how many earlier bans the netblock had, by cause.
|
|
||||||
|
|
||||||
## `clients.json`
|
|
||||||
|
|
||||||
Per client, one per line: its minute, hour and day counters, and its history
|
|
||||||
since it was first seen:
|
|
||||||
|
|
||||||
- the first and last time seen;
|
|
||||||
- the AS number, AS name and country last looked up, and when;
|
|
||||||
- total requests, and how many were forwarded and how many refused;
|
|
||||||
- bytes in each direction;
|
|
||||||
- responses by status class;
|
|
||||||
- offences by kind: `limit`, `attack`, `rule_blocked` and `waf_blocked`. A file
|
|
||||||
may also hold `token_refused`, which is kept, and which nothing adds to.
|
|
||||||
|
|
||||||
## `lookups.json`
|
|
||||||
|
|
||||||
GeoJS answers, one per line:
|
|
||||||
|
|
||||||
- `client`;
|
|
||||||
- `asn`, `as_name` and `country`;
|
|
||||||
- `answered`: when GeoJS answered;
|
|
||||||
- `used`: when the answer was last used.
|
|
||||||
|
|
||||||
## `reputation.json`
|
|
||||||
|
|
||||||
- the last good copy of each list fetched from a URL, with when it was last
|
|
||||||
tried;
|
|
||||||
- each DNS blocklist verdict still in use;
|
|
||||||
- AbuseIPDB's scores, and the checks spent today.
|
|
||||||
|
|
||||||
## `alerts.json`
|
|
||||||
|
|
||||||
- the anomaly counters;
|
|
||||||
- the alert cooldowns: for each event and netblock, source, file or anomaly
|
|
||||||
scope, when it was last sent and the count of repeats held back since;
|
|
||||||
- the alerts sent and held back in the current hour;
|
|
||||||
- the alerts waiting for each destination.
|
|
||||||
|
|
||||||
## Sizes
|
|
||||||
|
|
||||||
- `clients.json`: about 1 KiB per client, about 20 MiB at the default
|
|
||||||
`SWWAF_MAX_TRACKED_CLIENTS`. Written every 15 minutes, that is under 2 GiB of
|
|
||||||
disk writes a day.
|
|
||||||
- `bans.json`: about 1.2 KiB per ban, at most about 2.5 KiB, so about 6 MiB at
|
|
||||||
the default `SWWAF_MAX_BANS`. Written every 15 minutes, a full file comes to
|
|
||||||
about 600 MiB of writes a day, and a hundred new bans a day add about 600 MiB
|
|
||||||
more.
|
|
||||||
- `lookups.json`: about 150 bytes per answer, under 1.5 GiB of writes a day when
|
|
||||||
full.
|
|
||||||
- `reputation.json` and `alerts.json`: usually a few MiB or less.
|
|
||||||
@@ -1,8 +1,8 @@
|
|||||||
// Package alerts sends alerts on bans, on traffic over an anomaly
|
// Package alerts sends alerts on bans, on traffic over an anomaly
|
||||||
// threshold, on a source that fails and on a file with an error to each
|
// threshold, on a source that fails and on a file with an error to each
|
||||||
// destination set: to the webhook
|
// destination set: to the webhook
|
||||||
// SWWAF_ALERT_WEBHOOK_URL names, each as one JSON object, as "The JSON
|
// SWWAF_ALERT_WEBHOOK_URL names, each as one JSON object, as the "Alert
|
||||||
// object" in docs/alerts.md describes, to the Slack incoming
|
// webhook schema" section of SPEC.md describes, to the Slack incoming
|
||||||
// webhook SWWAF_ALERT_SLACK_WEBHOOK_URL names, as a message, and to the
|
// webhook SWWAF_ALERT_SLACK_WEBHOOK_URL names, as a message, and to the
|
||||||
// ntfy topic SWWAF_ALERT_NTFY_URL names. A repeat within
|
// ntfy topic SWWAF_ALERT_NTFY_URL names. A repeat within
|
||||||
// SWWAF_ALERT_COOLDOWN is held back, and so is an alert past
|
// SWWAF_ALERT_COOLDOWN is held back, and so is an alert past
|
||||||
@@ -149,17 +149,15 @@ type Params struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Alert is one alert, as the webhook is sent it and alerts.json holds it,
|
// Alert is one alert, as the webhook is sent it and alerts.json holds it,
|
||||||
// with the fields of "The JSON object" in docs/alerts.md. ASN, ASName and
|
// with the fields of the "Alert webhook schema" section of SPEC.md. ASN,
|
||||||
// Country are, for a ban, the client's as the ban's notes give them. Host is
|
// ASName and Country are, for a ban, the client's as the ban's notes give
|
||||||
// the site of the request that raised the alert, while one servewaf serves
|
// them.
|
||||||
// many sites, and "" otherwise.
|
|
||||||
//
|
//
|
||||||
//nolint:tagliatelle // docs/alerts.md names the alert's fields in snake_case
|
//nolint:tagliatelle // SPEC.md's alert webhook schema names its fields in snake_case
|
||||||
type Alert struct {
|
type Alert struct {
|
||||||
Instance string `json:"instance"`
|
Instance string `json:"instance"`
|
||||||
Time time.Time `json:"time"`
|
Time time.Time `json:"time"`
|
||||||
Event string `json:"event"`
|
Event string `json:"event"`
|
||||||
Host string `json:"host,omitempty"`
|
|
||||||
Client netip.Addr `json:"client"`
|
Client netip.Addr `json:"client"`
|
||||||
Netblock netip.Prefix `json:"netblock"`
|
Netblock netip.Prefix `json:"netblock"`
|
||||||
ASN string `json:"asn"`
|
ASN string `json:"asn"`
|
||||||
@@ -902,16 +900,12 @@ func title(alert *Alert) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// text returns the text of alert in Slack and ntfy: its reason, then a
|
// text returns the text of alert in Slack and ntfy: its reason, then a
|
||||||
// line for each of its host, client, netblock and country, the file,
|
// line for each of its client, netblock and country, the file, source,
|
||||||
// source, error and mode its detail gives, and its suppressed repeats, that
|
// error and mode its detail gives, and its suppressed repeats, that it
|
||||||
// it has.
|
// has.
|
||||||
func text(alert *Alert) string {
|
func text(alert *Alert) string {
|
||||||
lines := []string{alert.Reason}
|
lines := []string{alert.Reason}
|
||||||
|
|
||||||
if alert.Host != "" {
|
|
||||||
lines = append(lines, "host: "+alert.Host)
|
|
||||||
}
|
|
||||||
|
|
||||||
if alert.Client.IsValid() {
|
if alert.Client.IsValid() {
|
||||||
lines = append(lines, "client: "+alert.Client.String())
|
lines = append(lines, "client: "+alert.Client.String())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1284,11 +1284,9 @@ func anAlertForEachEvent() []eventMessage {
|
|||||||
Event: alerts.EventWAFBlock, Client: netip.MustParseAddr("192.0.2.1"),
|
Event: alerts.EventWAFBlock, Client: netip.MustParseAddr("192.0.2.1"),
|
||||||
Netblock: netip.MustParsePrefix("192.0.2.1/32"),
|
Netblock: netip.MustParsePrefix("192.0.2.1/32"),
|
||||||
Reason: "refused by the Core Rule Set",
|
Reason: "refused by the Core Rule Set",
|
||||||
Host: "app.example",
|
|
||||||
},
|
},
|
||||||
"fsn1app1/gitea: waf_block", "default shield",
|
"fsn1app1/gitea: waf_block", "default shield",
|
||||||
"refused by the Core Rule Set\nhost: app.example\nclient: 192.0.2.1\n" +
|
"refused by the Core Rule Set\nclient: 192.0.2.1\nnetblock: 192.0.2.1/32",
|
||||||
"netblock: 192.0.2.1/32",
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
alerts.Alert{
|
alerts.Alert{
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
// Package anomaly counts requests and bytes over a minute and an hour, per
|
// Package anomaly counts requests and bytes over a minute and an hour, per
|
||||||
// client, per surrounding netblock, per AS number, for the whole service
|
// client, per surrounding netblock, per AS number, for the whole service
|
||||||
// and per named netblock, and raises an anomaly alert for a count over its
|
// and per named netblock, and raises an anomaly alert for a count over its
|
||||||
// threshold, as "Anomaly thresholds" under "Settings" in SPEC.md
|
// threshold, as "Anomaly thresholds" under "Configuration surface" in
|
||||||
// describes. It refuses and bans nothing. At most 20,000 counters
|
// SPEC.md describes. It refuses and bans nothing. At most 20,000 counters
|
||||||
// are kept, in memory, and written to alerts.json and read from it by the
|
// are kept, in memory, and written to alerts.json and read from it by the
|
||||||
// state package.
|
// state package.
|
||||||
package anomaly
|
package anomaly
|
||||||
@@ -103,8 +103,6 @@ type Counter struct {
|
|||||||
|
|
||||||
// Request is a request that has ended, as the counters count it.
|
// Request is a request that has ended, as the counters count it.
|
||||||
type Request struct {
|
type Request struct {
|
||||||
// Host is the site the alert names, "" for none.
|
|
||||||
Host string
|
|
||||||
// Client is the client's address, and ClientGroup the client it is
|
// Client is the client's address, and ClientGroup the client it is
|
||||||
// counted as: its IPv4 address, or the IPv6 netblock of
|
// counted as: its IPv4 address, or the IPv6 netblock of
|
||||||
// SWWAF_IPV6_GROUP_PREFIX its address is in.
|
// SWWAF_IPV6_GROUP_PREFIX its address is in.
|
||||||
@@ -398,7 +396,6 @@ func alertFor(r Request, k key, o overThreshold) alerts.Alert {
|
|||||||
|
|
||||||
return alerts.Alert{
|
return alerts.Alert{
|
||||||
Event: alerts.EventAnomaly,
|
Event: alerts.EventAnomaly,
|
||||||
Host: r.Host,
|
|
||||||
Client: r.Client,
|
Client: r.Client,
|
||||||
Netblock: k.netblock,
|
Netblock: k.netblock,
|
||||||
ASN: r.ASN,
|
ASN: r.ASN,
|
||||||
|
|||||||
@@ -61,59 +61,6 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
|
|||||||
wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 1, second: 0, third: 1})
|
wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 1, second: 0, third: 1})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEarlierBansCountABanDroppedAfterAnAdminsBan(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
rules := defaultRules()
|
|
||||||
rules.MaxBans = 2
|
|
||||||
ledger := bans.New(rules)
|
|
||||||
netblock := netip.MustParsePrefix("203.0.113.1/32")
|
|
||||||
other := netip.MustParsePrefix("203.0.113.2/32")
|
|
||||||
|
|
||||||
// An admin bans the netblock for a minute, and it then breaks a limit
|
|
||||||
// twice, each time after its last ban has ended.
|
|
||||||
ledger.BanForAdmin(netblock, midnight(), midnight().Add(time.Minute),
|
|
||||||
"probes for logins")
|
|
||||||
first, _ := ledger.BanForLimit(netblock, midnight().Add(time.Minute), bans.Notes{})
|
|
||||||
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
|
|
||||||
|
|
||||||
// A ban on another netblock fills the ledger, and the netblock's first
|
|
||||||
// ban for a limit, its earliest that is not an admin's, goes.
|
|
||||||
ledger.BanForLimit(other, second.Start, bans.Notes{})
|
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{netblock: 2, other: 1})
|
|
||||||
|
|
||||||
// The netblock's next ban still counts it.
|
|
||||||
third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
|
|
||||||
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 2, Admin: 1}) {
|
|
||||||
t.Errorf("the third ban for a limit has earlier bans %+v, "+
|
|
||||||
"want 2 for a limit and 1 an admin's", third.Notes.EarlierBans)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEarlierBansCountABanAnAdminKeepsOnce(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
ledger := bans.New(defaultRules())
|
|
||||||
netblock := netip.MustParsePrefix("203.0.113.1/32")
|
|
||||||
|
|
||||||
// The netblock breaks a limit twice, the second time after its first
|
|
||||||
// ban has ended, and the second ban's notes count the first. An admin
|
|
||||||
// then keeps the first ban.
|
|
||||||
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
|
||||||
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
|
|
||||||
|
|
||||||
kept := first
|
|
||||||
kept.Cause = bans.CauseAdmin
|
|
||||||
ledger.LoadEdit([]bans.Ban{kept, second})
|
|
||||||
|
|
||||||
// The netblock's next ban counts the kept ban once, as an admin's.
|
|
||||||
third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
|
|
||||||
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Admin: 1}) {
|
|
||||||
t.Errorf("the third ban for a limit has earlier bans %+v, "+
|
|
||||||
"want 1 for a limit and 1 an admin's", third.Notes.EarlierBans)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReasonOfTheBansSmallwebwafMakes(t *testing.T) {
|
func TestReasonOfTheBansSmallwebwafMakes(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
+11
-37
@@ -123,8 +123,7 @@ type Notes struct {
|
|||||||
LimitPercentSetting string `json:"limit_percent_setting,omitempty"`
|
LimitPercentSetting string `json:"limit_percent_setting,omitempty"`
|
||||||
// RuleID and Target are, for a ban for a clear sign of attack, the id
|
// RuleID and Target are, for a ban for a clear sign of attack, the id
|
||||||
// of the rule file rule that matched, and its target; TrapPath is, for
|
// of the rule file rule that matched, and its target; TrapPath is, for
|
||||||
// one for a request for a path in SWWAF_TRAP_PATHS, that path as the
|
// one for a request for a path in SWWAF_TRAP_PATHS, that path.
|
||||||
// client sent it.
|
|
||||||
RuleID string `json:"rule_id,omitempty"`
|
RuleID string `json:"rule_id,omitempty"`
|
||||||
Target string `json:"target,omitempty"`
|
Target string `json:"target,omitempty"`
|
||||||
TrapPath string `json:"trap_path,omitempty"`
|
TrapPath string `json:"trap_path,omitempty"`
|
||||||
@@ -687,51 +686,26 @@ func (l *Ledger) ban(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// earlierBans returns how many bans a netblock with the bans held, oldest
|
// earlierBans returns how many bans a netblock with the bans held, oldest
|
||||||
// first, has had, by cause. Each ban held gives a count: the bans its
|
// first, has had, by cause: the first ban held counts the bans the
|
||||||
// notes count before it, and one for it and each ban held after it. A
|
// netblock had before that one, since dropped to make room, and each ban
|
||||||
// ban's count misses a ban dropped after it to make room, and that of an
|
// held adds one.
|
||||||
// admin's ban added to bans.json without notes misses the bans before it,
|
|
||||||
// so the count that names the most bans is taken, whole: a ban an admin
|
|
||||||
// keeps by giving it CauseAdmin is still counted under its old cause in
|
|
||||||
// later bans' notes, and taking each cause from another count would count
|
|
||||||
// it twice.
|
|
||||||
func earlierBans(held []Ban) EarlierBans {
|
func earlierBans(held []Ban) EarlierBans {
|
||||||
// fromHere counts ban and the bans held after it.
|
earlier := held[0].Notes.EarlierBans
|
||||||
var most, fromHere EarlierBans
|
|
||||||
|
|
||||||
for _, ban := range slices.Backward(held) {
|
for _, ban := range held {
|
||||||
switch ban.Cause {
|
switch ban.Cause {
|
||||||
case CauseLimit:
|
case CauseLimit:
|
||||||
fromHere.Limit++
|
earlier.Limit++
|
||||||
case CauseAttack:
|
case CauseAttack:
|
||||||
fromHere.Attack++
|
earlier.Attack++
|
||||||
case CauseAdmin:
|
case CauseAdmin:
|
||||||
fromHere.Admin++
|
earlier.Admin++
|
||||||
case CauseCrowdSec:
|
case CauseCrowdSec:
|
||||||
fromHere.CrowdSec++
|
earlier.CrowdSec++
|
||||||
}
|
|
||||||
|
|
||||||
notes := ban.Notes.EarlierBans
|
|
||||||
count := EarlierBans{
|
|
||||||
Limit: notes.Limit + fromHere.Limit,
|
|
||||||
Attack: notes.Attack + fromHere.Attack,
|
|
||||||
Admin: notes.Admin + fromHere.Admin,
|
|
||||||
CrowdSec: notes.CrowdSec + fromHere.CrowdSec,
|
|
||||||
}
|
|
||||||
|
|
||||||
// Of counts that name as many bans, the earlier ban's is taken: it
|
|
||||||
// counts more of the bans held by the cause they have now.
|
|
||||||
if count.total() >= most.total() {
|
|
||||||
most = count
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return most
|
return earlier
|
||||||
}
|
|
||||||
|
|
||||||
// total returns how many bans e counts, of all causes.
|
|
||||||
func (e EarlierBans) total() int {
|
|
||||||
return e.Limit + e.Attack + e.Admin + e.CrowdSec
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// markChanged has Changed receive a value, unless one is waiting already.
|
// markChanged has Changed receive a value, unless one is waiting already.
|
||||||
|
|||||||
+122
-72
@@ -1,10 +1,7 @@
|
|||||||
// Package config reads smallwebwaf's settings, and is the one place they are
|
// Package config reads smallwebwaf's settings. Every setting is an
|
||||||
// read. Every setting is an environment variable. The settings of `smallwebwaf
|
// environment variable whose name starts with SWWAF_, or a file such a
|
||||||
// servewaf` start with SWWAF_, every one of them has a default, and most can be
|
// variable names, every setting has a default, and this package is the
|
||||||
// given as a file instead. Those of `smallwebwaf rproxy` start with SWRP_. Each
|
// one place they are read.
|
||||||
// of them but SWRP_WAF_BACKEND has a default, and none can be given as a file,
|
|
||||||
// since rproxy never touches disk. SMALLWEBWAF_COMMAND chooses which of the two
|
|
||||||
// the image runs.
|
|
||||||
package config
|
package config
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -31,11 +28,13 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Config is the settings of `smallwebwaf servewaf`. A time or size limit of
|
// Config is smallwebwaf's settings. A timeout, size or rate limit of zero
|
||||||
// zero is off.
|
// is off.
|
||||||
type Config struct {
|
type Config struct {
|
||||||
// ListenAddr is where servewaf listens (SWWAF_LISTEN_ADDR).
|
// ListenAddr is where smallwebwaf listens (SWWAF_LISTEN_ADDR).
|
||||||
ListenAddr string
|
ListenAddr string
|
||||||
|
// UpstreamURL is the app (SWWAF_UPSTREAM_URL).
|
||||||
|
UpstreamURL *url.URL
|
||||||
// InstanceName is the name every log line and alert gives as instance,
|
// InstanceName is the name every log line and alert gives as instance,
|
||||||
// and every metric carries as its label instance (SWWAF_INSTANCE_NAME),
|
// and every metric carries as its label instance (SWWAF_INSTANCE_NAME),
|
||||||
// by default the host's name, which docker sets to the first 12
|
// by default the host's name, which docker sets to the first 12
|
||||||
@@ -44,14 +43,42 @@ type Config struct {
|
|||||||
// Observe is true in observe mode, when SWWAF_MODE is observe rather
|
// Observe is true in observe mode, when SWWAF_MODE is observe rather
|
||||||
// than enforce: a request that SWWAF_DENY_NETS, a ban, the country
|
// than enforce: a request that SWWAF_DENY_NETS, a ban, the country
|
||||||
// lists, a rate limit, a rule or the Core Rule Set would refuse is
|
// lists, a rate limit, a rule or the Core Rule Set would refuse is
|
||||||
// allowed instead, and no ban is made.
|
// passed to the app instead, and no ban is made.
|
||||||
Observe bool
|
Observe bool
|
||||||
|
// TrustedProxies are the netblocks whose X-Forwarded-For is
|
||||||
|
// believed (SWWAF_TRUSTED_PROXIES).
|
||||||
|
TrustedProxies []netip.Prefix
|
||||||
// IPv6GroupPrefix is the length of the IPv6 netblock that is one client
|
// IPv6GroupPrefix is the length of the IPv6 netblock that is one client
|
||||||
// (SWWAF_IPV6_GROUP_PREFIX), from 32 to 128.
|
// (SWWAF_IPV6_GROUP_PREFIX), from 32 to 128.
|
||||||
IPv6GroupPrefix int
|
IPv6GroupPrefix int
|
||||||
// MaxTrackedClients is the most clients the table of clients holds, in
|
// MaxTrackedClients is the most clients the table of clients holds, in
|
||||||
// memory and in clients.json (SWWAF_MAX_TRACKED_CLIENTS).
|
// memory and in clients.json (SWWAF_MAX_TRACKED_CLIENTS).
|
||||||
MaxTrackedClients int
|
MaxTrackedClients int
|
||||||
|
// ClientRequestTimeout bounds reading the whole request from the
|
||||||
|
// client (SWWAF_CLIENT_REQUEST_TIMEOUT).
|
||||||
|
ClientRequestTimeout time.Duration
|
||||||
|
// ClientRequestHeaderMaxBytes is the largest request line and headers
|
||||||
|
// a client may send (SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES). It is
|
||||||
|
// never off, and always more than 4K.
|
||||||
|
ClientRequestHeaderMaxBytes int64
|
||||||
|
// ClientIdleTimeout bounds how long a kept-open client connection
|
||||||
|
// may wait for its next request (SWWAF_CLIENT_IDLE_TIMEOUT).
|
||||||
|
ClientIdleTimeout time.Duration
|
||||||
|
// ClientResponseTimeout bounds writing the whole response to the
|
||||||
|
// client (SWWAF_CLIENT_RESPONSE_TIMEOUT).
|
||||||
|
ClientResponseTimeout time.Duration
|
||||||
|
// UpstreamRequestTimeout bounds connecting to the app and writing
|
||||||
|
// the whole request to it (SWWAF_UPSTREAM_REQUEST_TIMEOUT).
|
||||||
|
UpstreamRequestTimeout time.Duration
|
||||||
|
// UpstreamResponseTimeout bounds reading the whole response from
|
||||||
|
// the app (SWWAF_UPSTREAM_RESPONSE_TIMEOUT).
|
||||||
|
UpstreamResponseTimeout time.Duration
|
||||||
|
// RequestMaxBytes is the largest request body
|
||||||
|
// (SWWAF_REQUEST_MAX_BYTES).
|
||||||
|
RequestMaxBytes int64
|
||||||
|
// ResponseMaxBytes is the largest response body
|
||||||
|
// (SWWAF_RESPONSE_MAX_BYTES).
|
||||||
|
ResponseMaxBytes int64
|
||||||
// AllowNets are the netblocks whose clients skip every check
|
// AllowNets are the netblocks whose clients skip every check
|
||||||
// (SWWAF_ALLOW_NETS). RateLimitExemptNets are those whose clients the
|
// (SWWAF_ALLOW_NETS). RateLimitExemptNets are those whose clients the
|
||||||
// rate limits neither count nor refuse (SWWAF_RATE_LIMIT_EXEMPT_NETS).
|
// rate limits neither count nor refuse (SWWAF_RATE_LIMIT_EXEMPT_NETS).
|
||||||
@@ -231,8 +258,9 @@ type Config struct {
|
|||||||
// attack (SWWAF_TRAP_PATHS), each starting with / and without a ?.
|
// attack (SWWAF_TRAP_PATHS), each starting with / and without a ?.
|
||||||
TrapPaths []string
|
TrapPaths []string
|
||||||
// ErrorBurstThreshold is the most requests of a client within a minute
|
// ErrorBurstThreshold is the most requests of a client within a minute
|
||||||
// that servewaf may refuse after a rule file or Core Rule Set match; one
|
// that smallwebwaf may refuse after a rule file or Core Rule Set match
|
||||||
// more breaks a limit (SWWAF_ERROR_BURST_THRESHOLD). 0 is off.
|
// or for a missing or wrong token; one more breaks a limit
|
||||||
|
// (SWWAF_ERROR_BURST_THRESHOLD). 0 is off.
|
||||||
ErrorBurstThreshold int64
|
ErrorBurstThreshold int64
|
||||||
// LogRemoteURL is where every line on stdout is also sent
|
// LogRemoteURL is where every line on stdout is also sent
|
||||||
// (SWWAF_LOG_REMOTE_URL), nil while it is unset and nothing is sent.
|
// (SWWAF_LOG_REMOTE_URL), nil while it is unset and nothing is sent.
|
||||||
@@ -335,8 +363,8 @@ const (
|
|||||||
// masked is what the log shows for a token that is set, and in place of
|
// masked is what the log shows for a token that is set, and in place of
|
||||||
// a secret in another setting.
|
// a secret in another setting.
|
||||||
masked = "********"
|
masked = "********"
|
||||||
// defaultListenAddr is the default of SWWAF_LISTEN_ADDR and
|
// defaultListenAddr and defaultUpstreamURL are the defaults of
|
||||||
// SWRP_LISTEN_ADDR, and defaultUpstreamURL that of SWRP_UPSTREAM_URL.
|
// SWWAF_LISTEN_ADDR and SWWAF_UPSTREAM_URL.
|
||||||
defaultListenAddr = ":8080"
|
defaultListenAddr = ":8080"
|
||||||
defaultUpstreamURL = "http://127.0.0.1:8081"
|
defaultUpstreamURL = "http://127.0.0.1:8081"
|
||||||
)
|
)
|
||||||
@@ -364,14 +392,13 @@ var (
|
|||||||
errHeaderTakenOut = errors.New(
|
errHeaderTakenOut = errors.New(
|
||||||
"is taken out of every request by Go's HTTP server, so it can never " +
|
"is taken out of every request by Go's HTTP server, so it can never " +
|
||||||
"be logged")
|
"be logged")
|
||||||
errOnBothLists = errors.New("is in SWWAF_DENIED_COUNTRIES too")
|
errOnBothLists = errors.New("is in SWWAF_DENIED_COUNTRIES too")
|
||||||
errNotLookupSource = errors.New("is not geojs, file or off")
|
errNotLookupSource = errors.New("is not geojs, file or off")
|
||||||
errNeedsLookups = errors.New("it needs each client looked up")
|
errNeedsLookups = errors.New("it needs each client looked up")
|
||||||
errNeedsDBPath = errors.New("it names the file to look clients up in")
|
errNeedsDBPath = errors.New("it names the file to look clients up in")
|
||||||
errDBPathUnused = errors.New("only file reads it")
|
errDBPathUnused = errors.New("only file reads it")
|
||||||
errOver64K = errors.New(
|
errNotOver4K = errors.New("is not a size of more than 4K, such as 32K")
|
||||||
"is more than 64K, the most servewaf gets of a body")
|
errOver1G = errors.New("is more than 1G, the most Coraza reads")
|
||||||
errNotCommand = errors.New("is not rproxy or servewaf")
|
|
||||||
errNotDurationAboveZero = errors.New(
|
errNotDurationAboveZero = errors.New(
|
||||||
"is not a duration above zero, such as 1h or 7d")
|
"is not a duration above zero, such as 1h or 7d")
|
||||||
errNotNumberAboveZero = errors.New(
|
errNotNumberAboveZero = errors.New(
|
||||||
@@ -460,27 +487,38 @@ var (
|
|||||||
func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
|
func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
|
||||||
env := &environment{lookupEnv: lookupEnv}
|
env := &environment{lookupEnv: lookupEnv}
|
||||||
cfg := &Config{
|
cfg := &Config{
|
||||||
ListenAddr: env.address("SWWAF_LISTEN_ADDR"),
|
ListenAddr: env.address("SWWAF_LISTEN_ADDR", defaultListenAddr),
|
||||||
|
UpstreamURL: env.appURL("SWWAF_UPSTREAM_URL", defaultUpstreamURL),
|
||||||
InstanceName: env.instanceName(),
|
InstanceName: env.instanceName(),
|
||||||
Observe: env.observe("SWWAF_MODE", "enforce"),
|
Observe: env.observe("SWWAF_MODE", "enforce"),
|
||||||
|
TrustedProxies: env.netblocks("SWWAF_TRUSTED_PROXIES", privateRanges),
|
||||||
IPv6GroupPrefix: env.ipv6GroupPrefix("SWWAF_IPV6_GROUP_PREFIX", "64"),
|
IPv6GroupPrefix: env.ipv6GroupPrefix("SWWAF_IPV6_GROUP_PREFIX", "64"),
|
||||||
MaxTrackedClients: env.numberNotOff("SWWAF_MAX_TRACKED_CLIENTS", "20000"),
|
MaxTrackedClients: env.numberNotOff("SWWAF_MAX_TRACKED_CLIENTS", "20000"),
|
||||||
AllowNets: env.netblocks("SWWAF_ALLOW_NETS", ""),
|
ClientRequestTimeout: env.duration("SWWAF_CLIENT_REQUEST_TIMEOUT", "60s"),
|
||||||
RateLimitExemptNets: env.netblocks("SWWAF_RATE_LIMIT_EXEMPT_NETS", ""),
|
ClientRequestHeaderMaxBytes: env.headerSize(
|
||||||
DenyNets: env.netblocks("SWWAF_DENY_NETS", ""),
|
"SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES", "32K"),
|
||||||
RateLimitPerMinute: env.count("SWWAF_RATE_LIMIT_PER_MINUTE", "500"),
|
ClientIdleTimeout: env.duration("SWWAF_CLIENT_IDLE_TIMEOUT", "120s"),
|
||||||
RateLimitPerHour: env.count("SWWAF_RATE_LIMIT_PER_HOUR", "20000"),
|
ClientResponseTimeout: env.duration("SWWAF_CLIENT_RESPONSE_TIMEOUT", "30m"),
|
||||||
RateLimitPerDay: env.count("SWWAF_RATE_LIMIT_PER_DAY", "300000"),
|
UpstreamRequestTimeout: env.duration("SWWAF_UPSTREAM_REQUEST_TIMEOUT", "60s"),
|
||||||
RateLimitExemptPaths: env.pathPrefixes("SWWAF_RATE_LIMIT_EXEMPT_PATHS", ""),
|
UpstreamResponseTimeout: env.duration("SWWAF_UPSTREAM_RESPONSE_TIMEOUT", "30m"),
|
||||||
BytesLimitPerMinute: env.size("SWWAF_BYTES_LIMIT_PER_MINUTE", "4G"),
|
RequestMaxBytes: env.size("SWWAF_REQUEST_MAX_BYTES", "100M"),
|
||||||
BytesLimitPerHour: env.size("SWWAF_BYTES_LIMIT_PER_HOUR", "20G"),
|
ResponseMaxBytes: env.size("SWWAF_RESPONSE_MAX_BYTES", "5G"),
|
||||||
BytesLimitPerDay: env.size("SWWAF_BYTES_LIMIT_PER_DAY", "200G"),
|
AllowNets: env.netblocks("SWWAF_ALLOW_NETS", ""),
|
||||||
BytesCount: env.bytesCount("SWWAF_BYTES_COUNT", "both"),
|
RateLimitExemptNets: env.netblocks("SWWAF_RATE_LIMIT_EXEMPT_NETS", ""),
|
||||||
LookupSource: env.lookupSource("SWWAF_LOOKUP_SOURCE", "geojs"),
|
DenyNets: env.netblocks("SWWAF_DENY_NETS", ""),
|
||||||
LookupDBPath: env.value("SWWAF_LOOKUP_DB_PATH", ""),
|
RateLimitPerMinute: env.count("SWWAF_RATE_LIMIT_PER_MINUTE", "1000"),
|
||||||
LookupTimeout: env.durationNotOff("SWWAF_LOOKUP_TIMEOUT", "800ms"),
|
RateLimitPerHour: env.count("SWWAF_RATE_LIMIT_PER_HOUR", "10000"),
|
||||||
AddLookupHeaders: env.boolean("SWWAF_ADD_LOOKUP_HEADERS", "false"),
|
RateLimitPerDay: env.count("SWWAF_RATE_LIMIT_PER_DAY", "50000"),
|
||||||
DeniedCountries: env.countries("SWWAF_DENIED_COUNTRIES", ""),
|
RateLimitExemptPaths: env.pathPrefixes("SWWAF_RATE_LIMIT_EXEMPT_PATHS", ""),
|
||||||
|
BytesLimitPerMinute: env.size("SWWAF_BYTES_LIMIT_PER_MINUTE", "10G"),
|
||||||
|
BytesLimitPerHour: env.size("SWWAF_BYTES_LIMIT_PER_HOUR", "20G"),
|
||||||
|
BytesLimitPerDay: env.size("SWWAF_BYTES_LIMIT_PER_DAY", "50G"),
|
||||||
|
BytesCount: env.bytesCount("SWWAF_BYTES_COUNT", "both"),
|
||||||
|
LookupSource: env.lookupSource("SWWAF_LOOKUP_SOURCE", "geojs"),
|
||||||
|
LookupDBPath: env.value("SWWAF_LOOKUP_DB_PATH", ""),
|
||||||
|
LookupTimeout: env.durationNotOff("SWWAF_LOOKUP_TIMEOUT", "1s"),
|
||||||
|
AddLookupHeaders: env.boolean("SWWAF_ADD_LOOKUP_HEADERS", "false"),
|
||||||
|
DeniedCountries: env.countries("SWWAF_DENIED_COUNTRIES", ""),
|
||||||
ExclusivelyAllowedCountries: env.countries(
|
ExclusivelyAllowedCountries: env.countries(
|
||||||
"SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""),
|
"SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""),
|
||||||
ASNLimitPercent: env.percents("SWWAF_ASN_LIMIT_PERCENT", ParseASN),
|
ASNLimitPercent: env.percents("SWWAF_ASN_LIMIT_PERCENT", ParseASN),
|
||||||
@@ -572,30 +610,22 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
|
|||||||
return cfg, nil
|
return cfg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Command reads only SMALLWEBWAF_COMMAND, the command the image runs and its
|
// ListenAddrAndUpstreamURL reads only SWWAF_LISTEN_ADDR and
|
||||||
// health check checks: rproxy when it is unset or empty, or servewaf. It cannot
|
// SWWAF_UPSTREAM_URL, either of which may be given as a file, as
|
||||||
// be given as a file. Any other value is an error that names the setting.
|
// FromEnvironment does. The health check needs no other setting, so it
|
||||||
func Command(lookupEnv func(string) (string, bool)) (string, error) {
|
// reads no other, nor a file that another names.
|
||||||
command, _ := lookupEnv("SMALLWEBWAF_COMMAND")
|
func ListenAddrAndUpstreamURL(
|
||||||
|
lookupEnv func(string) (string, bool),
|
||||||
switch command {
|
) (string, *url.URL, error) {
|
||||||
case "", "rproxy":
|
|
||||||
return "rproxy", nil
|
|
||||||
case "servewaf":
|
|
||||||
return command, nil
|
|
||||||
default:
|
|
||||||
return "", fmt.Errorf("SMALLWEBWAF_COMMAND: %q %w", command, errNotCommand)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ServeWAFListenAddr reads only SWWAF_LISTEN_ADDR, which may be given as a
|
|
||||||
// file, as FromEnvironment does. The health check in servewaf's container needs
|
|
||||||
// no other setting, so it reads no other, nor a file that another names.
|
|
||||||
func ServeWAFListenAddr(lookupEnv func(string) (string, bool)) (string, error) {
|
|
||||||
env := &environment{lookupEnv: lookupEnv}
|
env := &environment{lookupEnv: lookupEnv}
|
||||||
listenAddr := env.address("SWWAF_LISTEN_ADDR")
|
listenAddr := env.address("SWWAF_LISTEN_ADDR", defaultListenAddr)
|
||||||
|
upstreamURL := env.appURL("SWWAF_UPSTREAM_URL", defaultUpstreamURL)
|
||||||
|
|
||||||
return listenAddr, env.err
|
if env.err != nil {
|
||||||
|
return "", nil, env.err
|
||||||
|
}
|
||||||
|
|
||||||
|
return listenAddr, upstreamURL, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// InstanceName reads only SWWAF_INSTANCE_NAME, which may be given as a
|
// InstanceName reads only SWWAF_INSTANCE_NAME, which may be given as a
|
||||||
@@ -677,10 +707,9 @@ func (e *environment) check(name string, err error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// address reads a setting that is an address to listen on, by default
|
// address reads a setting that is an address to listen on.
|
||||||
// defaultListenAddr.
|
func (e *environment) address(name, defaultValue string) string {
|
||||||
func (e *environment) address(name string) string {
|
address, err := parseListenAddr(e.value(name, defaultValue))
|
||||||
address, err := parseListenAddr(e.value(name, defaultListenAddr))
|
|
||||||
e.check(name, err)
|
e.check(name, err)
|
||||||
|
|
||||||
return address
|
return address
|
||||||
@@ -748,6 +777,15 @@ func (e *environment) wafBodyLimit(name, defaultValue string) int64 {
|
|||||||
return limit
|
return limit
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// headerSize reads the setting that is the largest request line and
|
||||||
|
// headers.
|
||||||
|
func (e *environment) headerSize(name, defaultValue string) int64 {
|
||||||
|
size, err := parseHeaderSize(e.value(name, defaultValue))
|
||||||
|
e.check(name, err)
|
||||||
|
|
||||||
|
return size
|
||||||
|
}
|
||||||
|
|
||||||
// count reads a setting that is a number of requests.
|
// count reads a setting that is a number of requests.
|
||||||
func (e *environment) count(name, defaultValue string) int64 {
|
func (e *environment) count(name, defaultValue string) int64 {
|
||||||
count, err := parseCount(e.value(name, defaultValue))
|
count, err := parseCount(e.value(name, defaultValue))
|
||||||
@@ -1394,17 +1432,30 @@ func parseSize(value string) (int64, error) {
|
|||||||
return n * unit, nil
|
return n * unit, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// parseHeaderSize reads the largest request line and headers: a size as
|
||||||
|
// parseSize reads it, but more than 4K and never off. Go's server reads 4K
|
||||||
|
// past the limit it is given before it refuses, so proxy.New gives it this
|
||||||
|
// size less 4K, which must leave a limit.
|
||||||
|
func parseHeaderSize(value string) (int64, error) {
|
||||||
|
size, err := parseSize(value)
|
||||||
|
if err != nil || size <= 4*kibibyte {
|
||||||
|
return 0, fmt.Errorf("%q %w", value, errNotOver4K)
|
||||||
|
}
|
||||||
|
|
||||||
|
return size, nil
|
||||||
|
}
|
||||||
|
|
||||||
// parseWAFBodyLimit reads the most of a request body the Core Rule Set
|
// parseWAFBodyLimit reads the most of a request body the Core Rule Set
|
||||||
// reads: a size as parseSize reads it, or off, but at most 64K, since servewaf
|
// reads: a size as parseSize reads it, or off, but at most 1G, since
|
||||||
// gets no more of a body than that.
|
// Coraza, which runs the Core Rule Set, refuses to load with more.
|
||||||
func parseWAFBodyLimit(value string) (int64, error) {
|
func parseWAFBodyLimit(value string) (int64, error) {
|
||||||
limit, err := parseSize(value)
|
limit, err := parseSize(value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if limit > 64*kibibyte {
|
if limit > gibibyte {
|
||||||
return 0, fmt.Errorf("%q %w", value, errOver64K)
|
return 0, fmt.Errorf("%q %w", value, errOver1G)
|
||||||
}
|
}
|
||||||
|
|
||||||
return limit, nil
|
return limit, nil
|
||||||
@@ -1614,8 +1665,7 @@ func parsePathPrefixes(value string) ([]string, error) {
|
|||||||
|
|
||||||
// parseTrapPaths reads a comma-separated list of trap paths. Each is
|
// parseTrapPaths reads a comma-separated list of trap paths. Each is
|
||||||
// matched against a request's path as a path rule is, without the query,
|
// matched against a request's path as a path rule is, without the query,
|
||||||
// so a path that does not start with / would never match, and one that
|
// so a path that does not start with / or holds a ? would never match.
|
||||||
// holds a ? is written with a query.
|
|
||||||
func parseTrapPaths(value string) ([]string, error) {
|
func parseTrapPaths(value string) ([]string, error) {
|
||||||
paths, err := parseList(value)
|
paths, err := parseList(value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
+377
-264
@@ -23,88 +23,98 @@ import (
|
|||||||
|
|
||||||
// The settings, by name.
|
// The settings, by name.
|
||||||
const (
|
const (
|
||||||
listenAddr = "SWWAF_LISTEN_ADDR"
|
listenAddr = "SWWAF_LISTEN_ADDR"
|
||||||
mode = "SWWAF_MODE"
|
upstreamURL = "SWWAF_UPSTREAM_URL"
|
||||||
ipv6GroupPrefix = "SWWAF_IPV6_GROUP_PREFIX"
|
mode = "SWWAF_MODE"
|
||||||
maxTrackedClients = "SWWAF_MAX_TRACKED_CLIENTS"
|
trustedProxies = "SWWAF_TRUSTED_PROXIES"
|
||||||
allowNets = "SWWAF_ALLOW_NETS"
|
ipv6GroupPrefix = "SWWAF_IPV6_GROUP_PREFIX"
|
||||||
rateLimitExemptNets = "SWWAF_RATE_LIMIT_EXEMPT_NETS"
|
maxTrackedClients = "SWWAF_MAX_TRACKED_CLIENTS"
|
||||||
denyNets = "SWWAF_DENY_NETS"
|
clientRequestTimeout = "SWWAF_CLIENT_REQUEST_TIMEOUT"
|
||||||
rateLimitPerMinute = "SWWAF_RATE_LIMIT_PER_MINUTE"
|
clientHeaderMaxBytes = "SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES"
|
||||||
rateLimitPerHour = "SWWAF_RATE_LIMIT_PER_HOUR"
|
clientIdleTimeout = "SWWAF_CLIENT_IDLE_TIMEOUT"
|
||||||
rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY"
|
clientResponseTimeout = "SWWAF_CLIENT_RESPONSE_TIMEOUT"
|
||||||
rateLimitExemptPaths = "SWWAF_RATE_LIMIT_EXEMPT_PATHS"
|
upstreamRequestTimeout = "SWWAF_UPSTREAM_REQUEST_TIMEOUT"
|
||||||
bytesLimitPerMinute = "SWWAF_BYTES_LIMIT_PER_MINUTE"
|
upstreamResponseTimeout = "SWWAF_UPSTREAM_RESPONSE_TIMEOUT"
|
||||||
bytesLimitPerHour = "SWWAF_BYTES_LIMIT_PER_HOUR"
|
requestMaxBytes = "SWWAF_REQUEST_MAX_BYTES"
|
||||||
bytesLimitPerDay = "SWWAF_BYTES_LIMIT_PER_DAY"
|
responseMaxBytes = "SWWAF_RESPONSE_MAX_BYTES"
|
||||||
bytesCount = "SWWAF_BYTES_COUNT"
|
allowNets = "SWWAF_ALLOW_NETS"
|
||||||
lookupSource = "SWWAF_LOOKUP_SOURCE"
|
rateLimitExemptNets = "SWWAF_RATE_LIMIT_EXEMPT_NETS"
|
||||||
lookupDBPath = "SWWAF_LOOKUP_DB_PATH"
|
denyNets = "SWWAF_DENY_NETS"
|
||||||
lookupTimeout = "SWWAF_LOOKUP_TIMEOUT"
|
rateLimitPerMinute = "SWWAF_RATE_LIMIT_PER_MINUTE"
|
||||||
addLookupHeaders = "SWWAF_ADD_LOOKUP_HEADERS"
|
rateLimitPerHour = "SWWAF_RATE_LIMIT_PER_HOUR"
|
||||||
deniedCountries = "SWWAF_DENIED_COUNTRIES"
|
rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY"
|
||||||
allowedCountries = "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES"
|
rateLimitExemptPaths = "SWWAF_RATE_LIMIT_EXEMPT_PATHS"
|
||||||
asnLimitPercent = "SWWAF_ASN_LIMIT_PERCENT"
|
bytesLimitPerMinute = "SWWAF_BYTES_LIMIT_PER_MINUTE"
|
||||||
countryLimitPercent = "SWWAF_COUNTRY_LIMIT_PERCENT"
|
bytesLimitPerHour = "SWWAF_BYTES_LIMIT_PER_HOUR"
|
||||||
asnBytesPercent = "SWWAF_ASN_BYTES_PERCENT"
|
bytesLimitPerDay = "SWWAF_BYTES_LIMIT_PER_DAY"
|
||||||
countryBytesPercent = "SWWAF_COUNTRY_BYTES_PERCENT"
|
bytesCount = "SWWAF_BYTES_COUNT"
|
||||||
unknownLimitPercent = "SWWAF_UNKNOWN_LIMIT_PERCENT"
|
lookupSource = "SWWAF_LOOKUP_SOURCE"
|
||||||
asnLimitPercentURL = "SWWAF_ASN_LIMIT_PERCENT_URL"
|
lookupDBPath = "SWWAF_LOOKUP_DB_PATH"
|
||||||
blocklistURLs = "SWWAF_BLOCKLIST_URLS"
|
lookupTimeout = "SWWAF_LOOKUP_TIMEOUT"
|
||||||
blocklistRefresh = "SWWAF_BLOCKLIST_REFRESH"
|
addLookupHeaders = "SWWAF_ADD_LOOKUP_HEADERS"
|
||||||
blocklistAction = "SWWAF_BLOCKLIST_ACTION"
|
deniedCountries = "SWWAF_DENIED_COUNTRIES"
|
||||||
dnsblZones = "SWWAF_DNSBL_ZONES"
|
allowedCountries = "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES"
|
||||||
dnsblResolver = "SWWAF_DNSBL_RESOLVER"
|
asnLimitPercent = "SWWAF_ASN_LIMIT_PERCENT"
|
||||||
abuseIPDBKey = "SWWAF_ABUSEIPDB_KEY"
|
countryLimitPercent = "SWWAF_COUNTRY_LIMIT_PERCENT"
|
||||||
abuseIPDBMinScore = "SWWAF_ABUSEIPDB_MIN_SCORE"
|
asnBytesPercent = "SWWAF_ASN_BYTES_PERCENT"
|
||||||
abuseIPDBDailyBudget = "SWWAF_ABUSEIPDB_DAILY_BUDGET"
|
countryBytesPercent = "SWWAF_COUNTRY_BYTES_PERCENT"
|
||||||
reputationAction = "SWWAF_REPUTATION_ACTION"
|
unknownLimitPercent = "SWWAF_UNKNOWN_LIMIT_PERCENT"
|
||||||
reputationCacheTTL = "SWWAF_REPUTATION_CACHE_TTL"
|
asnLimitPercentURL = "SWWAF_ASN_LIMIT_PERCENT_URL"
|
||||||
reputationTimeout = "SWWAF_REPUTATION_TIMEOUT"
|
blocklistURLs = "SWWAF_BLOCKLIST_URLS"
|
||||||
crowdSecURL = "SWWAF_CROWDSEC_LAPI_URL"
|
blocklistRefresh = "SWWAF_BLOCKLIST_REFRESH"
|
||||||
crowdSecKey = "SWWAF_CROWDSEC_LAPI_KEY"
|
blocklistAction = "SWWAF_BLOCKLIST_ACTION"
|
||||||
banResponse = "SWWAF_BAN_RESPONSE"
|
dnsblZones = "SWWAF_DNSBL_ZONES"
|
||||||
limitBanDuration = "SWWAF_LIMIT_BAN_DURATION"
|
dnsblResolver = "SWWAF_DNSBL_RESOLVER"
|
||||||
limitBanRepeatWindow = "SWWAF_LIMIT_BAN_REPEAT_WINDOW"
|
abuseIPDBKey = "SWWAF_ABUSEIPDB_KEY"
|
||||||
maxBanDuration = "SWWAF_MAX_BAN_DURATION"
|
abuseIPDBMinScore = "SWWAF_ABUSEIPDB_MIN_SCORE"
|
||||||
attackBanDuration = "SWWAF_ATTACK_BAN_DURATION"
|
abuseIPDBDailyBudget = "SWWAF_ABUSEIPDB_DAILY_BUDGET"
|
||||||
maxBans = "SWWAF_MAX_BANS"
|
reputationAction = "SWWAF_REPUTATION_ACTION"
|
||||||
banScopeV4Prefix = "SWWAF_BAN_SCOPE_V4_PREFIX"
|
reputationCacheTTL = "SWWAF_REPUTATION_CACHE_TTL"
|
||||||
stateDir = "SWWAF_STATE_DIR"
|
reputationTimeout = "SWWAF_REPUTATION_TIMEOUT"
|
||||||
stateWriteDelay = "SWWAF_STATE_WRITE_DELAY"
|
crowdSecURL = "SWWAF_CROWDSEC_LAPI_URL"
|
||||||
stateCounterInterval = "SWWAF_STATE_COUNTER_INTERVAL"
|
crowdSecKey = "SWWAF_CROWDSEC_LAPI_KEY"
|
||||||
adminToken = "SWWAF_ADMIN_TOKEN" //nolint:gosec // the setting's name
|
banResponse = "SWWAF_BAN_RESPONSE"
|
||||||
metricsToken = "SWWAF_METRICS_TOKEN" //nolint:gosec // the setting's name
|
limitBanDuration = "SWWAF_LIMIT_BAN_DURATION"
|
||||||
metricsTopN = "SWWAF_METRICS_TOP_N"
|
limitBanRepeatWindow = "SWWAF_LIMIT_BAN_REPEAT_WINDOW"
|
||||||
instanceName = "SWWAF_INSTANCE_NAME"
|
maxBanDuration = "SWWAF_MAX_BAN_DURATION"
|
||||||
logRequestHeaders = "SWWAF_LOG_REQUEST_HEADERS"
|
attackBanDuration = "SWWAF_ATTACK_BAN_DURATION"
|
||||||
logLevel = "SWWAF_LOG_LEVEL"
|
maxBans = "SWWAF_MAX_BANS"
|
||||||
rulesDir = "SWWAF_RULES_DIR"
|
banScopeV4Prefix = "SWWAF_BAN_SCOPE_V4_PREFIX"
|
||||||
rulesEnabled = "SWWAF_RULES_ENABLED"
|
stateDir = "SWWAF_STATE_DIR"
|
||||||
wafMode = "SWWAF_WAF_MODE"
|
stateWriteDelay = "SWWAF_STATE_WRITE_DELAY"
|
||||||
wafParanoiaLevel = "SWWAF_WAF_PARANOIA_LEVEL"
|
stateCounterInterval = "SWWAF_STATE_COUNTER_INTERVAL"
|
||||||
wafAnomalyThreshold = "SWWAF_WAF_ANOMALY_THRESHOLD"
|
adminToken = "SWWAF_ADMIN_TOKEN" //nolint:gosec // the setting's name
|
||||||
wafDisabledRules = "SWWAF_WAF_DISABLED_RULES"
|
metricsToken = "SWWAF_METRICS_TOKEN" //nolint:gosec // the setting's name
|
||||||
wafExemptPaths = "SWWAF_WAF_EXEMPT_PATHS"
|
metricsTopN = "SWWAF_METRICS_TOP_N"
|
||||||
wafBodyLimit = "SWWAF_WAF_BODY_LIMIT"
|
instanceName = "SWWAF_INSTANCE_NAME"
|
||||||
trapPaths = "SWWAF_TRAP_PATHS"
|
logRequestHeaders = "SWWAF_LOG_REQUEST_HEADERS"
|
||||||
errorBurstThreshold = "SWWAF_ERROR_BURST_THRESHOLD"
|
logLevel = "SWWAF_LOG_LEVEL"
|
||||||
logRemoteURL = "SWWAF_LOG_REMOTE_URL"
|
rulesDir = "SWWAF_RULES_DIR"
|
||||||
logRemoteTLSCAFile = "SWWAF_LOG_REMOTE_TLS_CA_FILE"
|
rulesEnabled = "SWWAF_RULES_ENABLED"
|
||||||
logRemoteBuffer = "SWWAF_LOG_REMOTE_BUFFER"
|
wafMode = "SWWAF_WAF_MODE"
|
||||||
logRemoteFacility = "SWWAF_LOG_REMOTE_FACILITY"
|
wafParanoiaLevel = "SWWAF_WAF_PARANOIA_LEVEL"
|
||||||
logRemoteAppName = "SWWAF_LOG_REMOTE_APP_NAME"
|
wafAnomalyThreshold = "SWWAF_WAF_ANOMALY_THRESHOLD"
|
||||||
alertWebhookURL = "SWWAF_ALERT_WEBHOOK_URL"
|
wafDisabledRules = "SWWAF_WAF_DISABLED_RULES"
|
||||||
alertWebhookHeaders = "SWWAF_ALERT_WEBHOOK_HEADERS"
|
wafExemptPaths = "SWWAF_WAF_EXEMPT_PATHS"
|
||||||
alertSlackWebhookURL = "SWWAF_ALERT_SLACK_WEBHOOK_URL"
|
wafBodyLimit = "SWWAF_WAF_BODY_LIMIT"
|
||||||
alertNtfyURL = "SWWAF_ALERT_NTFY_URL"
|
trapPaths = "SWWAF_TRAP_PATHS"
|
||||||
alertNtfyToken = "SWWAF_ALERT_NTFY_TOKEN" //nolint:gosec // the setting's name
|
errorBurstThreshold = "SWWAF_ERROR_BURST_THRESHOLD"
|
||||||
alertEvents = "SWWAF_ALERT_EVENTS"
|
logRemoteURL = "SWWAF_LOG_REMOTE_URL"
|
||||||
alertCooldown = "SWWAF_ALERT_COOLDOWN"
|
logRemoteTLSCAFile = "SWWAF_LOG_REMOTE_TLS_CA_FILE"
|
||||||
alertMaxPerHour = "SWWAF_ALERT_MAX_PER_HOUR"
|
logRemoteBuffer = "SWWAF_LOG_REMOTE_BUFFER"
|
||||||
anomalyNetV4Prefix = "SWWAF_ANOMALY_NET_V4_PREFIX"
|
logRemoteFacility = "SWWAF_LOG_REMOTE_FACILITY"
|
||||||
anomalyNetV6Prefix = "SWWAF_ANOMALY_NET_V6_PREFIX"
|
logRemoteAppName = "SWWAF_LOG_REMOTE_APP_NAME"
|
||||||
watchNets = "SWWAF_WATCH_NETS"
|
alertWebhookURL = "SWWAF_ALERT_WEBHOOK_URL"
|
||||||
|
alertWebhookHeaders = "SWWAF_ALERT_WEBHOOK_HEADERS"
|
||||||
|
alertSlackWebhookURL = "SWWAF_ALERT_SLACK_WEBHOOK_URL"
|
||||||
|
alertNtfyURL = "SWWAF_ALERT_NTFY_URL"
|
||||||
|
alertNtfyToken = "SWWAF_ALERT_NTFY_TOKEN" //nolint:gosec // the setting's name
|
||||||
|
alertEvents = "SWWAF_ALERT_EVENTS"
|
||||||
|
alertCooldown = "SWWAF_ALERT_COOLDOWN"
|
||||||
|
alertMaxPerHour = "SWWAF_ALERT_MAX_PER_HOUR"
|
||||||
|
anomalyNetV4Prefix = "SWWAF_ANOMALY_NET_V4_PREFIX"
|
||||||
|
anomalyNetV6Prefix = "SWWAF_ANOMALY_NET_V6_PREFIX"
|
||||||
|
watchNets = "SWWAF_WATCH_NETS"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The anomaly thresholds: each of the prefixes below, which name a scope,
|
// The anomaly thresholds: each of the prefixes below, which name a scope,
|
||||||
@@ -242,34 +252,48 @@ func TestDefaults(t *testing.T) {
|
|||||||
cfg := fromEnvironment(t, environment{})
|
cfg := fromEnvironment(t, environment{})
|
||||||
|
|
||||||
wantSettings(t, cfg, config.Config{
|
wantSettings(t, cfg, config.Config{
|
||||||
ListenAddr: ":8080",
|
ListenAddr: ":8080",
|
||||||
Observe: false,
|
Observe: false,
|
||||||
RateLimitPerMinute: 500,
|
ClientRequestTimeout: time.Minute,
|
||||||
RateLimitPerHour: 20000,
|
ClientRequestHeaderMaxBytes: 32 << 10,
|
||||||
RateLimitPerDay: 300000,
|
ClientIdleTimeout: 2 * time.Minute,
|
||||||
BanResponse: 403,
|
ClientResponseTimeout: 30 * time.Minute,
|
||||||
LimitBanDuration: time.Hour,
|
UpstreamRequestTimeout: time.Minute,
|
||||||
LimitBanRepeatWindow: 24 * time.Hour,
|
UpstreamResponseTimeout: 30 * time.Minute,
|
||||||
MaxBanDuration: 7 * 24 * time.Hour,
|
RequestMaxBytes: 100 << 20,
|
||||||
AttackBanDuration: 7 * 24 * time.Hour,
|
ResponseMaxBytes: 5 << 30,
|
||||||
MaxBans: 5000,
|
RateLimitPerMinute: 1000,
|
||||||
BanScopeV4Prefix: 32,
|
RateLimitPerHour: 10000,
|
||||||
StateDir: "/var/lib/smallwebwaf",
|
RateLimitPerDay: 50000,
|
||||||
StateWriteDelay: 10 * time.Second,
|
BanResponse: 403,
|
||||||
StateCounterInterval: 15 * time.Minute,
|
LimitBanDuration: time.Hour,
|
||||||
MetricsToken: "",
|
LimitBanRepeatWindow: 24 * time.Hour,
|
||||||
MetricsTopN: 50,
|
MaxBanDuration: 7 * 24 * time.Hour,
|
||||||
RulesDir: "/etc/smallwebwaf/rules.d",
|
AttackBanDuration: 7 * 24 * time.Hour,
|
||||||
RulesEnabled: true,
|
MaxBans: 5000,
|
||||||
|
BanScopeV4Prefix: 32,
|
||||||
|
StateDir: "/var/lib/smallwebwaf",
|
||||||
|
StateWriteDelay: 10 * time.Second,
|
||||||
|
StateCounterInterval: 15 * time.Minute,
|
||||||
|
MetricsToken: "",
|
||||||
|
MetricsTopN: 50,
|
||||||
|
RulesDir: "/etc/smallwebwaf/rules.d",
|
||||||
|
RulesEnabled: true,
|
||||||
})
|
})
|
||||||
wantLookupSettings(t, cfg, config.Config{
|
wantLookupSettings(t, cfg, config.Config{
|
||||||
LookupSource: defaultLookupSource, LookupTimeout: 800 * time.Millisecond,
|
LookupSource: defaultLookupSource, LookupTimeout: time.Second,
|
||||||
})
|
})
|
||||||
wantByteLimitSettings(t, cfg, config.Config{
|
wantByteLimitSettings(t, cfg, config.Config{
|
||||||
BytesLimitPerMinute: 4 << 30, BytesLimitPerHour: 20 << 30,
|
BytesLimitPerMinute: 10 << 30, BytesLimitPerHour: 20 << 30,
|
||||||
BytesLimitPerDay: 200 << 30, BytesCount: "both",
|
BytesLimitPerDay: 50 << 30, BytesCount: "both",
|
||||||
})
|
})
|
||||||
|
|
||||||
|
if cfg.UpstreamURL.String() != "http://127.0.0.1:8081" {
|
||||||
|
t.Errorf("%s is %s", upstreamURL, cfg.UpstreamURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantNetblocks(t, cfg.TrustedProxies,
|
||||||
|
"10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16")
|
||||||
wantNetblocks(t, cfg.AllowNets)
|
wantNetblocks(t, cfg.AllowNets)
|
||||||
wantNetblocks(t, cfg.RateLimitExemptNets)
|
wantNetblocks(t, cfg.RateLimitExemptNets)
|
||||||
wantNetblocks(t, cfg.DenyNets)
|
wantNetblocks(t, cfg.DenyNets)
|
||||||
@@ -298,13 +322,8 @@ func TestNoSingleRequestBreaksAByteLimitAtTheDefaults(t *testing.T) {
|
|||||||
|
|
||||||
cfg := fromEnvironment(t, environment{})
|
cfg := fromEnvironment(t, environment{})
|
||||||
|
|
||||||
rproxy, err := rproxyConfig(map[string]string{wafBackend: backendURL})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("settings: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The largest request body and the largest response, both counted.
|
// The largest request body and the largest response, both counted.
|
||||||
largest := rproxy.MaxRequestSize + rproxy.MaxResponseSize
|
largest := cfg.RequestMaxBytes + cfg.ResponseMaxBytes
|
||||||
for _, limit := range []int64{
|
for _, limit := range []int64{
|
||||||
cfg.BytesLimitPerMinute, cfg.BytesLimitPerHour, cfg.BytesLimitPerDay,
|
cfg.BytesLimitPerMinute, cfg.BytesLimitPerHour, cfg.BytesLimitPerDay,
|
||||||
} {
|
} {
|
||||||
@@ -318,54 +337,77 @@ func TestValuesAsSet(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
cfg := fromEnvironment(t, environment{
|
cfg := fromEnvironment(t, environment{
|
||||||
listenAddr: "127.0.0.1:9000",
|
listenAddr: "127.0.0.1:9000",
|
||||||
mode: "observe",
|
upstreamURL: "https://app.internal:8443/",
|
||||||
allowNets: "192.0.2.7",
|
mode: "observe",
|
||||||
rateLimitExemptNets: "2001:db8::/48, 10.9.8.7",
|
trustedProxies: " 192.0.2.1, 10.1.2.3/8 ,2001:db8::/32",
|
||||||
denyNets: "198.51.100.0/24",
|
clientRequestTimeout: "90s",
|
||||||
rateLimitPerMinute: "60",
|
clientHeaderMaxBytes: "8K",
|
||||||
rateLimitPerHour: "600",
|
clientIdleTimeout: "5m",
|
||||||
rateLimitPerDay: "6000",
|
clientResponseTimeout: "7d",
|
||||||
deniedCountries: "cn, RU,kp,Xk",
|
upstreamRequestTimeout: "1h30m",
|
||||||
allowedCountries: "de",
|
upstreamResponseTimeout: off,
|
||||||
banResponse: "429",
|
requestMaxBytes: "512K",
|
||||||
limitBanDuration: "15m",
|
responseMaxBytes: "1234",
|
||||||
limitBanRepeatWindow: "2d",
|
allowNets: "192.0.2.7",
|
||||||
maxBanDuration: "30d",
|
rateLimitExemptNets: "2001:db8::/48, 10.9.8.7",
|
||||||
attackBanDuration: "1d",
|
denyNets: "198.51.100.0/24",
|
||||||
maxBans: "100",
|
rateLimitPerMinute: "60",
|
||||||
banScopeV4Prefix: "24",
|
rateLimitPerHour: "600",
|
||||||
stateDir: "/srv/waf-state",
|
rateLimitPerDay: "6000",
|
||||||
stateWriteDelay: "500ms",
|
deniedCountries: "cn, RU,kp,Xk",
|
||||||
stateCounterInterval: "1h",
|
allowedCountries: "de",
|
||||||
metricsToken: token,
|
banResponse: "429",
|
||||||
metricsTopN: "10",
|
limitBanDuration: "15m",
|
||||||
rulesDir: "/srv/waf-rules",
|
limitBanRepeatWindow: "2d",
|
||||||
rulesEnabled: "false",
|
maxBanDuration: "30d",
|
||||||
|
attackBanDuration: "1d",
|
||||||
|
maxBans: "100",
|
||||||
|
banScopeV4Prefix: "24",
|
||||||
|
stateDir: "/srv/waf-state",
|
||||||
|
stateWriteDelay: "500ms",
|
||||||
|
stateCounterInterval: "1h",
|
||||||
|
metricsToken: token,
|
||||||
|
metricsTopN: "10",
|
||||||
|
rulesDir: "/srv/waf-rules",
|
||||||
|
rulesEnabled: "false",
|
||||||
})
|
})
|
||||||
|
|
||||||
wantSettings(t, cfg, config.Config{
|
wantSettings(t, cfg, config.Config{
|
||||||
ListenAddr: "127.0.0.1:9000",
|
ListenAddr: "127.0.0.1:9000",
|
||||||
Observe: true,
|
Observe: true,
|
||||||
RateLimitPerMinute: 60,
|
ClientRequestTimeout: 90 * time.Second,
|
||||||
RateLimitPerHour: 600,
|
ClientRequestHeaderMaxBytes: 8 << 10,
|
||||||
RateLimitPerDay: 6000,
|
ClientIdleTimeout: 5 * time.Minute,
|
||||||
BanResponse: 429,
|
ClientResponseTimeout: 7 * 24 * time.Hour,
|
||||||
LimitBanDuration: 15 * time.Minute,
|
UpstreamRequestTimeout: 90 * time.Minute,
|
||||||
LimitBanRepeatWindow: 48 * time.Hour,
|
UpstreamResponseTimeout: 0,
|
||||||
MaxBanDuration: 30 * 24 * time.Hour,
|
RequestMaxBytes: 512 << 10,
|
||||||
AttackBanDuration: 24 * time.Hour,
|
ResponseMaxBytes: 1234,
|
||||||
MaxBans: 100,
|
RateLimitPerMinute: 60,
|
||||||
BanScopeV4Prefix: 24,
|
RateLimitPerHour: 600,
|
||||||
StateDir: "/srv/waf-state",
|
RateLimitPerDay: 6000,
|
||||||
StateWriteDelay: 500 * time.Millisecond,
|
BanResponse: 429,
|
||||||
StateCounterInterval: time.Hour,
|
LimitBanDuration: 15 * time.Minute,
|
||||||
MetricsToken: token,
|
LimitBanRepeatWindow: 48 * time.Hour,
|
||||||
MetricsTopN: 10,
|
MaxBanDuration: 30 * 24 * time.Hour,
|
||||||
RulesDir: "/srv/waf-rules",
|
AttackBanDuration: 24 * time.Hour,
|
||||||
RulesEnabled: false,
|
MaxBans: 100,
|
||||||
|
BanScopeV4Prefix: 24,
|
||||||
|
StateDir: "/srv/waf-state",
|
||||||
|
StateWriteDelay: 500 * time.Millisecond,
|
||||||
|
StateCounterInterval: time.Hour,
|
||||||
|
MetricsToken: token,
|
||||||
|
MetricsTopN: 10,
|
||||||
|
RulesDir: "/srv/waf-rules",
|
||||||
|
RulesEnabled: false,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
if cfg.UpstreamURL.String() != "https://app.internal:8443/" {
|
||||||
|
t.Errorf("%s is %s", upstreamURL, cfg.UpstreamURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantNetblocks(t, cfg.TrustedProxies, "192.0.2.1/32", "10.0.0.0/8", "2001:db8::/32")
|
||||||
wantNetblocks(t, cfg.AllowNets, "192.0.2.7/32")
|
wantNetblocks(t, cfg.AllowNets, "192.0.2.7/32")
|
||||||
wantNetblocks(t, cfg.RateLimitExemptNets, "2001:db8::/48", "10.9.8.7/32")
|
wantNetblocks(t, cfg.RateLimitExemptNets, "2001:db8::/48", "10.9.8.7/32")
|
||||||
wantNetblocks(t, cfg.DenyNets, "198.51.100.0/24")
|
wantNetblocks(t, cfg.DenyNets, "198.51.100.0/24")
|
||||||
@@ -539,13 +581,13 @@ func TestCoreRuleSetSettings(t *testing.T) {
|
|||||||
environment{
|
environment{
|
||||||
wafMode: config.WAFModeDetect, wafParanoiaLevel: "4", wafAnomalyThreshold: "10",
|
wafMode: config.WAFModeDetect, wafParanoiaLevel: "4", wafAnomalyThreshold: "10",
|
||||||
wafDisabledRules: "942100, 920350", wafExemptPaths: "/api/, /static/",
|
wafDisabledRules: "942100, 920350", wafExemptPaths: "/api/, /static/",
|
||||||
wafBodyLimit: "32K",
|
wafBodyLimit: "128K",
|
||||||
},
|
},
|
||||||
config.Config{
|
config.Config{
|
||||||
WAFMode: config.WAFModeDetect, WAFParanoiaLevel: 4, WAFAnomalyThreshold: 10,
|
WAFMode: config.WAFModeDetect, WAFParanoiaLevel: 4, WAFAnomalyThreshold: 10,
|
||||||
WAFDisabledRules: []int{942100, 920350},
|
WAFDisabledRules: []int{942100, 920350},
|
||||||
WAFExemptPaths: []string{"/api/", "/static/"},
|
WAFExemptPaths: []string{"/api/", "/static/"},
|
||||||
WAFBodyLimit: 32 << 10,
|
WAFBodyLimit: 128 << 10,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -573,12 +615,12 @@ func TestCoreRuleSetSettings(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWAFBodyLimitOf64K(t *testing.T) {
|
func TestWAFBodyLimitOf1G(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
cfg := fromEnvironment(t, environment{wafBodyLimit: "64K"})
|
cfg := fromEnvironment(t, environment{wafBodyLimit: "1G"})
|
||||||
if cfg.WAFBodyLimit != 64<<10 {
|
if cfg.WAFBodyLimit != 1<<30 {
|
||||||
t.Errorf("64K read as %d", cfg.WAFBodyLimit)
|
t.Errorf("1G read as %d", cfg.WAFBodyLimit)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -627,10 +669,10 @@ func TestInvalidCoreRuleSetSettingStopsTheStart(t *testing.T) {
|
|||||||
wafBodyLimit, "128KB",
|
wafBodyLimit, "128KB",
|
||||||
`"128KB" is not a size such as 512K, 100M or 5G, or off`,
|
`"128KB" is not a size such as 512K, 100M or 5G, or off`,
|
||||||
},
|
},
|
||||||
{wafBodyLimit, "65K", `"65K" is more than 64K, the most servewaf gets of a body`},
|
{wafBodyLimit, "2G", `"2G" is more than 1G, the most Coraza reads`},
|
||||||
{
|
{
|
||||||
wafBodyLimit, "65537",
|
wafBodyLimit, "1073741825",
|
||||||
`"65537" is more than 64K, the most servewaf gets of a body`,
|
`"1073741825" is more than 1G, the most Coraza reads`,
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name+"="+tc.value, func(t *testing.T) {
|
t.Run(tc.name+"="+tc.value, func(t *testing.T) {
|
||||||
@@ -1123,7 +1165,7 @@ func TestLookupSettingsAsSet(t *testing.T) {
|
|||||||
|
|
||||||
cfg = fromEnvironment(t, environment{lookupSource: off})
|
cfg = fromEnvironment(t, environment{lookupSource: off})
|
||||||
wantLookupSettings(t, cfg, config.Config{
|
wantLookupSettings(t, cfg, config.Config{
|
||||||
LookupSource: off, LookupTimeout: 800 * time.Millisecond,
|
LookupSource: off, LookupTimeout: time.Second,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1925,15 +1967,45 @@ func TestSizesAndOff(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
cfg := fromEnvironment(t, environment{
|
cfg := fromEnvironment(t, environment{
|
||||||
bytesLimitPerMinute: "3G",
|
requestMaxBytes: "3G",
|
||||||
bytesLimitPerHour: off,
|
responseMaxBytes: off,
|
||||||
alertCooldown: off,
|
clientRequestTimeout: off,
|
||||||
|
clientIdleTimeout: off,
|
||||||
})
|
})
|
||||||
|
|
||||||
if cfg.BytesLimitPerMinute != 3<<30 || cfg.BytesLimitPerHour != 0 ||
|
if cfg.RequestMaxBytes != 3<<30 || cfg.ResponseMaxBytes != 0 ||
|
||||||
cfg.AlertCooldown != 0 {
|
cfg.ClientRequestTimeout != 0 || cfg.ClientIdleTimeout != 0 {
|
||||||
t.Errorf("3G, off and off read as %d, %d and %s",
|
t.Errorf("3G, off, off and off read as %d, %d, %s and %s",
|
||||||
cfg.BytesLimitPerMinute, cfg.BytesLimitPerHour, cfg.AlertCooldown)
|
cfg.RequestMaxBytes, cfg.ResponseMaxBytes, cfg.ClientRequestTimeout,
|
||||||
|
cfg.ClientIdleTimeout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRequestHeaderMaxBytesJustOver4K(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cfg := fromEnvironment(t, environment{clientHeaderMaxBytes: "4097"})
|
||||||
|
if cfg.ClientRequestHeaderMaxBytes != 4097 {
|
||||||
|
t.Errorf("4097 read as %d", cfg.ClientRequestHeaderMaxBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRequestHeaderMaxBytesRefusalNeverOffersOff(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, value := range []string{"32KB", "0", "4K", off} {
|
||||||
|
t.Run(value, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
_, err := config.FromEnvironment(
|
||||||
|
environment{clientHeaderMaxBytes: value}.lookupEnv)
|
||||||
|
|
||||||
|
want := clientHeaderMaxBytes + `: "` + value +
|
||||||
|
`" is not a size of more than 4K, such as 32K`
|
||||||
|
if err == nil || err.Error() != want {
|
||||||
|
t.Errorf("error %v, want %s", err, want)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1962,12 +2034,34 @@ func TestBanResponseCloseIsZero(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestTrustedProxiesSetButEmptyTrustNothing(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cfg := fromEnvironment(t, environment{trustedProxies: ""})
|
||||||
|
if len(cfg.TrustedProxies) != 0 {
|
||||||
|
t.Errorf("trusted proxies %v, want none", cfg.TrustedProxies)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestInvalidValueStopsTheStart(t *testing.T) {
|
func TestInvalidValueStopsTheStart(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
wantStartStopped(t, []struct{ name, value string }{
|
wantStartStopped(t, []struct{ name, value string }{
|
||||||
{listenAddr, "8080"}, {listenAddr, ":http"}, {listenAddr, ":65536"},
|
{listenAddr, "8080"}, {listenAddr, ":http"}, {listenAddr, ":65536"},
|
||||||
|
{upstreamURL, "127.0.0.1:8081"},
|
||||||
|
{upstreamURL, "ftp://127.0.0.1:8081"},
|
||||||
|
{upstreamURL, "http://"},
|
||||||
|
{upstreamURL, "http://:8081"},
|
||||||
|
{upstreamURL, "http://127.0.0.1:0"},
|
||||||
|
{upstreamURL, "http://127.0.0.1:99999"},
|
||||||
|
{upstreamURL, "http://127.0.0.1:8081/app"},
|
||||||
|
{upstreamURL, "http://127.0.0.1:8081/?a=1"},
|
||||||
|
{upstreamURL, "http://user:secret@127.0.0.1:8081"},
|
||||||
{mode, "Observe"}, {mode, "block"}, {mode, ""},
|
{mode, "Observe"}, {mode, "block"}, {mode, ""},
|
||||||
|
{trustedProxies, "10.0.0.0/33"},
|
||||||
|
{trustedProxies, "traefik"},
|
||||||
|
{trustedProxies, "10.0.0.0/8,,192.168.0.0/16"},
|
||||||
|
{trustedProxies, "fe80::1%eth0"},
|
||||||
{ipv6GroupPrefix, "31"}, {ipv6GroupPrefix, "129"}, {ipv6GroupPrefix, "/64"},
|
{ipv6GroupPrefix, "31"}, {ipv6GroupPrefix, "129"}, {ipv6GroupPrefix, "/64"},
|
||||||
{ipv6GroupPrefix, off}, {ipv6GroupPrefix, ""},
|
{ipv6GroupPrefix, off}, {ipv6GroupPrefix, ""},
|
||||||
{maxTrackedClients, "0"}, {maxTrackedClients, "-1"}, {maxTrackedClients, off},
|
{maxTrackedClients, "0"}, {maxTrackedClients, "-1"}, {maxTrackedClients, off},
|
||||||
@@ -1975,18 +2069,19 @@ func TestInvalidValueStopsTheStart(t *testing.T) {
|
|||||||
{allowNets, "192.0.2.0/24,monitoring"},
|
{allowNets, "192.0.2.0/24,monitoring"},
|
||||||
{rateLimitExemptNets, "2001:db8::/129"},
|
{rateLimitExemptNets, "2001:db8::/129"},
|
||||||
{denyNets, "198.51.100.0/24,"},
|
{denyNets, "198.51.100.0/24,"},
|
||||||
{alertCooldown, "60"}, {alertCooldown, ""},
|
{clientRequestTimeout, "60"}, {clientRequestTimeout, ""},
|
||||||
{alertCooldown, "0s"},
|
{clientIdleTimeout, "0s"},
|
||||||
{alertCooldown, "2 minutes"},
|
{clientIdleTimeout, "2 minutes"},
|
||||||
{alertCooldown, "1y"},
|
{clientResponseTimeout, "1y"},
|
||||||
{alertCooldown, "-1s"},
|
{upstreamRequestTimeout, "-1s"},
|
||||||
{alertCooldown, "1.5d"},
|
{upstreamResponseTimeout, "0s"},
|
||||||
{bytesLimitPerMinute, "100MB"},
|
{upstreamResponseTimeout, "1.5d"},
|
||||||
{bytesLimitPerMinute, "100m"},
|
{requestMaxBytes, "100MB"},
|
||||||
{bytesLimitPerMinute, "1.5M"},
|
{requestMaxBytes, "100m"},
|
||||||
{bytesLimitPerHour, "0"},
|
{requestMaxBytes, "1.5M"},
|
||||||
{bytesLimitPerHour, "-5"},
|
{responseMaxBytes, "0"},
|
||||||
{bytesLimitPerHour, "99999999999G"},
|
{responseMaxBytes, "-5"},
|
||||||
|
{responseMaxBytes, "99999999999G"},
|
||||||
{rateLimitPerMinute, ""},
|
{rateLimitPerMinute, ""},
|
||||||
{rateLimitPerMinute, "1K"},
|
{rateLimitPerMinute, "1K"},
|
||||||
{rateLimitPerHour, "0"},
|
{rateLimitPerHour, "0"},
|
||||||
@@ -2151,10 +2246,10 @@ func TestSettingFromFileIsCheckedAsTheSettingItself(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
_, err := config.FromEnvironment(environment{
|
_, err := config.FromEnvironment(environment{
|
||||||
bytesLimitPerMinute + "_FILE": writeFile(t, "lots\n"),
|
requestMaxBytes + "_FILE": writeFile(t, "lots\n"),
|
||||||
}.lookupEnv)
|
}.lookupEnv)
|
||||||
|
|
||||||
want := bytesLimitPerMinute + `: "lots" is not a size such as 512K, 100M or 5G, or off`
|
want := requestMaxBytes + `: "lots" is not a size such as 512K, 100M or 5G, or off`
|
||||||
if err == nil || err.Error() != want {
|
if err == nil || err.Error() != want {
|
||||||
t.Errorf("error %v, want %s", err, want)
|
t.Errorf("error %v, want %s", err, want)
|
||||||
}
|
}
|
||||||
@@ -2259,89 +2354,99 @@ func writeFile(t *testing.T, contents string) string {
|
|||||||
func TestLogsEachSettingWithItsValue(t *testing.T) {
|
func TestLogsEachSettingWithItsValue(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
cfg := fromEnvironment(t, environment{rateLimitPerMinute: "45"})
|
cfg := fromEnvironment(t, environment{clientRequestTimeout: "45s"})
|
||||||
hostname, _ := os.Hostname()
|
hostname, _ := os.Hostname()
|
||||||
|
|
||||||
want := map[string]string{
|
want := map[string]string{
|
||||||
listenAddr: ":8080",
|
listenAddr: ":8080",
|
||||||
mode: "enforce",
|
upstreamURL: "http://127.0.0.1:8081",
|
||||||
ipv6GroupPrefix: "64",
|
mode: "enforce",
|
||||||
maxTrackedClients: "20000",
|
trustedProxies: "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16",
|
||||||
allowNets: "",
|
ipv6GroupPrefix: "64",
|
||||||
rateLimitExemptNets: "",
|
maxTrackedClients: "20000",
|
||||||
denyNets: "",
|
clientRequestTimeout: "45s",
|
||||||
rateLimitPerMinute: "45",
|
clientHeaderMaxBytes: "32K",
|
||||||
rateLimitPerHour: "20000",
|
clientIdleTimeout: "120s",
|
||||||
rateLimitPerDay: "300000",
|
clientResponseTimeout: "30m",
|
||||||
rateLimitExemptPaths: "",
|
upstreamRequestTimeout: "60s",
|
||||||
bytesLimitPerMinute: "4G",
|
upstreamResponseTimeout: "30m",
|
||||||
bytesLimitPerHour: "20G",
|
requestMaxBytes: "100M",
|
||||||
bytesLimitPerDay: "200G",
|
responseMaxBytes: "5G",
|
||||||
bytesCount: "both",
|
allowNets: "",
|
||||||
lookupSource: defaultLookupSource,
|
rateLimitExemptNets: "",
|
||||||
lookupDBPath: "",
|
denyNets: "",
|
||||||
lookupTimeout: "800ms",
|
rateLimitPerMinute: "1000",
|
||||||
addLookupHeaders: "false",
|
rateLimitPerHour: "10000",
|
||||||
deniedCountries: "",
|
rateLimitPerDay: "50000",
|
||||||
allowedCountries: "",
|
rateLimitExemptPaths: "",
|
||||||
asnLimitPercent: "",
|
bytesLimitPerMinute: "10G",
|
||||||
countryLimitPercent: "",
|
bytesLimitPerHour: "20G",
|
||||||
asnBytesPercent: "",
|
bytesLimitPerDay: "50G",
|
||||||
countryBytesPercent: "",
|
bytesCount: "both",
|
||||||
unknownLimitPercent: "100",
|
lookupSource: defaultLookupSource,
|
||||||
asnLimitPercentURL: "",
|
lookupDBPath: "",
|
||||||
blocklistURLs: "",
|
lookupTimeout: "1s",
|
||||||
blocklistRefresh: "24h",
|
addLookupHeaders: "false",
|
||||||
blocklistAction: actionDeny,
|
deniedCountries: "",
|
||||||
dnsblZones: "",
|
allowedCountries: "",
|
||||||
dnsblResolver: "",
|
asnLimitPercent: "",
|
||||||
abuseIPDBKey: "",
|
countryLimitPercent: "",
|
||||||
abuseIPDBMinScore: "75",
|
asnBytesPercent: "",
|
||||||
abuseIPDBDailyBudget: "900",
|
countryBytesPercent: "",
|
||||||
reputationAction: "limit:25",
|
unknownLimitPercent: "100",
|
||||||
reputationCacheTTL: defaultReputationCacheTTL,
|
asnLimitPercentURL: "",
|
||||||
reputationTimeout: "2s",
|
blocklistURLs: "",
|
||||||
crowdSecURL: "",
|
blocklistRefresh: "24h",
|
||||||
crowdSecKey: "",
|
blocklistAction: actionDeny,
|
||||||
banResponse: "403",
|
dnsblZones: "",
|
||||||
limitBanDuration: "1h",
|
dnsblResolver: "",
|
||||||
limitBanRepeatWindow: "24h",
|
abuseIPDBKey: "",
|
||||||
maxBanDuration: "7d",
|
abuseIPDBMinScore: "75",
|
||||||
attackBanDuration: "7d",
|
abuseIPDBDailyBudget: "900",
|
||||||
maxBans: "5000",
|
reputationAction: "limit:25",
|
||||||
banScopeV4Prefix: "32",
|
reputationCacheTTL: defaultReputationCacheTTL,
|
||||||
stateDir: "/var/lib/smallwebwaf",
|
reputationTimeout: "2s",
|
||||||
stateWriteDelay: "10s",
|
crowdSecURL: "",
|
||||||
stateCounterInterval: "15m",
|
crowdSecKey: "",
|
||||||
adminToken: "",
|
banResponse: "403",
|
||||||
metricsToken: "",
|
limitBanDuration: "1h",
|
||||||
metricsTopN: "50",
|
limitBanRepeatWindow: "24h",
|
||||||
instanceName: hostname,
|
maxBanDuration: "7d",
|
||||||
logRequestHeaders: defaultLogRequestHeaders,
|
attackBanDuration: "7d",
|
||||||
logLevel: "info",
|
maxBans: "5000",
|
||||||
rulesDir: "/etc/smallwebwaf/rules.d",
|
banScopeV4Prefix: "32",
|
||||||
rulesEnabled: "true",
|
stateDir: "/var/lib/smallwebwaf",
|
||||||
wafMode: config.WAFModeBlock,
|
stateWriteDelay: "10s",
|
||||||
wafParanoiaLevel: "1",
|
stateCounterInterval: "15m",
|
||||||
wafAnomalyThreshold: "5",
|
adminToken: "",
|
||||||
wafDisabledRules: defaultWAFDisabledRules,
|
metricsToken: "",
|
||||||
wafExemptPaths: "",
|
metricsTopN: "50",
|
||||||
wafBodyLimit: off,
|
instanceName: hostname,
|
||||||
trapPaths: "",
|
logRequestHeaders: defaultLogRequestHeaders,
|
||||||
errorBurstThreshold: "30",
|
logLevel: "info",
|
||||||
logRemoteURL: "",
|
rulesDir: "/etc/smallwebwaf/rules.d",
|
||||||
logRemoteTLSCAFile: "",
|
rulesEnabled: "true",
|
||||||
logRemoteBuffer: "10000",
|
wafMode: config.WAFModeBlock,
|
||||||
logRemoteFacility: "local0",
|
wafParanoiaLevel: "1",
|
||||||
logRemoteAppName: hostname,
|
wafAnomalyThreshold: "5",
|
||||||
alertWebhookURL: "",
|
wafDisabledRules: defaultWAFDisabledRules,
|
||||||
alertWebhookHeaders: "",
|
wafExemptPaths: "",
|
||||||
alertSlackWebhookURL: "",
|
wafBodyLimit: off,
|
||||||
alertNtfyURL: "",
|
trapPaths: "",
|
||||||
alertNtfyToken: "",
|
errorBurstThreshold: "30",
|
||||||
alertEvents: defaultAlertEvents,
|
logRemoteURL: "",
|
||||||
alertCooldown: defaultAlertCooldown,
|
logRemoteTLSCAFile: "",
|
||||||
alertMaxPerHour: "60",
|
logRemoteBuffer: "10000",
|
||||||
|
logRemoteFacility: "local0",
|
||||||
|
logRemoteAppName: hostname,
|
||||||
|
alertWebhookURL: "",
|
||||||
|
alertWebhookHeaders: "",
|
||||||
|
alertSlackWebhookURL: "",
|
||||||
|
alertNtfyURL: "",
|
||||||
|
alertNtfyToken: "",
|
||||||
|
alertEvents: defaultAlertEvents,
|
||||||
|
alertCooldown: defaultAlertCooldown,
|
||||||
|
alertMaxPerHour: "60",
|
||||||
}
|
}
|
||||||
maps.Copy(want, loggedAnomalyDefaults())
|
maps.Copy(want, loggedAnomalyDefaults())
|
||||||
|
|
||||||
@@ -2376,6 +2481,14 @@ func wantSettings(t *testing.T, got *config.Config, want config.Config) {
|
|||||||
|
|
||||||
if got.ListenAddr != want.ListenAddr ||
|
if got.ListenAddr != want.ListenAddr ||
|
||||||
got.Observe != want.Observe ||
|
got.Observe != want.Observe ||
|
||||||
|
got.ClientRequestTimeout != want.ClientRequestTimeout ||
|
||||||
|
got.ClientRequestHeaderMaxBytes != want.ClientRequestHeaderMaxBytes ||
|
||||||
|
got.ClientIdleTimeout != want.ClientIdleTimeout ||
|
||||||
|
got.ClientResponseTimeout != want.ClientResponseTimeout ||
|
||||||
|
got.UpstreamRequestTimeout != want.UpstreamRequestTimeout ||
|
||||||
|
got.UpstreamResponseTimeout != want.UpstreamResponseTimeout ||
|
||||||
|
got.RequestMaxBytes != want.RequestMaxBytes ||
|
||||||
|
got.ResponseMaxBytes != want.ResponseMaxBytes ||
|
||||||
got.RateLimitPerMinute != want.RateLimitPerMinute ||
|
got.RateLimitPerMinute != want.RateLimitPerMinute ||
|
||||||
got.RateLimitPerHour != want.RateLimitPerHour ||
|
got.RateLimitPerHour != want.RateLimitPerHour ||
|
||||||
got.RateLimitPerDay != want.RateLimitPerDay {
|
got.RateLimitPerDay != want.RateLimitPerDay {
|
||||||
|
|||||||
@@ -1,192 +0,0 @@
|
|||||||
package config
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"log/slog"
|
|
||||||
"math"
|
|
||||||
"net/netip"
|
|
||||||
"net/url"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// RProxy is the settings of `smallwebwaf rproxy`. A size or time limit of zero
|
|
||||||
// is off.
|
|
||||||
type RProxy struct {
|
|
||||||
// ListenAddr is where rproxy listens (SWRP_LISTEN_ADDR).
|
|
||||||
ListenAddr string
|
|
||||||
// UpstreamURL is the app (SWRP_UPSTREAM_URL).
|
|
||||||
UpstreamURL *url.URL
|
|
||||||
// TrustedProxies are the netblocks whose X-Forwarded-For is believed
|
|
||||||
// (SWRP_TRUSTED_PROXIES).
|
|
||||||
TrustedProxies []netip.Prefix
|
|
||||||
// WAFBackend is servewaf's URL (SWRP_WAF_BACKEND).
|
|
||||||
WAFBackend *url.URL
|
|
||||||
// Timeout is how long rproxy waits for servewaf's answer to a question,
|
|
||||||
// and to a report (SWRP_TIMEOUT_MS). It cannot be off.
|
|
||||||
Timeout time.Duration
|
|
||||||
// DefaultDisposition is what is done with a request when servewaf cannot be
|
|
||||||
// connected to, does not answer at all, or does not answer within Timeout:
|
|
||||||
// allow or deny (SWRP_DEFAULT_DISPOSITION).
|
|
||||||
DefaultDisposition string
|
|
||||||
// MaxRequestSize is the largest request body (SWRP_MAX_REQUEST_SIZE), and
|
|
||||||
// MaxResponseSize the largest response body (SWRP_MAX_RESPONSE_SIZE).
|
|
||||||
MaxRequestSize int64
|
|
||||||
MaxResponseSize int64
|
|
||||||
// MaxRequestTime bounds sending the whole request: the visitor's request
|
|
||||||
// line and headers, and then, from their end, its body on its way to the
|
|
||||||
// app (SWRP_MAX_REQUEST_TIME_MS). MaxResponseTime bounds the answer, from
|
|
||||||
// the end of the request to its last byte reaching the visitor
|
|
||||||
// (SWRP_MAX_RESPONSE_TIME_MS).
|
|
||||||
MaxRequestTime time.Duration
|
|
||||||
MaxResponseTime time.Duration
|
|
||||||
|
|
||||||
// settings are the values read, as given or by default, for the log line
|
|
||||||
// at start.
|
|
||||||
settings []slog.Attr
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
errNeedsWAFBackend = errors.New(
|
|
||||||
"is required: it is servewaf's URL, such as http://servewaf:8080")
|
|
||||||
errNotDisposition = errors.New("is not allow or deny")
|
|
||||||
errNotMilliseconds = errors.New(
|
|
||||||
"is not a whole number of milliseconds above zero, such as 2000")
|
|
||||||
errNotMillisecondsOrOff = errors.New(
|
|
||||||
"is not a whole number of milliseconds above zero, such as 60000, or off")
|
|
||||||
)
|
|
||||||
|
|
||||||
// RProxyFromEnvironment reads rproxy's settings with lookupEnv, normally
|
|
||||||
// os.LookupEnv. A setting that is not set takes its default, and
|
|
||||||
// SWRP_WAF_BACKEND, which has none, must be set. A setting that is set but
|
|
||||||
// invalid is an error that names it. Since rproxy never touches disk, its
|
|
||||||
// settings have no _FILE form.
|
|
||||||
func RProxyFromEnvironment(lookupEnv func(string) (string, bool)) (*RProxy, error) {
|
|
||||||
env := &environment{lookupEnv: withoutFileForms(lookupEnv)}
|
|
||||||
cfg := &RProxy{
|
|
||||||
ListenAddr: env.address("SWRP_LISTEN_ADDR"),
|
|
||||||
UpstreamURL: env.appURL("SWRP_UPSTREAM_URL", defaultUpstreamURL),
|
|
||||||
TrustedProxies: env.netblocks("SWRP_TRUSTED_PROXIES", privateRanges),
|
|
||||||
WAFBackend: env.wafBackend("SWRP_WAF_BACKEND"),
|
|
||||||
Timeout: env.milliseconds("SWRP_TIMEOUT_MS", "2000"),
|
|
||||||
DefaultDisposition: env.disposition("SWRP_DEFAULT_DISPOSITION", "allow"),
|
|
||||||
MaxRequestSize: env.size("SWRP_MAX_REQUEST_SIZE", "100M"),
|
|
||||||
MaxResponseSize: env.size("SWRP_MAX_RESPONSE_SIZE", "1G"),
|
|
||||||
MaxRequestTime: env.millisecondsOrOff("SWRP_MAX_REQUEST_TIME_MS", "60000"),
|
|
||||||
MaxResponseTime: env.millisecondsOrOff("SWRP_MAX_RESPONSE_TIME_MS", "1800000"),
|
|
||||||
}
|
|
||||||
|
|
||||||
if env.err != nil {
|
|
||||||
return nil, env.err
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg.settings = env.settings
|
|
||||||
|
|
||||||
return cfg, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// RProxyListenAddrAndUpstreamURL reads only SWRP_LISTEN_ADDR and
|
|
||||||
// SWRP_UPSTREAM_URL. The health check in rproxy's container needs no other
|
|
||||||
// setting, so it reads no other.
|
|
||||||
func RProxyListenAddrAndUpstreamURL(
|
|
||||||
lookupEnv func(string) (string, bool),
|
|
||||||
) (string, *url.URL, error) {
|
|
||||||
env := &environment{lookupEnv: withoutFileForms(lookupEnv)}
|
|
||||||
listenAddr := env.address("SWRP_LISTEN_ADDR")
|
|
||||||
upstreamURL := env.appURL("SWRP_UPSTREAM_URL", defaultUpstreamURL)
|
|
||||||
|
|
||||||
if env.err != nil {
|
|
||||||
return "", nil, env.err
|
|
||||||
}
|
|
||||||
|
|
||||||
return listenAddr, upstreamURL, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// withoutFileForms returns lookupEnv finding no setting named with _FILE, since
|
|
||||||
// rproxy never touches disk.
|
|
||||||
func withoutFileForms(
|
|
||||||
lookupEnv func(string) (string, bool),
|
|
||||||
) func(string) (string, bool) {
|
|
||||||
return func(name string) (string, bool) {
|
|
||||||
if strings.HasSuffix(name, "_FILE") {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
|
|
||||||
return lookupEnv(name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// LogValue makes an RProxy log as each setting's name with the value it was
|
|
||||||
// given, or its default.
|
|
||||||
func (c *RProxy) LogValue() slog.Value {
|
|
||||||
return slog.GroupValue(c.settings...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// wafBackend reads the setting that is servewaf's URL, which must be set.
|
|
||||||
func (e *environment) wafBackend(name string) *url.URL {
|
|
||||||
value := e.value(name, "")
|
|
||||||
if value == "" {
|
|
||||||
e.check(name, errNeedsWAFBackend)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
backend, err := parseUpstreamURL(value)
|
|
||||||
e.check(name, err)
|
|
||||||
|
|
||||||
return backend
|
|
||||||
}
|
|
||||||
|
|
||||||
// disposition reads the setting that is what is done with a request when servewaf
|
|
||||||
// cannot be connected to, does not answer at all, or does not answer in time:
|
|
||||||
// allow or deny.
|
|
||||||
func (e *environment) disposition(name, defaultValue string) string {
|
|
||||||
value := e.value(name, defaultValue)
|
|
||||||
if value != "allow" && value != "deny" {
|
|
||||||
e.check(name, fmt.Errorf("%q %w", value, errNotDisposition))
|
|
||||||
}
|
|
||||||
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
|
|
||||||
// milliseconds reads a setting that is a whole number of milliseconds above
|
|
||||||
// zero.
|
|
||||||
func (e *environment) milliseconds(name, defaultValue string) time.Duration {
|
|
||||||
value := e.value(name, defaultValue)
|
|
||||||
|
|
||||||
duration, valid := parseMilliseconds(value)
|
|
||||||
if !valid {
|
|
||||||
e.check(name, fmt.Errorf("%q %w", value, errNotMilliseconds))
|
|
||||||
}
|
|
||||||
|
|
||||||
return duration
|
|
||||||
}
|
|
||||||
|
|
||||||
// millisecondsOrOff reads a setting that is a whole number of milliseconds
|
|
||||||
// above zero, or off, which is 0.
|
|
||||||
func (e *environment) millisecondsOrOff(name, defaultValue string) time.Duration {
|
|
||||||
value := e.value(name, defaultValue)
|
|
||||||
if value == off {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
duration, valid := parseMilliseconds(value)
|
|
||||||
if !valid {
|
|
||||||
e.check(name, fmt.Errorf("%q %w", value, errNotMillisecondsOrOff))
|
|
||||||
}
|
|
||||||
|
|
||||||
return duration
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseMilliseconds reads a whole number of milliseconds above zero, and
|
|
||||||
// reports whether it is one.
|
|
||||||
func parseMilliseconds(value string) (time.Duration, bool) {
|
|
||||||
n, err := strconv.ParseInt(value, 10, 64)
|
|
||||||
if err != nil || n <= 0 || n > math.MaxInt64/int64(time.Millisecond) {
|
|
||||||
return 0, false
|
|
||||||
}
|
|
||||||
|
|
||||||
return time.Duration(n) * time.Millisecond, true
|
|
||||||
}
|
|
||||||
@@ -1,231 +0,0 @@
|
|||||||
package config_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/netip"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
// rproxy's settings, by name, and their values in these tests.
|
|
||||||
const (
|
|
||||||
wafBackend = "SWRP_WAF_BACKEND"
|
|
||||||
rproxyTimeout = "SWRP_TIMEOUT_MS"
|
|
||||||
rproxyDefault = "SWRP_DEFAULT_DISPOSITION"
|
|
||||||
rproxyListen = "SWRP_LISTEN_ADDR"
|
|
||||||
rproxyUpstream = "SWRP_UPSTREAM_URL"
|
|
||||||
rproxyTrusted = "SWRP_TRUSTED_PROXIES"
|
|
||||||
rproxyMaxRequest = "SWRP_MAX_REQUEST_SIZE"
|
|
||||||
rproxyMaxResponse = "SWRP_MAX_RESPONSE_SIZE"
|
|
||||||
rproxyRequestTime = "SWRP_MAX_REQUEST_TIME_MS"
|
|
||||||
rproxyAnswerTime = "SWRP_MAX_RESPONSE_TIME_MS"
|
|
||||||
backendURL = "http://servewaf:8080"
|
|
||||||
rproxyDefaultAddr = ":8080"
|
|
||||||
rproxyDefaultApp = "http://127.0.0.1:8081"
|
|
||||||
notASize = "a lot"
|
|
||||||
)
|
|
||||||
|
|
||||||
// rproxyConfig reads rproxy's settings from env.
|
|
||||||
func rproxyConfig(env map[string]string) (*config.RProxy, error) {
|
|
||||||
return config.RProxyFromEnvironment(func(name string) (string, bool) {
|
|
||||||
value, ok := env[name]
|
|
||||||
|
|
||||||
return value, ok
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRProxyDefaults(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cfg, err := rproxyConfig(map[string]string{wafBackend: backendURL})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("settings: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.ListenAddr != rproxyDefaultAddr ||
|
|
||||||
cfg.UpstreamURL.String() != rproxyDefaultApp ||
|
|
||||||
cfg.WAFBackend.String() != backendURL {
|
|
||||||
t.Errorf("addresses %q, %v, %v", cfg.ListenAddr, cfg.UpstreamURL, cfg.WAFBackend)
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.Timeout != 2*time.Second || cfg.DefaultDisposition != "allow" {
|
|
||||||
t.Errorf("timeout %v, default disposition %q", cfg.Timeout, cfg.DefaultDisposition)
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.MaxRequestSize != 100<<20 || cfg.MaxResponseSize != 1<<30 {
|
|
||||||
t.Errorf("sizes %d and %d", cfg.MaxRequestSize, cfg.MaxResponseSize)
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.MaxRequestTime != time.Minute || cfg.MaxResponseTime != 30*time.Minute {
|
|
||||||
t.Errorf("times %v and %v", cfg.MaxRequestTime, cfg.MaxResponseTime)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := []netip.Prefix{
|
|
||||||
netip.MustParsePrefix("10.0.0.0/8"),
|
|
||||||
netip.MustParsePrefix("172.16.0.0/12"),
|
|
||||||
netip.MustParsePrefix("192.168.0.0/16"),
|
|
||||||
}
|
|
||||||
if !slices.Equal(cfg.TrustedProxies, want) {
|
|
||||||
t.Errorf("trusted proxies %v, want %v", cfg.TrustedProxies, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRProxySettingsAreRead(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cfg, err := rproxyConfig(map[string]string{
|
|
||||||
wafBackend: "https://waf.example:8443",
|
|
||||||
rproxyListen: ":9000",
|
|
||||||
rproxyUpstream: "http://app:3000",
|
|
||||||
rproxyTrusted: "192.0.2.0/24",
|
|
||||||
rproxyTimeout: "150",
|
|
||||||
rproxyDefault: "deny",
|
|
||||||
rproxyMaxRequest: "1K",
|
|
||||||
rproxyMaxResponse: off,
|
|
||||||
rproxyRequestTime: off,
|
|
||||||
rproxyAnswerTime: "250",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("settings: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.ListenAddr != ":9000" || cfg.UpstreamURL.Host != "app:3000" ||
|
|
||||||
cfg.WAFBackend.Host != "waf.example:8443" ||
|
|
||||||
cfg.TrustedProxies[0] != netip.MustParsePrefix("192.0.2.0/24") {
|
|
||||||
t.Errorf("addresses %+v", cfg)
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.Timeout != 150*time.Millisecond || cfg.DefaultDisposition != "deny" ||
|
|
||||||
cfg.MaxRequestSize != 1024 || cfg.MaxResponseSize != 0 ||
|
|
||||||
cfg.MaxRequestTime != 0 || cfg.MaxResponseTime != 250*time.Millisecond {
|
|
||||||
t.Errorf("limits %+v", cfg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRProxyInvalidSettingsAreRefused(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name, value string
|
|
||||||
}{
|
|
||||||
{wafBackend, ""},
|
|
||||||
{wafBackend, "servewaf:8080"},
|
|
||||||
{wafBackend, "http://servewaf:8080/path"},
|
|
||||||
{rproxyTimeout, "0"},
|
|
||||||
{rproxyTimeout, "-1"},
|
|
||||||
{rproxyTimeout, off},
|
|
||||||
{rproxyTimeout, "2s"},
|
|
||||||
{rproxyTimeout, "1.25"},
|
|
||||||
{rproxyTimeout, "99999999999999999"},
|
|
||||||
{rproxyDefault, "maybe"},
|
|
||||||
{rproxyListen, "8080"},
|
|
||||||
{rproxyUpstream, "app"},
|
|
||||||
{rproxyUpstream, "127.0.0.1:8081"},
|
|
||||||
{rproxyUpstream, "ftp://127.0.0.1:8081"},
|
|
||||||
{rproxyUpstream, "http://"},
|
|
||||||
{rproxyUpstream, "http://:8081"},
|
|
||||||
{rproxyUpstream, "http://127.0.0.1:0"},
|
|
||||||
{rproxyUpstream, "http://127.0.0.1:99999"},
|
|
||||||
{rproxyUpstream, "http://127.0.0.1:8081/app"},
|
|
||||||
{rproxyUpstream, "http://127.0.0.1:8081/?a=1"},
|
|
||||||
{rproxyUpstream, "http://user:secret@127.0.0.1:8081"},
|
|
||||||
{rproxyTrusted, "not a netblock"},
|
|
||||||
{rproxyTrusted, "10.0.0.0/33"},
|
|
||||||
{rproxyTrusted, "10.0.0.0/8,,192.168.0.0/16"},
|
|
||||||
{rproxyTrusted, "fe80::1%eth0"},
|
|
||||||
{rproxyMaxRequest, notASize},
|
|
||||||
{rproxyMaxResponse, notASize},
|
|
||||||
{rproxyRequestTime, "0"},
|
|
||||||
{rproxyRequestTime, "1m"},
|
|
||||||
{rproxyAnswerTime, "0"},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name+"="+tc.value, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, err := rproxyConfig(map[string]string{wafBackend: backendURL, tc.name: tc.value})
|
|
||||||
if err == nil || !strings.HasPrefix(err.Error(), tc.name+": ") {
|
|
||||||
t.Errorf("error %v, want one naming %s", err, tc.name)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRProxyTrustedProxiesSetButEmptyTrustNothing(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cfg, err := rproxyConfig(map[string]string{wafBackend: backendURL, rproxyTrusted: ""})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("settings: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(cfg.TrustedProxies) != 0 {
|
|
||||||
t.Errorf("trusted proxies %v, want none", cfg.TrustedProxies)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRProxyNeedsItsBackend(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, err := rproxyConfig(nil)
|
|
||||||
if err == nil || !strings.HasPrefix(err.Error(), wafBackend+": is required") {
|
|
||||||
t.Errorf("error %v, want one saying %s is required", err, wafBackend)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRProxyReadsNoFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "backend")
|
|
||||||
|
|
||||||
err := os.WriteFile(path, []byte("http://from-a-file:8080\n"), 0o600)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("write: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = rproxyConfig(map[string]string{wafBackend + "_FILE": path})
|
|
||||||
if err == nil || !strings.HasPrefix(err.Error(), wafBackend+": is required") {
|
|
||||||
t.Errorf("error %v, want %s still required", err, wafBackend)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := rproxyConfig(map[string]string{
|
|
||||||
wafBackend: backendURL,
|
|
||||||
rproxyTimeout + "_FILE": "/nonexistent/timeout",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("settings: %v, with the file left unread", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.Timeout != 2*time.Second {
|
|
||||||
t.Errorf("timeout %v, want the default", cfg.Timeout)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRProxySettingsAreLoggedByName(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cfg, err := rproxyConfig(map[string]string{wafBackend: backendURL})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("settings: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
group := cfg.LogValue().Group()
|
|
||||||
|
|
||||||
names := make([]string, 0, len(group))
|
|
||||||
for _, attr := range group {
|
|
||||||
names = append(names, attr.Key)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, name := range []string{
|
|
||||||
rproxyListen, rproxyUpstream, rproxyTrusted, wafBackend, rproxyTimeout, rproxyDefault,
|
|
||||||
rproxyMaxRequest, rproxyMaxResponse, rproxyRequestTime, rproxyAnswerTime,
|
|
||||||
} {
|
|
||||||
if !slices.Contains(names, name) {
|
|
||||||
t.Errorf("%s is not logged, only %v", name, names)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -179,7 +179,7 @@ func New(params Params) *GeoJS {
|
|||||||
// own address for IPv4, and an address in the same place for an IPv6
|
// own address for IPv4, and an address in the same place for an IPv6
|
||||||
// netblock.
|
// netblock.
|
||||||
func (g *GeoJS) LookUp(ctx context.Context, client netip.Prefix) Answer {
|
func (g *GeoJS) LookUp(ctx context.Context, client netip.Prefix) Answer {
|
||||||
answer, asked := g.answerOrWait(ctx, client, g.wait)
|
answer, asked := g.answerOrWait(ctx, client)
|
||||||
if asked == nil {
|
if asked == nil {
|
||||||
return answer
|
return answer
|
||||||
}
|
}
|
||||||
@@ -209,14 +209,6 @@ func (g *GeoJS) LookUp(ctx context.Context, client netip.Prefix) Answer {
|
|||||||
return answer
|
return answer
|
||||||
}
|
}
|
||||||
|
|
||||||
// LookUpWithoutWaiting is LookUp for a request that does not need the answer,
|
|
||||||
// whatever Wait says: it returns the zero Answer at once when none is kept.
|
|
||||||
func (g *GeoJS) LookUpWithoutWaiting(ctx context.Context, client netip.Prefix) Answer {
|
|
||||||
answer, _ := g.answerOrWait(ctx, client, false)
|
|
||||||
|
|
||||||
return answer
|
|
||||||
}
|
|
||||||
|
|
||||||
// Kept returns client's answer, if one is kept, without asking GeoJS.
|
// Kept returns client's answer, if one is kept, without asking GeoJS.
|
||||||
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
|
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
|
||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
@@ -271,9 +263,9 @@ func (g *GeoJS) Load(answers []Answer) {
|
|||||||
// answerOrWait returns client's kept answer if it has one. Otherwise it
|
// answerOrWait returns client's kept answer if it has one. Otherwise it
|
||||||
// puts the client among those waiting if there is room, has GeoJS asked
|
// puts the client among those waiting if there is room, has GeoJS asked
|
||||||
// about them if it can be, and returns what to wait on for the answer, or
|
// about them if it can be, and returns what to wait on for the answer, or
|
||||||
// nil when there is nothing to wait for or mustWait is false.
|
// nil when there is nothing to wait for.
|
||||||
func (g *GeoJS) answerOrWait(
|
func (g *GeoJS) answerOrWait(
|
||||||
ctx context.Context, client netip.Prefix, mustWait bool,
|
ctx context.Context, client netip.Prefix,
|
||||||
) (Answer, <-chan struct{}) {
|
) (Answer, <-chan struct{}) {
|
||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
defer g.mu.Unlock()
|
defer g.mu.Unlock()
|
||||||
@@ -291,7 +283,7 @@ func (g *GeoJS) answerOrWait(
|
|||||||
|
|
||||||
g.ask(ctx)
|
g.ask(ctx)
|
||||||
|
|
||||||
if !mustWait {
|
if !g.wait {
|
||||||
return Answer{}, nil // the answer is not needed before the request goes on
|
return Answer{}, nil // the answer is not needed before the request goes on
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -72,39 +72,6 @@ func TestKeptAnswerIsUsedFor7DaysThenAskedAgain(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLookUpWithoutWaitingAsksGeoJSAndGoesOnAtOnce(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
geojs, clock, g := start()
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
|
||||||
|
|
||||||
// Wait is set, and GeoJS does not answer.
|
|
||||||
geojs.set(hanging)
|
|
||||||
|
|
||||||
began := time.Now()
|
|
||||||
answer := g.LookUpWithoutWaiting(t.Context(), client)
|
|
||||||
|
|
||||||
if waited := time.Since(began); waited != 0 || answer != (lookup.Answer{}) {
|
|
||||||
t.Errorf("got %+v after %s, want no answer at once", answer, waited)
|
|
||||||
}
|
|
||||||
|
|
||||||
waitForRequests(t, geojs, 1)
|
|
||||||
wantAsked(t, geojs, 0, "203.0.113.9")
|
|
||||||
|
|
||||||
// That request is abandoned, and has failed once this returns. GeoJS
|
|
||||||
// answers when it is asked again, and the answer is kept.
|
|
||||||
time.Sleep(timeout)
|
|
||||||
synctest.Wait()
|
|
||||||
geojs.set(answering)
|
|
||||||
waitForCountry(t, g, clock, client, germany)
|
|
||||||
|
|
||||||
if got := g.LookUpWithoutWaiting(t.Context(), client).Country; got != germany {
|
|
||||||
t.Errorf("the kept answer gives the country %q, want %q", got, germany)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewClientWaitsAtMostOneSecondThenCountsAsNotFound(t *testing.T) {
|
func TestNewClientWaitsAtMostOneSecondThenCountsAsNotFound(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -156,8 +123,8 @@ func TestRequestWaitsAsLongAsTheTimeoutSaysAndGeoJSIsAbandonedAfterIt(t *testing
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
synctest.Test(t, func(t *testing.T) {
|
||||||
// A timeout longer than the default, and a GeoJS that does not
|
// A timeout longer than the default second, and a GeoJS that does
|
||||||
// answer.
|
// not answer.
|
||||||
const longerTimeout = 3 * time.Second
|
const longerTimeout = 3 * time.Second
|
||||||
|
|
||||||
m := metrics.New(1, "app")
|
m := metrics.New(1, "app")
|
||||||
|
|||||||
+51
-61
@@ -1,7 +1,6 @@
|
|||||||
// Package metrics keeps smallwebwaf's Prometheus metrics, as
|
// Package metrics keeps smallwebwaf's Prometheus metrics, as the "Metrics
|
||||||
// docs/metrics.md lists them, and serves them in the Prometheus text
|
// endpoint" section of SPEC.md lists them, and serves them in the
|
||||||
// format, as the "Metrics endpoint" section of SPEC.md requires. No
|
// Prometheus text format. No metric carries a client's address.
|
||||||
// metric carries a client's address.
|
|
||||||
package metrics
|
package metrics
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -28,15 +27,16 @@ type Metrics struct {
|
|||||||
registry prometheus.Registerer
|
registry prometheus.Registerer
|
||||||
handler http.Handler
|
handler http.Handler
|
||||||
|
|
||||||
inFlight prometheus.Gauge
|
inFlight prometheus.Gauge
|
||||||
requests *prometheus.CounterVec
|
requests *prometheus.CounterVec
|
||||||
requestBytes *prometheus.CounterVec
|
requestBytes *prometheus.CounterVec
|
||||||
responseBytes *prometheus.CounterVec
|
responseBytes *prometheus.CounterVec
|
||||||
requestDuration prometheus.Histogram
|
requestDuration prometheus.Histogram
|
||||||
upstreamDuration prometheus.Histogram
|
upstreamDuration prometheus.Histogram
|
||||||
rateLimitHits *prometheus.CounterVec
|
rateLimitHits *prometheus.CounterVec
|
||||||
offences *prometheus.CounterVec
|
sizeAndTimeLimitHits *prometheus.CounterVec
|
||||||
wafMatches *prometheus.CounterVec
|
offences *prometheus.CounterVec
|
||||||
|
wafMatches *prometheus.CounterVec
|
||||||
// ruleMatches are made by AddRules, and reputationHits by
|
// ruleMatches are made by AddRules, and reputationHits by
|
||||||
// AddReputation.
|
// AddReputation.
|
||||||
ruleMatches *prometheus.CounterVec
|
ruleMatches *prometheus.CounterVec
|
||||||
@@ -99,6 +99,9 @@ func New(topN int, instanceName string) *Metrics {
|
|||||||
"Requests that broke a rate limit, a byte limit or the error burst, by "+
|
"Requests that broke a rate limit, a byte limit or the error burst, by "+
|
||||||
"its window and its kind, requests, bytes or refusals.",
|
"its window and its kind, requests, bytes or refusals.",
|
||||||
[]string{"window", "kind"}),
|
[]string{"window", "kind"}),
|
||||||
|
sizeAndTimeLimitHits: counterVec("smallwebwaf_size_and_time_limit_hits_total",
|
||||||
|
"Requests that passed a size or time limit, by its setting.",
|
||||||
|
[]string{"limit"}),
|
||||||
offences: counterVec("smallwebwaf_offences_total",
|
offences: counterVec("smallwebwaf_offences_total",
|
||||||
"Offences, by kind.", []string{"kind"}),
|
"Offences, by kind.", []string{"kind"}),
|
||||||
wafMatches: counterVec("smallwebwaf_waf_matches_total",
|
wafMatches: counterVec("smallwebwaf_waf_matches_total",
|
||||||
@@ -139,7 +142,7 @@ func New(topN int, instanceName string) *Metrics {
|
|||||||
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
|
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
|
||||||
m.inFlight, m.requests, m.requestBytes, m.responseBytes,
|
m.inFlight, m.requests, m.requestBytes, m.responseBytes,
|
||||||
m.requestDuration, m.upstreamDuration,
|
m.requestDuration, m.upstreamDuration,
|
||||||
m.rateLimitHits, m.offences, m.wafMatches,
|
m.rateLimitHits, m.sizeAndTimeLimitHits, m.offences, m.wafMatches,
|
||||||
m.countries, m.asns,
|
m.countries, m.asns,
|
||||||
m.GeoJSRequests, m.GeoJSFailures, m.GeoJSUnanswered,
|
m.GeoJSRequests, m.GeoJSFailures, m.GeoJSUnanswered,
|
||||||
m.stateFileWrites, m.stateFileWriteFailures,
|
m.stateFileWrites, m.stateFileWriteFailures,
|
||||||
@@ -391,26 +394,36 @@ func (m *Metrics) RequestStarted() {
|
|||||||
m.inFlight.Inc()
|
m.inFlight.Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
// RequestEnded counts a request that has ended without being passed to the app,
|
// RequestEnded counts a request that has ended, from its log line. limit
|
||||||
// from its log line. duration is how long the request took.
|
// is the setting whose size or time limit the request passed, "" if none.
|
||||||
func (m *Metrics) RequestEnded(line *requestlog.Line, duration time.Duration) {
|
// duration is how long the request took, and upstreamDuration how long it
|
||||||
m.inFlight.Dec()
|
// took from when it was passed to the app, zero if it was not.
|
||||||
m.count(line, duration, 0)
|
func (m *Metrics) RequestEnded(
|
||||||
}
|
line *requestlog.Line, limit string, duration, upstreamDuration time.Duration,
|
||||||
|
|
||||||
// RequestAllowed counts a question of rproxy's, answered allow, as no longer
|
|
||||||
// under way. RequestReported counts the request it was about.
|
|
||||||
func (m *Metrics) RequestAllowed() {
|
|
||||||
m.inFlight.Dec()
|
|
||||||
}
|
|
||||||
|
|
||||||
// RequestReported counts a request rproxy passed to the app, from what its
|
|
||||||
// report says, which line holds: duration is how long the request took, and
|
|
||||||
// upstreamDuration how long it took from when it was passed to the app.
|
|
||||||
func (m *Metrics) RequestReported(
|
|
||||||
line *requestlog.Line, duration, upstreamDuration time.Duration,
|
|
||||||
) {
|
) {
|
||||||
m.count(line, duration, upstreamDuration)
|
m.inFlight.Dec()
|
||||||
|
|
||||||
|
class := statusClass(line.Status)
|
||||||
|
m.requests.WithLabelValues(class, line.Action).Inc()
|
||||||
|
m.requestBytes.WithLabelValues(class, line.Action).Add(float64(line.RequestBytes))
|
||||||
|
m.responseBytes.WithLabelValues(class, line.Action).Add(float64(line.ResponseBytes))
|
||||||
|
m.requestDuration.Observe(duration.Seconds())
|
||||||
|
|
||||||
|
if upstreamDuration > 0 {
|
||||||
|
m.upstreamDuration.Observe(upstreamDuration.Seconds())
|
||||||
|
}
|
||||||
|
|
||||||
|
if limit != "" {
|
||||||
|
m.sizeAndTimeLimitHits.WithLabelValues(limit).Inc()
|
||||||
|
}
|
||||||
|
|
||||||
|
if line.Country != "" {
|
||||||
|
m.countries.add(line.Country, line)
|
||||||
|
}
|
||||||
|
|
||||||
|
if line.ASN != "" {
|
||||||
|
m.asns.add(line.ASN, line)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// LimitHit counts a request that broke a rate limit, a byte limit or the
|
// LimitHit counts a request that broke a rate limit, a byte limit or the
|
||||||
@@ -424,10 +437,11 @@ func (m *Metrics) LimitHit(hit ratelimit.Hit) {
|
|||||||
// them.
|
// them.
|
||||||
func (m *Metrics) Offences(r ratelimit.Request) {
|
func (m *Metrics) Offences(r ratelimit.Request) {
|
||||||
for kind, committed := range map[string]bool{
|
for kind, committed := range map[string]bool{
|
||||||
"limit": r.BrokeLimit,
|
"limit": r.BrokeLimit,
|
||||||
"attack": r.Attack,
|
"attack": r.Attack,
|
||||||
"rule_blocked": r.RuleBlocked,
|
"rule_blocked": r.RuleBlocked,
|
||||||
"waf_blocked": r.WAFBlocked,
|
"waf_blocked": r.WAFBlocked,
|
||||||
|
"token_refused": r.TokenRefused,
|
||||||
} {
|
} {
|
||||||
if committed {
|
if committed {
|
||||||
m.offences.WithLabelValues(kind).Inc()
|
m.offences.WithLabelValues(kind).Inc()
|
||||||
@@ -478,30 +492,6 @@ func (m *Metrics) StateFileEditSetAside(name string) {
|
|||||||
m.stateFileEditsSetAside.WithLabelValues(name).Inc()
|
m.stateFileEditsSetAside.WithLabelValues(name).Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
// count counts a request that has ended, as RequestEnded and RequestReported
|
|
||||||
// describe.
|
|
||||||
func (m *Metrics) count(
|
|
||||||
line *requestlog.Line, duration, upstreamDuration time.Duration,
|
|
||||||
) {
|
|
||||||
class := statusClass(line.Status)
|
|
||||||
m.requests.WithLabelValues(class, line.Action).Inc()
|
|
||||||
m.requestBytes.WithLabelValues(class, line.Action).Add(float64(line.RequestBytes))
|
|
||||||
m.responseBytes.WithLabelValues(class, line.Action).Add(float64(line.ResponseBytes))
|
|
||||||
m.requestDuration.Observe(duration.Seconds())
|
|
||||||
|
|
||||||
if upstreamDuration > 0 {
|
|
||||||
m.upstreamDuration.Observe(upstreamDuration.Seconds())
|
|
||||||
}
|
|
||||||
|
|
||||||
if line.Country != "" {
|
|
||||||
m.countries.add(line.Country, line)
|
|
||||||
}
|
|
||||||
|
|
||||||
if line.ASN != "" {
|
|
||||||
m.asns.add(line.ASN, line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// addReputationQueries adds the counter of the queries to source, a DNSBL
|
// addReputationQueries adds the counter of the queries to source, a DNSBL
|
||||||
// zone, or of the checks of clients with AbuseIPDB, which count tells.
|
// zone, or of the checks of clients with AbuseIPDB, which count tells.
|
||||||
func (m *Metrics) addReputationQueries(source prometheus.Labels, count func() int) {
|
func (m *Metrics) addReputationQueries(source prometheus.Labels, count func() int) {
|
||||||
|
|||||||
+41
-20
@@ -9,6 +9,7 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -40,22 +41,16 @@ var (
|
|||||||
errNotAddress = errors.New("is not an address, such as 203.0.113.9")
|
errNotAddress = errors.New("is not an address, such as 203.0.113.9")
|
||||||
)
|
)
|
||||||
|
|
||||||
// answerHealth answers a request for the health endpoint.
|
// answerAdmin answers a request for smallwebwaf itself, under
|
||||||
func (rq *request) answerHealth() {
|
// /_smallwebwaf/, once it has passed the checks. Each endpoint needs a
|
||||||
rq.line.Action = requestlog.ActionAdmin
|
// token, sent as Authorization: Bearer <token>: the metrics
|
||||||
// Set here rather than left to Go's server, which would set it only
|
// SWWAF_METRICS_TOKEN, the others SWWAF_ADMIN_TOKEN. A request without
|
||||||
// after the log line has taken the response's headers.
|
// it is refused with 401, which counts toward the error burst. An
|
||||||
rq.out.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
// endpoint whose token is unset answers 404, as any other request under
|
||||||
_, _ = io.WriteString(rq.out, "ok\n")
|
// /_smallwebwaf/ does.
|
||||||
}
|
|
||||||
|
|
||||||
// answerAdmin answers a request for one of servewaf's own endpoints. Each
|
|
||||||
// needs a token, sent as Authorization: Bearer <token>: the metrics
|
|
||||||
// SWWAF_METRICS_TOKEN, the others SWWAF_ADMIN_TOKEN. A request without it is
|
|
||||||
// refused with 401. An endpoint whose token is unset answers 404, as any other
|
|
||||||
// path does.
|
|
||||||
func (rq *request) answerAdmin() {
|
func (rq *request) answerAdmin() {
|
||||||
rq.line.Action = requestlog.ActionAdmin
|
rq.line.Action = requestlog.ActionAdmin
|
||||||
|
rq.startClientResponseTimeout()
|
||||||
|
|
||||||
token, answer := rq.endpoint()
|
token, answer := rq.endpoint()
|
||||||
|
|
||||||
@@ -63,8 +58,12 @@ func (rq *request) answerAdmin() {
|
|||||||
case token == "":
|
case token == "":
|
||||||
http.Error(rq.out, http.StatusText(http.StatusNotFound), http.StatusNotFound)
|
http.Error(rq.out, http.StatusText(http.StatusNotFound), http.StatusNotFound)
|
||||||
case !hasToken(rq.in, token):
|
case !hasToken(rq.in, token):
|
||||||
|
rq.tokenRefused = true
|
||||||
rq.out.Header().Set("WWW-Authenticate", "Bearer")
|
rq.out.Header().Set("WWW-Authenticate", "Bearer")
|
||||||
http.Error(rq.out, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
|
rq.answer(refusal{
|
||||||
|
status: http.StatusUnauthorized,
|
||||||
|
action: requestlog.ActionAdmin,
|
||||||
|
})
|
||||||
default:
|
default:
|
||||||
answer()
|
answer()
|
||||||
}
|
}
|
||||||
@@ -121,7 +120,26 @@ type banToAdd struct {
|
|||||||
// an admin, from now for the duration the body gives, with its reason,
|
// an admin, from now for the duration the body gives, with its reason,
|
||||||
// and answers with that ban.
|
// and answers with that ban.
|
||||||
func (rq *request) addBan() {
|
func (rq *request) addBan() {
|
||||||
|
// The body must arrive within SWWAF_CLIENT_REQUEST_TIMEOUT, as any
|
||||||
|
// other request's must.
|
||||||
|
rq.stopReadingBody(rq.clientRequestDeadline())
|
||||||
|
|
||||||
toAdd, err := rq.readBanToAdd()
|
toAdd, err := rq.readBanToAdd()
|
||||||
|
if refused := rq.refused.Load(); refused != nil {
|
||||||
|
rq.answer(*refused) // the body is over SWWAF_REQUEST_MAX_BYTES
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if errors.Is(err, os.ErrDeadlineExceeded) {
|
||||||
|
rq.answer(refusal{
|
||||||
|
status: http.StatusRequestTimeout,
|
||||||
|
action: requestlog.ActionTimedOut,
|
||||||
|
limit: "SWWAF_CLIENT_REQUEST_TIMEOUT",
|
||||||
|
})
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
netblock netip.Prefix
|
netblock netip.Prefix
|
||||||
@@ -150,9 +168,12 @@ func (rq *request) addBan() {
|
|||||||
// readBanToAdd reads the body of POST BansPath: a JSON object with
|
// readBanToAdd reads the body of POST BansPath: a JSON object with
|
||||||
// nothing but whitespace after it, in at most banBodyMaxBytes.
|
// nothing but whitespace after it, in at most banBodyMaxBytes.
|
||||||
func (rq *request) readBanToAdd() (banToAdd, error) {
|
func (rq *request) readBanToAdd() (banToAdd, error) {
|
||||||
data, err := io.ReadAll(http.MaxBytesReader(nil, rq.in.Body, banBodyMaxBytes))
|
var body io.ReadCloser = http.NoBody
|
||||||
rq.requestBytes = int64(len(data))
|
if rq.body != nil {
|
||||||
|
body = rq.body
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := io.ReadAll(http.MaxBytesReader(nil, body, banBodyMaxBytes))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return banToAdd{}, fmt.Errorf("%w: %w", errNotBanToAdd, err)
|
return banToAdd{}, fmt.Errorf("%w: %w", errNotBanToAdd, err)
|
||||||
}
|
}
|
||||||
@@ -250,9 +271,9 @@ type clientAnswer struct {
|
|||||||
Bans []state.BanEntry `json:"bans"`
|
Bans []state.BanEntry `json:"bans"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// showClient answers GET ClientsPath<ip> with what servewaf knows of the client:
|
// showClient answers GET ClientsPath<ip> with what smallwebwaf knows of
|
||||||
// its counters, its history, which holds its country as last looked up and its
|
// the client: its counters, its history, which holds its country as last
|
||||||
// offences, and its bans with their notes.
|
// looked up and its offences, and its bans with their notes.
|
||||||
func (rq *request) showClient() {
|
func (rq *request) showClient() {
|
||||||
addr, err := pathAddress(rq.in.URL.Path, ClientsPath)
|
addr, err := pathAddress(rq.in.URL.Path, ClientsPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
+138
-20
@@ -5,6 +5,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -22,6 +23,8 @@ const (
|
|||||||
// how a request carries it.
|
// how a request carries it.
|
||||||
adminSecret = "fedcba9876543210fedcba9876543210"
|
adminSecret = "fedcba9876543210fedcba9876543210"
|
||||||
adminBearer = "Bearer " + adminSecret
|
adminBearer = "Bearer " + adminSecret
|
||||||
|
// adminClient is the client the tests' admin sends its requests from.
|
||||||
|
adminClient = "192.0.2.10"
|
||||||
// banOtherClient is the body of a request to ban otherClient for an
|
// banOtherClient is the body of a request to ban otherClient for an
|
||||||
// hour.
|
// hour.
|
||||||
banOtherClient = `{"netblock": "` + otherClient + `", "duration": "1h", ` +
|
banOtherClient = `{"netblock": "` + otherClient + `", "duration": "1h", ` +
|
||||||
@@ -40,7 +43,8 @@ func TestAdminEndpointsAreOffWhileTheTokenIsUnset(t *testing.T) {
|
|||||||
// An empty token does not match the unset one either.
|
// An empty token does not match the unset one either.
|
||||||
for _, authorization := range []string{adminBearer, bearer, "Bearer ", ""} {
|
for _, authorization := range []string{adminBearer, bearer, "Bearer ", ""} {
|
||||||
for _, e := range adminEndpoints() {
|
for _, e := range adminEndpoints() {
|
||||||
s.adminRequest(authorization, e.method, e.path, e.body, http.StatusNotFound)
|
s.adminRequest(adminClient, authorization, e.method, e.path, e.body,
|
||||||
|
http.StatusNotFound, requestlog.ActionAdmin)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -67,8 +71,8 @@ func TestAdminEndpointsNeedTheAdminToken(t *testing.T) {
|
|||||||
for _, authorization := range []string{
|
for _, authorization := range []string{
|
||||||
"", bearer, "Bearer " + strings.ToUpper(adminSecret), "Basic " + adminSecret,
|
"", bearer, "Bearer " + strings.ToUpper(adminSecret), "Basic " + adminSecret,
|
||||||
} {
|
} {
|
||||||
got := s.adminRequest(authorization, e.method, e.path, e.body,
|
got := s.adminRequest(adminClient, authorization, e.method, e.path, e.body,
|
||||||
http.StatusUnauthorized)
|
http.StatusUnauthorized, requestlog.ActionAdmin)
|
||||||
if got.header.Get("WWW-Authenticate") != "Bearer" {
|
if got.header.Get("WWW-Authenticate") != "Bearer" {
|
||||||
t.Errorf("%s %s with %q was answered without WWW-Authenticate: Bearer",
|
t.Errorf("%s %s with %q was answered without WWW-Authenticate: Bearer",
|
||||||
e.method, e.path, authorization)
|
e.method, e.path, authorization)
|
||||||
@@ -261,12 +265,64 @@ func TestBanToAddThatCannotBeReadIsRefused(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBanToAddOverTheRequestSizeLimitIsRefused(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s, _, server := startWithClock(t, "", map[string]string{
|
||||||
|
adminToken: adminSecret,
|
||||||
|
requestMaxBytes: "16",
|
||||||
|
})
|
||||||
|
|
||||||
|
// Sent in a chunk, its length is not announced, so that it is found
|
||||||
|
// over SWWAF_REQUEST_MAX_BYTES only as it is read.
|
||||||
|
chunk := `{"netblock": "203.0.113.9", "duration": "1h"}`
|
||||||
|
s.adminRequest(adminClient, adminBearer+"\r\nTransfer-Encoding: chunked",
|
||||||
|
http.MethodPost, proxy.BansPath,
|
||||||
|
strconv.FormatInt(int64(len(chunk)), 16)+"\r\n"+chunk+"\r\n0\r\n\r\n",
|
||||||
|
http.StatusRequestEntityTooLarge, requestlog.ActionTooLarge)
|
||||||
|
|
||||||
|
if held := server.Ledger.Snapshot(); len(held) != 0 {
|
||||||
|
t.Errorf("the ledger holds %+v, want no ban", held)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBanToAddSlowerThanTheClientRequestTimeoutIsRefused(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s, _, server := startWithClock(t, "", map[string]string{
|
||||||
|
adminToken: adminSecret,
|
||||||
|
metricsToken: token,
|
||||||
|
clientRequestTimeout: shortTimeoutSetting,
|
||||||
|
})
|
||||||
|
|
||||||
|
// The chunk announces 256 bytes and the rest of it never comes, so only
|
||||||
|
// the timeout ends the wait. A hold-up of the test process can only
|
||||||
|
// make the answer later, so the time is checked only for not being
|
||||||
|
// shorter than the timeout.
|
||||||
|
start := time.Now()
|
||||||
|
|
||||||
|
s.adminRequest(adminClient, adminBearer+"\r\nTransfer-Encoding: chunked",
|
||||||
|
http.MethodPost, proxy.BansPath, "100\r\n"+`{"netblock": "203.0.113.9", `,
|
||||||
|
http.StatusRequestTimeout, requestlog.ActionTimedOut)
|
||||||
|
|
||||||
|
if took := time.Since(start); took < shortTimeout {
|
||||||
|
t.Errorf("answered after %s, before the timeout of %s ran out", took, shortTimeout)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantLimitHits(t, s.addr, clientRequestTimeout, 1)
|
||||||
|
|
||||||
|
if held := server.Ledger.Snapshot(); len(held) != 0 {
|
||||||
|
t.Errorf("the ledger holds %+v, want no ban", held)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestClientEndpointShowsTheClientAndItsBans(t *testing.T) {
|
func TestClientEndpointShowsTheClientAndItsBans(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
s, clk, _ := startWithClock(t, "", map[string]string{
|
s, clk, _ := startWithClock(t, "", map[string]string{
|
||||||
adminToken: adminSecret,
|
adminToken: adminSecret,
|
||||||
rateLimitPerMinute: "2",
|
rateLimitPerMinute: "2",
|
||||||
|
rateLimitExemptNets: adminClient,
|
||||||
})
|
})
|
||||||
start := clk.Now()
|
start := clk.Now()
|
||||||
|
|
||||||
@@ -315,6 +371,61 @@ func TestClientEndpointShowsTheClientAndItsBans(t *testing.T) {
|
|||||||
http.StatusBadRequest)
|
http.StatusBadRequest)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBannedClientIsRefusedAtTheEndpointsEvenWithTheToken(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s, _, _ := startWithClock(t, "", map[string]string{adminToken: adminSecret})
|
||||||
|
|
||||||
|
s.admin(http.MethodPost, proxy.BansPath, banOtherClient, http.StatusOK)
|
||||||
|
|
||||||
|
// otherClient cannot lift its own ban either.
|
||||||
|
for _, e := range adminEndpoints() {
|
||||||
|
s.adminRequest(otherClient, adminBearer, e.method, e.path, e.body,
|
||||||
|
http.StatusForbidden, requestlog.ActionBanned)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAdminRequestsCountTowardTheLimits(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s, _, _ := startWithClock(t, "", map[string]string{
|
||||||
|
adminToken: adminSecret,
|
||||||
|
rateLimitPerMinute: "2",
|
||||||
|
})
|
||||||
|
|
||||||
|
// A request refused for a missing token and one answered count toward
|
||||||
|
// the limit of two a minute, so the next breaks it.
|
||||||
|
s.adminRequest(client, "", http.MethodGet, proxy.BansPath, "",
|
||||||
|
http.StatusUnauthorized, requestlog.ActionAdmin)
|
||||||
|
s.adminRequest(client, adminBearer, http.MethodGet, proxy.BansPath, "",
|
||||||
|
http.StatusOK, requestlog.ActionAdmin)
|
||||||
|
s.adminRequest(client, adminBearer, http.MethodGet, proxy.BansPath, "",
|
||||||
|
http.StatusForbidden, requestlog.ActionRateLimited)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClientInAllowNetsSkipsTheChecksButNeedsTheToken(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const allowed = "192.0.2.60" // in SWWAF_ALLOW_NETS
|
||||||
|
|
||||||
|
s, clk, server := startWithClock(t, "", map[string]string{
|
||||||
|
adminToken: adminSecret,
|
||||||
|
allowNets: allowed,
|
||||||
|
rateLimitPerMinute: "1",
|
||||||
|
})
|
||||||
|
|
||||||
|
// A ban on it refuses nothing, and its requests are not counted.
|
||||||
|
server.Ledger.BanForAdmin(netip.MustParsePrefix(allowed+"/32"), clk.Now(),
|
||||||
|
time.Time{}, "")
|
||||||
|
|
||||||
|
for range 2 {
|
||||||
|
s.adminRequest(allowed, "", http.MethodGet, proxy.BansPath, "",
|
||||||
|
http.StatusUnauthorized, requestlog.ActionAdmin)
|
||||||
|
s.adminRequest(allowed, adminBearer, http.MethodGet, proxy.BansPath, "",
|
||||||
|
http.StatusOK, requestlog.ActionAdmin)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestAdminEndpointsNeedTheTokenInObserveMode(t *testing.T) {
|
func TestAdminEndpointsNeedTheTokenInObserveMode(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -324,7 +435,8 @@ func TestAdminEndpointsNeedTheTokenInObserveMode(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
for _, e := range adminEndpoints() {
|
for _, e := range adminEndpoints() {
|
||||||
s.adminRequest("", e.method, e.path, e.body, http.StatusUnauthorized)
|
s.adminRequest(adminClient, "", e.method, e.path, e.body,
|
||||||
|
http.StatusUnauthorized, requestlog.ActionAdmin)
|
||||||
}
|
}
|
||||||
|
|
||||||
if held := server.Ledger.Snapshot(); len(held) != 0 {
|
if held := server.Ledger.Snapshot(); len(held) != 0 {
|
||||||
@@ -349,33 +461,39 @@ func adminEndpoints() []adminEndpoint {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// admin sends a request with method for path, with body, to servewaf's own
|
// admin sends a request with method for path, with body, from
|
||||||
// address, with the admin token, and checks that it is answered with status, its
|
// adminClient, with the admin token, and checks that it is answered with
|
||||||
// log line's action admin. It returns the answer.
|
// status, its log line's action admin. It returns the answer.
|
||||||
func (s *sender) admin(method, path, body string, status int) answer {
|
func (s *sender) admin(method, path, body string, status int) answer {
|
||||||
s.t.Helper()
|
s.t.Helper()
|
||||||
|
|
||||||
return s.adminRequest(adminBearer, method, path, body, status)
|
return s.adminRequest(adminClient, adminBearer, method, path, body, status,
|
||||||
|
requestlog.ActionAdmin)
|
||||||
}
|
}
|
||||||
|
|
||||||
// adminRequest is admin with authorization as the request's Authorization header
|
// adminRequest sends a request with method for path, with body, from the
|
||||||
// unless it is "".
|
// client at from, with authorization as its Authorization header unless
|
||||||
|
// it is "", and checks its answer's status and its log line's action, as
|
||||||
|
// request does. authorization may end in more header lines. A body that
|
||||||
|
// is not "" has its length announced, unless authorization names
|
||||||
|
// Transfer-Encoding. It returns the answer.
|
||||||
func (s *sender) adminRequest(
|
func (s *sender) adminRequest(
|
||||||
authorization, method, path, body string, status int,
|
from, authorization, method, path, body string, status int, action string,
|
||||||
) answer {
|
) answer {
|
||||||
s.t.Helper()
|
s.t.Helper()
|
||||||
|
|
||||||
req := newRequest(s.t, method, s.serveWAFAddr, path, strings.NewReader(body))
|
var header []string
|
||||||
|
|
||||||
if authorization != "" {
|
if authorization != "" {
|
||||||
req.Header.Set("Authorization", authorization)
|
header = append(header, "Authorization: "+authorization)
|
||||||
}
|
}
|
||||||
|
|
||||||
got := do(s.t, req)
|
if body != "" && !strings.Contains(authorization, "Transfer-Encoding") {
|
||||||
wantStatus(s.t, got, status)
|
header = append(header, "Content-Length: "+strconv.Itoa(len(body)))
|
||||||
|
}
|
||||||
|
|
||||||
line := s.out.endedLines(s.t, s.sent+1)[s.sent]
|
_, got := s.requestWithBody(method, from, path, strings.Join(header, "\r\n"),
|
||||||
s.sent++
|
body, status, action)
|
||||||
wantLine(s.t, line, status, requestlog.ActionAdmin)
|
|
||||||
|
|
||||||
return got
|
return got
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -224,9 +224,7 @@ func startAppWithAlerts(
|
|||||||
|
|
||||||
addr, out, server, queue := startProxyWithAlerts(t, app.URL, "", clk.Now, settings)
|
addr, out, server, queue := startProxyWithAlerts(t, app.URL, "", clk.Now, settings)
|
||||||
|
|
||||||
s := &sender{t: t, addr: addr, serveWAFAddr: server.Addr, out: out}
|
return &sender{t: t, addr: addr, out: out}, clk, server, queue
|
||||||
|
|
||||||
return s, clk, server, queue
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// banAlert returns the alert for event, raised by a request from client at
|
// banAlert returns the alert for event, raised by a request from client at
|
||||||
@@ -239,7 +237,6 @@ func banAlert(
|
|||||||
Instance: alertInstance,
|
Instance: alertInstance,
|
||||||
Time: raised,
|
Time: raised,
|
||||||
Event: event,
|
Event: event,
|
||||||
Host: appHost,
|
|
||||||
Client: netip.MustParseAddr(client),
|
Client: netip.MustParseAddr(client),
|
||||||
Netblock: ban.Netblock,
|
Netblock: ban.Netblock,
|
||||||
Reason: ban.Reason,
|
Reason: ban.Reason,
|
||||||
|
|||||||
@@ -15,11 +15,9 @@ func TestWithEveryAnomalyThresholdOffARequestIsNotCounted(t *testing.T) {
|
|||||||
// anomaly counters, nor a clock, and the request no response: reading
|
// anomaly counters, nor a clock, and the request no response: reading
|
||||||
// any of them to count the request panics.
|
// any of them to count the request panics.
|
||||||
rq := &request{
|
rq := &request{
|
||||||
decision: decision{
|
h: &handler{config: &config.Config{LookupSource: "geojs"}},
|
||||||
client: netip.MustParseAddr("203.0.113.9"),
|
client: netip.MustParseAddr("203.0.113.9"),
|
||||||
lookedUp: true,
|
lookedUp: true,
|
||||||
},
|
|
||||||
h: &handler{config: &config.Config{LookupSource: "geojs"}},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
|
|||||||
@@ -117,7 +117,6 @@ func TestEachScopeAndWindowOverItsThresholdAlertsOncePerCooldown(t *testing.T) {
|
|||||||
Instance: alertInstance,
|
Instance: alertInstance,
|
||||||
Time: start,
|
Time: start,
|
||||||
Event: alerts.EventAnomaly,
|
Event: alerts.EventAnomaly,
|
||||||
Host: appHost,
|
|
||||||
Client: netip.MustParseAddr(client),
|
Client: netip.MustParseAddr(client),
|
||||||
Netblock: scope.netblock,
|
Netblock: scope.netblock,
|
||||||
ASN: asnDE,
|
ASN: asnDE,
|
||||||
|
|||||||
@@ -1,564 +0,0 @@
|
|||||||
package proxy_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"maps"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/netip"
|
|
||||||
"slices"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"testing/synctest"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/proxy"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// answerTimeLimit is the default of rproxy's SWRP_TIMEOUT_MS, the time
|
|
||||||
// servewaf has to answer.
|
|
||||||
answerTimeLimit = 2 * time.Second
|
|
||||||
// defaultLookupWait is the default of SWWAF_LOOKUP_TIMEOUT.
|
|
||||||
defaultLookupWait = 800 * time.Millisecond
|
|
||||||
// The header limits SPEC.md gives.
|
|
||||||
cookieLimit = 50
|
|
||||||
headerFieldLimit = 100
|
|
||||||
// cookieHeader is the header the cookie limit counts.
|
|
||||||
cookieHeader = "Cookie"
|
|
||||||
// tooLarge is how servewaf refuses a request over the header limits.
|
|
||||||
tooLarge = http.StatusRequestHeaderFieldsTooLarge
|
|
||||||
// silentAbuseIPDBURL is where AbuseIPDB never answers: at the stand-in
|
|
||||||
// that unansweredGeoJSURL names too.
|
|
||||||
silentAbuseIPDBURL = "unanswered://abuseipdb/api/v2/check"
|
|
||||||
// wholeLimit is the percentage of a rate limit that lowers it not at all.
|
|
||||||
wholeLimit = 100
|
|
||||||
// clientsDecision is the CrowdSec decision list holding client alone.
|
|
||||||
clientsDecision = `[{"duration": "4h0m0s", "scenario": "crowdsecurity/ssh-bf", ` +
|
|
||||||
`"scope": "Ip", "type": "ban", "value": "` + client + `"}]`
|
|
||||||
)
|
|
||||||
|
|
||||||
// cookies returns a Cookie header's value with count cookies.
|
|
||||||
func cookies(count int) string {
|
|
||||||
parts := make([]string, 0, count)
|
|
||||||
for i := range count {
|
|
||||||
parts = append(parts, "c"+strconv.Itoa(i)+"=1")
|
|
||||||
}
|
|
||||||
|
|
||||||
return strings.Join(parts, "; ")
|
|
||||||
}
|
|
||||||
|
|
||||||
// fillHeaders adds headers to a question until the checks see total header
|
|
||||||
// fields: all but ClientHeader and PeerHeader, which servewaf removes. Go's
|
|
||||||
// client sends nothing else but Host, which the limit does not count, once
|
|
||||||
// User-Agent is set.
|
|
||||||
func fillHeaders(req *http.Request, total int) {
|
|
||||||
req.Header.Set("User-Agent", "test")
|
|
||||||
|
|
||||||
for i := len(req.Header) - 2; i < total; i++ {
|
|
||||||
req.Header.Set("X-Filler-"+strconv.Itoa(i), "1")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSixteenThousandCookiesAreRefusedWellWithinTheTimeLimit(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := startServeWAF(t, "", map[string]string{wafMode: block})
|
|
||||||
req := s.question(siteA, client, "/")
|
|
||||||
req.Header.Set(cookieHeader, strings.Repeat("a;", 16000))
|
|
||||||
|
|
||||||
began := time.Now()
|
|
||||||
got := do(t, req)
|
|
||||||
|
|
||||||
if took := time.Since(began); took >= answerTimeLimit/2 {
|
|
||||||
t.Errorf("answered after %s, want well within %s", took, answerTimeLimit)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantDisposition(t, got, tooLarge, proxy.DispositionDeny)
|
|
||||||
|
|
||||||
line := s.out.requestLine(t)
|
|
||||||
wantLine(t, line, tooLarge, requestlog.ActionTooLarge)
|
|
||||||
|
|
||||||
if line.DurationWAF != nil {
|
|
||||||
t.Errorf("the Core Rule Set read the request, for %f ms", *line.DurationWAF)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
repeated := make([]string, headerFieldLimit)
|
|
||||||
for i := range repeated {
|
|
||||||
repeated[i] = "1"
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
make func(*http.Request)
|
|
||||||
// refused is true for a request over a limit, and false for one at it.
|
|
||||||
refused bool
|
|
||||||
}{
|
|
||||||
{"cookies at the limit", func(req *http.Request) {
|
|
||||||
req.Header.Set(cookieHeader, cookies(cookieLimit))
|
|
||||||
}, false},
|
|
||||||
{"cookies over the limit", func(req *http.Request) {
|
|
||||||
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
|
||||||
}, true},
|
|
||||||
{"parts empty or only spaces and tabs are no cookies", func(req *http.Request) {
|
|
||||||
req.Header.Set(cookieHeader, cookies(cookieLimit)+"; \t;;")
|
|
||||||
}, false},
|
|
||||||
{"parts of no-break spaces count", func(req *http.Request) {
|
|
||||||
req.Header.Set(cookieHeader, strings.Repeat(" ;", cookieLimit+1))
|
|
||||||
}, true},
|
|
||||||
{"cookies over the limit across two headers", func(req *http.Request) {
|
|
||||||
req.Header[cookieHeader] = []string{cookies(cookieLimit), "last=1"}
|
|
||||||
}, true},
|
|
||||||
{"cookies without a value count", func(req *http.Request) {
|
|
||||||
req.Header.Set(cookieHeader, strings.Repeat("a;", cookieLimit+1))
|
|
||||||
}, true},
|
|
||||||
{"header fields at the limit", func(req *http.Request) {
|
|
||||||
fillHeaders(req, headerFieldLimit)
|
|
||||||
}, false},
|
|
||||||
{"header fields over the limit", func(req *http.Request) {
|
|
||||||
fillHeaders(req, headerFieldLimit+1)
|
|
||||||
}, true},
|
|
||||||
{"a header sent many times counts each time", func(req *http.Request) {
|
|
||||||
req.Header["X-Filler"] = repeated
|
|
||||||
}, true},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := startServeWAF(t, "", map[string]string{wafMode: block})
|
|
||||||
req := s.question(siteA, client, "/")
|
|
||||||
tc.make(req)
|
|
||||||
|
|
||||||
got := do(t, req)
|
|
||||||
line := s.out.requestLine(t)
|
|
||||||
|
|
||||||
if tc.refused {
|
|
||||||
wantDisposition(t, got, tooLarge, proxy.DispositionDeny)
|
|
||||||
wantLine(t, line, tooLarge, requestlog.ActionTooLarge)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
wantDisposition(t, got, http.StatusOK, proxy.DispositionAllow)
|
|
||||||
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
|
|
||||||
|
|
||||||
if line.DurationWAF == nil {
|
|
||||||
t.Error("the Core Rule Set did not read a request within the limits")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHeaderLimitsHoldWhateverTheCoreRuleSetSettingsSay(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, env := range []map[string]string{
|
|
||||||
{wafMode: off},
|
|
||||||
{wafMode: "detect"},
|
|
||||||
{wafMode: block, "SWWAF_WAF_EXEMPT_PATHS": "/"},
|
|
||||||
} {
|
|
||||||
t.Run(fmt.Sprint(env), func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := startServeWAF(t, "", env)
|
|
||||||
req := s.question(siteA, client, "/")
|
|
||||||
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
|
||||||
|
|
||||||
wantDisposition(t, do(t, req), tooLarge, proxy.DispositionDeny)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestClientInTheAllowNetsSkipsTheHeaderLimits(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := startServeWAF(t, "", map[string]string{wafMode: block, allowNets: client})
|
|
||||||
req := s.question(siteA, client, "/")
|
|
||||||
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
|
||||||
|
|
||||||
wantDisposition(t, do(t, req), http.StatusOK, proxy.DispositionAllow)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestObserveModeAllowsARequestOverTheHeaderLimitsUnread(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := startServeWAF(t, "", map[string]string{mode: observe, wafMode: block})
|
|
||||||
req := s.question(siteA, client, "/")
|
|
||||||
req.Header.Set(cookieHeader, strings.Repeat("a;", 16000))
|
|
||||||
|
|
||||||
began := time.Now()
|
|
||||||
got := do(t, req)
|
|
||||||
|
|
||||||
if took := time.Since(began); took >= answerTimeLimit/2 {
|
|
||||||
t.Errorf("answered after %s, want well within %s", took, answerTimeLimit)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantDisposition(t, got, http.StatusOK, proxy.DispositionAllow)
|
|
||||||
|
|
||||||
line := s.out.requestLine(t)
|
|
||||||
if line.WouldAction != requestlog.ActionTooLarge || line.DurationWAF != nil {
|
|
||||||
t.Errorf("would_action %q and duration_waf %v, want %q and none",
|
|
||||||
line.WouldAction, line.DurationWAF, requestlog.ActionTooLarge)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRequestsOverTheHeaderLimitsCountTowardTheRateLimits(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := startServeWAF(t, "", map[string]string{rateLimitPerMinute: "1"})
|
|
||||||
ask := func() answer {
|
|
||||||
req := s.question(siteA, client, "/")
|
|
||||||
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
|
||||||
|
|
||||||
return do(t, req)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantDisposition(t, ask(), tooLarge, proxy.DispositionDeny)
|
|
||||||
wantDisposition(t, ask(), http.StatusForbidden, proxy.DispositionDeny)
|
|
||||||
|
|
||||||
lines := s.out.requestLines(t, 2)
|
|
||||||
wantLine(t, lines[1], http.StatusForbidden, requestlog.ActionRateLimited)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTheHeaderLimitsComeAfterTheTrapPathsAndTheRuleFiles(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name, path, action string
|
|
||||||
}{
|
|
||||||
{"a trap path", "/wp-login.php", requestlog.ActionBanned},
|
|
||||||
{"a block rule", "/blocked", requestlog.ActionRuleBlocked},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := startServeWAF(t, "", map[string]string{
|
|
||||||
trapPaths: trapPathList, rulesDir: writeRules(t, testRules),
|
|
||||||
})
|
|
||||||
req := s.question(siteA, client, tc.path)
|
|
||||||
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
|
||||||
|
|
||||||
wantDisposition(t, do(t, req), http.StatusForbidden, proxy.DispositionDeny)
|
|
||||||
wantLine(t, s.out.requestLine(t), http.StatusForbidden, tc.action)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNotWaitingForGeoJSNeverLowersAClientsLimit(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
geojsURL, _ := startGeoJS(t)
|
|
||||||
s := startServeWAF(t, geojsURL, map[string]string{
|
|
||||||
unknownLimitPercent: "0", lookupTimeout: "1h",
|
|
||||||
})
|
|
||||||
req := s.question(siteA, fromDE, "/")
|
|
||||||
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
|
||||||
|
|
||||||
wantDisposition(t, do(t, req), tooLarge, proxy.DispositionDeny)
|
|
||||||
|
|
||||||
line := s.out.requestLine(t)
|
|
||||||
wantLine(t, line, tooLarge, requestlog.ActionTooLarge)
|
|
||||||
|
|
||||||
if line.LimitPercent != nil {
|
|
||||||
t.Errorf("limit percentage %d, want none", *line.LimitPercent)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The client is under no ban, and GeoJS places it in a country.
|
|
||||||
wantDisposition(t, s.ask(fromDE, "/"), http.StatusOK, proxy.DispositionAllow)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAClientGeoJSPlacedInNoCountryCountsAsUnknownWithoutWaiting(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
geojsURL, _ := startGeoJS(t)
|
|
||||||
s := startServeWAF(t, geojsURL, map[string]string{unknownLimitPercent: "0"})
|
|
||||||
s.server.GeoJS.Load([]lookup.Answer{{
|
|
||||||
Client: netip.MustParsePrefix(client + "/32"), ASN: "AS64512",
|
|
||||||
Answered: s.clock.Now(), Used: s.clock.Now(),
|
|
||||||
}})
|
|
||||||
|
|
||||||
req := s.question(siteA, client, "/")
|
|
||||||
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
|
||||||
|
|
||||||
wantDisposition(t, do(t, req), http.StatusForbidden, proxy.DispositionDeny)
|
|
||||||
wantLine(t, s.out.requestLine(t), http.StatusForbidden, requestlog.ActionRateLimited)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestObserveModeWaitsForGeoJSForARequestOverTheHeaderLimits(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
geojsURL, _ := startGeoJS(t)
|
|
||||||
s := startServeWAF(t, geojsURL, map[string]string{
|
|
||||||
mode: observe, addLookupHeaders: on, lookupTimeout: "1h",
|
|
||||||
})
|
|
||||||
req := s.question(siteA, fromDE, "/")
|
|
||||||
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
|
||||||
|
|
||||||
got := do(t, req)
|
|
||||||
wantDisposition(t, got, http.StatusOK, proxy.DispositionAllow)
|
|
||||||
|
|
||||||
asn, country := got.header.Get("X-Client-Asn"), got.header.Get("X-Client-Country")
|
|
||||||
if asn != asnDE || country != "DE" {
|
|
||||||
t.Errorf("answered with the AS number %q and the country %q, want %q and DE",
|
|
||||||
asn, country, asnDE)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// newServeWAFInBubble makes a servewaf for a synctest bubble: GeoJS never
|
|
||||||
// answers, a setting needs its answer, and everything is at its default but
|
|
||||||
// for env. No request to it may use the network.
|
|
||||||
func newServeWAFInBubble(t *testing.T, env map[string]string) (*proxy.Server, *output) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
settings := map[string]string{deniedCountries: "KP"}
|
|
||||||
maps.Copy(settings, env)
|
|
||||||
|
|
||||||
server, out, _ := newServeWAF(t, unansweredGeoJSURL, time.Now, settings)
|
|
||||||
|
|
||||||
return server, out
|
|
||||||
}
|
|
||||||
|
|
||||||
// askDirectly hands a question about a request from client straight to server,
|
|
||||||
// as change leaves it, and returns the answer's status and how long it took.
|
|
||||||
func askDirectly(
|
|
||||||
t *testing.T, server *proxy.Server, change func(*http.Request),
|
|
||||||
) (int, time.Duration) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", http.NoBody)
|
|
||||||
req.RemoteAddr = net.JoinHostPort(rproxyPeer, "1234")
|
|
||||||
req.Header.Set(proxy.ClientHeader, client)
|
|
||||||
req.Header.Set(requestIDHeader, "the-id")
|
|
||||||
change(req)
|
|
||||||
|
|
||||||
recorder := httptest.NewRecorder()
|
|
||||||
began := time.Now()
|
|
||||||
|
|
||||||
server.Handler.ServeHTTP(recorder, req)
|
|
||||||
|
|
||||||
return recorder.Code, time.Since(began)
|
|
||||||
}
|
|
||||||
|
|
||||||
// banClient bans client at now.
|
|
||||||
func banClient(_ *testing.T, server *proxy.Server, now time.Time) {
|
|
||||||
server.Ledger.BanForLimit(netip.MustParsePrefix(client+"/32"), now, bans.Notes{})
|
|
||||||
}
|
|
||||||
|
|
||||||
// countClient counts one request from client at now for the rate limits.
|
|
||||||
func countClient(_ *testing.T, server *proxy.Server, now time.Time) {
|
|
||||||
server.Limiter.Count(netip.MustParsePrefix(client+"/32"), now, wholeLimit)
|
|
||||||
}
|
|
||||||
|
|
||||||
// listClient puts client on the blocklist at dropURL.
|
|
||||||
func listClient(t *testing.T, server *proxy.Server, now time.Time) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
err := server.Lists.Load([]reputation.List{{
|
|
||||||
URL: dropURL, Tried: now, Fetched: now, Lines: []string{client},
|
|
||||||
}})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("load the blocklist: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// decideOnClient puts client on the CrowdSec decision list.
|
|
||||||
func decideOnClient(t *testing.T, server *proxy.Server, now time.Time) {
|
|
||||||
t.Helper()
|
|
||||||
loadDecisions(t, server, now, clientsDecision)
|
|
||||||
}
|
|
||||||
|
|
||||||
// leaveAsItIs prepares nothing.
|
|
||||||
func leaveAsItIs(*testing.T, *proxy.Server, time.Time) {}
|
|
||||||
|
|
||||||
func TestARefusedRequestDoesNotWaitForASlowGeoJS(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const forbidden = http.StatusForbidden
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
env map[string]string
|
|
||||||
// prepare gets the server ready, at now, before the question.
|
|
||||||
prepare func(t *testing.T, server *proxy.Server, now time.Time)
|
|
||||||
// cookie is the request's Cookie header.
|
|
||||||
cookie string
|
|
||||||
status int
|
|
||||||
action string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "SWWAF_DENY_NETS", env: map[string]string{denyNets: client},
|
|
||||||
prepare: leaveAsItIs, status: forbidden, action: requestlog.ActionDenied,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a ban",
|
|
||||||
prepare: banClient, status: forbidden, action: requestlog.ActionBanned,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "the rate limits", env: map[string]string{rateLimitPerMinute: "1"},
|
|
||||||
prepare: countClient, status: forbidden, action: requestlog.ActionRateLimited,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "the rate limits, with a biased threshold",
|
|
||||||
env: map[string]string{rateLimitPerMinute: "1", unknownLimitPercent: "50"},
|
|
||||||
prepare: countClient, status: forbidden, action: requestlog.ActionRateLimited,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a blocklist", env: map[string]string{blocklistURLs: dropURL},
|
|
||||||
prepare: listClient, status: forbidden, action: requestlog.ActionDenied,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "the CrowdSec decision list",
|
|
||||||
env: map[string]string{crowdSecURL: lapi, crowdSecKey: bouncerKey},
|
|
||||||
prepare: decideOnClient, status: forbidden, action: requestlog.ActionBanned,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "the header limits", cookie: cookies(cookieLimit + 1),
|
|
||||||
prepare: leaveAsItIs, status: tooLarge, action: requestlog.ActionTooLarge,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// In a synctest bubble, as TestRequestWaitsAsLongAsTheLookupTimeoutSays
|
|
||||||
// says, so that a request that does not wait takes no time at all.
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
server, out := newServeWAFInBubble(t, tc.env)
|
|
||||||
tc.prepare(t, server, time.Now())
|
|
||||||
|
|
||||||
status, took := askDirectly(t, server, func(req *http.Request) {
|
|
||||||
req.Header.Set(cookieHeader, tc.cookie)
|
|
||||||
})
|
|
||||||
|
|
||||||
if status != tc.status || took != 0 {
|
|
||||||
t.Errorf("answered %d after %s, want %d at once; output:\n%s",
|
|
||||||
status, took, tc.status, out.text())
|
|
||||||
}
|
|
||||||
|
|
||||||
wantLine(t, out.requestLine(t), tc.status, tc.action)
|
|
||||||
|
|
||||||
// GeoJS's request is abandoned before the bubble ends.
|
|
||||||
time.Sleep(defaultLookupWait)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAQuestionWaitsForASlowGeoJSOnlyAsLongAsTheDefaultSays(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
env map[string]string
|
|
||||||
// prepare gets the server ready, at now, before the question.
|
|
||||||
prepare func(t *testing.T, server *proxy.Server, now time.Time)
|
|
||||||
status int
|
|
||||||
action string
|
|
||||||
}{
|
|
||||||
{"nothing refuses it", nil, leaveAsItIs, http.StatusOK, requestlog.ActionForward},
|
|
||||||
{
|
|
||||||
"a country list needs the answer", map[string]string{allowedCountries: "DE"},
|
|
||||||
leaveAsItIs, http.StatusForbidden, requestlog.ActionCountryDenied,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"a blocklist that only logs",
|
|
||||||
map[string]string{blocklistURLs: dropURL, blocklistAction: actionLog},
|
|
||||||
listClient, http.StatusOK, requestlog.ActionForward,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"a blocklist that only limits",
|
|
||||||
map[string]string{blocklistURLs: dropURL, blocklistAction: limitQuarter},
|
|
||||||
listClient, http.StatusOK, requestlog.ActionForward,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"over a rate limit on an exempt path",
|
|
||||||
map[string]string{rateLimitPerMinute: "1", rateLimitExemptPaths: "/"},
|
|
||||||
countClient, http.StatusOK, requestlog.ActionForward,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
server, out := newServeWAFInBubble(t, tc.env)
|
|
||||||
tc.prepare(t, server, time.Now())
|
|
||||||
|
|
||||||
status, took := askDirectly(t, server, func(*http.Request) {})
|
|
||||||
|
|
||||||
if status != tc.status || took != defaultLookupWait {
|
|
||||||
t.Errorf("answered %d after %s, want %d after %s; output:\n%s",
|
|
||||||
status, took, tc.status, defaultLookupWait, out.text())
|
|
||||||
}
|
|
||||||
|
|
||||||
wantLine(t, out.requestLine(t), tc.status, tc.action)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestServeWAFAnswersWithinTheTimeLimitWhileEveryOutsideSourceHangs(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
held, askedGeoJS, _ := startHeldGeoJS(t)
|
|
||||||
|
|
||||||
// A resolver that takes every query and answers none.
|
|
||||||
resolver, err := (&net.ListenConfig{}).ListenPacket(t.Context(), "udp", localhost+":0")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("listen: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = resolver.Close() })
|
|
||||||
|
|
||||||
server, out, _ := newServeWAFWith(t, func(params proxy.Params) *proxy.Server {
|
|
||||||
params.AbuseIPDBURL = silentAbuseIPDBURL
|
|
||||||
|
|
||||||
return proxy.NewServeWAF(params)
|
|
||||||
}, held, time.Now, map[string]string{
|
|
||||||
deniedCountries: "KP", wafMode: block,
|
|
||||||
dnsblZones: dnsblZone, dnsblResolver: resolver.LocalAddr().String(),
|
|
||||||
abuseIPDBKey: accountKey,
|
|
||||||
blocklistURLs: held + "/drop.txt", asnLimitPercentURL: held + "/asn.txt",
|
|
||||||
crowdSecURL: held, crowdSecKey: bouncerKey,
|
|
||||||
})
|
|
||||||
|
|
||||||
// AbuseIPDB is asked only about a client with an offence.
|
|
||||||
group := netip.MustParsePrefix(client + "/32")
|
|
||||||
server.Limiter.AddToHistory(group, time.Now(), ratelimit.Request{Attack: true})
|
|
||||||
|
|
||||||
go server.Lists.Run(t.Context())
|
|
||||||
|
|
||||||
status, took := askDirectly(t, server, func(req *http.Request) {
|
|
||||||
req.Header.Set("User-Agent", "test")
|
|
||||||
req.Header.Set("Accept", "*/*")
|
|
||||||
req.Host = siteA
|
|
||||||
})
|
|
||||||
|
|
||||||
if status != http.StatusOK || took >= answerTimeLimit {
|
|
||||||
t.Errorf("answered %d after %s, want %d within %s; output:\n%s",
|
|
||||||
status, took, http.StatusOK, answerTimeLimit, out.text())
|
|
||||||
}
|
|
||||||
|
|
||||||
// Each outside source was asked, and none has answered. The stand-in for
|
|
||||||
// GeoJS is the lists' too, which name no address.
|
|
||||||
if !slices.Contains(askedGeoJS(), client) {
|
|
||||||
t.Errorf("GeoJS was asked about %q, not %s", askedGeoJS(), client)
|
|
||||||
}
|
|
||||||
|
|
||||||
if queries := server.DNSBL.Queries(dnsblZone); queries != 1 {
|
|
||||||
t.Errorf("%d queries to the DNS blocklist, want 1", queries)
|
|
||||||
}
|
|
||||||
|
|
||||||
if checked := server.AbuseIPDB.Checked(); checked != 1 {
|
|
||||||
t.Errorf("%d checks with AbuseIPDB, want 1", checked)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+29
-17
@@ -58,12 +58,13 @@ func (rq *request) limitBroken(now time.Time) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// countBytes counts the request's bytes, as countedBytes gives them, for
|
// countBytes counts the request's bytes, as countedBytes gives them, for
|
||||||
// the byte limits, once its report has come, and notes the client's byte totals
|
// the byte limits, once its response has ended, and notes the client's
|
||||||
// for the log line; its requests stay there as the rate limits counted them.
|
// byte totals for the log line; its requests stay there as the rate limits
|
||||||
// Only an allowed request has them counted, and only one the rate limits
|
// counted them. Only a request passed to the app has them counted, and
|
||||||
// counted; in observe mode, not one that enforce mode would have refused. Bytes
|
// only one the rate limits counted; in observe mode, not one that enforce
|
||||||
// that take the client over a byte limit, as its limit percentage for the byte
|
// mode would have refused. Bytes that take the client over a byte limit,
|
||||||
// limits lowers it, break it; the response was passed on whole.
|
// as its limit percentage for the byte limits lowers it, break it; the
|
||||||
|
// response was passed on whole.
|
||||||
func (rq *request) countBytes() {
|
func (rq *request) countBytes() {
|
||||||
if !rq.counted || rq.line.WouldAction != "" {
|
if !rq.counted || rq.line.WouldAction != "" {
|
||||||
return
|
return
|
||||||
@@ -83,8 +84,9 @@ func (rq *request) countBytes() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// countRefusal counts the request for the error burst once it has been
|
// countRefusal counts the request for the error burst once it has been
|
||||||
// answered, if servewaf refused it after a rule file match, a trap path or a
|
// answered, if smallwebwaf refused it after a rule file match, a trap path
|
||||||
// Core Rule Set match, and in observe mode if enforce mode would have: more than
|
// or a Core Rule Set match, or for a missing or wrong token, and in
|
||||||
|
// observe mode if enforce mode would have: more than
|
||||||
// SWWAF_ERROR_BURST_THRESHOLD such refusals of the client within a minute
|
// SWWAF_ERROR_BURST_THRESHOLD such refusals of the client within a minute
|
||||||
// break a limit. A client in SWWAF_ALLOW_NETS, which the checks skip, is
|
// break a limit. A client in SWWAF_ALLOW_NETS, which the checks skip, is
|
||||||
// not counted, and nothing is while the threshold is off.
|
// not counted, and nothing is while the threshold is off.
|
||||||
@@ -94,7 +96,11 @@ func (rq *request) countRefusal() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if !rq.attack && !rq.ruleBlocked && !rq.wafBlocked {
|
// In observe mode, a request that enforce mode would have refused
|
||||||
|
// before it reached the endpoint has had no token refused there.
|
||||||
|
tokenRefused := rq.tokenRefused && rq.line.WouldAction == "" &&
|
||||||
|
!isInside(rq.client, cfg.AllowNets)
|
||||||
|
if !rq.attack && !rq.ruleBlocked && !rq.wafBlocked && !tokenRefused {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -121,9 +127,16 @@ func (rq *request) countRefusal() {
|
|||||||
|
|
||||||
// countedBytes returns the request's bytes, once it has ended, as the
|
// countedBytes returns the request's bytes, once it has ended, as the
|
||||||
// byte limits and the anomaly thresholds count them: the response's body
|
// byte limits and the anomaly thresholds count them: the response's body
|
||||||
// bytes, the request's, or both, as SWWAF_BYTES_COUNT says.
|
// bytes, the request's, or both, as SWWAF_BYTES_COUNT says. For an
|
||||||
|
// upgraded connection, such as a WebSocket, which has closed by then, what
|
||||||
|
// it carried from the app counts with the response's and what it carried
|
||||||
|
// from the client with the request's.
|
||||||
func (rq *request) countedBytes() int64 {
|
func (rq *request) countedBytes() int64 {
|
||||||
response, request := rq.out.bytes, rq.requestBytes
|
response, request := rq.out.bytes, rq.requestBytes()
|
||||||
|
if rq.upgraded != nil {
|
||||||
|
response += rq.upgraded.fromApp.Load()
|
||||||
|
request += rq.upgraded.toApp.Load()
|
||||||
|
}
|
||||||
|
|
||||||
switch rq.h.config.BytesCount {
|
switch rq.h.config.BytesCount {
|
||||||
case "response":
|
case "response":
|
||||||
@@ -138,7 +151,7 @@ func (rq *request) countedBytes() int64 {
|
|||||||
// banForLimit bans the client's netblock at now for a broken limit, the
|
// banForLimit bans the client's netblock at now for a broken limit, the
|
||||||
// one hit names, notes the offence for the log line and counts the hit in
|
// one hit names, notes the offence for the log line and counts the hit in
|
||||||
// the metrics. status is what the client was sent, or is sent:
|
// the metrics. status is what the client was sent, or is sent:
|
||||||
// SWWAF_BAN_RESPONSE for a request over a rate limit, what the report says for
|
// SWWAF_BAN_RESPONSE for a request over a rate limit, the app's answer for
|
||||||
// one whose bytes broke a byte limit, the refusal for one that broke the
|
// one whose bytes broke a byte limit, the refusal for one that broke the
|
||||||
// error burst. The ban's notes give the client's limit percentage for a
|
// error burst. The ban's notes give the client's limit percentage for a
|
||||||
// rate limit or a byte limit; the error burst is not lowered. The ban sets
|
// rate limit or a byte limit; the error burst is not lowered. The ban sets
|
||||||
@@ -310,7 +323,6 @@ func (rq *request) alertBan(ban bans.Ban) {
|
|||||||
|
|
||||||
rq.h.alerts.Raise(alerts.Alert{
|
rq.h.alerts.Raise(alerts.Alert{
|
||||||
Event: event,
|
Event: event,
|
||||||
Host: rq.site,
|
|
||||||
Client: rq.client,
|
Client: rq.client,
|
||||||
Netblock: ban.Netblock,
|
Netblock: ban.Netblock,
|
||||||
ASN: ban.Notes.ASN,
|
ASN: ban.Notes.ASN,
|
||||||
@@ -327,11 +339,11 @@ func (rq *request) alertBan(ban bans.Ban) {
|
|||||||
func (rq *request) noted(now time.Time, status int) bans.Request {
|
func (rq *request) noted(now time.Time, status int) bans.Request {
|
||||||
return bans.Request{
|
return bans.Request{
|
||||||
Time: now,
|
Time: now,
|
||||||
Method: rq.line.Method,
|
Method: rq.in.Method,
|
||||||
Host: rq.line.Host,
|
Host: rq.in.Host,
|
||||||
Path: rq.requestURI,
|
Path: rq.in.URL.RequestURI(),
|
||||||
Status: status,
|
Status: status,
|
||||||
UserAgent: rq.line.UserAgent,
|
UserAgent: rq.in.UserAgent(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+23
-28
@@ -9,7 +9,6 @@ import (
|
|||||||
"net/netip"
|
"net/netip"
|
||||||
"reflect"
|
"reflect"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -26,8 +25,6 @@ const (
|
|||||||
userAgent = "ban-test/1.0"
|
userAgent = "ban-test/1.0"
|
||||||
// permanent is the log line's ban_expires for a permanent ban.
|
// permanent is the log line's ban_expires for a permanent ban.
|
||||||
permanent = "permanent"
|
permanent = "permanent"
|
||||||
// banResponseClose is the SWWAF_BAN_RESPONSE that closes the connection.
|
|
||||||
banResponseClose = "close"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestBrokenLimitBansTheClient(t *testing.T) {
|
func TestBrokenLimitBansTheClient(t *testing.T) {
|
||||||
@@ -240,7 +237,7 @@ func TestBanResponseAnswersEveryRefusalButTheSizeLimits(t *testing.T) {
|
|||||||
{"", http.StatusForbidden},
|
{"", http.StatusForbidden},
|
||||||
{"403", http.StatusForbidden},
|
{"403", http.StatusForbidden},
|
||||||
{"429", http.StatusTooManyRequests},
|
{"429", http.StatusTooManyRequests},
|
||||||
{banResponseClose, 0},
|
{"close", 0},
|
||||||
} {
|
} {
|
||||||
t.Run(banResponse+"="+tc.setting, func(t *testing.T) {
|
t.Run(banResponse+"="+tc.setting, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
@@ -375,7 +372,7 @@ func (c *clock) advance(d time.Duration) {
|
|||||||
c.now = c.now.Add(d)
|
c.now = c.now.Add(d)
|
||||||
}
|
}
|
||||||
|
|
||||||
// startWithClock starts servewaf, and rproxy in front of an app that answers 200,
|
// startWithClock starts smallwebwaf in front of an app that answers 200,
|
||||||
// with the settings in env on top of trusting localhost's
|
// with the settings in env on top of trusting localhost's
|
||||||
// X-Forwarded-For, clients' AS numbers and countries looked up at
|
// X-Forwarded-For, clients' AS numbers and countries looked up at
|
||||||
// geojsURL, and a clock set to midnight, the start of a bucket in every
|
// geojsURL, and a clock set to midnight, the start of a bucket in every
|
||||||
@@ -392,19 +389,18 @@ func startWithClock(
|
|||||||
|
|
||||||
addr, out, server := startProxyWithClock(t, app.URL, geojsURL, clk.Now, settings)
|
addr, out, server := startProxyWithClock(t, app.URL, geojsURL, clk.Now, settings)
|
||||||
|
|
||||||
return &sender{t: t, addr: addr, serveWAFAddr: server.Addr, out: out}, clk, server
|
return &sender{t: t, addr: addr, out: out}, clk, server
|
||||||
}
|
}
|
||||||
|
|
||||||
// sender sends requests to rproxy one after another, each on a connection of its
|
// sender sends requests to smallwebwaf one after another, each on a
|
||||||
// own, and checks each one's answer and log line. serveWAFAddr is where servewaf's
|
// connection of its own, and checks each one's answer and log line. They
|
||||||
// own endpoints are, which adminRequest asks. Every request servewaf gets must
|
// must be the only requests smallwebwaf is sent, since the log lines are
|
||||||
// come through the sender, since the log lines are matched to them in order.
|
// matched to them in order.
|
||||||
type sender struct {
|
type sender struct {
|
||||||
t *testing.T
|
t *testing.T
|
||||||
addr string
|
addr string
|
||||||
serveWAFAddr string
|
out *output
|
||||||
out *output
|
sent int
|
||||||
sent int
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// get sends a GET request for / from the client at from.
|
// get sends a GET request for / from the client at from.
|
||||||
@@ -421,19 +417,22 @@ func (s *sender) get(from string, status int, action string) logLine {
|
|||||||
func (s *sender) request(from, path string, status int, action string) logLine {
|
func (s *sender) request(from, path string, status int, action string) logLine {
|
||||||
s.t.Helper()
|
s.t.Helper()
|
||||||
|
|
||||||
return s.requestWithHeader(from, path, "", status, action)
|
line, _ := s.requestWithHeader(from, path, "", status, action)
|
||||||
|
|
||||||
|
return line
|
||||||
}
|
}
|
||||||
|
|
||||||
// requestWithHeader is request with header, such as "Authorization:
|
// requestWithHeader is request with header, such as "Authorization:
|
||||||
// Bearer x", added to the request unless it is "".
|
// Bearer x", added to the request unless it is "". It returns the body of
|
||||||
|
// the answer too.
|
||||||
func (s *sender) requestWithHeader(
|
func (s *sender) requestWithHeader(
|
||||||
from, path, header string, status int, action string,
|
from, path, header string, status int, action string,
|
||||||
) logLine {
|
) (logLine, string) {
|
||||||
s.t.Helper()
|
s.t.Helper()
|
||||||
|
|
||||||
line, _ := s.requestWithBody(http.MethodGet, from, path, header, "", status, action)
|
line, got := s.requestWithBody(http.MethodGet, from, path, header, "", status, action)
|
||||||
|
|
||||||
return line
|
return line, string(got.body)
|
||||||
}
|
}
|
||||||
|
|
||||||
// requestWithBody is requestWithHeader for a request with method, whose
|
// requestWithBody is requestWithHeader for a request with method, whose
|
||||||
@@ -449,14 +448,10 @@ func (s *sender) requestWithBody(
|
|||||||
header += "\r\n"
|
header += "\r\n"
|
||||||
}
|
}
|
||||||
|
|
||||||
// rproxy refuses a request with more than one.
|
|
||||||
if !strings.Contains(header, "User-Agent:") {
|
|
||||||
header += "User-Agent: " + userAgent + "\r\n"
|
|
||||||
}
|
|
||||||
|
|
||||||
conn := dial(s.t, s.addr)
|
conn := dial(s.t, s.addr)
|
||||||
send(s.t, conn, method+" "+path+" HTTP/1.1\r\nHost: "+appHost+"\r\n"+
|
send(s.t, conn, method+" "+path+" HTTP/1.1\r\nHost: "+appHost+
|
||||||
forwardedFor+": "+from+"\r\n"+header+"\r\n"+body)
|
"\r\nUser-Agent: "+userAgent+"\r\n"+forwardedFor+": "+from+"\r\n"+
|
||||||
|
header+"\r\n"+body)
|
||||||
|
|
||||||
err := conn.SetReadDeadline(time.Now().Add(waitLimit))
|
err := conn.SetReadDeadline(time.Now().Add(waitLimit))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -481,7 +476,7 @@ func (s *sender) requestWithBody(
|
|||||||
got.status, status)
|
got.status, status)
|
||||||
}
|
}
|
||||||
|
|
||||||
line := s.out.endedLines(s.t, s.sent+1)[s.sent]
|
line := s.out.requestLines(s.t, s.sent+1)[s.sent]
|
||||||
s.sent++
|
s.sent++
|
||||||
wantLine(s.t, line, status, action)
|
wantLine(s.t, line, status, action)
|
||||||
|
|
||||||
|
|||||||
@@ -34,19 +34,19 @@ func biasedThresholdsSet(cfg *config.Config) bool {
|
|||||||
// percentage SWWAF_ASN_LIMIT_PERCENT gives its AS number, the one the file
|
// percentage SWWAF_ASN_LIMIT_PERCENT gives its AS number, the one the file
|
||||||
// SWWAF_ASN_LIMIT_PERCENT_URL names gives it, the one
|
// SWWAF_ASN_LIMIT_PERCENT_URL names gives it, the one
|
||||||
// SWWAF_COUNTRY_LIMIT_PERCENT gives its country, for a client without a
|
// SWWAF_COUNTRY_LIMIT_PERCENT gives its country, for a client without a
|
||||||
// country, unless its request skipped the wait for GeoJS,
|
// country, SWWAF_UNKNOWN_LIMIT_PERCENT, for a client a blocklist lists,
|
||||||
// SWWAF_UNKNOWN_LIMIT_PERCENT, for a client a blocklist lists, the
|
// the percentage of SWWAF_BLOCKLIST_ACTION while it is limit, and for a
|
||||||
// percentage of SWWAF_BLOCKLIST_ACTION while it is limit, and for a client
|
// client a DNSBL zone's verdict lists, or whose AbuseIPDB score is a hit,
|
||||||
// a DNSBL zone's verdict lists, or whose AbuseIPDB score is a hit, the
|
// the percentage of SWWAF_REPUTATION_ACTION while it is limit. For the
|
||||||
// percentage of SWWAF_REPUTATION_ACTION while it is limit. For the byte
|
// byte limits, SWWAF_ASN_BYTES_PERCENT and SWWAF_COUNTRY_BYTES_PERCENT
|
||||||
// limits, SWWAF_ASN_BYTES_PERCENT and SWWAF_COUNTRY_BYTES_PERCENT take the
|
// take the place of the first three for an AS number or a country they
|
||||||
// place of the first three for an AS number or a country they list.
|
// list.
|
||||||
func (rq *request) limitPercentages() (percentage, percentage) {
|
func (rq *request) limitPercentages() (percentage, percentage) {
|
||||||
cfg := rq.h.config
|
cfg := rq.h.config
|
||||||
asn, country := rq.line.ASN, rq.line.Country
|
asn, country := rq.line.ASN, rq.line.Country
|
||||||
|
|
||||||
unknown := percentage{percent: whole}
|
unknown := percentage{percent: whole}
|
||||||
if country == "" && !rq.skippedLookupWait {
|
if country == "" {
|
||||||
unknown = percentage{cfg.UnknownLimitPercent, "SWWAF_UNKNOWN_LIMIT_PERCENT"}
|
unknown = percentage{cfg.UnknownLimitPercent, "SWWAF_UNKNOWN_LIMIT_PERCENT"}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"maps"
|
"maps"
|
||||||
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
@@ -280,12 +281,12 @@ func TestClientWithoutAnAnswerInTimeHasTheUnknownLimitPercent(t *testing.T) {
|
|||||||
// In a synctest bubble, as TestRequestWaitsAsLongAsTheLookupTimeoutSays
|
// In a synctest bubble, as TestRequestWaitsAsLongAsTheLookupTimeoutSays
|
||||||
// says, with a GeoJS that never answers.
|
// says, with a GeoJS that never answers.
|
||||||
synctest.Test(t, func(t *testing.T) {
|
synctest.Test(t, func(t *testing.T) {
|
||||||
server, out, _ := newServeWAF(t, unansweredGeoJSURL, time.Now,
|
server, out, _ := newProxy(t, "http://app.invalid", unansweredGeoJSURL,
|
||||||
map[string]string{unknownLimitPercent: "0"})
|
time.Now, map[string]string{unknownLimitPercent: "0"})
|
||||||
|
|
||||||
// Once the time the request waits for its answer is up, the client
|
// Once the second the request waits for its answer is up, the client
|
||||||
// counts as without a country, and its zero allowance refuses the
|
// counts as without a country, and its zero allowance refuses the
|
||||||
// request.
|
// request before it reaches the app.
|
||||||
serveFromDE(t, server, http.MethodGet, http.NoBody)
|
serveFromDE(t, server, http.MethodGet, http.NoBody)
|
||||||
|
|
||||||
line := out.requestLine(t)
|
line := out.requestLine(t)
|
||||||
@@ -319,8 +320,13 @@ func TestRequestWaitsForItsLookupWhileABiasedThresholdIsSet(t *testing.T) {
|
|||||||
// In a synctest bubble, as TestRequestWaitsAsLongAsTheLookupTimeoutSays
|
// In a synctest bubble, as TestRequestWaitsAsLongAsTheLookupTimeoutSays
|
||||||
// says, with a GeoJS that never answers.
|
// says, with a GeoJS that never answers.
|
||||||
synctest.Test(t, func(t *testing.T) {
|
synctest.Test(t, func(t *testing.T) {
|
||||||
server, out, _ := newServeWAF(t, unansweredGeoJSURL, time.Now,
|
// The request's body is over SWWAF_REQUEST_MAX_BYTES, so that it
|
||||||
map[string]string{lookupTimeout: timeout.String(), tc.setting: tc.value})
|
// is refused after the checks, and never reaches the app.
|
||||||
|
server, out, _ := newProxy(t, "http://app.invalid", unansweredGeoJSURL,
|
||||||
|
time.Now, map[string]string{
|
||||||
|
lookupTimeout: timeout.String(), requestMaxBytes: "1",
|
||||||
|
tc.setting: tc.value,
|
||||||
|
})
|
||||||
began := time.Now()
|
began := time.Now()
|
||||||
|
|
||||||
serveFromDE(t, server, http.MethodPost, strings.NewReader("ab"))
|
serveFromDE(t, server, http.MethodPost, strings.NewReader("ab"))
|
||||||
@@ -334,7 +340,8 @@ func TestRequestWaitsForItsLookupWhileABiasedThresholdIsSet(t *testing.T) {
|
|||||||
t.Errorf("the request waited %s for its answer, want %s", waited, want)
|
t.Errorf("the request waited %s for its answer, want %s", waited, want)
|
||||||
}
|
}
|
||||||
|
|
||||||
wantLine(t, out.requestLine(t), http.StatusOK, requestlog.ActionForward)
|
wantLine(t, out.requestLine(t), http.StatusRequestEntityTooLarge,
|
||||||
|
requestlog.ActionTooLarge)
|
||||||
|
|
||||||
// The bubble's clock stops once this function returns, so the
|
// The bubble's clock stops once this function returns, so the
|
||||||
// request to GeoJS, which a request that did not wait leaves
|
// request to GeoJS, which a request that did not wait leaves
|
||||||
@@ -460,15 +467,13 @@ func (s *sender) uploadFrom(from string) logLine {
|
|||||||
return line
|
return line
|
||||||
}
|
}
|
||||||
|
|
||||||
// serveFromDE hands a question about a request from fromDE with method and body
|
// serveFromDE hands a request from fromDE with method and body straight to
|
||||||
// straight to server's handler, without the network, and returns once it is
|
// server's handler, without the network, and returns once it is answered.
|
||||||
// answered.
|
|
||||||
func serveFromDE(t *testing.T, server *proxy.Server, method string, body io.Reader) {
|
func serveFromDE(t *testing.T, server *proxy.Server, method string, body io.Reader) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(t.Context(), method, "/", body)
|
req := httptest.NewRequestWithContext(t.Context(), method, "/", body)
|
||||||
req.Header.Set(proxy.ClientHeader, fromDE)
|
req.RemoteAddr = net.JoinHostPort(fromDE, "1234")
|
||||||
req.Header.Set(requestIDHeader, "the-id")
|
|
||||||
|
|
||||||
server.Handler.ServeHTTP(httptest.NewRecorder(), req)
|
server.Handler.ServeHTTP(httptest.NewRecorder(), req)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,44 +1,43 @@
|
|||||||
package rproxy
|
package proxy
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"net"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// errResponseTooLarge ends an app's response body that is longer than
|
// errResponseTooLarge ends an app's response body that is longer than
|
||||||
// SWRP_MAX_RESPONSE_SIZE.
|
// SWWAF_RESPONSE_MAX_BYTES.
|
||||||
var errResponseTooLarge = errors.New(
|
var errResponseTooLarge = errors.New(
|
||||||
"the response body is over SWRP_MAX_RESPONSE_SIZE")
|
"the response body is over SWWAF_RESPONSE_MAX_BYTES")
|
||||||
|
|
||||||
// requestBody is the visitor's request body. rproxy first reads up to the first
|
// requestBody is the client's request body on its way to the app. The
|
||||||
// 64 KiB of it into held, and then, on its way to the app, the transport reads
|
// transport reads it on a goroutine of its own.
|
||||||
// it on a goroutine of its own.
|
|
||||||
type requestBody struct {
|
type requestBody struct {
|
||||||
// body is the visitor's body, ending in an *http.MaxBytesError past
|
// body is the client's body, ending in an *http.MaxBytesError past
|
||||||
// SWRP_MAX_REQUEST_SIZE.
|
// SWWAF_REQUEST_MAX_BYTES.
|
||||||
body io.ReadCloser
|
body io.ReadCloser
|
||||||
rq *request
|
rq *request
|
||||||
// held is the part read before the request goes to the app, which Read gives
|
// readByCoreRuleSet is what the Core Rule Set read of the body before
|
||||||
// first.
|
// the request went to the app, and Read gives first.
|
||||||
held []byte
|
readByCoreRuleSet []byte
|
||||||
// waiting is true while a Read waits for the visitor to send more.
|
// waiting is true while a Read waits for the client to send more.
|
||||||
waiting atomic.Bool
|
waiting atomic.Bool
|
||||||
// received is true once the visitor has sent the whole body.
|
// received is true once the client has sent the whole body.
|
||||||
received atomic.Bool
|
received atomic.Bool
|
||||||
// bytes is how much of the body has been read.
|
// bytes is how much of the body has been read.
|
||||||
bytes atomic.Int64
|
bytes atomic.Int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read reads from the visitor's body, after the part held, which has been
|
// Read reads from the client's body, after what the Core Rule Set read of
|
||||||
// counted already.
|
// it, which has been counted already.
|
||||||
func (b *requestBody) Read(p []byte) (int, error) {
|
func (b *requestBody) Read(p []byte) (int, error) {
|
||||||
if len(b.held) > 0 {
|
if len(b.readByCoreRuleSet) > 0 {
|
||||||
n := copy(p, b.held)
|
n := copy(p, b.readByCoreRuleSet)
|
||||||
b.held = b.held[n:]
|
b.readByCoreRuleSet = b.readByCoreRuleSet[n:]
|
||||||
|
|
||||||
return n, nil
|
return n, nil
|
||||||
}
|
}
|
||||||
@@ -53,22 +52,27 @@ func (b *requestBody) Read(p []byte) (int, error) {
|
|||||||
switch {
|
switch {
|
||||||
case errors.Is(err, io.EOF):
|
case errors.Is(err, io.EOF):
|
||||||
b.received.Store(true)
|
b.received.Store(true)
|
||||||
|
b.rq.bodyReceived()
|
||||||
case errors.As(err, &tooLarge):
|
case errors.As(err, &tooLarge):
|
||||||
b.rq.refuse(http.StatusRequestEntityTooLarge)
|
b.rq.refuse(refusal{
|
||||||
|
status: http.StatusRequestEntityTooLarge,
|
||||||
|
action: requestlog.ActionTooLarge,
|
||||||
|
limit: "SWWAF_REQUEST_MAX_BYTES",
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
return n, err
|
return n, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Close closes the visitor's body.
|
// Close closes the client's body.
|
||||||
func (b *requestBody) Close() error {
|
func (b *requestBody) Close() error {
|
||||||
return b.body.Close()
|
return b.body.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
// responseBody is the app's response body on its way to the visitor.
|
// responseBody is the app's response body on its way to the client.
|
||||||
type responseBody struct {
|
type responseBody struct {
|
||||||
// body is the app's body, ending in an *http.MaxBytesError past
|
// body is the app's body, ending in an *http.MaxBytesError past
|
||||||
// SWRP_MAX_RESPONSE_SIZE.
|
// SWWAF_RESPONSE_MAX_BYTES.
|
||||||
body io.ReadCloser
|
body io.ReadCloser
|
||||||
rq *request
|
rq *request
|
||||||
}
|
}
|
||||||
@@ -86,13 +90,20 @@ func (b *responseBody) Read(p []byte) (int, error) {
|
|||||||
case errors.Is(err, io.EOF):
|
case errors.Is(err, io.EOF):
|
||||||
b.rq.responseReceived()
|
b.rq.responseReceived()
|
||||||
case errors.As(err, &tooLarge):
|
case errors.As(err, &tooLarge):
|
||||||
b.rq.refuse(http.StatusBadGateway)
|
b.rq.refuse(refusal{
|
||||||
|
status: http.StatusBadGateway,
|
||||||
|
action: requestlog.ActionTooLarge,
|
||||||
|
limit: "SWWAF_RESPONSE_MAX_BYTES",
|
||||||
|
})
|
||||||
|
|
||||||
return n, errResponseTooLarge
|
return n, errResponseTooLarge
|
||||||
case b.rq.in.Context().Err() == nil:
|
case b.rq.in.Context().Err() == nil:
|
||||||
// The answer broke off, not because the visitor went away. If a time
|
// The answer broke off, not because the client went away. If a
|
||||||
// limit cut it, that refusal came first and is the one kept.
|
// timeout cut it, that refusal came first and is the one kept.
|
||||||
b.rq.refuse(http.StatusBadGateway)
|
b.rq.refuse(refusal{
|
||||||
|
status: http.StatusBadGateway,
|
||||||
|
action: requestlog.ActionUpstreamError,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
return n, err
|
return n, err
|
||||||
@@ -104,14 +115,15 @@ func (b *responseBody) Close() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// upgradedConn is the connection to the app once the app has switched
|
// upgradedConn is the connection to the app once the app has switched
|
||||||
// protocols, as for a WebSocket. ReverseProxy writes to it what the visitor
|
// protocols, as for a WebSocket. ReverseProxy writes to it what the client
|
||||||
// sends and reads from it what the app sends, on goroutines of its own, until
|
// sends and reads from it what the app sends, on goroutines of its own,
|
||||||
// the connection closes; it counts the bytes each way, for the report.
|
// until the connection closes; it counts the bytes each way, for the byte
|
||||||
|
// limits.
|
||||||
type upgradedConn struct {
|
type upgradedConn struct {
|
||||||
io.ReadWriteCloser
|
io.ReadWriteCloser
|
||||||
|
|
||||||
// fromApp is how many bytes the app has sent, and toApp how many the
|
// fromApp is how many bytes the app has sent, and toApp how many the
|
||||||
// visitor has.
|
// client has.
|
||||||
fromApp atomic.Int64
|
fromApp atomic.Int64
|
||||||
toApp atomic.Int64
|
toApp atomic.Int64
|
||||||
}
|
}
|
||||||
@@ -124,7 +136,7 @@ func (c *upgradedConn) Read(p []byte) (int, error) {
|
|||||||
return n, err
|
return n, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write sends the app what the visitor sent.
|
// Write sends the app what the client sent.
|
||||||
func (c *upgradedConn) Write(p []byte) (int, error) {
|
func (c *upgradedConn) Write(p []byte) (int, error) {
|
||||||
n, err := c.ReadWriteCloser.Write(p)
|
n, err := c.ReadWriteCloser.Write(p)
|
||||||
c.toApp.Add(int64(n))
|
c.toApp.Add(int64(n))
|
||||||
@@ -132,9 +144,9 @@ func (c *upgradedConn) Write(p []byte) (int, error) {
|
|||||||
return n, err
|
return n, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// CloseWrite tells the app that the visitor sends no more, while what the app
|
// CloseWrite tells the app that the client sends no more, while what the
|
||||||
// sends still passes. ReverseProxy calls it once the visitor has stopped
|
// app sends still passes. ReverseProxy calls it once the client has
|
||||||
// sending, and closes the connection there if it is not supported.
|
// stopped sending, and closes the connection there if it is not supported.
|
||||||
func (c *upgradedConn) CloseWrite() error {
|
func (c *upgradedConn) CloseWrite() error {
|
||||||
conn, ok := c.ReadWriteCloser.(interface{ CloseWrite() error })
|
conn, ok := c.ReadWriteCloser.(interface{ CloseWrite() error })
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -144,8 +156,8 @@ func (c *upgradedConn) CloseWrite() error {
|
|||||||
return conn.CloseWrite()
|
return conn.CloseWrite()
|
||||||
}
|
}
|
||||||
|
|
||||||
// limitBody returns body, cut off with an *http.MaxBytesError after maxBytes, or
|
// limitBody returns body, cut off with an *http.MaxBytesError after
|
||||||
// unchanged if maxBytes is zero, which is off.
|
// maxBytes, or unchanged if maxBytes is zero, which is off.
|
||||||
func limitBody(body io.ReadCloser, maxBytes int64) io.ReadCloser {
|
func limitBody(body io.ReadCloser, maxBytes int64) io.ReadCloser {
|
||||||
if maxBytes == 0 {
|
if maxBytes == 0 {
|
||||||
return body
|
return body
|
||||||
@@ -155,18 +167,19 @@ func limitBody(body io.ReadCloser, maxBytes int64) io.ReadCloser {
|
|||||||
return http.MaxBytesReader(nil, body, maxBytes)
|
return http.MaxBytesReader(nil, body, maxBytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
// responseWriter is the response to the visitor. It notes the status and size
|
// responseWriter is the response to the client. It notes the status and
|
||||||
// for the log line and the report.
|
// size for the log line, and the first error writing to the client.
|
||||||
type responseWriter struct {
|
type responseWriter struct {
|
||||||
http.ResponseWriter
|
http.ResponseWriter
|
||||||
|
|
||||||
// status is the final status sent, or zero before one is.
|
// status is the final status sent, or zero before one is.
|
||||||
status int
|
status int
|
||||||
bytes int64
|
bytes int64
|
||||||
|
err error
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteHeader sends the status and headers. An informational 1xx status is
|
// WriteHeader sends the status and headers. An informational 1xx status
|
||||||
// passed on and the final status still comes later.
|
// is passed on and the final status still comes later.
|
||||||
func (w *responseWriter) WriteHeader(status int) {
|
func (w *responseWriter) WriteHeader(status int) {
|
||||||
if status >= http.StatusOK && w.status == 0 {
|
if status >= http.StatusOK && w.status == 0 {
|
||||||
w.status = status
|
w.status = status
|
||||||
@@ -181,31 +194,33 @@ func (w *responseWriter) Write(p []byte) (int, error) {
|
|||||||
w.status = http.StatusOK
|
w.status = http.StatusOK
|
||||||
}
|
}
|
||||||
|
|
||||||
n, err := w.ResponseWriter.Write(p) //nolint:gosec // the answer, as it was given
|
n, err := w.ResponseWriter.Write(p)
|
||||||
w.bytes += int64(n)
|
w.bytes += int64(n)
|
||||||
|
w.noteError(err)
|
||||||
|
|
||||||
return n, err
|
return n, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// FlushError sends what has been written so far. http.ResponseController calls
|
// FlushError sends what has been written so far.
|
||||||
// it, as ReverseProxy does after each write.
|
// http.ResponseController calls it, as ReverseProxy does after each
|
||||||
|
// write.
|
||||||
func (w *responseWriter) FlushError() error {
|
func (w *responseWriter) FlushError() error {
|
||||||
return http.NewResponseController(w.ResponseWriter).Flush()
|
err := http.NewResponseController(w.ResponseWriter).Flush()
|
||||||
|
w.noteError(err)
|
||||||
|
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Hijack takes over the connection to the visitor, which ReverseProxy does only
|
// Unwrap lets http.ResponseController reach net/http's own
|
||||||
// to pass on the app's 101 Switching Protocols: that is then the status sent.
|
// ResponseWriter, which is how ReverseProxy takes over the connection of
|
||||||
func (w *responseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
|
// an upgraded request.
|
||||||
conn, buffered, err := http.NewResponseController(w.ResponseWriter).Hijack()
|
|
||||||
if err == nil {
|
|
||||||
w.status = http.StatusSwitchingProtocols
|
|
||||||
}
|
|
||||||
|
|
||||||
return conn, buffered, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Unwrap lets http.ResponseController reach net/http's own ResponseWriter for
|
|
||||||
// what responseWriter does not do itself.
|
|
||||||
func (w *responseWriter) Unwrap() http.ResponseWriter {
|
func (w *responseWriter) Unwrap() http.ResponseWriter {
|
||||||
return w.ResponseWriter
|
return w.ResponseWriter
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// noteError keeps the first error writing to the client.
|
||||||
|
func (w *responseWriter) noteError(err error) {
|
||||||
|
if w.err == nil {
|
||||||
|
w.err = err
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -47,6 +47,8 @@ const (
|
|||||||
func TestEachByteLimitBansOnceTheResponseHasEnded(t *testing.T) {
|
func TestEachByteLimitBansOnceTheResponseHasEnded(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
const scraper = "192.0.2.200"
|
||||||
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
setting, window string
|
setting, window string
|
||||||
// apart is the time between the two requests, which the window
|
// apart is the time between the two requests, which the window
|
||||||
@@ -92,7 +94,7 @@ func TestEachByteLimitBansOnceTheResponseHasEnded(t *testing.T) {
|
|||||||
|
|
||||||
s.get(client, http.StatusForbidden, requestlog.ActionBanned)
|
s.get(client, http.StatusForbidden, requestlog.ActionBanned)
|
||||||
|
|
||||||
wantMetric(t, s.scrape(), `smallwebwaf_rate_limit_hits_total{`+
|
wantMetric(t, s.scrape(scraper), `smallwebwaf_rate_limit_hits_total{`+
|
||||||
`instance="`+alertInstance+`",kind="bytes",window="`+tc.window+`"}`, 1)
|
`instance="`+alertInstance+`",kind="bytes",window="`+tc.window+`"}`, 1)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -124,7 +126,7 @@ func TestBytesOfAnAnswerThatBreaksOffAreCounted(t *testing.T) {
|
|||||||
// The 70 bytes passed on before the app broke off are over the limit of
|
// The 70 bytes passed on before the app broke off are over the limit of
|
||||||
// 50, and ban the client for an hour.
|
// 50, and ban the client for an hour.
|
||||||
line, got := s.requestWithBody(http.MethodGet, client, "/", "", "", http.StatusOK,
|
line, got := s.requestWithBody(http.MethodGet, client, "/", "", "", http.StatusOK,
|
||||||
requestlog.ActionForward)
|
requestlog.ActionUpstreamError)
|
||||||
if len(got.body) != answerBytes || line.LimitHit != minuteBytes ||
|
if len(got.body) != answerBytes || line.LimitHit != minuteBytes ||
|
||||||
line.BanExpires != expires {
|
line.BanExpires != expires {
|
||||||
t.Errorf("got %d bytes, and the log line has limit_hit %q and ban_expires %q, "+
|
t.Errorf("got %d bytes, and the log line has limit_hit %q and ban_expires %q, "+
|
||||||
@@ -166,6 +168,39 @@ func TestWebSocketBytesAreCountedOnceItCloses(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestWebSocketPassesTheAnswerAfterTheClientStopsSending(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
app := startApp(t, echoOnceTheClientStops)
|
||||||
|
addr, out := startProxy(t, app.URL,
|
||||||
|
map[string]string{trustedProxies: trustLocalhost})
|
||||||
|
s := &sender{t: t, addr: addr, out: out}
|
||||||
|
|
||||||
|
conn, reader := s.openWebSocket()
|
||||||
|
send(t, conn, uploadBody)
|
||||||
|
|
||||||
|
// The client closes its sending side and waits for the answer, which the
|
||||||
|
// app sends only once it has seen the client stop. smallwebwaf passes the
|
||||||
|
// close on to the app through CloseWrite on upgradedConn; without that,
|
||||||
|
// it closes both connections, and the answer is lost.
|
||||||
|
tcp, ok := conn.(*net.TCPConn)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("connection is a %T, want a *net.TCPConn", conn)
|
||||||
|
}
|
||||||
|
|
||||||
|
err := tcp.CloseWrite()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("close the sending side: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := io.ReadAll(reader)
|
||||||
|
if err != nil || string(got) != uploadBody {
|
||||||
|
t.Errorf("got %q (%v), want %q", got, err, uploadBody)
|
||||||
|
}
|
||||||
|
|
||||||
|
s.closeWebSocket(conn)
|
||||||
|
}
|
||||||
|
|
||||||
func TestBytesCountSaysWhichBytesCount(t *testing.T) {
|
func TestBytesCountSaysWhichBytesCount(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -430,6 +465,28 @@ func answerAfterUpgrade(w http.ResponseWriter, _ *http.Request) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// echoOnceTheClientStops is an app that switches protocols, as for a
|
||||||
|
// WebSocket, reads what the client sends until the client stops sending,
|
||||||
|
// and then sends it all back.
|
||||||
|
func echoOnceTheClientStops(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
conn, buffered, err := http.NewResponseController(w).Hijack()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
_ = conn.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
|
_, _ = buffered.WriteString("HTTP/1.1 101 Switching Protocols\r\n" +
|
||||||
|
"Connection: Upgrade\r\nUpgrade: websocket\r\n\r\n")
|
||||||
|
_ = buffered.Flush()
|
||||||
|
|
||||||
|
received, _ := io.ReadAll(buffered)
|
||||||
|
_, _ = buffered.Write(received)
|
||||||
|
_ = buffered.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
// webSocket opens a WebSocket from client to answerAfterUpgrade, sends a
|
// webSocket opens a WebSocket from client to answerAfterUpgrade, sends a
|
||||||
// line of bodyBytes on it, reads the answer, and closes it. It checks the
|
// line of bodyBytes on it, reads the answer, and closes it. It checks the
|
||||||
// answer, and the log line as request does, and returns the log line.
|
// answer, and the log line as request does, and returns the log line.
|
||||||
@@ -486,7 +543,7 @@ func (s *sender) closeWebSocket(conn net.Conn) logLine {
|
|||||||
|
|
||||||
_ = conn.Close()
|
_ = conn.Close()
|
||||||
|
|
||||||
line := s.out.endedLines(s.t, s.sent+1)[s.sent]
|
line := s.out.requestLines(s.t, s.sent+1)[s.sent]
|
||||||
s.sent++
|
s.sent++
|
||||||
wantLine(s.t, line, http.StatusSwitchingProtocols, requestlog.ActionForward)
|
wantLine(s.t, line, http.StatusSwitchingProtocols, requestlog.ActionForward)
|
||||||
|
|
||||||
|
|||||||
+94
-14
@@ -5,9 +5,10 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"slices"
|
"slices"
|
||||||
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
// peerAddress is the address of the request's TCP peer.
|
// peerAddress is the address of the request's TCP peer, normally traefik.
|
||||||
func peerAddress(r *http.Request) netip.Addr {
|
func peerAddress(r *http.Request) netip.Addr {
|
||||||
addrPort, err := netip.ParseAddrPort(r.RemoteAddr)
|
addrPort, err := netip.ParseAddrPort(r.RemoteAddr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -17,24 +18,58 @@ func peerAddress(r *http.Request) netip.Addr {
|
|||||||
return addrPort.Addr().Unmap()
|
return addrPort.Addr().Unmap()
|
||||||
}
|
}
|
||||||
|
|
||||||
// requestIDHeader carries the request's id, from rproxy.
|
// clientAddress works out who the client is. A peer outside the trusted
|
||||||
const requestIDHeader = "X-Request-ID"
|
// proxies is the client, and what it says in X-Forwarded-For is ignored.
|
||||||
|
// For a peer inside them, X-Forwarded-For is read from the right, and the
|
||||||
// requestID is the request's id: the one a question gives, or for any other
|
// first address outside them is the client; if every address in it is
|
||||||
// request a new random one.
|
// inside, the leftmost is, and with no header, the peer. An entry that is
|
||||||
func requestID(r *http.Request, isQuestion bool) string {
|
// not an address ends the reading, since nothing to its left can be
|
||||||
if isQuestion {
|
// believed.
|
||||||
return r.Header.Get(requestIDHeader)
|
func clientAddress(
|
||||||
|
peer netip.Addr, forwardedFor []string, trusted []netip.Prefix,
|
||||||
|
) netip.Addr {
|
||||||
|
client := peer
|
||||||
|
if !isInside(peer, trusted) {
|
||||||
|
return client
|
||||||
}
|
}
|
||||||
|
|
||||||
return rand.Text()
|
entries := strings.Split(strings.Join(forwardedFor, ","), ",")
|
||||||
|
for _, entry := range slices.Backward(entries) {
|
||||||
|
addr, err := netip.ParseAddr(strings.TrimSpace(entry))
|
||||||
|
if err != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
client = addr.Unmap()
|
||||||
|
if !isInside(client, trusted) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return client
|
||||||
}
|
}
|
||||||
|
|
||||||
// scheme is how the visitor reached rproxy, or the trusted proxy in front of
|
// requestIDHeader carries the request's id, from traefik and to the app.
|
||||||
// it, as a question says in X-Forwarded-Proto, or otherwise http.
|
const requestIDHeader = "X-Request-ID"
|
||||||
func scheme(r *http.Request, isQuestion bool) string {
|
|
||||||
|
// requestID is the request's id: the one a trusted proxy sent, or a new
|
||||||
|
// random one. A peer outside the trusted proxies did not come through
|
||||||
|
// traefik, so the id it sends is its own claim, and is replaced.
|
||||||
|
func requestID(r *http.Request, peerTrusted bool) string {
|
||||||
|
id := r.Header.Get(requestIDHeader)
|
||||||
|
if !peerTrusted || id == "" {
|
||||||
|
id = rand.Text()
|
||||||
|
}
|
||||||
|
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
|
||||||
|
// scheme is how the client reached traefik, as a trusted proxy says in
|
||||||
|
// X-Forwarded-Proto, or otherwise http, the only scheme smallwebwaf
|
||||||
|
// serves.
|
||||||
|
func scheme(r *http.Request, peerTrusted bool) string {
|
||||||
proto := r.Header.Get("X-Forwarded-Proto")
|
proto := r.Header.Get("X-Forwarded-Proto")
|
||||||
if !isQuestion || proto == "" {
|
if !peerTrusted || proto == "" {
|
||||||
return "http"
|
return "http"
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -60,3 +95,48 @@ func isInside(addr netip.Addr, netblocks []netip.Prefix) bool {
|
|||||||
return netblock.Contains(addr)
|
return netblock.Contains(addr)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// setForwardedHeaders sets the headers in which the app learns about the
|
||||||
|
// client, so that it sees what it would see from traefik directly. A
|
||||||
|
// trusted proxy's forwarded headers pass on, with the proxy's own address
|
||||||
|
// added to X-Forwarded-For. Those of any other peer are its own claims and
|
||||||
|
// are replaced: X-Forwarded-For names the peer, X-Forwarded-Host the host
|
||||||
|
// it asked for, and X-Forwarded-Proto plain http, which is how it reached
|
||||||
|
// smallwebwaf.
|
||||||
|
func setForwardedHeaders(in, out *http.Request, peer netip.Addr, trusted bool) {
|
||||||
|
forwardedFor := peer.String()
|
||||||
|
|
||||||
|
if trusted {
|
||||||
|
// ReverseProxy removes these from out before Rewrite.
|
||||||
|
for _, name := range []string{"Forwarded", "X-Forwarded-Host", "X-Forwarded-Proto"} {
|
||||||
|
values, ok := in.Header[name]
|
||||||
|
if ok {
|
||||||
|
out.Header[name] = values
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
prior := in.Header.Values("X-Forwarded-For")
|
||||||
|
if len(prior) > 0 {
|
||||||
|
forwardedFor = strings.Join(prior, ", ") + ", " + forwardedFor
|
||||||
|
}
|
||||||
|
|
||||||
|
out.Header.Set("X-Forwarded-For", forwardedFor)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReverseProxy has removed Forwarded and the three set below; these
|
||||||
|
// are the other headers in which traefik tells the app about the
|
||||||
|
// client and its request.
|
||||||
|
for _, name := range []string{
|
||||||
|
"X-Forwarded-Port", "X-Forwarded-Server", "X-Forwarded-Uri",
|
||||||
|
"X-Forwarded-Method", "X-Forwarded-Prefix", "X-Forwarded-Tls-Client-Cert",
|
||||||
|
"X-Forwarded-Tls-Client-Cert-Info", "X-Real-Ip",
|
||||||
|
} {
|
||||||
|
out.Header.Del(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
out.Header.Set("X-Forwarded-For", forwardedFor)
|
||||||
|
out.Header.Set("X-Forwarded-Host", in.Host)
|
||||||
|
out.Header.Set("X-Forwarded-Proto", "http")
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,165 @@
|
|||||||
|
package proxy_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// trustLocalhost trusts the address every test connects from, and a
|
||||||
|
// network for proxies in front of it.
|
||||||
|
trustLocalhost = localhost + "/32,10.0.0.0/8"
|
||||||
|
// appHost is the host every test asks for.
|
||||||
|
appHost = "app.example"
|
||||||
|
// client is the client's address, as a proxy names it.
|
||||||
|
client = "203.0.113.9"
|
||||||
|
// forwardedFor is the header that lists the client and its proxies,
|
||||||
|
// and forwardedProto the one that gives the scheme the client used.
|
||||||
|
forwardedFor = "X-Forwarded-For"
|
||||||
|
forwardedProto = "X-Forwarded-Proto"
|
||||||
|
// secure is the scheme a client reached traefik with, and plain the
|
||||||
|
// one smallwebwaf serves.
|
||||||
|
secure = "https"
|
||||||
|
plain = "http"
|
||||||
|
)
|
||||||
|
|
||||||
|
// appHeaders is what the app tells about the headers it received.
|
||||||
|
type appHeaders struct {
|
||||||
|
Host string `json:"host"`
|
||||||
|
ForwardedFor string `json:"forwardedFor"`
|
||||||
|
ForwardedHost string `json:"forwardedHost"`
|
||||||
|
ForwardedProto string `json:"forwardedProto"`
|
||||||
|
RealIP string `json:"realIp"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// clientAddressCase is a request and what smallwebwaf makes of it.
|
||||||
|
type clientAddressCase struct {
|
||||||
|
name string
|
||||||
|
env map[string]string
|
||||||
|
header http.Header
|
||||||
|
wantClient string
|
||||||
|
wantApp appHeaders
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClientAddressAndForwardedHeaders(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, tc := range clientAddressCases() {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
got, line := requestWithHeaders(t, tc.env, tc.header)
|
||||||
|
|
||||||
|
tc.wantApp.Host = appHost
|
||||||
|
if got != tc.wantApp {
|
||||||
|
t.Errorf("app received %+v, want %+v", got, tc.wantApp)
|
||||||
|
}
|
||||||
|
|
||||||
|
if line.ClientIP != tc.wantClient || line.PeerIP != localhost {
|
||||||
|
t.Errorf("log line has client_ip %q and peer_ip %q, want %q and %q",
|
||||||
|
line.ClientIP, line.PeerIP, tc.wantClient, localhost)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// clientAddressCases are the requests TestClientAddressAndForwardedHeaders
|
||||||
|
// sends, from 127.0.0.1, which the default trusted proxies leave out.
|
||||||
|
func clientAddressCases() []clientAddressCase {
|
||||||
|
trusted := map[string]string{trustedProxies: trustLocalhost}
|
||||||
|
forged := http.Header{
|
||||||
|
forwardedFor: {client},
|
||||||
|
"X-Forwarded-Host": {"forged.example"},
|
||||||
|
forwardedProto: {secure},
|
||||||
|
"X-Real-Ip": {client},
|
||||||
|
}
|
||||||
|
replaced := appHeaders{
|
||||||
|
ForwardedFor: localhost, ForwardedHost: appHost, ForwardedProto: plain,
|
||||||
|
}
|
||||||
|
|
||||||
|
return []clientAddressCase{{
|
||||||
|
name: "a peer outside the trusted proxies is the client, " +
|
||||||
|
"and its forwarded headers are replaced",
|
||||||
|
header: forged, wantClient: localhost, wantApp: replaced,
|
||||||
|
}, {
|
||||||
|
name: "set but empty, the trusted proxies trust nothing",
|
||||||
|
env: map[string]string{trustedProxies: ""},
|
||||||
|
header: forged, wantClient: localhost, wantApp: replaced,
|
||||||
|
}, {
|
||||||
|
name: "behind a trusted peer, the client is the first address " +
|
||||||
|
"outside the trusted proxies from the right",
|
||||||
|
env: trusted,
|
||||||
|
header: http.Header{
|
||||||
|
forwardedFor: {"198.51.100.7, " + client + ", 10.0.0.2"},
|
||||||
|
"X-Forwarded-Host": {appHost},
|
||||||
|
forwardedProto: {secure},
|
||||||
|
"X-Real-Ip": {client},
|
||||||
|
},
|
||||||
|
wantClient: client,
|
||||||
|
wantApp: appHeaders{
|
||||||
|
ForwardedFor: "198.51.100.7, " + client + ", 10.0.0.2, " + localhost,
|
||||||
|
ForwardedHost: appHost, ForwardedProto: secure, RealIP: client,
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
name: "when every address is a trusted proxy, the leftmost is the client",
|
||||||
|
env: trusted,
|
||||||
|
header: http.Header{forwardedFor: {"10.0.0.5, 10.0.0.2"}},
|
||||||
|
wantClient: "10.0.0.5",
|
||||||
|
wantApp: appHeaders{ForwardedFor: "10.0.0.5, 10.0.0.2, " + localhost},
|
||||||
|
}, {
|
||||||
|
name: "with no header, a trusted peer is the client",
|
||||||
|
env: trusted,
|
||||||
|
wantClient: localhost,
|
||||||
|
wantApp: appHeaders{ForwardedFor: localhost},
|
||||||
|
}, {
|
||||||
|
name: "an entry that is not an address ends the reading",
|
||||||
|
env: trusted,
|
||||||
|
header: http.Header{forwardedFor: {client + ", unknown, 10.0.0.2"}},
|
||||||
|
wantClient: "10.0.0.2",
|
||||||
|
wantApp: appHeaders{
|
||||||
|
ForwardedFor: client + ", unknown, 10.0.0.2, " + localhost,
|
||||||
|
},
|
||||||
|
}, {
|
||||||
|
name: "several header lines are read as one list",
|
||||||
|
env: trusted,
|
||||||
|
header: http.Header{forwardedFor: {"2001:db8::7", "10.0.0.2"}},
|
||||||
|
wantClient: "2001:db8::7",
|
||||||
|
wantApp: appHeaders{ForwardedFor: "2001:db8::7, 10.0.0.2, " + localhost},
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// requestWithHeaders sends a request for appHost with header through
|
||||||
|
// smallwebwaf, with the settings in env, and returns the headers the app
|
||||||
|
// received and the request's log line.
|
||||||
|
func requestWithHeaders(
|
||||||
|
t *testing.T, env map[string]string, header http.Header,
|
||||||
|
) (appHeaders, logLine) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_ = json.NewEncoder(w).Encode(appHeaders{
|
||||||
|
Host: r.Host,
|
||||||
|
ForwardedFor: r.Header.Get(forwardedFor),
|
||||||
|
ForwardedHost: r.Header.Get("X-Forwarded-Host"),
|
||||||
|
ForwardedProto: r.Header.Get(forwardedProto),
|
||||||
|
RealIP: r.Header.Get("X-Real-IP"),
|
||||||
|
})
|
||||||
|
})
|
||||||
|
addr, out := startProxy(t, app.URL, env)
|
||||||
|
|
||||||
|
req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
|
||||||
|
req.Host = appHost
|
||||||
|
req.Header = header.Clone()
|
||||||
|
|
||||||
|
answered := do(t, req)
|
||||||
|
|
||||||
|
var got appHeaders
|
||||||
|
|
||||||
|
err := json.Unmarshal(answered.body, &got)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decode the app's answer %q: %v", answered.body, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return got, out.requestLine(t)
|
||||||
|
}
|
||||||
@@ -1,6 +1,9 @@
|
|||||||
package proxy
|
package proxy
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
@@ -16,19 +19,20 @@ import (
|
|||||||
// SWWAF_WAF_ANOMALY_THRESHOLD is a match: it raises the waf_block alert,
|
// SWWAF_WAF_ANOMALY_THRESHOLD is a match: it raises the waf_block alert,
|
||||||
// and in block mode refuses the request, which is an offence its client's
|
// and in block mode refuses the request, which is an offence its client's
|
||||||
// history counts, and so returns ActionWAFBlocked. It returns "" for a
|
// history counts, and so returns ActionWAFBlocked. It returns "" for a
|
||||||
// request it does not refuse.
|
// request it does not refuse, and for one whose body meets a size or time
|
||||||
|
// limit while the Core Rule Set reads it, which it notes nothing of.
|
||||||
func (rq *request) checkCoreRuleSet() string {
|
func (rq *request) checkCoreRuleSet() string {
|
||||||
cfg := rq.h.config
|
cfg := rq.h.config
|
||||||
if cfg.WAFMode == config.WAFModeOff || pathExempt(rq.in, cfg.WAFExemptPaths) {
|
if cfg.WAFMode == config.WAFModeOff || pathExempt(rq.in.URL, cfg.WAFExemptPaths) {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
|
|
||||||
// A question's body has been read into memory, so that reading it here
|
result := rq.inspect()
|
||||||
// cannot fail.
|
if rq.refused.Load() != nil {
|
||||||
result, read, _ := rq.h.coreRuleSet.Inspect(rq.in, rq.client, rq.in.Body)
|
return "" // the refusal for that limit, which check returns
|
||||||
rq.requestBytes = int64(len(read))
|
}
|
||||||
|
|
||||||
rq.line.DurationWAF = new(requestlog.Milliseconds(time.Since(start)))
|
rq.line.DurationWAF = new(requestlog.Milliseconds(time.Since(start)))
|
||||||
rq.line.WAFRuleIDs = result.RuleIDs
|
rq.line.WAFRuleIDs = result.RuleIDs
|
||||||
@@ -54,6 +58,39 @@ func (rq *request) checkCoreRuleSet() string {
|
|||||||
return requestlog.ActionWAFBlocked
|
return requestlog.ActionWAFBlocked
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// inspect runs the Core Rule Set on the request, which reads the part of
|
||||||
|
// its body it inspects within SWWAF_CLIENT_REQUEST_TIMEOUT, and keeps that
|
||||||
|
// part for the app. A client that runs out of time is refused with 408
|
||||||
|
// here, and a body over SWWAF_REQUEST_MAX_BYTES with 413 as it is read;
|
||||||
|
// check returns the refusal. A body that breaks off for any other reason
|
||||||
|
// is passed on as far as it came, and the request to the app fails there,
|
||||||
|
// as it would have without the Core Rule Set.
|
||||||
|
func (rq *request) inspect() waf.Result {
|
||||||
|
if rq.body == nil {
|
||||||
|
// Nothing is read of no body, so nothing can go wrong reading it.
|
||||||
|
result, _, _ := rq.h.coreRuleSet.Inspect(rq.in, rq.client, http.NoBody)
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = rq.rc.SetReadDeadline(rq.clientRequestDeadline())
|
||||||
|
result, read, err := rq.h.coreRuleSet.Inspect(rq.in, rq.client, rq.body)
|
||||||
|
// The timeouts that run while the request goes to the app take over.
|
||||||
|
_ = rq.rc.SetReadDeadline(time.Time{})
|
||||||
|
|
||||||
|
rq.body.readByCoreRuleSet = read
|
||||||
|
|
||||||
|
if errors.Is(err, os.ErrDeadlineExceeded) {
|
||||||
|
rq.refuse(refusal{
|
||||||
|
status: http.StatusRequestTimeout,
|
||||||
|
action: requestlog.ActionTimedOut,
|
||||||
|
limit: "SWWAF_CLIENT_REQUEST_TIMEOUT",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
// alertWAFBlock raises the waf_block alert for the request, which the Core
|
// alertWAFBlock raises the waf_block alert for the request, which the Core
|
||||||
// Rule Set scored at result, at or over SWWAF_WAF_ANOMALY_THRESHOLD. Its
|
// Rule Set scored at result, at or over SWWAF_WAF_ANOMALY_THRESHOLD. Its
|
||||||
// detail gives the rule ids, the score, the method and the path with the
|
// detail gives the rule ids, the score, the method and the path with the
|
||||||
@@ -76,7 +113,6 @@ func (rq *request) alertWAFBlock(result waf.Result) {
|
|||||||
|
|
||||||
rq.h.alerts.Raise(alerts.Alert{
|
rq.h.alerts.Raise(alerts.Alert{
|
||||||
Event: alerts.EventWAFBlock,
|
Event: alerts.EventWAFBlock,
|
||||||
Host: rq.site,
|
|
||||||
Client: rq.client,
|
Client: rq.client,
|
||||||
Netblock: rq.h.clientGroup(rq.client),
|
Netblock: rq.h.clientGroup(rq.client),
|
||||||
ASN: rq.line.ASN,
|
ASN: rq.line.ASN,
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
@@ -28,13 +29,6 @@ const (
|
|||||||
// formData is the type of a form's body.
|
// formData is the type of a form's body.
|
||||||
const formData = "application/x-www-form-urlencoded"
|
const formData = "application/x-www-form-urlencoded"
|
||||||
|
|
||||||
// sizeLimit is SWWAF_WAF_BODY_LIMIT in the tests that read bodies, 1K as a
|
|
||||||
// setting.
|
|
||||||
const (
|
|
||||||
sizeLimit = 1 << 10
|
|
||||||
sizeLimitSetting = "1K"
|
|
||||||
)
|
|
||||||
|
|
||||||
// sqlInjection asks for / with an SQL injection in its query, which only
|
// sqlInjection asks for / with an SQL injection in its query, which only
|
||||||
// the Core Rule Set's rule 942100 matches, with a score of 5, the default
|
// the Core Rule Set's rule 942100 matches, with a score of 5, the default
|
||||||
// SWWAF_WAF_ANOMALY_THRESHOLD.
|
// SWWAF_WAF_ANOMALY_THRESHOLD.
|
||||||
@@ -102,7 +96,7 @@ func TestCoreRuleSetRefusesAttacksInBlockModeAndOnlyLogsThemInDetectMode(t *test
|
|||||||
} {
|
} {
|
||||||
s, _, _ := startWithClock(t, "", map[string]string{wafMode: tc.mode})
|
s, _, _ := startWithClock(t, "", map[string]string{wafMode: tc.mode})
|
||||||
|
|
||||||
line := s.requestWithHeader(client, attack.path, attack.header,
|
line, _ := s.requestWithHeader(client, attack.path, attack.header,
|
||||||
tc.status, tc.action)
|
tc.status, tc.action)
|
||||||
wantWAF(t, line, &attack.score, attack.ruleIDs...)
|
wantWAF(t, line, &attack.score, attack.ruleIDs...)
|
||||||
}
|
}
|
||||||
@@ -305,7 +299,7 @@ func TestBodiesReachTheAppAsSentWhileBodiesAreRead(t *testing.T) {
|
|||||||
tc.name, len(got.body), got.status, len(tc.body))
|
tc.name, len(got.body), got.status, len(tc.body))
|
||||||
}
|
}
|
||||||
|
|
||||||
line := out.endedLines(t, i+1)[i]
|
line := out.requestLines(t, i+1)[i]
|
||||||
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
|
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
|
||||||
|
|
||||||
if line.RequestBytes != int64(len(tc.body)) {
|
if line.RequestBytes != int64(len(tc.body)) {
|
||||||
@@ -315,6 +309,119 @@ func TestBodiesReachTheAppAsSentWhileBodiesAreRead(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFormBodyLongerThanTheLimitStreamsOnToTheApp(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const (
|
||||||
|
first = "a=" // and twice the limit of b's, then the rest
|
||||||
|
rest = 64 * sizeLimit
|
||||||
|
)
|
||||||
|
|
||||||
|
// past is closed once the app has received twice what the Core Rule
|
||||||
|
// Set reads, and got is the length of the whole body it received.
|
||||||
|
past := make(chan struct{})
|
||||||
|
got := make(chan int64, 1)
|
||||||
|
app := startApp(t, func(_ http.ResponseWriter, r *http.Request) {
|
||||||
|
n, _ := io.CopyN(io.Discard, r.Body, 2*sizeLimit)
|
||||||
|
|
||||||
|
close(past)
|
||||||
|
|
||||||
|
m, _ := io.Copy(io.Discard, r.Body)
|
||||||
|
got <- n + m
|
||||||
|
})
|
||||||
|
addr, out := startProxy(t, app.URL, map[string]string{
|
||||||
|
wafMode: block, wafBodyLimit: sizeLimitSetting,
|
||||||
|
})
|
||||||
|
|
||||||
|
// The client sends the rest only once the app has received the first
|
||||||
|
// part: were smallwebwaf to hold the body until the end, it would
|
||||||
|
// never come.
|
||||||
|
body, sender := io.Pipe()
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
_, _ = io.WriteString(sender, first+strings.Repeat("b", 2*sizeLimit))
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-past:
|
||||||
|
case <-time.After(waitLimit):
|
||||||
|
t.Error("the app got no more than the Core Rule Set reads " +
|
||||||
|
"before the whole body was sent")
|
||||||
|
|
||||||
|
_ = sender.CloseWithError(io.ErrUnexpectedEOF)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
_, _ = io.WriteString(sender, strings.Repeat("b", rest))
|
||||||
|
_ = sender.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
|
req := newRequest(t, http.MethodPost, addr, "/", body)
|
||||||
|
req.Header.Set("Content-Type", formData)
|
||||||
|
wantStatus(t, do(t, req), http.StatusOK)
|
||||||
|
|
||||||
|
want := int64(len(first) + 2*sizeLimit + rest)
|
||||||
|
if n := <-got; n != want {
|
||||||
|
t.Errorf("the app got %d bytes, want %d", n, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantLine(t, out.requestLine(t), http.StatusOK, requestlog.ActionForward)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClientTooSlowToSendWhatTheCoreRuleSetReads(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
app := startApp(t, func(http.ResponseWriter, *http.Request) {})
|
||||||
|
addr, out := startProxy(t, app.URL, map[string]string{
|
||||||
|
wafMode: block, wafBodyLimit: sizeLimitSetting,
|
||||||
|
clientRequestTimeout: shortTimeoutSetting, metricsToken: token,
|
||||||
|
})
|
||||||
|
|
||||||
|
conn := dial(t, addr)
|
||||||
|
send(t, conn, "POST /comment HTTP/1.1\r\nHost: app\r\nContent-Type: "+formData+
|
||||||
|
"\r\nContent-Length: 100\r\n\r\ncontent=the first bytes")
|
||||||
|
|
||||||
|
wantStatus(t, readResponse(t, conn), http.StatusRequestTimeout)
|
||||||
|
|
||||||
|
line := out.requestLine(t)
|
||||||
|
wantLine(t, line, http.StatusRequestTimeout, requestlog.ActionTimedOut)
|
||||||
|
wantNotSentToTheApp(t, line)
|
||||||
|
wantLimitHits(t, addr, clientRequestTimeout, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBodyOverTheSizeLimitWhileTheCoreRuleSetReadsIt(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
app := startApp(t, func(http.ResponseWriter, *http.Request) {})
|
||||||
|
addr, out := startProxy(t, app.URL, map[string]string{
|
||||||
|
wafMode: block, wafBodyLimit: "4K",
|
||||||
|
requestMaxBytes: sizeLimitSetting, metricsToken: token,
|
||||||
|
})
|
||||||
|
|
||||||
|
// Sent in chunks, its length is not announced, and is found to be over
|
||||||
|
// the limit as the Core Rule Set reads it.
|
||||||
|
body := io.MultiReader(strings.NewReader("a=" + strings.Repeat("b", 2*sizeLimit)))
|
||||||
|
req := newRequest(t, http.MethodPost, addr, "/", body)
|
||||||
|
req.Header.Set("Content-Type", formData)
|
||||||
|
wantStatus(t, do(t, req), http.StatusRequestEntityTooLarge)
|
||||||
|
|
||||||
|
line := out.requestLine(t)
|
||||||
|
wantLine(t, line, http.StatusRequestEntityTooLarge, requestlog.ActionTooLarge)
|
||||||
|
wantNotSentToTheApp(t, line)
|
||||||
|
wantLimitHits(t, addr, requestMaxBytes, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// wantNotSentToTheApp checks that the request of line was not sent to the
|
||||||
|
// app at all.
|
||||||
|
func wantNotSentToTheApp(t *testing.T, line logLine) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
_, sent := line.fields["duration_upstream_total"]
|
||||||
|
if sent {
|
||||||
|
t.Error("log line has duration_upstream_total, for a request sent to the app")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestResponsesAreNotInspected(t *testing.T) {
|
func TestResponsesAreNotInspected(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -334,12 +441,14 @@ func TestResponsesAreNotInspected(t *testing.T) {
|
|||||||
t.Errorf("answered %d with %q, want 200 with the app's page", got.status, got.body)
|
t.Errorf("answered %d with %q, want 200 with the app's page", got.status, got.body)
|
||||||
}
|
}
|
||||||
|
|
||||||
wantLine(t, out.endedLine(t), http.StatusOK, requestlog.ActionForward)
|
wantLine(t, out.requestLine(t), http.StatusOK, requestlog.ActionForward)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCoreRuleSetRefusalIsAnOffenceAndCountsTowardTheErrorBurst(t *testing.T) {
|
func TestCoreRuleSetRefusalIsAnOffenceAndCountsTowardTheErrorBurst(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
const scraper = "192.0.2.200"
|
||||||
|
|
||||||
s, _, server := startWithClock(t, "", map[string]string{
|
s, _, server := startWithClock(t, "", map[string]string{
|
||||||
wafMode: block, errorBurstThreshold: "2", metricsToken: token,
|
wafMode: block, errorBurstThreshold: "2", metricsToken: token,
|
||||||
})
|
})
|
||||||
@@ -365,7 +474,7 @@ func TestCoreRuleSetRefusalIsAnOffenceAndCountsTowardTheErrorBurst(t *testing.T)
|
|||||||
t.Errorf("history counts the offences %+v, want %+v", offences, want)
|
t.Errorf("history counts the offences %+v, want %+v", offences, want)
|
||||||
}
|
}
|
||||||
|
|
||||||
metrics := s.scrape()
|
metrics := s.scrape(scraper)
|
||||||
wantMetric(t, metrics,
|
wantMetric(t, metrics,
|
||||||
`smallwebwaf_waf_matches_total{instance="app",mode="block",rule_id="942100"}`, 3)
|
`smallwebwaf_waf_matches_total{instance="app",mode="block",rule_id="942100"}`, 3)
|
||||||
wantMetric(t, metrics,
|
wantMetric(t, metrics,
|
||||||
@@ -377,6 +486,8 @@ func TestCoreRuleSetRefusalIsAnOffenceAndCountsTowardTheErrorBurst(t *testing.T)
|
|||||||
func TestDetectModeMatchIsNoOffenceAndNotCountedTowardTheErrorBurst(t *testing.T) {
|
func TestDetectModeMatchIsNoOffenceAndNotCountedTowardTheErrorBurst(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
const scraper = "192.0.2.200"
|
||||||
|
|
||||||
s, _, server := startWithClock(t, "", map[string]string{
|
s, _, server := startWithClock(t, "", map[string]string{
|
||||||
wafMode: detect, errorBurstThreshold: "2", metricsToken: token,
|
wafMode: detect, errorBurstThreshold: "2", metricsToken: token,
|
||||||
})
|
})
|
||||||
@@ -392,7 +503,7 @@ func TestDetectModeMatchIsNoOffenceAndNotCountedTowardTheErrorBurst(t *testing.T
|
|||||||
t.Errorf("history counts the offences %+v, want none", offences)
|
t.Errorf("history counts the offences %+v, want none", offences)
|
||||||
}
|
}
|
||||||
|
|
||||||
metrics := s.scrape()
|
metrics := s.scrape(scraper)
|
||||||
wantMetric(t, metrics,
|
wantMetric(t, metrics,
|
||||||
`smallwebwaf_waf_matches_total{instance="app",mode="detect",rule_id="942100"}`, 3)
|
`smallwebwaf_waf_matches_total{instance="app",mode="detect",rule_id="942100"}`, 3)
|
||||||
wantNoSeries(t, metrics,
|
wantNoSeries(t, metrics,
|
||||||
@@ -465,7 +576,6 @@ func TestCoreRuleSetMatchRaisesTheWAFBlockAlert(t *testing.T) {
|
|||||||
Instance: alertInstance,
|
Instance: alertInstance,
|
||||||
Time: clk.Now(),
|
Time: clk.Now(),
|
||||||
Event: alerts.EventWAFBlock,
|
Event: alerts.EventWAFBlock,
|
||||||
Host: appHost,
|
|
||||||
Client: netip.MustParseAddr(client),
|
Client: netip.MustParseAddr(client),
|
||||||
Netblock: netip.MustParsePrefix(client + "/32"),
|
Netblock: netip.MustParsePrefix(client + "/32"),
|
||||||
Reason: "scored by the Core Rule Set at or over SWWAF_WAF_ANOMALY_THRESHOLD",
|
Reason: "scored by the Core Rule Set at or over SWWAF_WAF_ANOMALY_THRESHOLD",
|
||||||
|
|||||||
@@ -65,7 +65,7 @@ func TestCountryLists(t *testing.T) {
|
|||||||
req.Header.Set(forwardedFor, sent.client)
|
req.Header.Set(forwardedFor, sent.client)
|
||||||
got := do(t, req)
|
got := do(t, req)
|
||||||
|
|
||||||
line := out.endedLines(t, i+1)[i]
|
line := out.requestLines(t, i+1)[i]
|
||||||
if line.ASN != sent.asn || line.ASName != sent.asName ||
|
if line.ASN != sent.asn || line.ASName != sent.asName ||
|
||||||
line.Country != sent.country {
|
line.Country != sent.country {
|
||||||
t.Errorf("log line has %q, %q and %q, want %q, %q and %q",
|
t.Errorf("log line has %q, %q and %q, want %q, %q and %q",
|
||||||
@@ -90,7 +90,7 @@ func TestCountryLists(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCountryRefusalKeepsARequestFromTheApp(t *testing.T) {
|
func TestCountryRefusalComesBeforeTheBody(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
var calls atomic.Int32
|
var calls atomic.Int32
|
||||||
@@ -109,9 +109,13 @@ func TestCountryRefusalKeepsARequestFromTheApp(t *testing.T) {
|
|||||||
req.Header.Set(forwardedFor, fromKP)
|
req.Header.Set(forwardedFor, fromKP)
|
||||||
wantStatus(t, do(t, req), http.StatusForbidden)
|
wantStatus(t, do(t, req), http.StatusForbidden)
|
||||||
|
|
||||||
line := out.endedLine(t)
|
line := out.requestLine(t)
|
||||||
wantLine(t, line, http.StatusForbidden, requestlog.ActionCountryDenied)
|
wantLine(t, line, http.StatusForbidden, requestlog.ActionCountryDenied)
|
||||||
|
|
||||||
|
if line.RequestBytes != 0 {
|
||||||
|
t.Errorf("log line has request_bytes %d, want 0", line.RequestBytes)
|
||||||
|
}
|
||||||
|
|
||||||
if calls.Load() != 0 {
|
if calls.Load() != 0 {
|
||||||
t.Errorf("the app was called %d times, want none", calls.Load())
|
t.Errorf("the app was called %d times, want none", calls.Load())
|
||||||
}
|
}
|
||||||
@@ -207,7 +211,7 @@ func TestPrivateAddressIsNeverLookedUp(t *testing.T) {
|
|||||||
|
|
||||||
wantStatus(t, do(t, req), http.StatusOK)
|
wantStatus(t, do(t, req), http.StatusOK)
|
||||||
|
|
||||||
line := out.endedLines(t, i+1)[i]
|
line := out.requestLines(t, i+1)[i]
|
||||||
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
|
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
|
||||||
|
|
||||||
for _, field := range []string{"asn", "as_name", "country"} {
|
for _, field := range []string{"asn", "as_name", "country"} {
|
||||||
@@ -280,9 +284,9 @@ func TestExclusiveListRefusesAPrivateAddressUnlessAllowed(t *testing.T) {
|
|||||||
// each in an AS of its own, and no other address. It returns its URL, and
|
// each in an AS of its own, and no other address. It returns its URL, and
|
||||||
// what returns the addresses it has been asked about. A test that needs
|
// what returns the addresses it has been asked about. A test that needs
|
||||||
// the stand-in to be asked or to answer sets SWWAF_LOOKUP_TIMEOUT to an
|
// the stand-in to be asked or to answer sets SWWAF_LOOKUP_TIMEOUT to an
|
||||||
// hour, whether or not a request waits for the answer: on the default, a
|
// hour, whether or not a request waits for the answer: on the default
|
||||||
// hold-up of the test process can abandon the request to the stand-in, and
|
// second, a hold-up of the test process can abandon the request to the
|
||||||
// leave the client unknown.
|
// stand-in, and leave the client unknown.
|
||||||
func startGeoJS(t *testing.T) (string, func() []string) {
|
func startGeoJS(t *testing.T) (string, func() []string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
|||||||
@@ -84,7 +84,7 @@ func TestClientTheCrowdSecDecisionListListsIsBannedUntilTheDecisionEnds(t *testi
|
|||||||
|
|
||||||
// The ban and the hit are counted, and the list has the metrics of any
|
// The ban and the hit are counted, and the list has the metrics of any
|
||||||
// list fetched from a URL.
|
// list fetched from a URL.
|
||||||
metrics := s.scrape()
|
metrics := s.scrape(unplaced)
|
||||||
labels := `{instance="` + alertInstance + `",source="` + decisionsURL + `"}`
|
labels := `{instance="` + alertInstance + `",source="` + decisionsURL + `"}`
|
||||||
|
|
||||||
wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="crowdsec",`+
|
wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="crowdsec",`+
|
||||||
|
|||||||
@@ -17,8 +17,8 @@ import (
|
|||||||
|
|
||||||
const errorBurstThreshold = "SWWAF_ERROR_BURST_THRESHOLD"
|
const errorBurstThreshold = "SWWAF_ERROR_BURST_THRESHOLD"
|
||||||
|
|
||||||
// refused is a request the tests here send, which servewaf refuses after a rule
|
// refused is a request the tests here send, which smallwebwaf refuses
|
||||||
// file match.
|
// after a rule file match, or for a missing or wrong token.
|
||||||
type refused int
|
type refused int
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -27,6 +27,11 @@ const (
|
|||||||
// banRule is one its ban rule refuses with 403, and bans the client
|
// banRule is one its ban rule refuses with 403, and bans the client
|
||||||
// for.
|
// for.
|
||||||
banRule
|
banRule
|
||||||
|
// noMetricsToken is one for the metrics without a token, and
|
||||||
|
// wrongAdminToken one for the bans with the metrics token, each
|
||||||
|
// refused with 401.
|
||||||
|
noMetricsToken
|
||||||
|
wrongAdminToken
|
||||||
)
|
)
|
||||||
|
|
||||||
// send sends r from the client at from, checks its answer and log line as
|
// send sends r from the client at from, checks its answer and log line as
|
||||||
@@ -40,6 +45,14 @@ func (r refused) send(s *sender, from string) logLine {
|
|||||||
requestlog.ActionRuleBlocked)
|
requestlog.ActionRuleBlocked)
|
||||||
case banRule:
|
case banRule:
|
||||||
return s.request(from, probePath, http.StatusForbidden, requestlog.ActionBanned)
|
return s.request(from, probePath, http.StatusForbidden, requestlog.ActionBanned)
|
||||||
|
case noMetricsToken:
|
||||||
|
return s.request(from, proxy.MetricsPath, http.StatusUnauthorized,
|
||||||
|
requestlog.ActionAdmin)
|
||||||
|
case wrongAdminToken:
|
||||||
|
line, _ := s.requestWithHeader(from, proxy.BansPath, "Authorization: "+bearer,
|
||||||
|
http.StatusUnauthorized, requestlog.ActionAdmin)
|
||||||
|
|
||||||
|
return line
|
||||||
}
|
}
|
||||||
|
|
||||||
s.t.Fatalf("no request for the refusal %d", r)
|
s.t.Fatalf("no request for the refusal %d", r)
|
||||||
@@ -47,7 +60,7 @@ func (r refused) send(s *sender, from string) logLine {
|
|||||||
return logLine{}
|
return logLine{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// startForErrorBurst is startWithClock with testRules, the metrics token and
|
// startForErrorBurst is startWithClock with testRules, both tokens and
|
||||||
// SWWAF_ERROR_BURST_THRESHOLD at threshold, and the settings in env.
|
// SWWAF_ERROR_BURST_THRESHOLD at threshold, and the settings in env.
|
||||||
func startForErrorBurst(
|
func startForErrorBurst(
|
||||||
t *testing.T, threshold string, env map[string]string,
|
t *testing.T, threshold string, env map[string]string,
|
||||||
@@ -57,6 +70,7 @@ func startForErrorBurst(
|
|||||||
settings := map[string]string{
|
settings := map[string]string{
|
||||||
errorBurstThreshold: threshold,
|
errorBurstThreshold: threshold,
|
||||||
rulesDir: writeRules(t, testRules),
|
rulesDir: writeRules(t, testRules),
|
||||||
|
adminToken: adminSecret,
|
||||||
metricsToken: token,
|
metricsToken: token,
|
||||||
}
|
}
|
||||||
maps.Copy(settings, env)
|
maps.Copy(settings, env)
|
||||||
@@ -73,7 +87,14 @@ func TestErrorBurstBreaksAtOneOverTheThreshold(t *testing.T) {
|
|||||||
refusals []refused
|
refusals []refused
|
||||||
}{
|
}{
|
||||||
{"block rule", []refused{blockRule, blockRule, blockRule, blockRule}},
|
{"block rule", []refused{blockRule, blockRule, blockRule, blockRule}},
|
||||||
{"a ban rule last", []refused{blockRule, blockRule, blockRule, banRule}},
|
{
|
||||||
|
"missing or wrong token",
|
||||||
|
[]refused{noMetricsToken, wrongAdminToken, noMetricsToken, wrongAdminToken},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"a mix ending in a ban rule",
|
||||||
|
[]refused{blockRule, noMetricsToken, blockRule, banRule},
|
||||||
|
},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
@@ -109,11 +130,13 @@ func TestErrorBurstBreaksAtOneOverTheThreshold(t *testing.T) {
|
|||||||
func TestErrorBurstBanNotesHistoryAndMetrics(t *testing.T) {
|
func TestErrorBurstBanNotesHistoryAndMetrics(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
const scraper = "192.0.2.200"
|
||||||
|
|
||||||
s, clk, server := startForErrorBurst(t, "2", nil)
|
s, clk, server := startForErrorBurst(t, "2", nil)
|
||||||
start := clk.Now()
|
start := clk.Now()
|
||||||
|
|
||||||
blockRule.send(s, client)
|
blockRule.send(s, client)
|
||||||
blockRule.send(s, client)
|
wrongAdminToken.send(s, client)
|
||||||
line := blockRule.send(s, client)
|
line := blockRule.send(s, client)
|
||||||
|
|
||||||
expires := start.Add(time.Hour)
|
expires := start.Add(time.Hour)
|
||||||
@@ -150,17 +173,19 @@ func TestErrorBurstBanNotesHistoryAndMetrics(t *testing.T) {
|
|||||||
t.Fatalf("bans\n%+v\nwant\n%+v", got, want)
|
t.Fatalf("bans\n%+v\nwant\n%+v", got, want)
|
||||||
}
|
}
|
||||||
|
|
||||||
wantOffences := ratelimit.Offences{Limit: 1, RuleBlocked: 3}
|
wantOffences := ratelimit.Offences{Limit: 1, RuleBlocked: 2, TokenRefused: 1}
|
||||||
if offences := historyOf(t, server, client).Offences; offences != wantOffences {
|
if offences := historyOf(t, server, client).Offences; offences != wantOffences {
|
||||||
t.Errorf("history counts the offences %+v, want %+v", offences, wantOffences)
|
t.Errorf("history counts the offences %+v, want %+v", offences, wantOffences)
|
||||||
}
|
}
|
||||||
|
|
||||||
metrics := s.scrape()
|
metrics := s.scrape(scraper)
|
||||||
wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{instance="app",`+
|
wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{instance="app",`+
|
||||||
`kind="refusals",window="minute"}`, 1)
|
`kind="refusals",window="minute"}`, 1)
|
||||||
wantMetric(t, metrics, `smallwebwaf_offences_total{instance="app",kind="limit"}`, 1)
|
wantMetric(t, metrics, `smallwebwaf_offences_total{instance="app",kind="limit"}`, 1)
|
||||||
wantMetric(t, metrics,
|
wantMetric(t, metrics,
|
||||||
`smallwebwaf_offences_total{instance="app",kind="rule_blocked"}`, 3)
|
`smallwebwaf_offences_total{instance="app",kind="rule_blocked"}`, 2)
|
||||||
|
wantMetric(t, metrics,
|
||||||
|
`smallwebwaf_offences_total{instance="app",kind="token_refused"}`, 1)
|
||||||
wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit",instance="app"}`, 1)
|
wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit",instance="app"}`, 1)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -266,15 +291,19 @@ func TestErrorBurstCountsEachClientTheChecksApplyTo(t *testing.T) {
|
|||||||
allowNets: allowed, rateLimitExemptNets: exempt,
|
allowNets: allowed, rateLimitExemptNets: exempt,
|
||||||
})
|
})
|
||||||
|
|
||||||
// A client in SWWAF_ALLOW_NETS is not refused, and so not counted.
|
// A client in SWWAF_ALLOW_NETS still needs the token, but is not
|
||||||
|
// counted.
|
||||||
for range 3 {
|
for range 3 {
|
||||||
s.request(allowed, blockedPath, http.StatusOK, requestlog.ActionForward)
|
line := noMetricsToken.send(s, allowed)
|
||||||
|
if line.LimitHit != "" {
|
||||||
|
t.Errorf("log line has limit_hit %q, want none", line.LimitHit)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// One the rate limits do not apply to is.
|
// One the rate limits do not apply to is.
|
||||||
blockRule.send(s, exempt)
|
noMetricsToken.send(s, exempt)
|
||||||
|
|
||||||
line := blockRule.send(s, exempt)
|
line := wrongAdminToken.send(s, exempt)
|
||||||
if line.LimitHit != requestlog.LimitHitErrorBurst {
|
if line.LimitHit != requestlog.LimitHitErrorBurst {
|
||||||
t.Errorf("log line has limit_hit %q, want error_burst", line.LimitHit)
|
t.Errorf("log line has limit_hit %q, want error_burst", line.LimitHit)
|
||||||
}
|
}
|
||||||
@@ -307,6 +336,7 @@ func TestObserveModeLogsAndAlertsTheErrorBurst(t *testing.T) {
|
|||||||
mode: observe,
|
mode: observe,
|
||||||
errorBurstThreshold: "1",
|
errorBurstThreshold: "1",
|
||||||
rulesDir: writeRules(t, testRules),
|
rulesDir: writeRules(t, testRules),
|
||||||
|
adminToken: adminSecret,
|
||||||
})
|
})
|
||||||
start := clk.Now()
|
start := clk.Now()
|
||||||
held := bans.Ban{
|
held := bans.Ban{
|
||||||
@@ -317,11 +347,10 @@ func TestObserveModeLogsAndAlertsTheErrorBurst(t *testing.T) {
|
|||||||
}
|
}
|
||||||
server.Ledger.Load([]bans.Ban{held})
|
server.Ledger.Load([]bans.Ban{held})
|
||||||
|
|
||||||
// Under a ban, enforce mode would have refused these before the rule, so
|
// Under a ban, enforce mode would have refused these before the
|
||||||
// they are not counted.
|
// endpoint, so their tokens are not counted.
|
||||||
for range 2 {
|
for range 2 {
|
||||||
line := s.request(otherClient, blockedPath, http.StatusOK,
|
line := wrongAdminToken.send(s, otherClient)
|
||||||
requestlog.ActionForward)
|
|
||||||
wantWouldAction(t, line, requestlog.ActionBanned)
|
wantWouldAction(t, line, requestlog.ActionBanned)
|
||||||
|
|
||||||
if line.LimitHit != "" {
|
if line.LimitHit != "" {
|
||||||
@@ -331,7 +360,7 @@ func TestObserveModeLogsAndAlertsTheErrorBurst(t *testing.T) {
|
|||||||
|
|
||||||
// The block rule's refusal, which enforce mode would have answered 403,
|
// The block rule's refusal, which enforce mode would have answered 403,
|
||||||
// is the second of the client's, and would have banned it.
|
// is the second of the client's, and would have banned it.
|
||||||
s.request(client, blockedPath, http.StatusOK, requestlog.ActionForward)
|
wrongAdminToken.send(s, client)
|
||||||
|
|
||||||
line := s.request(client, blockedPath, http.StatusOK, requestlog.ActionForward)
|
line := s.request(client, blockedPath, http.StatusOK, requestlog.ActionForward)
|
||||||
wantWouldAction(t, line, requestlog.ActionRuleBlocked)
|
wantWouldAction(t, line, requestlog.ActionRuleBlocked)
|
||||||
|
|||||||
@@ -2,18 +2,24 @@ package proxy_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/proxy"
|
"sneak.berlin/go/smallwebwaf/internal/proxy"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestHealthEndpointCountsTowardNoLimit(t *testing.T) {
|
func TestHealthEndpointIsAnsweredBeforeAnyCheck(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
var calls atomic.Int32
|
||||||
|
|
||||||
|
app := startApp(t, func(http.ResponseWriter, *http.Request) {
|
||||||
|
calls.Add(1)
|
||||||
|
})
|
||||||
// With a limit of one request a minute, any request counted before
|
// With a limit of one request a minute, any request counted before
|
||||||
// the last one would have it refused.
|
// the last one would have it refused.
|
||||||
s, _, _ := startWithClock(t, "", map[string]string{rateLimitPerMinute: "1"})
|
addr, out := startProxy(t, app.URL, map[string]string{rateLimitPerMinute: "1"})
|
||||||
|
|
||||||
const (
|
const (
|
||||||
healthChecks = 3
|
healthChecks = 3
|
||||||
@@ -21,7 +27,8 @@ func TestHealthEndpointCountsTowardNoLimit(t *testing.T) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
for range healthChecks {
|
for range healthChecks {
|
||||||
got := s.adminRequest("", http.MethodGet, proxy.HealthPath, "", http.StatusOK)
|
got := get(t, addr, proxy.HealthPath)
|
||||||
|
wantStatus(t, got, http.StatusOK)
|
||||||
|
|
||||||
if string(got.body) != "ok\n" || got.header.Get("Content-Type") != contentType {
|
if string(got.body) != "ok\n" || got.header.Get("Content-Type") != contentType {
|
||||||
t.Errorf("health endpoint answered %q with Content-Type %q, want ok "+
|
t.Errorf("health endpoint answered %q with Content-Type %q, want ok "+
|
||||||
@@ -29,12 +36,21 @@ func TestHealthEndpointCountsTowardNoLimit(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
s.get(client, http.StatusOK, requestlog.ActionForward)
|
wantStatus(t, get(t, addr, "/"), http.StatusOK)
|
||||||
|
|
||||||
|
lines := out.requestLines(t, healthChecks+1)
|
||||||
|
for _, line := range lines[:healthChecks] {
|
||||||
|
wantLine(t, line, http.StatusOK, requestlog.ActionAdmin)
|
||||||
|
|
||||||
for _, line := range s.out.endedLines(t, healthChecks)[:healthChecks] {
|
|
||||||
if line.ResponseContentType != contentType {
|
if line.ResponseContentType != contentType {
|
||||||
t.Errorf("health check's log line has response_content_type %q, "+
|
t.Errorf("health check's log line has response_content_type %q, "+
|
||||||
"want %q", line.ResponseContentType, contentType)
|
"want %q", line.ResponseContentType, contentType)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
wantLine(t, lines[healthChecks], http.StatusOK, requestlog.ActionForward)
|
||||||
|
|
||||||
|
if calls.Load() != 1 {
|
||||||
|
t.Errorf("the app was called %d times, want once", calls.Load())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -86,7 +86,7 @@ func TestHistoryCountsTheBodiesEachWay(t *testing.T) {
|
|||||||
|
|
||||||
got := do(t, newRequest(t, http.MethodPost, addr, "/", strings.NewReader("abc")))
|
got := do(t, newRequest(t, http.MethodPost, addr, "/", strings.NewReader("abc")))
|
||||||
wantStatus(t, got, http.StatusOK)
|
wantStatus(t, got, http.StatusOK)
|
||||||
out.endedLine(t)
|
out.requestLine(t)
|
||||||
|
|
||||||
history := historyOf(t, server, localhost)
|
history := historyOf(t, server, localhost)
|
||||||
if history.RequestBytes != 3 || history.ResponseBytes != 5 {
|
if history.RequestBytes != 3 || history.ResponseBytes != 5 {
|
||||||
@@ -95,33 +95,38 @@ func TestHistoryCountsTheBodiesEachWay(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOwnEndpointsAreNotInTheHistory(t *testing.T) {
|
func TestHealthEndpointIsNotInTheHistory(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
s, _, server := startWithClock(t, "", map[string]string{metricsToken: token})
|
app := startApp(t, func(http.ResponseWriter, *http.Request) {})
|
||||||
|
addr, out, server := startProxyWithClock(t, app.URL, "", time.Now, nil)
|
||||||
|
|
||||||
s.adminRequest("", http.MethodGet, proxy.HealthPath, "", http.StatusOK)
|
wantStatus(t, get(t, addr, proxy.HealthPath), http.StatusOK)
|
||||||
s.adminRequest("", http.MethodGet, proxy.MetricsPath, "", http.StatusUnauthorized)
|
out.requestLine(t)
|
||||||
s.scrape()
|
|
||||||
|
|
||||||
if clients := server.Limiter.Snapshot(); len(clients) != 0 {
|
if clients := server.Limiter.Snapshot(); len(clients) != 0 {
|
||||||
t.Errorf("the table holds %+v, want no client", clients)
|
t.Errorf("the table holds %+v, want no client", clients)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPathUnderTheOwnPrefixIsNeitherForwardedNorRefused(t *testing.T) {
|
func TestRequestForSmallwebwafIsRefusedOnlyWithoutTheToken(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
app := startApp(t, func(http.ResponseWriter, *http.Request) {})
|
app := startApp(t, func(http.ResponseWriter, *http.Request) {})
|
||||||
addr, out, server := startProxyWithClock(t, app.URL, "", time.Now, nil)
|
addr, out, server := startProxyWithClock(t, app.URL, "", time.Now,
|
||||||
|
map[string]string{metricsToken: token})
|
||||||
|
|
||||||
|
// The metrics and the 404 are neither forwarded nor refused; the 401
|
||||||
|
// is refused.
|
||||||
|
scrape(t, addr)
|
||||||
wantStatus(t, get(t, addr, "/_smallwebwaf/nothing"), http.StatusNotFound)
|
wantStatus(t, get(t, addr, "/_smallwebwaf/nothing"), http.StatusNotFound)
|
||||||
out.endedLine(t)
|
wantStatus(t, get(t, addr, proxy.MetricsPath), http.StatusUnauthorized)
|
||||||
|
out.requestLines(t, 3)
|
||||||
|
|
||||||
history := historyOf(t, server, localhost)
|
history := historyOf(t, server, localhost)
|
||||||
if history.Requests != 1 || history.Forwarded != 0 || history.Refused != 0 {
|
if history.Requests != 3 || history.Forwarded != 0 || history.Refused != 1 {
|
||||||
t.Errorf("history counts %d requests, %d forwarded and %d refused, "+
|
t.Errorf("history counts %d requests, %d forwarded and %d refused, "+
|
||||||
"want 1, 0 and 0", history.Requests, history.Forwarded, history.Refused)
|
"want 3, 0 and 1", history.Requests, history.Forwarded, history.Refused)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,162 @@
|
|||||||
|
package proxy_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
|
)
|
||||||
|
|
||||||
|
// sizeLimit is the size limit the tests set, 1K as a setting.
|
||||||
|
const (
|
||||||
|
sizeLimit = 1 << 10
|
||||||
|
sizeLimitSetting = "1K"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRequestBodyLimit(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
size int
|
||||||
|
// announced sends the size in Content-Length; otherwise the body
|
||||||
|
// is sent in chunks with no length given.
|
||||||
|
announced bool
|
||||||
|
want int
|
||||||
|
action string
|
||||||
|
// refusedBeforeApp is a refusal before anything reaches the app.
|
||||||
|
// A body over the limit with no length given has already partly
|
||||||
|
// reached the app when it is refused.
|
||||||
|
refusedBeforeApp bool
|
||||||
|
}{
|
||||||
|
{"announced, over the limit", 2 * sizeLimit, true,
|
||||||
|
http.StatusRequestEntityTooLarge, requestlog.ActionTooLarge, true},
|
||||||
|
{"announced, at the limit", sizeLimit, true,
|
||||||
|
http.StatusOK, requestlog.ActionForward, false},
|
||||||
|
{"not announced, over the limit", 4 * sizeLimit, false,
|
||||||
|
http.StatusRequestEntityTooLarge, requestlog.ActionTooLarge, false},
|
||||||
|
{"not announced, at the limit", sizeLimit, false,
|
||||||
|
http.StatusOK, requestlog.ActionForward, false},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var calls atomic.Int32
|
||||||
|
|
||||||
|
app := startApp(t, func(_ http.ResponseWriter, r *http.Request) {
|
||||||
|
calls.Add(1)
|
||||||
|
|
||||||
|
_, _ = io.Copy(io.Discard, r.Body)
|
||||||
|
})
|
||||||
|
addr, out := startProxy(t, app.URL, map[string]string{
|
||||||
|
requestMaxBytes: sizeLimitSetting,
|
||||||
|
metricsToken: token,
|
||||||
|
})
|
||||||
|
|
||||||
|
var body io.Reader = bytes.NewReader(make([]byte, tc.size))
|
||||||
|
if !tc.announced {
|
||||||
|
body = io.MultiReader(body) // hides the length
|
||||||
|
}
|
||||||
|
|
||||||
|
wantStatus(t, do(t, newRequest(t, http.MethodPost, addr, "/upload", body)),
|
||||||
|
tc.want)
|
||||||
|
wantLine(t, out.requestLine(t), tc.want, tc.action)
|
||||||
|
|
||||||
|
hits := 0
|
||||||
|
if tc.action == requestlog.ActionTooLarge {
|
||||||
|
hits = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
wantLimitHits(t, addr, requestMaxBytes, hits)
|
||||||
|
|
||||||
|
if tc.refusedBeforeApp && calls.Load() != 0 {
|
||||||
|
t.Errorf("the app was called %d times, want never", calls.Load())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResponseBodyLimit(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
size int
|
||||||
|
// announced sends the size in Content-Length; otherwise the body
|
||||||
|
// is sent in chunks with no length given.
|
||||||
|
announced bool
|
||||||
|
want int
|
||||||
|
action string
|
||||||
|
// received is how much of a body the client gets, and cutOff
|
||||||
|
// whether the connection is then cut.
|
||||||
|
received int
|
||||||
|
cutOff bool
|
||||||
|
}{
|
||||||
|
{"announced, over the limit", 2 * sizeLimit, true, http.StatusBadGateway,
|
||||||
|
requestlog.ActionTooLarge, len("Bad Gateway\n"), false},
|
||||||
|
{"announced, at the limit", sizeLimit, true, http.StatusOK,
|
||||||
|
requestlog.ActionForward, sizeLimit, false},
|
||||||
|
{"not announced, over the limit", 4 * sizeLimit, false, http.StatusOK,
|
||||||
|
requestlog.ActionTooLarge, sizeLimit, true},
|
||||||
|
{"not announced, at the limit", sizeLimit, false, http.StatusOK,
|
||||||
|
requestlog.ActionForward, sizeLimit, false},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
app := startApp(t, func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
answerWithSize(w, tc.size, tc.announced)
|
||||||
|
})
|
||||||
|
addr, out := startProxy(t, app.URL, map[string]string{
|
||||||
|
responseMaxBytes: sizeLimitSetting,
|
||||||
|
metricsToken: token,
|
||||||
|
})
|
||||||
|
|
||||||
|
got := get(t, addr, "/download")
|
||||||
|
wantStatus(t, got, tc.want)
|
||||||
|
|
||||||
|
if len(got.body) != tc.received ||
|
||||||
|
errors.Is(got.err, io.ErrUnexpectedEOF) != tc.cutOff {
|
||||||
|
t.Errorf("client got %d bytes (%v), want %d",
|
||||||
|
len(got.body), got.err, tc.received)
|
||||||
|
}
|
||||||
|
|
||||||
|
line := out.requestLine(t)
|
||||||
|
wantLine(t, line, tc.want, tc.action)
|
||||||
|
|
||||||
|
if line.UpstreamStatus != http.StatusOK {
|
||||||
|
t.Errorf("log line has upstream_status %d", line.UpstreamStatus)
|
||||||
|
}
|
||||||
|
|
||||||
|
hits := 0
|
||||||
|
if tc.action == requestlog.ActionTooLarge {
|
||||||
|
hits = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
wantLimitHits(t, addr, responseMaxBytes, hits)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// answerWithSize answers with a body of size bytes, announced in
|
||||||
|
// Content-Length or sent in chunks with no length given.
|
||||||
|
func answerWithSize(w http.ResponseWriter, size int, announced bool) {
|
||||||
|
body := make([]byte, size)
|
||||||
|
if announced {
|
||||||
|
w.Header().Set("Content-Length", strconv.Itoa(size))
|
||||||
|
_, _ = w.Write(body)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sending part of it before the end keeps Go's server from working
|
||||||
|
// out the length.
|
||||||
|
_, _ = w.Write(body[:size/2])
|
||||||
|
_ = http.NewResponseController(w).Flush()
|
||||||
|
_, _ = w.Write(body[size/2:])
|
||||||
|
}
|
||||||
+14
-21
@@ -4,15 +4,15 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The headers in which an answer of allow gives rproxy the client's AS number
|
// The headers in which the app is passed the client's AS number and
|
||||||
// and country, for the app, while SWWAF_ADD_LOOKUP_HEADERS is set. Go writes
|
// country while SWWAF_ADD_LOOKUP_HEADERS is set. Go writes every header
|
||||||
// every header name in this form, as it sends it and as it receives it, so
|
// name in this form, as it sends it and as it receives it, so X-Client-ASN
|
||||||
// X-Client-ASN arrives as X-Client-Asn; header names are not case-sensitive.
|
// arrives as X-Client-Asn, and Del removes a client's own whatever their
|
||||||
|
// case; header names are not case-sensitive.
|
||||||
const (
|
const (
|
||||||
asnHeader = "X-Client-Asn"
|
asnHeader = "X-Client-Asn"
|
||||||
countryHeader = "X-Client-Country"
|
countryHeader = "X-Client-Country"
|
||||||
@@ -23,24 +23,17 @@ const (
|
|||||||
// SWWAF_LOOKUP_SOURCE is off or the client is on a private, loopback or
|
// SWWAF_LOOKUP_SOURCE is off or the client is on a private, loopback or
|
||||||
// link-local address, which no lookup can place. The lookup database
|
// link-local address, which no lookup can place. The lookup database
|
||||||
// answers at once. With GeoJS, while a setting needs the answer, such as a
|
// answers at once. With GeoJS, while a setting needs the answer, such as a
|
||||||
// country list or a biased threshold, a new client's request waits for it,
|
// country list or a biased threshold, a new client's request waits for it.
|
||||||
// unless refusedWhateverTheAnswer reports true. ctx is the request's own
|
// ctx is the request's own context.
|
||||||
// context.
|
func (rq *request) lookUp(ctx context.Context) {
|
||||||
func (rq *request) lookUp(ctx context.Context, now time.Time) {
|
|
||||||
if rq.h.config.LookupSource == "off" || !canBePlaced(rq.client) {
|
if rq.h.config.LookupSource == "off" || !canBePlaced(rq.client) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
client := rq.h.clientGroup(rq.client)
|
if rq.h.config.LookupSource == "file" {
|
||||||
|
rq.lookupAnswer = rq.h.lookupFile.LookUp(rq.h.clientGroup(rq.client))
|
||||||
switch {
|
} else {
|
||||||
case rq.h.config.LookupSource == "file":
|
rq.lookupAnswer = rq.h.geojs.LookUp(ctx, rq.h.clientGroup(rq.client))
|
||||||
rq.lookupAnswer = rq.h.lookupFile.LookUp(client)
|
|
||||||
case rq.refusedWhateverTheAnswer(now):
|
|
||||||
rq.lookupAnswer = rq.h.geojs.LookUpWithoutWaiting(ctx, client)
|
|
||||||
rq.skippedLookupWait = rq.lookupAnswer.Answered.IsZero()
|
|
||||||
default:
|
|
||||||
rq.lookupAnswer = rq.h.geojs.LookUp(ctx, client)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.lookedUp = true
|
rq.lookedUp = true
|
||||||
@@ -59,8 +52,8 @@ func (h *handler) addLookup(answer lookup.Answer) {
|
|||||||
answer.ASN, answer.ASName, answer.Country)
|
answer.ASN, answer.ASName, answer.Country)
|
||||||
}
|
}
|
||||||
|
|
||||||
// setLookupHeaders sets the headers that give the client's AS number and
|
// setLookupHeaders sets the headers in which the app is passed the
|
||||||
// country, leaving out one that is unknown.
|
// client's AS number and country, leaving out one that is unknown.
|
||||||
func setLookupHeaders(header http.Header, asn, country string) {
|
func setLookupHeaders(header http.Header, asn, country string) {
|
||||||
if asn != "" {
|
if asn != "" {
|
||||||
header.Set(asnHeader, asn)
|
header.Set(asnHeader, asn)
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
package proxy_test
|
package proxy_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"slices"
|
"slices"
|
||||||
@@ -84,75 +86,6 @@ func TestEveryClientIsLookedUpWithoutWaitingWhileNoSettingNeedsIt(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRequestThatWaitedForItsLookupIsCountedWhenItGoesOn(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// The country list has a new client's request wait for its answer,
|
|
||||||
// which the stand-in for GeoJS gives only once released.
|
|
||||||
geojsURL, asked, release := startHeldGeoJS(t)
|
|
||||||
app := startApp(t, func(http.ResponseWriter, *http.Request) {})
|
|
||||||
clk := &clock{now: time.Date(2026, 10, 6, 0, 0, 58, 0, time.UTC)}
|
|
||||||
addr, out, server := startProxyWithClock(t, app.URL, geojsURL, clk.Now,
|
|
||||||
map[string]string{
|
|
||||||
trustedProxies: trustLocalhost,
|
|
||||||
lookupTimeout: "1h",
|
|
||||||
deniedCountries: "kp",
|
|
||||||
})
|
|
||||||
|
|
||||||
requestFromDE := func() *http.Request {
|
|
||||||
req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
|
|
||||||
req.Header.Set(forwardedFor, fromDE)
|
|
||||||
|
|
||||||
return req
|
|
||||||
}
|
|
||||||
|
|
||||||
// fromDE's first request arrives two seconds before a minute ends,
|
|
||||||
// and waits for GeoJS.
|
|
||||||
first, httpClient := requestFromDE(), newClient(t)
|
|
||||||
firstEnded := make(chan error, 1)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
res, err := httpClient.Do(first)
|
|
||||||
if err == nil {
|
|
||||||
err = readAnswer(res).err
|
|
||||||
}
|
|
||||||
|
|
||||||
firstEnded <- err
|
|
||||||
}()
|
|
||||||
|
|
||||||
waitUntil(func() bool { return len(asked()) == 1 })
|
|
||||||
|
|
||||||
// Half a second into the next minute, the answer is kept, as an edit
|
|
||||||
// of lookups.json would keep it, while the first request still waits
|
|
||||||
// for GeoJS. The second request goes on at once, and is counted.
|
|
||||||
clk.advance(2500 * time.Millisecond)
|
|
||||||
server.GeoJS.Load([]lookup.Answer{{
|
|
||||||
Client: netip.MustParsePrefix(fromDE + "/32"), ASN: asnDE, ASName: asNameDE,
|
|
||||||
Country: "DE", Answered: clk.Now(), Used: clk.Now(),
|
|
||||||
}})
|
|
||||||
wantStatus(t, do(t, requestFromDE()), http.StatusOK)
|
|
||||||
|
|
||||||
// GeoJS answers, and the first request goes on.
|
|
||||||
release()
|
|
||||||
|
|
||||||
err := <-firstEnded
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("first request: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantLine(t, out.endedLines(t, 2)[1], http.StatusOK, requestlog.ActionForward)
|
|
||||||
|
|
||||||
// The third request's count holds all three: the first was counted
|
|
||||||
// when it went on, in the minute under way, and the second's count
|
|
||||||
// was kept.
|
|
||||||
clk.advance(500 * time.Millisecond)
|
|
||||||
wantStatus(t, do(t, requestFromDE()), http.StatusOK)
|
|
||||||
|
|
||||||
if got := out.endedLines(t, 3)[2].Counts.Minute; got != 3 {
|
|
||||||
t.Errorf("the third request's count in the minute is %v, want 3", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestASNumberAndNameInTheLogLineTheHistoryTheBanNotesAndTheAlert(t *testing.T) {
|
func TestASNumberAndNameInTheLogLineTheHistoryTheBanNotesAndTheAlert(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -317,7 +250,7 @@ func TestLookupHeadersArePassedToTheAppAndTheClientsOwnRemoved(t *testing.T) {
|
|||||||
addr, out, _ := startProxyWithClock(t, app.URL, geojsURL, time.Now, map[string]string{
|
addr, out, _ := startProxyWithClock(t, app.URL, geojsURL, time.Now, map[string]string{
|
||||||
trustedProxies: trustLocalhost,
|
trustedProxies: trustLocalhost,
|
||||||
lookupTimeout: "1h",
|
lookupTimeout: "1h",
|
||||||
addLookupHeaders: on,
|
addLookupHeaders: "true",
|
||||||
})
|
})
|
||||||
s := &sender{t: t, addr: addr, out: out}
|
s := &sender{t: t, addr: addr, out: out}
|
||||||
|
|
||||||
@@ -381,19 +314,22 @@ func TestRequestWaitsAsLongAsTheLookupTimeoutSays(t *testing.T) {
|
|||||||
// network, which would keep that clock from moving on: the request is
|
// network, which would keep that clock from moving on: the request is
|
||||||
// handed to the proxy's handler, and GeoJS is one that never answers.
|
// handed to the proxy's handler, and GeoJS is one that never answers.
|
||||||
synctest.Test(t, func(t *testing.T) {
|
synctest.Test(t, func(t *testing.T) {
|
||||||
// Not the default. The exclusive list needs the answer, and the app
|
// Not the default second. The exclusive list needs the answer, and
|
||||||
// is never reached.
|
// the app is never reached.
|
||||||
const timeout = 3 * time.Second
|
const timeout = 3 * time.Second
|
||||||
|
|
||||||
server, out, _ := newServeWAF(t, unansweredGeoJSURL, time.Now,
|
server, out, _ := newProxy(t, "http://app.invalid", unansweredGeoJSURL,
|
||||||
map[string]string{
|
time.Now, map[string]string{
|
||||||
lookupTimeout: timeout.String(),
|
lookupTimeout: timeout.String(),
|
||||||
allowedCountries: "DE",
|
allowedCountries: "DE",
|
||||||
})
|
})
|
||||||
|
|
||||||
|
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/",
|
||||||
|
http.NoBody)
|
||||||
|
req.RemoteAddr = net.JoinHostPort(fromDE, "1234")
|
||||||
began := time.Now()
|
began := time.Now()
|
||||||
|
|
||||||
serveFromDE(t, server, http.MethodGet, http.NoBody)
|
server.Handler.ServeHTTP(httptest.NewRecorder(), req)
|
||||||
|
|
||||||
if waited := time.Since(began); waited != timeout {
|
if waited := time.Since(began); waited != timeout {
|
||||||
t.Errorf("the request waited %s for its answer, want %s", waited, timeout)
|
t.Errorf("the request waited %s for its answer, want %s", waited, timeout)
|
||||||
|
|||||||
+163
-49
@@ -7,6 +7,8 @@ import (
|
|||||||
"net/netip"
|
"net/netip"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -28,21 +30,41 @@ const (
|
|||||||
func TestMetricsAreOffWhileTheTokenIsUnset(t *testing.T) {
|
func TestMetricsAreOffWhileTheTokenIsUnset(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
s, _, _ := startWithClock(t, "", nil)
|
var calls atomic.Int32
|
||||||
|
|
||||||
|
app := startApp(t, func(http.ResponseWriter, *http.Request) {
|
||||||
|
calls.Add(1)
|
||||||
|
})
|
||||||
|
addr, out := startProxy(t, app.URL, nil)
|
||||||
|
|
||||||
// An empty token does not match the unset one either.
|
// An empty token does not match the unset one either.
|
||||||
for _, authorization := range []string{bearer, "Bearer ", ""} {
|
for i, authorization := range []string{bearer, "Bearer ", ""} {
|
||||||
s.adminRequest(authorization, http.MethodGet, proxy.MetricsPath, "",
|
req := newRequest(t, http.MethodGet, addr, proxy.MetricsPath, http.NoBody)
|
||||||
http.StatusNotFound)
|
if authorization != "" {
|
||||||
|
req.Header.Set("Authorization", authorization)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantStatus(t, do(t, req), http.StatusNotFound)
|
||||||
|
wantLine(t, out.requestLines(t, i+1)[i], http.StatusNotFound,
|
||||||
|
requestlog.ActionAdmin)
|
||||||
|
}
|
||||||
|
|
||||||
|
if calls.Load() != 0 {
|
||||||
|
t.Errorf("the app was called %d times, want never", calls.Load())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMetricsNeedTheTokenAndOtherPathsAreNotFound(t *testing.T) {
|
func TestMetricsNeedTheTokenAndOtherPathsAreNotFound(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
s, _, _ := startWithClock(t, "", map[string]string{metricsToken: token})
|
var calls atomic.Int32
|
||||||
|
|
||||||
for _, tc := range []struct {
|
app := startApp(t, func(http.ResponseWriter, *http.Request) {
|
||||||
|
calls.Add(1)
|
||||||
|
})
|
||||||
|
addr, out := startProxy(t, app.URL, map[string]string{metricsToken: token})
|
||||||
|
|
||||||
|
for i, tc := range []struct {
|
||||||
method, path, authorization string
|
method, path, authorization string
|
||||||
status int
|
status int
|
||||||
}{
|
}{
|
||||||
@@ -59,7 +81,14 @@ func TestMetricsNeedTheTokenAndOtherPathsAreNotFound(t *testing.T) {
|
|||||||
{http.MethodGet, "/_smallwebwaf/bans", bearer, http.StatusNotFound},
|
{http.MethodGet, "/_smallwebwaf/bans", bearer, http.StatusNotFound},
|
||||||
{http.MethodPost, proxy.HealthPath, "", http.StatusNotFound},
|
{http.MethodPost, proxy.HealthPath, "", http.StatusNotFound},
|
||||||
} {
|
} {
|
||||||
got := s.adminRequest(tc.authorization, tc.method, tc.path, "", tc.status)
|
req := newRequest(t, tc.method, addr, tc.path, http.NoBody)
|
||||||
|
if tc.authorization != "" {
|
||||||
|
req.Header.Set("Authorization", tc.authorization)
|
||||||
|
}
|
||||||
|
|
||||||
|
got := do(t, req)
|
||||||
|
wantStatus(t, got, tc.status)
|
||||||
|
wantLine(t, out.requestLines(t, i+1)[i], tc.status, requestlog.ActionAdmin)
|
||||||
|
|
||||||
if tc.status == http.StatusUnauthorized &&
|
if tc.status == http.StatusUnauthorized &&
|
||||||
got.header.Get("WWW-Authenticate") != "Bearer" {
|
got.header.Get("WWW-Authenticate") != "Bearer" {
|
||||||
@@ -72,28 +101,58 @@ func TestMetricsNeedTheTokenAndOtherPathsAreNotFound(t *testing.T) {
|
|||||||
t.Errorf("the metrics are\n%s", got.body)
|
t.Errorf("the metrics are\n%s", got.body)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if calls.Load() != 0 {
|
||||||
|
t.Errorf("the app was called %d times, want never", calls.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMetricsAreAskedForThroughTheChecks(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s, _, _ := startWithClock(t, "", map[string]string{
|
||||||
|
metricsToken: token,
|
||||||
|
rateLimitPerMinute: "1",
|
||||||
|
})
|
||||||
|
|
||||||
|
// Asking for the metrics counts toward the client's limit of one
|
||||||
|
// request a minute, so its next request breaks it, and bans it. A
|
||||||
|
// banned client is refused the metrics too.
|
||||||
|
s.scrape(client)
|
||||||
|
s.get(client, http.StatusForbidden, requestlog.ActionRateLimited)
|
||||||
|
s.requestWithHeader(client, proxy.MetricsPath, "Authorization: "+bearer,
|
||||||
|
http.StatusForbidden, requestlog.ActionBanned)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMetricsCountTheTraffic(t *testing.T) {
|
func TestMetricsCountTheTraffic(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
arrived, release := make(chan struct{}), make(chan struct{})
|
||||||
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
_, _ = io.Copy(io.Discard, r.Body)
|
_, _ = io.Copy(io.Discard, r.Body)
|
||||||
|
|
||||||
|
if r.URL.Path == "/held" {
|
||||||
|
close(arrived)
|
||||||
|
<-release
|
||||||
|
}
|
||||||
|
|
||||||
_, _ = io.WriteString(w, "hello")
|
_, _ = io.WriteString(w, "hello")
|
||||||
})
|
})
|
||||||
addr, out, server := startProxyWithClock(t, app.URL, "", time.Now,
|
releaseApp := sync.OnceFunc(func() { close(release) })
|
||||||
map[string]string{metricsToken: token})
|
t.Cleanup(releaseApp)
|
||||||
|
|
||||||
|
addr, out := startProxy(t, app.URL, map[string]string{metricsToken: token})
|
||||||
|
|
||||||
got := do(t, newRequest(t, http.MethodPost, addr, "/", strings.NewReader("abc")))
|
got := do(t, newRequest(t, http.MethodPost, addr, "/", strings.NewReader("abc")))
|
||||||
wantStatus(t, got, http.StatusOK)
|
wantStatus(t, got, http.StatusOK)
|
||||||
wantStatus(t, get(t, addr, "/_smallwebwaf/nothing"), http.StatusNotFound)
|
wantStatus(t, get(t, addr, "/_smallwebwaf/nothing"), http.StatusNotFound)
|
||||||
out.endedLines(t, 2)
|
out.requestLines(t, 2)
|
||||||
|
|
||||||
forward := `{action="forward",instance="app",status_class="2xx"}`
|
forward := `{action="forward",instance="app",status_class="2xx"}`
|
||||||
notFound := `{action="admin",instance="app",status_class="4xx"}`
|
notFound := `{action="admin",instance="app",status_class="4xx"}`
|
||||||
|
|
||||||
// The request for the metrics is itself under way.
|
// The request for the metrics is itself under way.
|
||||||
metrics := scrape(t, server.Addr)
|
metrics := scrape(t, addr)
|
||||||
wantMetric(t, metrics, "smallwebwaf_requests_total"+forward, 1)
|
wantMetric(t, metrics, "smallwebwaf_requests_total"+forward, 1)
|
||||||
wantMetric(t, metrics, "smallwebwaf_requests_total"+notFound, 1)
|
wantMetric(t, metrics, "smallwebwaf_requests_total"+notFound, 1)
|
||||||
wantMetric(t, metrics, "smallwebwaf_request_bytes_total"+forward, 3)
|
wantMetric(t, metrics, "smallwebwaf_request_bytes_total"+forward, 3)
|
||||||
@@ -107,20 +166,50 @@ func TestMetricsCountTheTraffic(t *testing.T) {
|
|||||||
wantMetric(t, metrics, `smallwebwaf_requests_in_flight{instance="app"}`, 1)
|
wantMetric(t, metrics, `smallwebwaf_requests_in_flight{instance="app"}`, 1)
|
||||||
metric(t, metrics, `go_goroutines{instance="app"}`)
|
metric(t, metrics, `go_goroutines{instance="app"}`)
|
||||||
metric(t, metrics, `process_start_time_seconds{instance="app"}`)
|
metric(t, metrics, `process_start_time_seconds{instance="app"}`)
|
||||||
|
|
||||||
|
// A request the app holds is under way until it ends.
|
||||||
|
httpClient := newClient(t)
|
||||||
|
held := newRequest(t, http.MethodGet, addr, "/held", http.NoBody)
|
||||||
|
ended := make(chan error, 1)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
res, err := httpClient.Do(held)
|
||||||
|
if err == nil {
|
||||||
|
err = readAnswer(res).err
|
||||||
|
}
|
||||||
|
|
||||||
|
ended <- err
|
||||||
|
}()
|
||||||
|
|
||||||
|
<-arrived
|
||||||
|
wantMetric(t, scrape(t, addr), `smallwebwaf_requests_in_flight{instance="app"}`, 2)
|
||||||
|
releaseApp()
|
||||||
|
|
||||||
|
err := <-ended
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("held request: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
out.requestLines(t, 5)
|
||||||
|
wantMetric(t, scrape(t, addr), `smallwebwaf_requests_in_flight{instance="app"}`, 1)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMetricsCountLimitsAndBans(t *testing.T) {
|
func TestMetricsCountLimitsAndBans(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const denied = "192.0.2.50" // in SWWAF_DENY_NETS
|
const (
|
||||||
|
scraper = "192.0.2.200" // in SWWAF_RATE_LIMIT_EXEMPT_NETS
|
||||||
|
denied = "192.0.2.50" // in SWWAF_DENY_NETS
|
||||||
|
)
|
||||||
|
|
||||||
s, clk, _ := startWithClock(t, "", map[string]string{
|
s, clk, _ := startWithClock(t, "", map[string]string{
|
||||||
metricsToken: token,
|
metricsToken: token,
|
||||||
rateLimitPerMinute: "1",
|
rateLimitPerMinute: "1",
|
||||||
denyNets: denied,
|
rateLimitExemptNets: scraper,
|
||||||
banResponse: banResponseClose,
|
denyNets: denied,
|
||||||
limitBanDuration: "1h",
|
banResponse: "close",
|
||||||
maxBanDuration: "2h",
|
limitBanDuration: "1h",
|
||||||
|
maxBanDuration: "2h",
|
||||||
})
|
})
|
||||||
|
|
||||||
// SWWAF_BAN_RESPONSE=close sends no status at all.
|
// SWWAF_BAN_RESPONSE=close sends no status at all.
|
||||||
@@ -130,7 +219,7 @@ func TestMetricsCountLimitsAndBans(t *testing.T) {
|
|||||||
s.get(client, http.StatusOK, requestlog.ActionForward)
|
s.get(client, http.StatusOK, requestlog.ActionForward)
|
||||||
s.get(client, 0, requestlog.ActionRateLimited)
|
s.get(client, 0, requestlog.ActionRateLimited)
|
||||||
|
|
||||||
metrics := s.scrape()
|
metrics := s.scrape(scraper)
|
||||||
wantMetric(t, metrics,
|
wantMetric(t, metrics,
|
||||||
`smallwebwaf_requests_total{action="denied",instance="app",status_class="none"}`, 1)
|
`smallwebwaf_requests_total{action="denied",instance="app",status_class="none"}`, 1)
|
||||||
wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{instance="app",`+
|
wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{instance="app",`+
|
||||||
@@ -141,35 +230,38 @@ func TestMetricsCountLimitsAndBans(t *testing.T) {
|
|||||||
wantMetric(t, metrics, `smallwebwaf_permanent_bans{instance="app"}`, 0)
|
wantMetric(t, metrics, `smallwebwaf_permanent_bans{instance="app"}`, 0)
|
||||||
|
|
||||||
clk.advance(time.Hour)
|
clk.advance(time.Hour)
|
||||||
wantMetric(t, s.scrape(), `smallwebwaf_active_bans{instance="app"}`, 0)
|
wantMetric(t, s.scrape(scraper), `smallwebwaf_active_bans{instance="app"}`, 0)
|
||||||
|
|
||||||
// A limit broken again right after would ban for three hours, longer
|
// A limit broken again right after would ban for three hours, longer
|
||||||
// than SWWAF_MAX_BAN_DURATION, so the ban is permanent.
|
// than SWWAF_MAX_BAN_DURATION, so the ban is permanent.
|
||||||
s.get(client, http.StatusOK, requestlog.ActionForward)
|
s.get(client, http.StatusOK, requestlog.ActionForward)
|
||||||
s.get(client, 0, requestlog.ActionRateLimited)
|
s.get(client, 0, requestlog.ActionRateLimited)
|
||||||
|
|
||||||
metrics = s.scrape()
|
metrics = s.scrape(scraper)
|
||||||
wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{instance="app",`+
|
wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{instance="app",`+
|
||||||
`kind="requests",window="minute"}`, 2)
|
`kind="requests",window="minute"}`, 2)
|
||||||
wantMetric(t, metrics, `smallwebwaf_offences_total{instance="app",kind="limit"}`, 2)
|
wantMetric(t, metrics, `smallwebwaf_offences_total{instance="app",kind="limit"}`, 2)
|
||||||
wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit",instance="app"}`, 2)
|
wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit",instance="app"}`, 2)
|
||||||
wantMetric(t, metrics, `smallwebwaf_active_bans{instance="app"}`, 1)
|
wantMetric(t, metrics, `smallwebwaf_active_bans{instance="app"}`, 1)
|
||||||
wantMetric(t, metrics, `smallwebwaf_permanent_bans{instance="app"}`, 1)
|
wantMetric(t, metrics, `smallwebwaf_permanent_bans{instance="app"}`, 1)
|
||||||
// denied and client.
|
// denied, client, and the scraper as of its earlier requests.
|
||||||
wantMetric(t, metrics, `smallwebwaf_tracked_clients{instance="app"}`, 2)
|
wantMetric(t, metrics, `smallwebwaf_tracked_clients{instance="app"}`, 3)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMetricsCountTheBansAnAdminMakes(t *testing.T) {
|
func TestMetricsCountTheBansAnAdminMakes(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
const scraper = "192.0.2.200" // in SWWAF_RATE_LIMIT_EXEMPT_NETS
|
||||||
|
|
||||||
s, clk, server := startWithClock(t, "", map[string]string{
|
s, clk, server := startWithClock(t, "", map[string]string{
|
||||||
metricsToken: token,
|
metricsToken: token,
|
||||||
adminToken: adminSecret,
|
adminToken: adminSecret,
|
||||||
|
rateLimitExemptNets: scraper,
|
||||||
})
|
})
|
||||||
|
|
||||||
const admins = `smallwebwaf_bans_made_total{cause="admin",instance="app"}`
|
const admins = `smallwebwaf_bans_made_total{cause="admin",instance="app"}`
|
||||||
|
|
||||||
wantMetric(t, s.scrape(), admins, 0)
|
wantMetric(t, s.scrape(scraper), admins, 0)
|
||||||
|
|
||||||
// As an admin's edit of bans.json that adds a ban is taken in.
|
// As an admin's edit of bans.json that adds a ban is taken in.
|
||||||
server.Ledger.LoadEdit([]bans.Ban{{
|
server.Ledger.LoadEdit([]bans.Ban{{
|
||||||
@@ -177,11 +269,11 @@ func TestMetricsCountTheBansAnAdminMakes(t *testing.T) {
|
|||||||
Start: clk.Now(),
|
Start: clk.Now(),
|
||||||
}})
|
}})
|
||||||
|
|
||||||
wantMetric(t, s.scrape(), admins, 1)
|
wantMetric(t, s.scrape(scraper), admins, 1)
|
||||||
|
|
||||||
// And a ban made through the endpoint.
|
// And a ban made through the endpoint.
|
||||||
s.admin(http.MethodPost, proxy.BansPath, banOtherClient, http.StatusOK)
|
s.admin(http.MethodPost, proxy.BansPath, banOtherClient, http.StatusOK)
|
||||||
wantMetric(t, s.scrape(), admins, 2)
|
wantMetric(t, s.scrape(scraper), admins, 2)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
|
func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
|
||||||
@@ -220,7 +312,7 @@ func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
|
|||||||
// Each is counted before the next is sent, so that the
|
// Each is counted before the next is sent, so that the
|
||||||
// countries are ranked in the order sent.
|
// countries are ranked in the order sent.
|
||||||
lines++
|
lines++
|
||||||
out.endedLines(t, lines)
|
out.requestLines(t, lines)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -229,7 +321,7 @@ func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
|
|||||||
send(fromDE, 2, http.StatusOK)
|
send(fromDE, 2, http.StatusOK)
|
||||||
send(fromFR, 1, http.StatusOK)
|
send(fromFR, 1, http.StatusOK)
|
||||||
|
|
||||||
metrics := scrape(t, server.Addr)
|
metrics := scrape(t, addr)
|
||||||
lines++
|
lines++
|
||||||
|
|
||||||
wantMetric(t, metrics,
|
wantMetric(t, metrics,
|
||||||
@@ -257,7 +349,7 @@ func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
|
|||||||
// from then on, and DE's is gone.
|
// from then on, and DE's is gone.
|
||||||
send(fromFR, 3, http.StatusOK)
|
send(fromFR, 3, http.StatusOK)
|
||||||
|
|
||||||
metrics = scrape(t, server.Addr)
|
metrics = scrape(t, addr)
|
||||||
wantMetric(t, metrics,
|
wantMetric(t, metrics,
|
||||||
`smallwebwaf_country_requests_total{country="KP",instance="app"}`, 3)
|
`smallwebwaf_country_requests_total{country="KP",instance="app"}`, 3)
|
||||||
wantMetric(t, metrics,
|
wantMetric(t, metrics,
|
||||||
@@ -291,12 +383,13 @@ func TestMetricsByASNumberKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
|
|||||||
answer(fromDE, "AS64501"), answer(fromKP, "AS64502"),
|
answer(fromDE, "AS64501"), answer(fromKP, "AS64502"),
|
||||||
})
|
})
|
||||||
|
|
||||||
// With one AS number of its own, the other is counted as other.
|
// With one AS number of its own, the other is counted as other. The
|
||||||
|
// metrics are asked for from a private address, which has no AS number.
|
||||||
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
||||||
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
||||||
s.get(fromKP, http.StatusOK, requestlog.ActionForward)
|
s.get(fromKP, http.StatusOK, requestlog.ActionForward)
|
||||||
|
|
||||||
metrics := s.scrape()
|
metrics := s.scrape("10.0.0.9")
|
||||||
wantMetric(t, metrics,
|
wantMetric(t, metrics,
|
||||||
`smallwebwaf_asn_requests_total{asn="AS64501",instance="app"}`, 2)
|
`smallwebwaf_asn_requests_total{asn="AS64501",instance="app"}`, 2)
|
||||||
wantMetric(t, metrics,
|
wantMetric(t, metrics,
|
||||||
@@ -319,12 +412,11 @@ func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) {
|
|||||||
t.Cleanup(geojs.Close)
|
t.Cleanup(geojs.Close)
|
||||||
|
|
||||||
app := startApp(t, func(http.ResponseWriter, *http.Request) {})
|
app := startApp(t, func(http.ResponseWriter, *http.Request) {})
|
||||||
addr, _, server := startProxyWithClock(t, app.URL, geojs.URL, time.Now,
|
addr, _ := startProxyWithGeoJS(t, app.URL, geojs.URL, map[string]string{
|
||||||
map[string]string{
|
trustedProxies: trustLocalhost,
|
||||||
trustedProxies: trustLocalhost,
|
metricsToken: token,
|
||||||
metricsToken: token,
|
deniedCountries: "kp",
|
||||||
deniedCountries: "kp",
|
})
|
||||||
})
|
|
||||||
|
|
||||||
// GeoJS fails, so the client counts as coming from an unknown country,
|
// GeoJS fails, so the client counts as coming from an unknown country,
|
||||||
// which SWWAF_DENIED_COUNTRIES does not refuse.
|
// which SWWAF_DENIED_COUNTRIES does not refuse.
|
||||||
@@ -332,16 +424,16 @@ func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) {
|
|||||||
req.Header.Set(forwardedFor, fromDE)
|
req.Header.Set(forwardedFor, fromDE)
|
||||||
wantStatus(t, do(t, req), http.StatusOK)
|
wantStatus(t, do(t, req), http.StatusOK)
|
||||||
|
|
||||||
// The client stops waiting for GeoJS once SWWAF_LOOKUP_TIMEOUT is up, so
|
// The client stops waiting for GeoJS after a second, so GeoJS's
|
||||||
// GeoJS's failure can come after its request has ended.
|
// failure can come after its request has ended.
|
||||||
deadline := time.Now().Add(waitLimit)
|
deadline := time.Now().Add(waitLimit)
|
||||||
metrics := scrape(t, server.Addr)
|
metrics := scrape(t, addr)
|
||||||
|
|
||||||
for metric(t, metrics, `smallwebwaf_geojs_failures_total{instance="app"}`) == 0 &&
|
for metric(t, metrics, `smallwebwaf_geojs_failures_total{instance="app"}`) == 0 &&
|
||||||
time.Now().Before(deadline) {
|
time.Now().Before(deadline) {
|
||||||
time.Sleep(pollInterval)
|
time.Sleep(pollInterval)
|
||||||
|
|
||||||
metrics = scrape(t, server.Addr)
|
metrics = scrape(t, addr)
|
||||||
}
|
}
|
||||||
|
|
||||||
wantMetric(t, metrics, `smallwebwaf_geojs_requests_total{instance="app"}`, 1)
|
wantMetric(t, metrics, `smallwebwaf_geojs_requests_total{instance="app"}`, 1)
|
||||||
@@ -360,8 +452,8 @@ func keptAnswer(addr, country string) lookup.Answer {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// scrape asks servewaf at addr for the metrics, with the token, and returns
|
// scrape asks smallwebwaf at addr for the metrics, with the token, and
|
||||||
// them.
|
// returns them.
|
||||||
func scrape(t *testing.T, addr string) string {
|
func scrape(t *testing.T, addr string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -376,12 +468,15 @@ func scrape(t *testing.T, addr string) string {
|
|||||||
return string(got.body)
|
return string(got.body)
|
||||||
}
|
}
|
||||||
|
|
||||||
// scrape asks for the metrics, with the token, and returns them.
|
// scrape asks for the metrics, with the token, from the client at from,
|
||||||
func (s *sender) scrape() string {
|
// and returns them.
|
||||||
|
func (s *sender) scrape(from string) string {
|
||||||
s.t.Helper()
|
s.t.Helper()
|
||||||
|
|
||||||
return string(s.adminRequest(bearer, http.MethodGet, proxy.MetricsPath, "",
|
_, metrics := s.requestWithHeader(from, proxy.MetricsPath, "Authorization: "+bearer,
|
||||||
http.StatusOK).body)
|
http.StatusOK, requestlog.ActionAdmin)
|
||||||
|
|
||||||
|
return metrics
|
||||||
}
|
}
|
||||||
|
|
||||||
// metric returns the value of series in metrics, which are in the
|
// metric returns the value of series in metrics, which are in the
|
||||||
@@ -429,3 +524,22 @@ func wantNoSeries(t *testing.T, metrics, series string) {
|
|||||||
t.Errorf("there is a series %s", series)
|
t.Errorf("there is a series %s", series)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// wantLimitHits checks that the metrics of smallwebwaf at addr count hits
|
||||||
|
// requests that passed the size or time limit of the setting limit, with
|
||||||
|
// no series for it when hits is 0.
|
||||||
|
func wantLimitHits(t *testing.T, addr, limit string, hits int) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
series := `smallwebwaf_size_and_time_limit_hits_total{instance="app",limit="` +
|
||||||
|
limit + `"}`
|
||||||
|
metrics := scrape(t, addr)
|
||||||
|
|
||||||
|
if hits == 0 {
|
||||||
|
wantNoSeries(t, metrics, series)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
wantMetric(t, metrics, series, float64(hits))
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,13 +1,17 @@
|
|||||||
package proxy_test
|
package proxy_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/proxy"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -72,6 +76,11 @@ func TestObserveModeForwardsWhatEnforceModeRefuses(t *testing.T) {
|
|||||||
// Passed to the app, which answered it.
|
// Passed to the app, which answered it.
|
||||||
line := s.get(sent.from, http.StatusOK, requestlog.ActionForward)
|
line := s.get(sent.from, http.StatusOK, requestlog.ActionForward)
|
||||||
wantWouldAction(t, line, sent.refusal)
|
wantWouldAction(t, line, sent.refusal)
|
||||||
|
|
||||||
|
if line.UpstreamStatus != http.StatusOK {
|
||||||
|
t.Errorf("log line has upstream_status %d, want 200",
|
||||||
|
line.UpstreamStatus)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -124,6 +133,62 @@ func TestObserveModeMakesNoBanAndKeepsTheBansItHas(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestObserveModeKeepsTheSizeLimitsAndTheToken(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const denied = "192.0.2.50" // in SWWAF_DENY_NETS
|
||||||
|
|
||||||
|
var calls atomic.Int32
|
||||||
|
|
||||||
|
app := startApp(t, func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
calls.Add(1)
|
||||||
|
answerWithSize(w, 2*sizeLimit, true)
|
||||||
|
})
|
||||||
|
addr, out := startProxy(t, app.URL, map[string]string{
|
||||||
|
mode: observe,
|
||||||
|
trustedProxies: trustLocalhost,
|
||||||
|
denyNets: denied,
|
||||||
|
requestMaxBytes: sizeLimitSetting,
|
||||||
|
responseMaxBytes: sizeLimitSetting,
|
||||||
|
metricsToken: token,
|
||||||
|
})
|
||||||
|
|
||||||
|
// SWWAF_DENY_NETS would refuse each request; instead a size limit or
|
||||||
|
// the missing token does.
|
||||||
|
for i, tc := range []struct {
|
||||||
|
method, path string
|
||||||
|
body io.Reader
|
||||||
|
status int
|
||||||
|
action string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
http.MethodPost, "/upload", bytes.NewReader(make([]byte, 2*sizeLimit)),
|
||||||
|
http.StatusRequestEntityTooLarge, requestlog.ActionTooLarge,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
http.MethodGet, "/download", http.NoBody,
|
||||||
|
http.StatusBadGateway, requestlog.ActionTooLarge,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
http.MethodGet, proxy.MetricsPath, http.NoBody,
|
||||||
|
http.StatusUnauthorized, requestlog.ActionAdmin,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
req := newRequest(t, tc.method, addr, tc.path, tc.body)
|
||||||
|
req.Header.Set(forwardedFor, denied)
|
||||||
|
wantStatus(t, do(t, req), tc.status)
|
||||||
|
|
||||||
|
line := out.requestLines(t, i+1)[i]
|
||||||
|
wantLine(t, line, tc.status, tc.action)
|
||||||
|
wantWouldAction(t, line, requestlog.ActionDenied)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The upload was refused before it reached the app.
|
||||||
|
if calls.Load() != 1 {
|
||||||
|
t.Errorf("the app was called %d times, want once", calls.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// wantWouldAction checks the request log line's would_action, and that a
|
// wantWouldAction checks the request log line's would_action, and that a
|
||||||
// line that should have none has no such field.
|
// line that should have none has no such field.
|
||||||
func wantWouldAction(t *testing.T, line logLine, want string) {
|
func wantWouldAction(t *testing.T, line logLine, want string) {
|
||||||
|
|||||||
@@ -0,0 +1,468 @@
|
|||||||
|
package proxy_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"reflect"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/proxy"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A request target with an escaped slash and space in its path, and a
|
||||||
|
// query with a parameter ReverseProxy cannot parse.
|
||||||
|
const (
|
||||||
|
rawPath = "/some%2Fpath/with%20space"
|
||||||
|
rawQuery = "b=2&a=1&bad=%zz;x"
|
||||||
|
)
|
||||||
|
|
||||||
|
// chunkSize is the size of each part of a body a test sends in parts.
|
||||||
|
const chunkSize = 1 << 10
|
||||||
|
|
||||||
|
var errNotStreamed = errors.New("the first part never reached the app")
|
||||||
|
|
||||||
|
// appSaw is what the app received.
|
||||||
|
type appSaw struct {
|
||||||
|
method string
|
||||||
|
target string
|
||||||
|
header http.Header
|
||||||
|
body []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPassesRequestAndAnswerUnchanged(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
requestBody := bytes.Repeat([]byte("request body "), 8000)
|
||||||
|
answerBody := bytes.Repeat([]byte("answer body "), 8000)
|
||||||
|
saw := make(chan appSaw, 1)
|
||||||
|
|
||||||
|
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
body, _ := io.ReadAll(r.Body)
|
||||||
|
saw <- appSaw{r.Method, r.RequestURI, r.Header.Clone(), body}
|
||||||
|
|
||||||
|
w.Header().Set("X-App", "yes")
|
||||||
|
w.Header().Add("Set-Cookie", "a=1")
|
||||||
|
w.Header().Add("Set-Cookie", "b=2")
|
||||||
|
w.WriteHeader(http.StatusTeapot)
|
||||||
|
_, _ = w.Write(answerBody)
|
||||||
|
})
|
||||||
|
addr, out := startProxy(t, app.URL, nil)
|
||||||
|
|
||||||
|
req := newRequest(t, http.MethodPatch, addr, rawPath+"?"+rawQuery,
|
||||||
|
bytes.NewReader(requestBody))
|
||||||
|
req.Header.Add("X-Test", "one")
|
||||||
|
req.Header.Add("X-Test", "two")
|
||||||
|
req.Header.Set("User-Agent", "test-agent")
|
||||||
|
|
||||||
|
got := do(t, req)
|
||||||
|
|
||||||
|
wantAppSaw(t, <-saw, requestBody)
|
||||||
|
wantAnswer(t, got, answerBody)
|
||||||
|
|
||||||
|
line := out.requestLine(t)
|
||||||
|
wantLine(t, line, http.StatusTeapot, requestlog.ActionForward)
|
||||||
|
wantRequestFields(t, line, addr, len(requestBody), len(answerBody))
|
||||||
|
}
|
||||||
|
|
||||||
|
// wantAppSaw checks that the app received the test's request unchanged.
|
||||||
|
func wantAppSaw(t *testing.T, saw appSaw, body []byte) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
if saw.method != http.MethodPatch || saw.target != rawPath+"?"+rawQuery {
|
||||||
|
t.Errorf("app saw %s %s, want %s %s", saw.method, saw.target,
|
||||||
|
http.MethodPatch, rawPath+"?"+rawQuery)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !slices.Equal(saw.header.Values("X-Test"), []string{"one", "two"}) {
|
||||||
|
t.Errorf("app saw X-Test %q", saw.header.Values("X-Test"))
|
||||||
|
}
|
||||||
|
|
||||||
|
if saw.header.Get("User-Agent") != "test-agent" {
|
||||||
|
t.Errorf("app saw User-Agent %q", saw.header.Get("User-Agent"))
|
||||||
|
}
|
||||||
|
|
||||||
|
if !bytes.Equal(saw.body, body) {
|
||||||
|
t.Errorf("app saw a body of %d bytes, want the %d sent",
|
||||||
|
len(saw.body), len(body))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// wantAnswer checks that the client received the app's answer unchanged.
|
||||||
|
func wantAnswer(t *testing.T, got answer, body []byte) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
wantStatus(t, got, http.StatusTeapot)
|
||||||
|
|
||||||
|
if got.header.Get("X-App") != "yes" {
|
||||||
|
t.Errorf("client got X-App %q", got.header.Get("X-App"))
|
||||||
|
}
|
||||||
|
|
||||||
|
if !slices.Equal(got.header.Values("Set-Cookie"), []string{"a=1", "b=2"}) {
|
||||||
|
t.Errorf("client got Set-Cookie %q", got.header.Values("Set-Cookie"))
|
||||||
|
}
|
||||||
|
|
||||||
|
if got.err != nil || !bytes.Equal(got.body, body) {
|
||||||
|
t.Errorf("client got %d bytes (%v), want the %d the app sent",
|
||||||
|
len(got.body), got.err, len(body))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// wantRequestFields checks the log line's fields about the request. Its
|
||||||
|
// time, its id and its timings are checked only for being there.
|
||||||
|
func wantRequestFields(t *testing.T, line logLine, host string, sent, received int) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
bytes := float64(sent + received)
|
||||||
|
|
||||||
|
want := withTimings(line, requestlog.Line{
|
||||||
|
Type: requestType, Time: line.Time, Instance: "app",
|
||||||
|
ClientIP: localhost, Method: http.MethodPatch, Scheme: plain, Host: host,
|
||||||
|
Path: rawPath, Query: rawQuery, Protocol: protocol,
|
||||||
|
Status: http.StatusTeapot, RequestBytes: int64(sent),
|
||||||
|
ResponseBytes: int64(received), UserAgent: "test-agent",
|
||||||
|
RequestID: line.RequestID, PeerIP: localhost, ClientGroup: localhost + "/32",
|
||||||
|
ContentLength: int64(sent), ResponseContentType: "text/plain; charset=utf-8",
|
||||||
|
UpstreamStatus: http.StatusTeapot, Action: requestlog.ActionForward,
|
||||||
|
Counts: ratelimit.Counts{
|
||||||
|
Minute: 1, Hour: 1, Day: 1,
|
||||||
|
MinuteBytes: bytes, HourBytes: bytes, DayBytes: bytes,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if !reflect.DeepEqual(line.Line, want) {
|
||||||
|
t.Errorf("log line\n%+v\nwant\n%+v", line.Line, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := time.Parse(time.RFC3339, line.Time)
|
||||||
|
if err != nil || line.RequestID == "" || line.DurationTotal <= 0 ||
|
||||||
|
line.DurationUpstreamTotal == nil || *line.DurationUpstreamTotal <= 0 {
|
||||||
|
t.Errorf("log line has time %q, request_id %q and durations %v and %v",
|
||||||
|
line.Time, line.RequestID, line.DurationTotal,
|
||||||
|
line.fields["duration_upstream_total"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStreamsTheRequestBody(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
chunk := bytes.Repeat([]byte("x"), chunkSize)
|
||||||
|
firstArrived := make(chan struct{})
|
||||||
|
|
||||||
|
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
first := make([]byte, len(chunk))
|
||||||
|
|
||||||
|
_, err := io.ReadFull(r.Body, first)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
close(firstArrived)
|
||||||
|
|
||||||
|
rest, _ := io.ReadAll(r.Body)
|
||||||
|
_, _ = w.Write(rest)
|
||||||
|
})
|
||||||
|
addr, _ := startProxy(t, app.URL, nil)
|
||||||
|
|
||||||
|
body, writer := io.Pipe()
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
_, _ = writer.Write(chunk)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-firstArrived:
|
||||||
|
_, _ = writer.Write(chunk)
|
||||||
|
_ = writer.Close()
|
||||||
|
case <-time.After(waitLimit):
|
||||||
|
_ = writer.CloseWithError(errNotStreamed)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
got := do(t, newRequest(t, http.MethodPost, addr, "/upload", body))
|
||||||
|
if got.err != nil || !bytes.Equal(got.body, chunk) {
|
||||||
|
t.Errorf("app read %d bytes after the first part (%v), want %d",
|
||||||
|
len(got.body), got.err, len(chunk))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStreamsTheAnswerBody(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
chunk := bytes.Repeat([]byte("y"), chunkSize)
|
||||||
|
firstArrived := make(chan struct{})
|
||||||
|
|
||||||
|
app := startApp(t, func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
_, _ = w.Write(chunk)
|
||||||
|
_ = http.NewResponseController(w).Flush()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-firstArrived:
|
||||||
|
_, _ = w.Write(chunk)
|
||||||
|
case <-time.After(waitLimit):
|
||||||
|
}
|
||||||
|
})
|
||||||
|
addr, _ := startProxy(t, app.URL, nil)
|
||||||
|
|
||||||
|
req := newRequest(t, http.MethodGet, addr, "/download", http.NoBody)
|
||||||
|
|
||||||
|
res, err := newClient(t).Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("request: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
first := make([]byte, len(chunk))
|
||||||
|
_, err = io.ReadFull(res.Body, first)
|
||||||
|
|
||||||
|
close(firstArrived)
|
||||||
|
|
||||||
|
got := readAnswer(res)
|
||||||
|
if err != nil || got.err != nil || !bytes.Equal(got.body, chunk) {
|
||||||
|
t.Errorf("client read %d bytes after the first part (%v, %v), want %d",
|
||||||
|
len(got.body), err, got.err, len(chunk))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUpgradedConnectionOutlastsTheTimeouts(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
app := startApp(t, echoAfterUpgrade)
|
||||||
|
addr, out := startProxy(t, app.URL, map[string]string{
|
||||||
|
clientRequestTimeout: shortTimeoutSetting,
|
||||||
|
clientResponseTimeout: shortTimeoutSetting,
|
||||||
|
upstreamRequestTimeout: shortTimeoutSetting,
|
||||||
|
upstreamResponseTimeout: shortTimeoutSetting,
|
||||||
|
})
|
||||||
|
|
||||||
|
conn := dial(t, addr)
|
||||||
|
send(t, conn, "GET /socket HTTP/1.1\r\nHost: app\r\n"+
|
||||||
|
"Connection: Upgrade\r\nUpgrade: websocket\r\n\r\n")
|
||||||
|
|
||||||
|
reader := bufio.NewReader(conn)
|
||||||
|
|
||||||
|
res, err := http.ReadResponse(reader, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read the answer to the upgrade: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
answered := time.Now()
|
||||||
|
_ = res.Body.Close()
|
||||||
|
|
||||||
|
if res.StatusCode != http.StatusSwitchingProtocols {
|
||||||
|
t.Fatalf("status %d, want %d", res.StatusCode, http.StatusSwitchingProtocols)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every timeout started before the upgrade was answered, the response
|
||||||
|
// timeouts last, at the end of the request: wait until just past
|
||||||
|
// shortTimeout after the answer was read, then use the connection.
|
||||||
|
time.Sleep(time.Until(answered.Add(shortTimeout + 100*time.Millisecond)))
|
||||||
|
send(t, conn, "still here\n")
|
||||||
|
|
||||||
|
echoed, err := reader.ReadString('\n')
|
||||||
|
if err != nil || echoed != "still here\n" {
|
||||||
|
t.Errorf("echo %q (%v), want %q", echoed, err, "still here\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = conn.Close()
|
||||||
|
|
||||||
|
wantLine(t, out.requestLine(t), http.StatusSwitchingProtocols,
|
||||||
|
requestlog.ActionForward)
|
||||||
|
}
|
||||||
|
|
||||||
|
// echoAfterUpgrade is an app that switches protocols on request, and then
|
||||||
|
// sends back each line it receives.
|
||||||
|
func echoAfterUpgrade(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Header.Get("Upgrade") != "websocket" {
|
||||||
|
http.Error(w, "not an upgrade", http.StatusBadRequest)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
conn, buffered, err := http.NewResponseController(w).Hijack()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
_ = conn.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
|
_, _ = buffered.WriteString("HTTP/1.1 101 Switching Protocols\r\n" +
|
||||||
|
"Connection: Upgrade\r\nUpgrade: websocket\r\n\r\n")
|
||||||
|
_ = buffered.Flush()
|
||||||
|
|
||||||
|
for {
|
||||||
|
line, err := buffered.ReadString('\n')
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
_, _ = buffered.WriteString(line)
|
||||||
|
_ = buffered.Flush()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestServerHasTheDefaultLimits(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cfg, err := config.FromEnvironment(func(string) (string, bool) { return "", false })
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("default settings: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
server := proxy.New(proxy.Params{
|
||||||
|
Config: cfg,
|
||||||
|
RequestLog: io.Discard,
|
||||||
|
ProcessLog: requestlog.NewProcessLogger(io.Discard, cfg.InstanceName, cfg.LogLevel),
|
||||||
|
})
|
||||||
|
|
||||||
|
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
|
||||||
|
server.IdleTimeout != 2*time.Minute || server.ReadHeaderTimeout != time.Minute {
|
||||||
|
t.Errorf("server listens on %q with header limit %d, idle time %s and "+
|
||||||
|
"header timeout %s", server.Addr, server.MaxHeaderBytes,
|
||||||
|
server.IdleTimeout, server.ReadHeaderTimeout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRefusesHeadersOverTheLimit(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
env map[string]string
|
||||||
|
limit int
|
||||||
|
}{
|
||||||
|
{"by default", nil, 32 << 10},
|
||||||
|
{"as set", map[string]string{clientHeaderMaxBytes: "8K"}, 8 << 10},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var calls atomic.Int32
|
||||||
|
|
||||||
|
app := startApp(t, func(http.ResponseWriter, *http.Request) {
|
||||||
|
calls.Add(1)
|
||||||
|
})
|
||||||
|
addr, _ := startProxy(t, app.URL, tc.env)
|
||||||
|
|
||||||
|
// size counts every byte of the request: the request line,
|
||||||
|
// the headers and the blank line that ends them.
|
||||||
|
const (
|
||||||
|
start = "GET / HTTP/1.1\r\nHost: app\r\nX-Large: "
|
||||||
|
end = "\r\n\r\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
for _, sent := range []struct{ size, want int }{
|
||||||
|
{tc.limit, http.StatusOK},
|
||||||
|
{tc.limit + 1, http.StatusRequestHeaderFieldsTooLarge},
|
||||||
|
} {
|
||||||
|
conn := dial(t, addr)
|
||||||
|
send(t, conn,
|
||||||
|
start+strings.Repeat("a", sent.size-len(start)-len(end))+end)
|
||||||
|
wantStatus(t, readResponse(t, conn), sent.want)
|
||||||
|
}
|
||||||
|
|
||||||
|
if calls.Load() != 1 {
|
||||||
|
t.Errorf("the app was called %d times, want once", calls.Load())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnswers502WhenTheAppCannotBeReached(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// No test can listen on port 1: listening on port 0 gets one from 32768 up.
|
||||||
|
addr, out := startProxy(t, "http://"+localhost+":1", nil)
|
||||||
|
|
||||||
|
wantStatus(t, get(t, addr, "/"), http.StatusBadGateway)
|
||||||
|
|
||||||
|
line := out.requestLine(t)
|
||||||
|
wantLine(t, line, http.StatusBadGateway, requestlog.ActionUpstreamError)
|
||||||
|
|
||||||
|
// There never was a connection to the app, nor an answer from it.
|
||||||
|
wantTimings(t, line, "duration_total", "duration_checks",
|
||||||
|
"duration_upstream_total")
|
||||||
|
|
||||||
|
logged := slices.ContainsFunc(out.lines(t), func(line map[string]any) bool {
|
||||||
|
return line["type"] == "process" && line["msg"] == "request to the app failed"
|
||||||
|
})
|
||||||
|
if !logged {
|
||||||
|
t.Errorf("no process line says the request to the app failed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLogLevelHoldsBackNoRequestLine(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// At error the warning that the request to the app failed is held back,
|
||||||
|
// and is written before the answer is.
|
||||||
|
addr, out := startProxy(t, "http://"+localhost+":1", map[string]string{
|
||||||
|
"SWWAF_LOG_LEVEL": "error",
|
||||||
|
})
|
||||||
|
|
||||||
|
wantStatus(t, get(t, addr, "/"), http.StatusBadGateway)
|
||||||
|
wantLine(t, out.requestLine(t), http.StatusBadGateway, requestlog.ActionUpstreamError)
|
||||||
|
|
||||||
|
for _, line := range out.lines(t) {
|
||||||
|
if line["type"] == "process" {
|
||||||
|
t.Errorf("process line %v, want none at error", line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLogsAnAnswerThatBrokeOff(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
app := startApp(t, func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
_, _ = io.WriteString(w, "the first part")
|
||||||
|
_ = http.NewResponseController(w).Flush()
|
||||||
|
|
||||||
|
panic(http.ErrAbortHandler) // drops the connection mid-answer
|
||||||
|
})
|
||||||
|
addr, out := startProxy(t, app.URL, nil)
|
||||||
|
|
||||||
|
got := get(t, addr, "/")
|
||||||
|
if string(got.body) != "the first part" || !errors.Is(got.err, io.ErrUnexpectedEOF) {
|
||||||
|
t.Errorf("client read %q (%v), want the first part cut off", got.body, got.err)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantLine(t, out.requestLine(t), http.StatusOK, requestlog.ActionUpstreamError)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLogsAClientThatWentAway(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
arrived := make(chan struct{})
|
||||||
|
|
||||||
|
app := startApp(t, func(_ http.ResponseWriter, r *http.Request) {
|
||||||
|
close(arrived)
|
||||||
|
<-r.Context().Done()
|
||||||
|
})
|
||||||
|
addr, out := startProxy(t, app.URL, nil)
|
||||||
|
|
||||||
|
conn := dial(t, addr)
|
||||||
|
send(t, conn, "GET /slow HTTP/1.1\r\nHost: app\r\n\r\n")
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-arrived:
|
||||||
|
case <-time.After(waitLimit):
|
||||||
|
t.Fatal("the request never reached the app")
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = conn.Close()
|
||||||
|
|
||||||
|
line := out.requestLine(t)
|
||||||
|
if !line.Aborted || line.Status != 0 || line.Action != requestlog.ActionForward {
|
||||||
|
t.Errorf("log line has aborted %v, status %d and action %q, "+
|
||||||
|
"want true, 0 and %q",
|
||||||
|
line.Aborted, line.Status, line.Action, requestlog.ActionForward)
|
||||||
|
}
|
||||||
|
}
|
||||||
+136
-50
@@ -1,19 +1,16 @@
|
|||||||
// Package proxy is servewaf's server. It runs the checks on the request each of
|
// Package proxy passes each request to the app and the app's answer back,
|
||||||
// rproxy's questions carries, and answers rproxy what to do with that request. It
|
// unchanged, within the size and time limits, and writes one request log
|
||||||
// takes rproxy's report once the visitor's answer has ended, which counts the
|
// line for each request.
|
||||||
// request's bytes and writes its last log line. It answers servewaf's own
|
|
||||||
// endpoints: the health check, the metrics, the bans and the clients. It passes
|
|
||||||
// nothing to an app: internal/rproxy does that.
|
|
||||||
package proxy
|
package proxy
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"io"
|
"io"
|
||||||
|
"log"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
lru "github.com/hashicorp/golang-lru/v2"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
@@ -22,16 +19,23 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
|
|
||||||
// adminPrefix starts the path of each of servewaf's own endpoints. The app never
|
// How smallwebwaf keeps connections to the app open between requests.
|
||||||
// sees a path under it.
|
const (
|
||||||
|
appIdleConns = 100
|
||||||
|
appIdleConnTimeout = 90 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// adminPrefix starts the path of every request for smallwebwaf itself,
|
||||||
|
// which never reaches the app.
|
||||||
const adminPrefix = "/_smallwebwaf/"
|
const adminPrefix = "/_smallwebwaf/"
|
||||||
|
|
||||||
// HealthPath is the health endpoint of servewaf and of rproxy, which the
|
// HealthPath is smallwebwaf's health endpoint, which the container's
|
||||||
// container's health check asks.
|
// health check asks.
|
||||||
const HealthPath = "/_smallwebwaf/healthz"
|
const HealthPath = "/_smallwebwaf/healthz"
|
||||||
|
|
||||||
// MetricsPath is where the metrics are, for a request that carries
|
// MetricsPath is where the metrics are, for a request that carries
|
||||||
@@ -42,15 +46,14 @@ const MetricsPath = "/_smallwebwaf/metrics"
|
|||||||
// a client's address, lifts them, with SWWAF_ADMIN_TOKEN.
|
// a client's address, lifts them, with SWWAF_ADMIN_TOKEN.
|
||||||
const BansPath = "/_smallwebwaf/bans"
|
const BansPath = "/_smallwebwaf/bans"
|
||||||
|
|
||||||
// ClientsPath is where an admin asks what servewaf knows of a client, by the
|
// ClientsPath is where an admin asks what smallwebwaf knows of a client,
|
||||||
// client's address after it, with SWWAF_ADMIN_TOKEN.
|
// by the client's address after it, with SWWAF_ADMIN_TOKEN.
|
||||||
const ClientsPath = "/_smallwebwaf/clients/"
|
const ClientsPath = "/_smallwebwaf/clients/"
|
||||||
|
|
||||||
// Params are what NewServeWAF needs.
|
// Params are what New needs.
|
||||||
type Params struct {
|
type Params struct {
|
||||||
Config *config.Config
|
Config *config.Config
|
||||||
// RequestLog receives two JSON lines per question, one at its decision and
|
// RequestLog receives one JSON line per request.
|
||||||
// one when its report comes, and one per request to servewaf's own endpoints.
|
|
||||||
RequestLog io.Writer
|
RequestLog io.Writer
|
||||||
// ProcessLog receives the process's own messages.
|
// ProcessLog receives the process's own messages.
|
||||||
ProcessLog *slog.Logger
|
ProcessLog *slog.Logger
|
||||||
@@ -70,7 +73,7 @@ type Params struct {
|
|||||||
// Rules are the rule files' rules, which each request is checked
|
// Rules are the rule files' rules, which each request is checked
|
||||||
// against.
|
// against.
|
||||||
Rules *rules.Files
|
Rules *rules.Files
|
||||||
// Alerts receive the alert for each ban servewaf makes or makes
|
// Alerts receive the alert for each ban the proxy makes or makes
|
||||||
// permanent, for each count over an anomaly threshold, for each request
|
// permanent, for each count over an anomaly threshold, for each request
|
||||||
// whose client a blocklist, the CrowdSec decision list, a DNSBL zone or
|
// whose client a blocklist, the CrowdSec decision list, a DNSBL zone or
|
||||||
// AbuseIPDB lists, for each request the Core Rule Set scores at or over
|
// AbuseIPDB lists, for each request the Core Rule Set scores at or over
|
||||||
@@ -80,10 +83,11 @@ type Params struct {
|
|||||||
Alerts *alerts.Queue
|
Alerts *alerts.Queue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Server is the server servewaf runs, with the parts whose state the state
|
// Server is the server smallwebwaf runs, with the parts of the proxy
|
||||||
// files keep, the lookup database, nil unless SWWAF_LOOKUP_SOURCE is file, the
|
// whose state the state files keep, the lookup database, nil unless
|
||||||
// lists fetched from URLs, which its Run fetches, the DNSBL zones' verdicts,
|
// SWWAF_LOOKUP_SOURCE is file, the lists fetched from URLs, which its Run
|
||||||
// AbuseIPDB's scores and checks spent, and the metrics.
|
// fetches, the DNSBL zones' verdicts, AbuseIPDB's scores and checks
|
||||||
|
// spent, and the metrics.
|
||||||
type Server struct {
|
type Server struct {
|
||||||
*http.Server
|
*http.Server
|
||||||
|
|
||||||
@@ -98,24 +102,24 @@ type Server struct {
|
|||||||
Metrics *metrics.Metrics
|
Metrics *metrics.Metrics
|
||||||
}
|
}
|
||||||
|
|
||||||
// newHandler returns the handler with what every request shares, as the
|
// New returns the server smallwebwaf runs: each request it reads passes
|
||||||
// settings in params make it.
|
// through the proxy. Go's server itself refuses a request line and
|
||||||
func newHandler(params Params) *handler {
|
// headers over SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES, with 431, closes a
|
||||||
|
// connection idle for SWWAF_CLIENT_IDLE_TIMEOUT, and applies
|
||||||
|
// SWWAF_CLIENT_REQUEST_TIMEOUT while the headers arrive; the proxy
|
||||||
|
// applies the timeouts and size limits from then on.
|
||||||
|
func New(params Params) *Server {
|
||||||
|
errorLog := slog.NewLogLogger(params.ProcessLog.Handler(), slog.LevelWarn)
|
||||||
m := metrics.New(params.Config.MetricsTopN, params.Config.InstanceName)
|
m := metrics.New(params.Config.MetricsTopN, params.Config.InstanceName)
|
||||||
lists, dnsbl, abuseIPDB := newReputation(params, m)
|
lists, dnsbl, abuseIPDB := newReputation(params, m)
|
||||||
|
|
||||||
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
|
|
||||||
if err != nil {
|
|
||||||
panic(err) // New fails only for a size below one
|
|
||||||
}
|
|
||||||
|
|
||||||
h := &handler{
|
h := &handler{
|
||||||
config: params.Config,
|
config: params.Config,
|
||||||
requestLog: params.RequestLog,
|
requestLog: params.RequestLog,
|
||||||
processLog: params.ProcessLog,
|
processLog: params.ProcessLog,
|
||||||
now: params.Now,
|
errorLog: errorLog,
|
||||||
metrics: m,
|
transport: newTransport(),
|
||||||
awaitingReport: awaitingReport,
|
now: params.Now,
|
||||||
|
metrics: m,
|
||||||
limiter: ratelimit.New(ratelimit.Limits{
|
limiter: ratelimit.New(ratelimit.Limits{
|
||||||
PerMinute: params.Config.RateLimitPerMinute,
|
PerMinute: params.Config.RateLimitPerMinute,
|
||||||
PerHour: params.Config.RateLimitPerHour,
|
PerHour: params.Config.RateLimitPerHour,
|
||||||
@@ -167,7 +171,29 @@ func newHandler(params Params) *handler {
|
|||||||
m.AddBansAndClients(h.ledger, h.limiter, params.Now)
|
m.AddBansAndClients(h.ledger, h.limiter, params.Now)
|
||||||
m.AddRules(params.Rules)
|
m.AddRules(params.Rules)
|
||||||
|
|
||||||
return h
|
return &Server{
|
||||||
|
Server: &http.Server{
|
||||||
|
Addr: params.Config.ListenAddr,
|
||||||
|
Handler: h,
|
||||||
|
ReadHeaderTimeout: params.Config.ClientRequestTimeout,
|
||||||
|
// Off is an IdleTimeout of 0, which Go's server replaces with
|
||||||
|
// ReadTimeout: no limit, as long as ReadTimeout stays unset.
|
||||||
|
IdleTimeout: params.Config.ClientIdleTimeout,
|
||||||
|
// Go's server reads 4 KiB past MaxHeaderBytes before it
|
||||||
|
// refuses, so the limit a client meets is the setting.
|
||||||
|
MaxHeaderBytes: int(params.Config.ClientRequestHeaderMaxBytes - 4<<10),
|
||||||
|
ErrorLog: errorLog,
|
||||||
|
},
|
||||||
|
Ledger: h.ledger,
|
||||||
|
Limiter: h.limiter,
|
||||||
|
GeoJS: h.geojs,
|
||||||
|
Anomalies: h.anomalies,
|
||||||
|
LookupFile: h.lookupFile,
|
||||||
|
Lists: h.lists,
|
||||||
|
DNSBL: h.dnsbl,
|
||||||
|
AbuseIPDB: h.abuseIPDB,
|
||||||
|
Metrics: m,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// newReputation returns the lists fetched from URLs, the DNSBL zones'
|
// newReputation returns the lists fetched from URLs, the DNSBL zones'
|
||||||
@@ -207,10 +233,7 @@ func newReputation(
|
|||||||
|
|
||||||
// newCoreRuleSet returns the Core Rule Set at SWWAF_WAF_PARANOIA_LEVEL,
|
// newCoreRuleSet returns the Core Rule Set at SWWAF_WAF_PARANOIA_LEVEL,
|
||||||
// without the rules SWWAF_WAF_DISABLED_RULES switches off, reading bodies
|
// without the rules SWWAF_WAF_DISABLED_RULES switches off, reading bodies
|
||||||
// up to SWWAF_WAF_BODY_LIMIT, or nil while SWWAF_WAF_MODE is off. It reads at
|
// up to SWWAF_WAF_BODY_LIMIT, or nil while SWWAF_WAF_MODE is off.
|
||||||
// most QuestionBodyMaxBytes less one byte: a question's body of
|
|
||||||
// QuestionBodyMaxBytes may be cut short, and the Core Rule Set does not read a
|
|
||||||
// JSON or XML body over its limit.
|
|
||||||
func newCoreRuleSet(cfg *config.Config) *waf.CoreRuleSet {
|
func newCoreRuleSet(cfg *config.Config) *waf.CoreRuleSet {
|
||||||
if cfg.WAFMode == config.WAFModeOff {
|
if cfg.WAFMode == config.WAFModeOff {
|
||||||
return nil
|
return nil
|
||||||
@@ -218,24 +241,26 @@ func newCoreRuleSet(cfg *config.Config) *waf.CoreRuleSet {
|
|||||||
|
|
||||||
coreRuleSet, err := waf.New(waf.Params{
|
coreRuleSet, err := waf.New(waf.Params{
|
||||||
ParanoiaLevel: cfg.WAFParanoiaLevel, DisabledRules: cfg.WAFDisabledRules,
|
ParanoiaLevel: cfg.WAFParanoiaLevel, DisabledRules: cfg.WAFDisabledRules,
|
||||||
BodyLimit: min(cfg.WAFBodyLimit, QuestionBodyMaxBytes-1),
|
BodyLimit: cfg.WAFBodyLimit,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// The Core Rule Set is built in, and the settings cannot break it:
|
// The Core Rule Set is built in, and the settings cannot break it:
|
||||||
// the paranoia level is from 1 to 4, and the id of no rule switches
|
// the paranoia level is from 1 to 4, the body limit at most 1G, and
|
||||||
// nothing off.
|
// the id of no rule switches nothing off.
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return coreRuleSet
|
return coreRuleSet
|
||||||
}
|
}
|
||||||
|
|
||||||
// handler is servewaf's handler. It holds what every request shares; what
|
// handler is the proxy. It holds what every request shares; what belongs
|
||||||
// belongs to one request is in a request.
|
// to one request is in a request.
|
||||||
type handler struct {
|
type handler struct {
|
||||||
config *config.Config
|
config *config.Config
|
||||||
requestLog io.Writer
|
requestLog io.Writer
|
||||||
processLog *slog.Logger
|
processLog *slog.Logger
|
||||||
|
errorLog *log.Logger
|
||||||
|
transport http.RoundTripper
|
||||||
now func() time.Time
|
now func() time.Time
|
||||||
metrics *metrics.Metrics
|
metrics *metrics.Metrics
|
||||||
limiter *ratelimit.Limiter
|
limiter *ratelimit.Limiter
|
||||||
@@ -249,7 +274,68 @@ type handler struct {
|
|||||||
rules *rules.Files
|
rules *rules.Files
|
||||||
coreRuleSet *waf.CoreRuleSet
|
coreRuleSet *waf.CoreRuleSet
|
||||||
alerts *alerts.Queue
|
alerts *alerts.Queue
|
||||||
// awaitingReport holds the decision on each answered question by its id
|
}
|
||||||
// until its report comes, and nil for a question still being handled.
|
|
||||||
awaitingReport *lru.Cache[string, *decision]
|
// newTransport returns what carries requests to the app. It never goes
|
||||||
|
// through a proxy named in the environment, and leaves the app's answers
|
||||||
|
// compressed or not as the app sent them.
|
||||||
|
func newTransport() *http.Transport {
|
||||||
|
return &http.Transport{
|
||||||
|
MaxIdleConns: appIdleConns,
|
||||||
|
MaxIdleConnsPerHost: appIdleConns,
|
||||||
|
IdleConnTimeout: appIdleConnTimeout,
|
||||||
|
DisableCompression: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ServeHTTP handles one request: it works out the client, runs the
|
||||||
|
// checks, passes the request to the app and the answer back within the
|
||||||
|
// limits, or answers it itself if it is for smallwebwaf, and writes the
|
||||||
|
// request's log line.
|
||||||
|
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
|
rq := h.newRequest(w, r)
|
||||||
|
defer rq.finish()
|
||||||
|
|
||||||
|
// The health endpoint is answered at once, before any check, so that
|
||||||
|
// a health checker is never refused. It does not ask the app.
|
||||||
|
if r.Method == http.MethodGet && r.URL.Path == HealthPath {
|
||||||
|
rq.line.Action = requestlog.ActionAdmin
|
||||||
|
// Set here rather than left to Go's server, which would set it only
|
||||||
|
// after the log line has taken the response's headers.
|
||||||
|
rq.out.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||||
|
_, _ = io.WriteString(rq.out, "ok\n")
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Once the request has ended, before its log line is written. The
|
||||||
|
// last deferred runs first: countRefusal before addToHistory, so that
|
||||||
|
// a broken error burst is in the client's history.
|
||||||
|
defer rq.addToHistory()
|
||||||
|
defer rq.countAnomalies()
|
||||||
|
defer rq.countRefusal()
|
||||||
|
|
||||||
|
refused := rq.check(r.Context())
|
||||||
|
rq.checked = time.Now()
|
||||||
|
|
||||||
|
if refused != nil {
|
||||||
|
rq.answer(*refused)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// A request for smallwebwaf itself is answered where another would be
|
||||||
|
// passed to the app, so that it goes through every check first.
|
||||||
|
if strings.HasPrefix(r.URL.Path, adminPrefix) {
|
||||||
|
rq.answerAdmin()
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Once the response has ended, before the request is added to its
|
||||||
|
// client's history. Deferred, since ReverseProxy panics to end a
|
||||||
|
// response it cannot finish.
|
||||||
|
defer rq.countBytes()
|
||||||
|
|
||||||
|
rq.forward(r.Context())
|
||||||
}
|
}
|
||||||
|
|||||||
+93
-211
@@ -5,7 +5,6 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
|
||||||
"maps"
|
"maps"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -20,76 +19,78 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/proxy"
|
"sneak.berlin/go/smallwebwaf/internal/proxy"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
// shortTimeout is what a test sets a timeout to, to see it run out.
|
||||||
|
// It starts before the test has set up its case, such as an upgrade
|
||||||
|
// or the app's buffers filling, so it is as long as the hold-up of the
|
||||||
|
// test process that wantTimedOut allows: a shorter one can run out
|
||||||
|
// first on a busy host.
|
||||||
|
shortTimeout = waitLimit / 2
|
||||||
|
// longTimeoutSetting is a timeout that does not run out in a test.
|
||||||
|
longTimeoutSetting = "1m"
|
||||||
// waitLimit bounds how long a test waits for what should happen.
|
// waitLimit bounds how long a test waits for what should happen.
|
||||||
waitLimit = 10 * time.Second
|
waitLimit = 10 * time.Second
|
||||||
// pollInterval is how often a test looks for a log line.
|
// pollInterval is how often a test looks for a log line.
|
||||||
pollInterval = 10 * time.Millisecond
|
pollInterval = 10 * time.Millisecond
|
||||||
// localhost is where every test server listens, and so the address rproxy
|
// localhost is where every test server listens, and so the address
|
||||||
// sees each test's requests come from.
|
// smallwebwaf sees each test's requests come from.
|
||||||
localhost = "127.0.0.1"
|
localhost = "127.0.0.1"
|
||||||
// requestType is the type that marks a request log line.
|
// requestType is the type that marks a request log line.
|
||||||
requestType = "request"
|
requestType = "request"
|
||||||
// protocol is the protocol of every question.
|
// protocol is the protocol of every test's requests.
|
||||||
protocol = "HTTP/1.1"
|
protocol = "HTTP/1.1"
|
||||||
// trustLocalhost trusts the address every test connects from, and a
|
|
||||||
// network for proxies in front of it.
|
|
||||||
trustLocalhost = localhost + "/32,10.0.0.0/8"
|
|
||||||
// appHost is the host every test asks for.
|
|
||||||
appHost = "app.example"
|
|
||||||
// client is the client's address, as a proxy names it.
|
|
||||||
client = "203.0.113.9"
|
|
||||||
// forwardedFor is the header that lists the client and its proxies,
|
|
||||||
// and forwardedProto the one that gives the scheme the client used.
|
|
||||||
forwardedFor = "X-Forwarded-For"
|
|
||||||
forwardedProto = "X-Forwarded-Proto"
|
|
||||||
// secure is the scheme a client reached traefik with, and plain the one
|
|
||||||
// rproxy serves.
|
|
||||||
secure = "https"
|
|
||||||
plain = "http"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// shortTimeoutSetting is shortTimeout as a setting's value.
|
||||||
|
//
|
||||||
|
//nolint:gochecknoglobals // a constant cannot call String
|
||||||
|
var shortTimeoutSetting = shortTimeout.String()
|
||||||
|
|
||||||
// The settings the tests set.
|
// The settings the tests set.
|
||||||
const (
|
const (
|
||||||
mode = "SWWAF_MODE"
|
clientRequestTimeout = "SWWAF_CLIENT_REQUEST_TIMEOUT"
|
||||||
trustedProxies = "SWRP_TRUSTED_PROXIES"
|
clientHeaderMaxBytes = "SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES"
|
||||||
ipv6GroupPrefix = "SWWAF_IPV6_GROUP_PREFIX"
|
clientIdleTimeout = "SWWAF_CLIENT_IDLE_TIMEOUT"
|
||||||
maxTrackedClients = "SWWAF_MAX_TRACKED_CLIENTS"
|
clientResponseTimeout = "SWWAF_CLIENT_RESPONSE_TIMEOUT"
|
||||||
allowNets = "SWWAF_ALLOW_NETS"
|
upstreamRequestTimeout = "SWWAF_UPSTREAM_REQUEST_TIMEOUT"
|
||||||
rateLimitExemptNets = "SWWAF_RATE_LIMIT_EXEMPT_NETS"
|
upstreamResponseTimeout = "SWWAF_UPSTREAM_RESPONSE_TIMEOUT"
|
||||||
denyNets = "SWWAF_DENY_NETS"
|
mode = "SWWAF_MODE"
|
||||||
rateLimitPerMinute = "SWWAF_RATE_LIMIT_PER_MINUTE"
|
requestMaxBytes = "SWWAF_REQUEST_MAX_BYTES"
|
||||||
rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY"
|
responseMaxBytes = "SWWAF_RESPONSE_MAX_BYTES"
|
||||||
rateLimitExemptPaths = "SWWAF_RATE_LIMIT_EXEMPT_PATHS"
|
trustedProxies = "SWWAF_TRUSTED_PROXIES"
|
||||||
lookupSource = "SWWAF_LOOKUP_SOURCE"
|
ipv6GroupPrefix = "SWWAF_IPV6_GROUP_PREFIX"
|
||||||
lookupDBPath = "SWWAF_LOOKUP_DB_PATH"
|
maxTrackedClients = "SWWAF_MAX_TRACKED_CLIENTS"
|
||||||
lookupTimeout = "SWWAF_LOOKUP_TIMEOUT"
|
allowNets = "SWWAF_ALLOW_NETS"
|
||||||
addLookupHeaders = "SWWAF_ADD_LOOKUP_HEADERS"
|
rateLimitExemptNets = "SWWAF_RATE_LIMIT_EXEMPT_NETS"
|
||||||
deniedCountries = "SWWAF_DENIED_COUNTRIES"
|
denyNets = "SWWAF_DENY_NETS"
|
||||||
allowedCountries = "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES"
|
rateLimitPerMinute = "SWWAF_RATE_LIMIT_PER_MINUTE"
|
||||||
banResponse = "SWWAF_BAN_RESPONSE"
|
rateLimitPerDay = "SWWAF_RATE_LIMIT_PER_DAY"
|
||||||
limitBanDuration = "SWWAF_LIMIT_BAN_DURATION"
|
rateLimitExemptPaths = "SWWAF_RATE_LIMIT_EXEMPT_PATHS"
|
||||||
limitBanRepeatWindow = "SWWAF_LIMIT_BAN_REPEAT_WINDOW"
|
lookupSource = "SWWAF_LOOKUP_SOURCE"
|
||||||
maxBanDuration = "SWWAF_MAX_BAN_DURATION"
|
lookupDBPath = "SWWAF_LOOKUP_DB_PATH"
|
||||||
maxBans = "SWWAF_MAX_BANS"
|
lookupTimeout = "SWWAF_LOOKUP_TIMEOUT"
|
||||||
banScopeV4Prefix = "SWWAF_BAN_SCOPE_V4_PREFIX"
|
addLookupHeaders = "SWWAF_ADD_LOOKUP_HEADERS"
|
||||||
instanceName = "SWWAF_INSTANCE_NAME"
|
deniedCountries = "SWWAF_DENIED_COUNTRIES"
|
||||||
logRequestHeaders = "SWWAF_LOG_REQUEST_HEADERS"
|
allowedCountries = "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES"
|
||||||
attackBanDuration = "SWWAF_ATTACK_BAN_DURATION"
|
banResponse = "SWWAF_BAN_RESPONSE"
|
||||||
rulesDir = "SWWAF_RULES_DIR"
|
limitBanDuration = "SWWAF_LIMIT_BAN_DURATION"
|
||||||
wafMode = "SWWAF_WAF_MODE"
|
limitBanRepeatWindow = "SWWAF_LIMIT_BAN_REPEAT_WINDOW"
|
||||||
|
maxBanDuration = "SWWAF_MAX_BAN_DURATION"
|
||||||
|
maxBans = "SWWAF_MAX_BANS"
|
||||||
|
banScopeV4Prefix = "SWWAF_BAN_SCOPE_V4_PREFIX"
|
||||||
|
instanceName = "SWWAF_INSTANCE_NAME"
|
||||||
|
logRequestHeaders = "SWWAF_LOG_REQUEST_HEADERS"
|
||||||
|
attackBanDuration = "SWWAF_ATTACK_BAN_DURATION"
|
||||||
|
rulesDir = "SWWAF_RULES_DIR"
|
||||||
|
wafMode = "SWWAF_WAF_MODE"
|
||||||
)
|
)
|
||||||
|
|
||||||
// off is the value that switches a setting off.
|
// off is the value that switches a setting off.
|
||||||
const off = "off"
|
const off = "off"
|
||||||
|
|
||||||
// on is the value that switches a setting that takes true or false on.
|
|
||||||
const on = "true"
|
|
||||||
|
|
||||||
// output collects what smallwebwaf writes on stdout.
|
// output collects what smallwebwaf writes on stdout.
|
||||||
type output struct {
|
type output struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
@@ -175,73 +176,6 @@ func (o *output) requestLine(t *testing.T) logLine {
|
|||||||
return o.requestLines(t, 1)[0]
|
return o.requestLines(t, 1)[0]
|
||||||
}
|
}
|
||||||
|
|
||||||
// endedLines waits for count requests that went through rproxy, or to servewaf's
|
|
||||||
// own endpoints, to have ended, and returns the last log line of each, in the
|
|
||||||
// order servewaf took them. A question's last line is the one its report writes.
|
|
||||||
func (o *output) endedLines(t *testing.T, count int) []logLine {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
deadline := time.Now().Add(waitLimit)
|
|
||||||
for time.Now().Before(deadline) {
|
|
||||||
found := o.lastLines(t)
|
|
||||||
|
|
||||||
ended := len(found) >= count
|
|
||||||
for _, line := range found[:min(count, len(found))] {
|
|
||||||
if line.ServeWAFLine == requestlog.ServeWAFLineDecision {
|
|
||||||
ended = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ended {
|
|
||||||
return found
|
|
||||||
}
|
|
||||||
|
|
||||||
time.Sleep(pollInterval)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Fatalf("fewer than %d requests have ended after %s", count, waitLimit)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// lastLines returns the last request log line written so far of each request,
|
|
||||||
// in the order servewaf took them.
|
|
||||||
func (o *output) lastLines(t *testing.T) []logLine {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var found []logLine
|
|
||||||
|
|
||||||
places := map[string]int{}
|
|
||||||
|
|
||||||
for _, fields := range o.lines(t) {
|
|
||||||
if fields["type"] != requestType {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
line := decodeLine(t, fields)
|
|
||||||
|
|
||||||
place, seen := places[line.RequestID]
|
|
||||||
if seen {
|
|
||||||
found[place] = line
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
places[line.RequestID] = len(found)
|
|
||||||
found = append(found, line)
|
|
||||||
}
|
|
||||||
|
|
||||||
return found
|
|
||||||
}
|
|
||||||
|
|
||||||
// endedLine waits for a test's one request to have ended, and returns its last
|
|
||||||
// log line.
|
|
||||||
func (o *output) endedLine(t *testing.T) logLine {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
return o.endedLines(t, 1)[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
// decodeLine reads a request log line's fields into a logLine.
|
// decodeLine reads a request log line's fields into a logLine.
|
||||||
func decodeLine(t *testing.T, fields map[string]any) logLine {
|
func decodeLine(t *testing.T, fields map[string]any) logLine {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
@@ -261,7 +195,7 @@ func decodeLine(t *testing.T, fields map[string]any) logLine {
|
|||||||
return line
|
return line
|
||||||
}
|
}
|
||||||
|
|
||||||
// startApp starts app as the app rproxy passes requests to.
|
// startApp starts app as the app smallwebwaf passes requests to.
|
||||||
func startApp(t *testing.T, app http.HandlerFunc) *httptest.Server {
|
func startApp(t *testing.T, app http.HandlerFunc) *httptest.Server {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -271,9 +205,9 @@ func startApp(t *testing.T, app http.HandlerFunc) *httptest.Server {
|
|||||||
return server
|
return server
|
||||||
}
|
}
|
||||||
|
|
||||||
// startProxy starts servewaf, and rproxy in front of the app at appURL, each with
|
// startProxy starts smallwebwaf in front of the app at appURL, with the
|
||||||
// the settings in env on top of the defaults, and returns where rproxy listens
|
// settings in env on top of the defaults, and returns where it listens and
|
||||||
// and what servewaf writes.
|
// what it writes.
|
||||||
func startProxy(t *testing.T, appURL string, env map[string]string) (string, *output) {
|
func startProxy(t *testing.T, appURL string, env map[string]string) (string, *output) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -293,10 +227,10 @@ func startProxyWithGeoJS(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// startProxyWithClock is startProxyWithGeoJS with requests counted and
|
// startProxyWithClock is startProxyWithGeoJS with requests counted and
|
||||||
// bans made by the time now tells, and returns servewaf's server as well, whose
|
// bans made by the time now tells, and returns the server as well. Unless
|
||||||
// Addr is where servewaf listens. Unless env sets SWWAF_RULES_DIR, it is an empty
|
// env sets SWWAF_RULES_DIR, it is an empty directory, of no rules, and
|
||||||
// directory, of no rules, and unless it sets SWWAF_INSTANCE_NAME, that is app,
|
// unless it sets SWWAF_INSTANCE_NAME, that is app, the label instance of
|
||||||
// the label instance of every metric.
|
// every metric.
|
||||||
func startProxyWithClock(
|
func startProxyWithClock(
|
||||||
t *testing.T, appURL, geojsURL string, now func() time.Time,
|
t *testing.T, appURL, geojsURL string, now func() time.Time,
|
||||||
env map[string]string,
|
env map[string]string,
|
||||||
@@ -309,70 +243,14 @@ func startProxyWithClock(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// startProxyWithAlerts is startProxyWithClock, and returns the queue of
|
// startProxyWithAlerts is startProxyWithClock, and returns the queue of
|
||||||
// the alerts servewaf raises as well, as newServeWAF makes them.
|
// the alerts the proxy raises as well, as newProxy makes them.
|
||||||
func startProxyWithAlerts(
|
func startProxyWithAlerts(
|
||||||
t *testing.T, appURL, geojsURL string, now func() time.Time,
|
t *testing.T, appURL, geojsURL string, now func() time.Time,
|
||||||
env map[string]string,
|
env map[string]string,
|
||||||
) (string, *output, *proxy.Server, *alerts.Queue) {
|
) (string, *output, *proxy.Server, *alerts.Queue) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
server, out, alertQueue := newServeWAF(t, geojsURL, now, env)
|
server, out, alertQueue := newProxy(t, appURL, geojsURL, now, env)
|
||||||
serve(t, server)
|
|
||||||
|
|
||||||
return startRProxy(t, server.Addr, appURL, env), out, server, alertQueue
|
|
||||||
}
|
|
||||||
|
|
||||||
// serve serves server on a port of its own until the test ends, and sets its
|
|
||||||
// Addr to where it listens.
|
|
||||||
func serve(t *testing.T, server *proxy.Server) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("listen: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
server.Addr = listener.Addr().String()
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
_ = server.Serve(listener)
|
|
||||||
}()
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
_ = server.Close()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// startRProxy starts rproxy in front of the app at appURL, asking the servewaf
|
|
||||||
// at serveWAFAddr, with the settings in env on top of the defaults, and returns
|
|
||||||
// where it listens. What it writes is dropped. SWRP_TIMEOUT_MS is a minute unless
|
|
||||||
// env sets it, so that on a busy host SWRP_DEFAULT_DISPOSITION does not decide.
|
|
||||||
func startRProxy(
|
|
||||||
t *testing.T, serveWAFAddr, appURL string, env map[string]string,
|
|
||||||
) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
settings := map[string]string{
|
|
||||||
"SWRP_WAF_BACKEND": "http://" + serveWAFAddr,
|
|
||||||
"SWRP_UPSTREAM_URL": appURL,
|
|
||||||
"SWRP_TIMEOUT_MS": "60000",
|
|
||||||
}
|
|
||||||
maps.Copy(settings, env)
|
|
||||||
|
|
||||||
cfg, err := config.RProxyFromEnvironment(func(name string) (string, bool) {
|
|
||||||
value, ok := settings[name]
|
|
||||||
|
|
||||||
return value, ok
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("settings %v: %v", settings, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
server := rproxy.New(rproxy.Params{
|
|
||||||
Config: cfg,
|
|
||||||
RequestLog: io.Discard,
|
|
||||||
ProcessLog: requestlog.NewProcessLogger(io.Discard, "rproxy", slog.LevelInfo),
|
|
||||||
})
|
|
||||||
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0")
|
listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -387,35 +265,28 @@ func startRProxy(
|
|||||||
_ = server.Close()
|
_ = server.Close()
|
||||||
})
|
})
|
||||||
|
|
||||||
return listener.Addr().String()
|
return listener.Addr().String(), out, server, alertQueue
|
||||||
}
|
}
|
||||||
|
|
||||||
// newServeWAF makes servewaf's server, without starting it, and returns it, what
|
// newProxy makes the server startProxyWithClock starts, without starting
|
||||||
// it writes, and the queue of the alerts it raises, as the settings in env make
|
// it, and returns it, what it writes, and the queue of the alerts the
|
||||||
// it. No alert is sent from the queue: they wait in it, for the test to look at.
|
// proxy raises, as the settings in env make it. No alert is sent from the
|
||||||
// With no geojsURL, there is no stand-in for GeoJS to look clients up at, and
|
// queue: they wait in it, for the test to look at. With no geojsURL, there
|
||||||
// SWWAF_LOOKUP_SOURCE is off unless env sets it. While it is file, the lookup
|
// is no stand-in for GeoJS to look clients up at, and SWWAF_LOOKUP_SOURCE
|
||||||
// database SWWAF_LOOKUP_DB_PATH names is read. Clients are checked with AbuseIPDB
|
// is off unless env sets it. While it is file, the lookup database
|
||||||
// at abuseIPDBURL while env sets SWWAF_ABUSEIPDB_KEY. SWWAF_WAF_MODE is off unless
|
// SWWAF_LOOKUP_DB_PATH names is read. Clients are checked with AbuseIPDB
|
||||||
// env sets it, so that only the tests of the Core Rule Set have their requests
|
// at abuseIPDBURL while env sets SWWAF_ABUSEIPDB_KEY. SWWAF_WAF_MODE is off
|
||||||
// inspected by it.
|
// unless env sets it, so that only the tests of the Core Rule Set have
|
||||||
func newServeWAF(
|
// their requests inspected by it.
|
||||||
t *testing.T, geojsURL string, now func() time.Time, env map[string]string,
|
func newProxy(
|
||||||
) (*proxy.Server, *output, *alerts.Queue) {
|
t *testing.T, appURL, geojsURL string, now func() time.Time,
|
||||||
t.Helper()
|
env map[string]string,
|
||||||
|
|
||||||
return newServeWAFWith(t, proxy.NewServeWAF, geojsURL, now, env)
|
|
||||||
}
|
|
||||||
|
|
||||||
// newServeWAFWith is newServeWAF for the server build makes.
|
|
||||||
func newServeWAFWith(
|
|
||||||
t *testing.T, build func(proxy.Params) *proxy.Server,
|
|
||||||
geojsURL string, now func() time.Time, env map[string]string,
|
|
||||||
) (*proxy.Server, *output, *alerts.Queue) {
|
) (*proxy.Server, *output, *alerts.Queue) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
settings := map[string]string{
|
settings := map[string]string{
|
||||||
rulesDir: t.TempDir(), instanceName: "app", wafMode: off,
|
"SWWAF_UPSTREAM_URL": appURL, rulesDir: t.TempDir(), instanceName: "app",
|
||||||
|
wafMode: off,
|
||||||
}
|
}
|
||||||
if geojsURL == "" {
|
if geojsURL == "" {
|
||||||
settings[lookupSource] = off
|
settings[lookupSource] = off
|
||||||
@@ -463,7 +334,7 @@ func newServeWAFWith(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
server := build(proxy.Params{
|
server := proxy.New(proxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: out,
|
RequestLog: out,
|
||||||
ProcessLog: processLog,
|
ProcessLog: processLog,
|
||||||
@@ -507,7 +378,7 @@ func readAnswer(res *http.Response) answer {
|
|||||||
return answer{status: res.StatusCode, header: res.Header, body: body, err: err}
|
return answer{status: res.StatusCode, header: res.Header, body: body, err: err}
|
||||||
}
|
}
|
||||||
|
|
||||||
// newRequest makes a request for path to addr, where rproxy or servewaf listens.
|
// newRequest makes a request for path to smallwebwaf at addr.
|
||||||
func newRequest(t *testing.T, method, addr, path string, body io.Reader) *http.Request {
|
func newRequest(t *testing.T, method, addr, path string, body io.Reader) *http.Request {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -531,15 +402,15 @@ func do(t *testing.T, req *http.Request) answer {
|
|||||||
return readAnswer(res)
|
return readAnswer(res)
|
||||||
}
|
}
|
||||||
|
|
||||||
// get sends a GET request for path to addr.
|
// get sends a GET request for path to smallwebwaf at addr.
|
||||||
func get(t *testing.T, addr, path string) answer {
|
func get(t *testing.T, addr, path string) answer {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
return do(t, newRequest(t, http.MethodGet, addr, path, http.NoBody))
|
return do(t, newRequest(t, http.MethodGet, addr, path, http.NoBody))
|
||||||
}
|
}
|
||||||
|
|
||||||
// dial opens a connection to addr, for requests the HTTP client cannot make,
|
// dial opens a connection to smallwebwaf at addr, for requests the HTTP
|
||||||
// such as one that stops sending halfway.
|
// client cannot make, such as one that stops sending halfway.
|
||||||
func dial(t *testing.T, addr string) net.Conn {
|
func dial(t *testing.T, addr string) net.Conn {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -600,3 +471,14 @@ func wantStatus(t *testing.T, got answer, status int) {
|
|||||||
t.Errorf("status %d, want %d", got.status, status)
|
t.Errorf("status %d, want %d", got.status, status)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// wantTimedOut checks that what began at start ended once shortTimeout
|
||||||
|
// had run out, and not much later.
|
||||||
|
func wantTimedOut(t *testing.T, start time.Time) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
took := time.Since(start)
|
||||||
|
if took < shortTimeout || took > shortTimeout+waitLimit/2 {
|
||||||
|
t.Errorf("took %s, want %s", took, shortTimeout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -51,7 +51,7 @@ func TestRateLimitRefusesBeforeTheApp(t *testing.T) {
|
|||||||
req.Header.Set(forwardedFor, sent.client)
|
req.Header.Set(forwardedFor, sent.client)
|
||||||
wantStatus(t, do(t, req), sent.want)
|
wantStatus(t, do(t, req), sent.want)
|
||||||
|
|
||||||
line := out.endedLines(t, i+1)[i]
|
line := out.requestLines(t, i+1)[i]
|
||||||
if line.ClientIP != sent.logged {
|
if line.ClientIP != sent.logged {
|
||||||
t.Errorf("log line has client_ip %q, want %q", line.ClientIP, sent.logged)
|
t.Errorf("log line has client_ip %q, want %q", line.ClientIP, sent.logged)
|
||||||
}
|
}
|
||||||
@@ -185,9 +185,6 @@ func TestRateLimitCountsPathsThatAreNotExempt(t *testing.T) {
|
|||||||
"/assets/x%2Fy",
|
"/assets/x%2Fy",
|
||||||
"/assets/x%2fy",
|
"/assets/x%2fy",
|
||||||
`/assets/x\y`,
|
`/assets/x\y`,
|
||||||
// The same with a character Go escapes, |, for which Go's
|
|
||||||
// URL.EscapedPath gives /assets/x/y%7Cz, with no encoded slash.
|
|
||||||
"/assets/x%2Fy|z",
|
|
||||||
} {
|
} {
|
||||||
t.Run(sent, func(t *testing.T) {
|
t.Run(sent, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|||||||
@@ -110,7 +110,6 @@ func (rq *request) noteHit(hit bans.ReputationHit, reason string) {
|
|||||||
rq.h.metrics.ReputationHit(hit.Source)
|
rq.h.metrics.ReputationHit(hit.Source)
|
||||||
rq.h.alerts.Raise(alerts.Alert{
|
rq.h.alerts.Raise(alerts.Alert{
|
||||||
Event: alerts.EventReputationHit,
|
Event: alerts.EventReputationHit,
|
||||||
Host: rq.site,
|
|
||||||
Client: rq.client,
|
Client: rq.client,
|
||||||
Netblock: rq.h.clientGroup(rq.client),
|
Netblock: rq.h.clientGroup(rq.client),
|
||||||
ASN: rq.line.ASN,
|
ASN: rq.line.ASN,
|
||||||
|
|||||||
@@ -272,7 +272,6 @@ func TestEachBlocklistThatListsAClientRaisesAnAlertOncePerCooldownAndIsCounted(
|
|||||||
Instance: alertInstance,
|
Instance: alertInstance,
|
||||||
Time: clk.Now(),
|
Time: clk.Now(),
|
||||||
Event: alerts.EventReputationHit,
|
Event: alerts.EventReputationHit,
|
||||||
Host: appHost,
|
|
||||||
Client: netip.MustParseAddr(fromDE),
|
Client: netip.MustParseAddr(fromDE),
|
||||||
Netblock: netip.MustParsePrefix(fromDE + "/32"),
|
Netblock: netip.MustParsePrefix(fromDE + "/32"),
|
||||||
ASN: asnDE,
|
ASN: asnDE,
|
||||||
@@ -290,7 +289,7 @@ func TestEachBlocklistThatListsAClientRaisesAnAlertOncePerCooldownAndIsCounted(
|
|||||||
|
|
||||||
// Each list's hits, none of its fetches failed, and when its copy was
|
// Each list's hits, none of its fetches failed, and when its copy was
|
||||||
// fetched, 0 for the one without.
|
// fetched, 0 for the one without.
|
||||||
metrics := s.scrape()
|
metrics := s.scrape(unplaced)
|
||||||
fetched := float64(listsFetched().Unix())
|
fetched := float64(listsFetched().Unix())
|
||||||
|
|
||||||
for listURL, want := range map[string]struct{ hits, fetched float64 }{
|
for listURL, want := range map[string]struct{ hits, fetched float64 }{
|
||||||
@@ -514,7 +513,6 @@ func TestEachZoneThatListsAClientRaisesAnAlertOncePerCooldownAndIsCounted(
|
|||||||
Instance: alertInstance,
|
Instance: alertInstance,
|
||||||
Time: clk.Now(),
|
Time: clk.Now(),
|
||||||
Event: alerts.EventReputationHit,
|
Event: alerts.EventReputationHit,
|
||||||
Host: appHost,
|
|
||||||
Client: netip.MustParseAddr(fromDE),
|
Client: netip.MustParseAddr(fromDE),
|
||||||
Netblock: netip.MustParsePrefix(fromDE + "/32"),
|
Netblock: netip.MustParsePrefix(fromDE + "/32"),
|
||||||
ASN: asnDE,
|
ASN: asnDE,
|
||||||
@@ -532,7 +530,7 @@ func TestEachZoneThatListsAClientRaisesAnAlertOncePerCooldownAndIsCounted(
|
|||||||
|
|
||||||
// Each zone's hits, and its queries and their failures, none, since
|
// Each zone's hits, and its queries and their failures, none, since
|
||||||
// every client had its verdicts.
|
// every client had its verdicts.
|
||||||
metrics := s.scrape()
|
metrics := s.scrape(unplaced)
|
||||||
|
|
||||||
for _, zone := range []string{dnsblZone, otherZone} {
|
for _, zone := range []string{dnsblZone, otherZone} {
|
||||||
labels := `{instance="` + alertInstance + `",source="` + zone + `"}`
|
labels := `{instance="` + alertInstance + `",source="` + zone + `"}`
|
||||||
@@ -566,7 +564,7 @@ func TestZoneKeyIsMaskedInTheLogTheAlertAndTheMetrics(t *testing.T) {
|
|||||||
masked)
|
masked)
|
||||||
}
|
}
|
||||||
|
|
||||||
metrics := s.scrape()
|
metrics := s.scrape(unplaced)
|
||||||
wantMetric(t, metrics, `smallwebwaf_reputation_hits_total{instance="`+
|
wantMetric(t, metrics, `smallwebwaf_reputation_hits_total{instance="`+
|
||||||
alertInstance+`",source="`+masked+`"}`, 1)
|
alertInstance+`",source="`+masked+`"}`, 1)
|
||||||
|
|
||||||
@@ -739,6 +737,10 @@ func TestClientRefusedForAnOffenceIsCheckedWithAbuseIPDBAtItsNextRequest(t *test
|
|||||||
"a trap path", "/xmlrpc.php", http.StatusForbidden, requestlog.ActionBanned,
|
"a trap path", "/xmlrpc.php", http.StatusForbidden, requestlog.ActionBanned,
|
||||||
ratelimit.Offences{Attack: 1},
|
ratelimit.Offences{Attack: 1},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"a missing token", proxy.MetricsPath, http.StatusUnauthorized,
|
||||||
|
requestlog.ActionAdmin, ratelimit.Offences{TokenRefused: 1},
|
||||||
|
},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
@@ -746,7 +748,7 @@ func TestClientRefusedForAnOffenceIsCheckedWithAbuseIPDBAtItsNextRequest(t *test
|
|||||||
s, clk, server := startWithClock(t, "", map[string]string{
|
s, clk, server := startWithClock(t, "", map[string]string{
|
||||||
abuseIPDBKey: accountKey, reputationAction: actionLog,
|
abuseIPDBKey: accountKey, reputationAction: actionLog,
|
||||||
rulesDir: writeRules(t, testRules), attackBanDuration: "1h",
|
rulesDir: writeRules(t, testRules), attackBanDuration: "1h",
|
||||||
trapPaths: trapPathList, wafMode: block,
|
trapPaths: trapPathList, metricsToken: token, wafMode: block,
|
||||||
})
|
})
|
||||||
|
|
||||||
s.request(client, tc.path, tc.status, tc.action)
|
s.request(client, tc.path, tc.status, tc.action)
|
||||||
@@ -860,7 +862,7 @@ func TestAbuseIPDBHitRaisesAnAlertWithTheScoreOncePerCooldownAndIsCounted(
|
|||||||
|
|
||||||
// The hits, and the checks, none, since no client committed an
|
// The hits, and the checks, none, since no client committed an
|
||||||
// offence, so that the whole budget is left.
|
// offence, so that the whole budget is left.
|
||||||
metrics := s.scrape()
|
metrics := s.scrape(unplaced)
|
||||||
labels := `{instance="` + alertInstance + `",source="` + abuseipdb + `"}`
|
labels := `{instance="` + alertInstance + `",source="` + abuseipdb + `"}`
|
||||||
|
|
||||||
wantMetric(t, metrics, "smallwebwaf_reputation_hits_total"+labels, 2)
|
wantMetric(t, metrics, "smallwebwaf_reputation_hits_total"+labels, 2)
|
||||||
@@ -887,7 +889,7 @@ func TestWithoutAnAbuseIPDBKeyNoClientIsCheckedNorAScoreUsed(t *testing.T) {
|
|||||||
wantReputation(t, s.get(fromDE, http.StatusOK, forward))
|
wantReputation(t, s.get(fromDE, http.StatusOK, forward))
|
||||||
wantAbuseIPDBChecks(t, server, 0)
|
wantAbuseIPDBChecks(t, server, 0)
|
||||||
|
|
||||||
wantNoSeries(t, s.scrape(), `smallwebwaf_reputation_daily_budget_remaining{`+
|
wantNoSeries(t, s.scrape(unplaced), `smallwebwaf_reputation_daily_budget_remaining{`+
|
||||||
`instance="`+alertInstance+`",source="`+abuseipdb+`"}`)
|
`instance="`+alertInstance+`",source="`+abuseipdb+`"}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+549
-169
@@ -2,10 +2,18 @@ package proxy
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/http/httptrace"
|
||||||
|
"net/http/httputil"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
||||||
@@ -14,39 +22,42 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// refusal is servewaf refusing a request: the status the visitor is to get, 0
|
// flushAfterEachWrite has ReverseProxy pass on each part of the app's
|
||||||
// to close its connection without an answer, and the action the log line names.
|
// answer as soon as it arrives.
|
||||||
|
const flushAfterEachWrite time.Duration = -1
|
||||||
|
|
||||||
|
// refusal is smallwebwaf refusing a request, or refusing to go on with it:
|
||||||
|
// the status the client is answered if the response has not started yet,
|
||||||
|
// 0 to close the connection without an answer, the action the log line
|
||||||
|
// names, and the setting whose size or time limit the request passed, if
|
||||||
|
// that is why.
|
||||||
type refusal struct {
|
type refusal struct {
|
||||||
status int
|
status int
|
||||||
action string
|
action string
|
||||||
|
limit string
|
||||||
}
|
}
|
||||||
|
|
||||||
// decision is what the checks note of a request: its log line, and what
|
// request is one request on its way through smallwebwaf, from the moment
|
||||||
// counting the request needs once it has ended. servewaf keeps a copy of it
|
// its headers have been read to its log line.
|
||||||
// until the request's report comes. It holds no credentials, since the log line
|
type request struct {
|
||||||
// never does, and neither the request's objects nor their bodies.
|
h *handler
|
||||||
type decision struct {
|
in *http.Request
|
||||||
line requestlog.Line
|
// rc sets the deadlines of the connection to the client.
|
||||||
client netip.Addr
|
rc *http.ResponseController
|
||||||
// requestURI is the path with the query, as a ban's notes keep it.
|
out *responseWriter
|
||||||
requestURI string
|
body *requestBody // nil for a request without a body
|
||||||
// site is, for a question, the host the visitor asked for, which the
|
line requestlog.Line
|
||||||
// alerts name, since one servewaf serves many sites.
|
|
||||||
site string
|
client netip.Addr
|
||||||
// allowed is true for a question answered with DispositionAllow.
|
peer netip.Addr
|
||||||
allowed bool
|
peerTrusted bool
|
||||||
// lookedUp is true once the client's AS number and country have been
|
// lookedUp is true once the client's AS number and country have been
|
||||||
// looked up, whether or not an answer was there, and lookupAnswer is
|
// looked up, whether or not an answer was there, and lookupAnswer is
|
||||||
// what the lookup gave then, the zero Answer while GeoJS had given none.
|
// what the lookup gave then, the zero Answer while GeoJS had given none.
|
||||||
lookedUp bool
|
lookedUp bool
|
||||||
lookupAnswer lookup.Answer
|
lookupAnswer lookup.Answer
|
||||||
// skippedLookupWait is true for a request that did not wait for GeoJS and
|
|
||||||
// found no answer kept. SWWAF_UNKNOWN_LIMIT_PERCENT does not apply to it,
|
|
||||||
// so that not waiting never lowers its client's limits.
|
|
||||||
skippedLookupWait bool
|
|
||||||
// counted is true for a request the rate limits counted, whose bytes
|
// counted is true for a request the rate limits counted, whose bytes
|
||||||
// the byte limits count once it has ended. limitPercent and
|
// the byte limits count once it has ended. limitPercent and
|
||||||
// bytesPercent are then its client's limit percentages for the rate
|
// bytesPercent are then its client's limit percentages for the rate
|
||||||
@@ -54,80 +65,93 @@ type decision struct {
|
|||||||
counted bool
|
counted bool
|
||||||
limitPercent, bytesPercent percentage
|
limitPercent, bytesPercent percentage
|
||||||
// attack is true for a request that matched a ban rule or asked for a
|
// attack is true for a request that matched a ban rule or asked for a
|
||||||
// trap path, ruleBlocked for one a block rule refused, and wafBlocked for
|
// trap path, ruleBlocked for one a block rule refused, wafBlocked for
|
||||||
// one the Core Rule Set refused, each an offence its client's history
|
// one the Core Rule Set refused, and tokenRefused for one refused for a
|
||||||
// counts.
|
// missing or wrong token, each an offence its client's history counts.
|
||||||
attack, ruleBlocked, wafBlocked bool
|
attack, ruleBlocked, wafBlocked, tokenRefused bool
|
||||||
// reputation is the reputation sources that list the client, for the
|
|
||||||
// notes of a ban the request makes.
|
|
||||||
reputation []bans.ReputationHit
|
|
||||||
}
|
|
||||||
|
|
||||||
// request is one request to servewaf, from the moment its headers have been
|
|
||||||
// read to its log line.
|
|
||||||
type request struct {
|
|
||||||
decision
|
|
||||||
|
|
||||||
h *handler
|
|
||||||
in *http.Request
|
|
||||||
out *responseWriter
|
|
||||||
|
|
||||||
// blocklisted is true once a blocklist is found to list the client,
|
// blocklisted is true once a blocklist is found to list the client,
|
||||||
// dnsblListed once a DNSBL zone's verdict is, and abuseIPDBHit once
|
// dnsblListed once a DNSBL zone's verdict is, and abuseIPDBHit once
|
||||||
// AbuseIPDB's score of it is a hit.
|
// AbuseIPDB's score of it is a hit.
|
||||||
blocklisted, dnsblListed, abuseIPDBHit bool
|
blocklisted, dnsblListed, abuseIPDBHit bool
|
||||||
start time.Time
|
// reputation is the reputation sources that list the client, for the
|
||||||
// checked is when the checks were done.
|
// notes of a ban the request makes.
|
||||||
checked time.Time
|
reputation []bans.ReputationHit
|
||||||
// refused is what the request was refused with, nil if it was not.
|
start time.Time
|
||||||
refused *refusal
|
// checked is when the checks were done, and upstreamStart when the
|
||||||
// requestBytes is how many bytes of the request's body have been read.
|
// request was handed to the app.
|
||||||
requestBytes int64
|
checked time.Time
|
||||||
|
upstreamStart time.Time
|
||||||
|
// cancel ends the request to the app.
|
||||||
|
cancel context.CancelFunc
|
||||||
|
// refused is the first refusal, from whichever goroutine meets it.
|
||||||
|
refused atomic.Pointer[refusal]
|
||||||
|
// complete is true once the app's whole answer has been passed on.
|
||||||
|
complete bool
|
||||||
|
// upgraded is the connection to the app once the app has switched
|
||||||
|
// protocols, as for a WebSocket, and nil otherwise.
|
||||||
|
upgraded *upgradedConn
|
||||||
|
|
||||||
|
// mu guards what follows. The timeouts run on goroutines of their
|
||||||
|
// own, and the transport starts and stops them, and notes the times
|
||||||
|
// below, from its own; once timersStopped is set, none of the timeouts
|
||||||
|
// acts any more.
|
||||||
|
mu sync.Mutex
|
||||||
|
timersStopped bool
|
||||||
|
clientRequestTimer *time.Timer
|
||||||
|
upstreamRequestTimer *time.Timer
|
||||||
|
upstreamResponseTimer *time.Timer
|
||||||
|
// connected is when there was a connection to the app, requestSent
|
||||||
|
// when the app had been sent the whole request, and answerStarted
|
||||||
|
// when the first byte of its answer arrived.
|
||||||
|
connected time.Time
|
||||||
|
requestSent time.Time
|
||||||
|
answerStarted time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// startRequest starts handling r from client, which came through peer: it
|
// newRequest starts handling r: it notes the time, counts the request as
|
||||||
// notes the time, counts the request as under way, and starts the log line
|
// under way, works out the client, and starts the log line with what is
|
||||||
// with what is known of the request. isQuestion says whether r is a question,
|
// known of the request.
|
||||||
// whose headers give the request's id and scheme.
|
func (h *handler) newRequest(w http.ResponseWriter, r *http.Request) *request {
|
||||||
func (h *handler) startRequest(
|
|
||||||
w http.ResponseWriter, r *http.Request, client, peer netip.Addr, isQuestion bool,
|
|
||||||
) *request {
|
|
||||||
h.metrics.RequestStarted()
|
h.metrics.RequestStarted()
|
||||||
|
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
|
peer := peerAddress(r)
|
||||||
|
trusted := h.config.TrustedProxies
|
||||||
|
peerTrusted := isInside(peer, trusted)
|
||||||
forwardedFor := r.Header.Values("X-Forwarded-For")
|
forwardedFor := r.Header.Values("X-Forwarded-For")
|
||||||
|
client := clientAddress(peer, forwardedFor, trusted)
|
||||||
|
|
||||||
rq := &request{
|
rq := &request{
|
||||||
decision: decision{
|
h: h,
|
||||||
client: client,
|
in: r,
|
||||||
requestURI: r.URL.RequestURI(),
|
rc: http.NewResponseController(w),
|
||||||
line: requestlog.Line{
|
out: &responseWriter{ResponseWriter: w},
|
||||||
Time: requestlog.FormatTime(start),
|
client: client,
|
||||||
Instance: h.config.InstanceName,
|
peer: peer,
|
||||||
ClientIP: client.String(),
|
peerTrusted: peerTrusted,
|
||||||
Method: r.Method,
|
start: start,
|
||||||
Scheme: scheme(r, isQuestion),
|
line: requestlog.Line{
|
||||||
Host: r.Host,
|
Time: requestlog.FormatTime(start),
|
||||||
Path: rules.Path(r),
|
Instance: h.config.InstanceName,
|
||||||
Query: r.URL.RawQuery,
|
ClientIP: client.String(),
|
||||||
Protocol: r.Proto,
|
Method: r.Method,
|
||||||
Referer: r.Referer(),
|
Scheme: scheme(r, peerTrusted),
|
||||||
UserAgent: r.UserAgent(),
|
Host: r.Host,
|
||||||
RequestID: requestID(r, isQuestion),
|
Path: r.URL.EscapedPath(),
|
||||||
PeerIP: peer.String(),
|
Query: r.URL.RawQuery,
|
||||||
ForwardedFor: strings.Join(forwardedFor, ", "),
|
Protocol: r.Proto,
|
||||||
ClientGroup: h.clientGroup(client).String(),
|
Referer: r.Referer(),
|
||||||
ContentType: r.Header.Get("Content-Type"),
|
UserAgent: r.UserAgent(),
|
||||||
RequestHeaders: requestHeaders(r, h.config.LogRequestHeaders),
|
RequestID: requestID(r, peerTrusted),
|
||||||
HasAuthorization: len(r.Header.Values("Authorization")) > 0,
|
PeerIP: peer.String(),
|
||||||
HasCookie: len(r.Header.Values("Cookie")) > 0,
|
ForwardedFor: strings.Join(forwardedFor, ", "),
|
||||||
Action: requestlog.ActionForward,
|
ClientGroup: h.clientGroup(client).String(),
|
||||||
},
|
ContentType: r.Header.Get("Content-Type"),
|
||||||
|
RequestHeaders: requestHeaders(r, h.config.LogRequestHeaders),
|
||||||
|
HasAuthorization: len(r.Header.Values("Authorization")) > 0,
|
||||||
|
HasCookie: len(r.Header.Values("Cookie")) > 0,
|
||||||
|
Action: requestlog.ActionForward,
|
||||||
},
|
},
|
||||||
h: h,
|
|
||||||
in: r,
|
|
||||||
out: &responseWriter{ResponseWriter: w},
|
|
||||||
start: start,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// A length of -1 is a body whose length was not announced.
|
// A length of -1 is a body whose length was not announced.
|
||||||
@@ -135,6 +159,10 @@ func (h *handler) startRequest(
|
|||||||
rq.line.ContentLength = r.ContentLength
|
rq.line.ContentLength = r.ContentLength
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if r.Body != http.NoBody {
|
||||||
|
rq.body = &requestBody{body: limitBody(r.Body, h.config.RequestMaxBytes), rq: rq}
|
||||||
|
}
|
||||||
|
|
||||||
return rq
|
return rq
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -159,28 +187,45 @@ func requestHeaders(r *http.Request, names []string) map[string]string {
|
|||||||
return headers
|
return headers
|
||||||
}
|
}
|
||||||
|
|
||||||
// check is the one place where a request can be refused. It returns nil to let
|
// check is the one place where a request can be refused once its client
|
||||||
// the request through. The checks of checkClient decide: a request they refuse
|
// is known, before anything reaches the app, and before its body is read,
|
||||||
// is answered with SWWAF_BAN_RESPONSE, 403 for a block rule or the Core Rule Set,
|
// but for the part the Core Rule Set reads. It returns nil to let the
|
||||||
// or 431 for the header limits. In observe mode nothing is refused, and the log
|
// request through. The checks of checkClient come first, answered with
|
||||||
// line names what enforce mode would have done. ctx is the request's own context.
|
// SWWAF_BAN_RESPONSE, or 403 for a block rule or the Core Rule Set, and
|
||||||
|
// then the size limit, so that a request the rate limits count is counted
|
||||||
|
// even when it is refused for its size. In observe mode a request
|
||||||
|
// checkClient refuses goes on to the size limit like any other. A size or
|
||||||
|
// time limit the Core Rule Set's reading of the body meets ends the
|
||||||
|
// request in either mode. ctx is the request's own context.
|
||||||
func (rq *request) check(ctx context.Context) *refusal {
|
func (rq *request) check(ctx context.Context) *refusal {
|
||||||
action := rq.checkClient(ctx)
|
action := rq.checkClient(ctx)
|
||||||
|
|
||||||
|
refused := rq.refused.Load()
|
||||||
|
if refused != nil {
|
||||||
|
return refused
|
||||||
|
}
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case action == "":
|
case action == "":
|
||||||
return nil
|
|
||||||
case rq.h.config.Observe:
|
case rq.h.config.Observe:
|
||||||
|
// The log line names what enforce mode would have done.
|
||||||
rq.line.WouldAction = action
|
rq.line.WouldAction = action
|
||||||
|
|
||||||
return nil
|
|
||||||
case action == requestlog.ActionRuleBlocked || action == requestlog.ActionWAFBlocked:
|
case action == requestlog.ActionRuleBlocked || action == requestlog.ActionWAFBlocked:
|
||||||
return &refusal{status: http.StatusForbidden, action: action}
|
return &refusal{status: http.StatusForbidden, action: action}
|
||||||
case action == requestlog.ActionTooLarge:
|
|
||||||
return &refusal{status: http.StatusRequestHeaderFieldsTooLarge, action: action}
|
|
||||||
default:
|
default:
|
||||||
return rq.banResponse(action)
|
return rq.banResponse(action)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
maxBytes := rq.h.config.RequestMaxBytes
|
||||||
|
if maxBytes > 0 && rq.in.ContentLength > maxBytes {
|
||||||
|
return &refusal{
|
||||||
|
status: http.StatusRequestEntityTooLarge,
|
||||||
|
action: requestlog.ActionTooLarge,
|
||||||
|
limit: "SWWAF_REQUEST_MAX_BYTES",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkClient runs the checks on the request's client, and returns the
|
// checkClient runs the checks on the request's client, and returns the
|
||||||
@@ -195,9 +240,9 @@ func (rq *request) check(ctx context.Context) *refusal {
|
|||||||
// rate limits, unless the client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the
|
// rate limits, unless the client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the
|
||||||
// request's path is exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that
|
// request's path is exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that
|
||||||
// every other request is counted, each of them by the client's limit
|
// every other request is counted, each of them by the client's limit
|
||||||
// percentages, then SWWAF_TRAP_PATHS, then the rule files, then the header
|
// percentages, then SWWAF_TRAP_PATHS, then the rule files, and last the
|
||||||
// limits, and last the Core Rule Set. A request exempt from the rate limits
|
// Core Rule Set. A request exempt from the rate limits is exempt from the
|
||||||
// is exempt from the byte limits too. ctx is the request's own context.
|
// byte limits too. ctx is the request's own context.
|
||||||
func (rq *request) checkClient(ctx context.Context) string {
|
func (rq *request) checkClient(ctx context.Context) string {
|
||||||
cfg := rq.h.config
|
cfg := rq.h.config
|
||||||
if isInside(rq.client, cfg.AllowNets) {
|
if isInside(rq.client, cfg.AllowNets) {
|
||||||
@@ -214,13 +259,7 @@ func (rq *request) checkClient(ctx context.Context) string {
|
|||||||
return requestlog.ActionBanned
|
return requestlog.ActionBanned
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.lookUp(ctx, now)
|
rq.lookUp(ctx)
|
||||||
|
|
||||||
// The lookup can wait for GeoJS, so the checks after it read the time
|
|
||||||
// again: a request counted at the time it arrived could be counted
|
|
||||||
// after a later one of its client's, which the rate limits would take
|
|
||||||
// for the clock set back.
|
|
||||||
now = rq.h.now()
|
|
||||||
|
|
||||||
if rq.countryDenied() {
|
if rq.countryDenied() {
|
||||||
return requestlog.ActionCountryDenied
|
return requestlog.ActionCountryDenied
|
||||||
@@ -238,7 +277,8 @@ func (rq *request) checkClient(ctx context.Context) string {
|
|||||||
return requestlog.ActionDenied
|
return requestlog.ActionDenied
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.counted = rq.countedByRateLimits()
|
rq.counted = !isInside(rq.client, cfg.RateLimitExemptNets) &&
|
||||||
|
!pathExempt(rq.in.URL, cfg.RateLimitExemptPaths)
|
||||||
if rq.counted {
|
if rq.counted {
|
||||||
rq.limitPercent, rq.bytesPercent = rq.limitPercentages()
|
rq.limitPercent, rq.bytesPercent = rq.limitPercentages()
|
||||||
rq.line.LimitPercent, rq.line.LimitPercentSetting = rq.limitPercent.logged()
|
rq.line.LimitPercent, rq.line.LimitPercentSetting = rq.limitPercent.logged()
|
||||||
@@ -258,24 +298,10 @@ func (rq *request) checkClient(ctx context.Context) string {
|
|||||||
return action
|
return action
|
||||||
}
|
}
|
||||||
|
|
||||||
if rq.overHeaderLimits() {
|
|
||||||
return requestlog.ActionTooLarge
|
|
||||||
}
|
|
||||||
|
|
||||||
return rq.checkCoreRuleSet()
|
return rq.checkCoreRuleSet()
|
||||||
}
|
}
|
||||||
|
|
||||||
// countedByRateLimits reports whether the rate limits count the request: its
|
// pathExempt reports whether a request for u is exempt under prefixes,
|
||||||
// client is not in SWWAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt
|
|
||||||
// under SWWAF_RATE_LIMIT_EXEMPT_PATHS.
|
|
||||||
func (rq *request) countedByRateLimits() bool {
|
|
||||||
cfg := rq.h.config
|
|
||||||
|
|
||||||
return !isInside(rq.client, cfg.RateLimitExemptNets) &&
|
|
||||||
!pathExempt(rq.in, cfg.RateLimitExemptPaths)
|
|
||||||
}
|
|
||||||
|
|
||||||
// pathExempt reports whether r is exempt under prefixes,
|
|
||||||
// SWWAF_RATE_LIMIT_EXEMPT_PATHS or SWWAF_WAF_EXEMPT_PATHS: whether its
|
// SWWAF_RATE_LIMIT_EXEMPT_PATHS or SWWAF_WAF_EXEMPT_PATHS: whether its
|
||||||
// path as sent, the path the app receives, not percent-decoded, starts
|
// path as sent, the path the app receives, not percent-decoded, starts
|
||||||
// with one of prefixes, so that /%61ssets/x is not under /assets/ for an
|
// with one of prefixes, so that /%61ssets/x is not under /assets/ for an
|
||||||
@@ -284,9 +310,10 @@ func (rq *request) countedByRateLimits() bool {
|
|||||||
// encoded slash (%2F or %2f), never is, since an app may act on it as a
|
// encoded slash (%2F or %2f), never is, since an app may act on it as a
|
||||||
// path outside every prefix: /assets/..%2Flogin as /login, or /assets%2Fx
|
// path outside every prefix: /assets/..%2Flogin as /login, or /assets%2Fx
|
||||||
// as one path segment, as Go's router does.
|
// as one path segment, as Go's router does.
|
||||||
func pathExempt(r *http.Request, prefixes []string) bool {
|
func pathExempt(u *url.URL, prefixes []string) bool {
|
||||||
decoded := r.URL.Path
|
decoded := u.Path
|
||||||
sent := rules.Path(r)
|
// EscapedPath is the path as the app receives it, not decoded.
|
||||||
|
sent := u.EscapedPath()
|
||||||
|
|
||||||
if strings.Contains(decoded, "..") || strings.Contains(decoded, `\`) ||
|
if strings.Contains(decoded, "..") || strings.Contains(decoded, `\`) ||
|
||||||
strings.Contains(strings.ToLower(sent), "%2f") {
|
strings.Contains(strings.ToLower(sent), "%2f") {
|
||||||
@@ -298,40 +325,222 @@ func pathExempt(r *http.Request, prefixes []string) bool {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// finish counts the request in the metrics and writes its log line.
|
// forward passes the request to the app and the app's answer back. ctx
|
||||||
|
// is the request's own context.
|
||||||
|
func (rq *request) forward(ctx context.Context) {
|
||||||
|
ctx, cancel := context.WithCancel(ctx)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
rq.cancel = cancel
|
||||||
|
ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{
|
||||||
|
GotConn: rq.gotConn,
|
||||||
|
WroteRequest: rq.wroteRequest,
|
||||||
|
GotFirstResponseByte: rq.gotFirstResponseByte,
|
||||||
|
})
|
||||||
|
|
||||||
|
out := rq.in.WithContext(ctx)
|
||||||
|
if rq.body != nil {
|
||||||
|
out.Body = rq.body
|
||||||
|
}
|
||||||
|
|
||||||
|
reverseProxy := &httputil.ReverseProxy{
|
||||||
|
Rewrite: rq.rewrite,
|
||||||
|
Transport: rq.h.transport,
|
||||||
|
FlushInterval: flushAfterEachWrite,
|
||||||
|
ErrorLog: rq.h.errorLog,
|
||||||
|
ModifyResponse: rq.modifyResponse,
|
||||||
|
ErrorHandler: rq.answerError,
|
||||||
|
}
|
||||||
|
|
||||||
|
rq.startRequestTimers()
|
||||||
|
rq.upstreamStart = time.Now()
|
||||||
|
reverseProxy.ServeHTTP(rq.out, out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// rewrite makes the request the app receives: the client's request,
|
||||||
|
// unchanged, sent to SWWAF_UPSTREAM_URL, with the forwarded headers and
|
||||||
|
// the request's id set, without any X-Client-ASN or X-Client-Country the
|
||||||
|
// client sent, whatever SWWAF_ADD_LOOKUP_HEADERS says, and, while it is
|
||||||
|
// set, with the client's AS number and country in them.
|
||||||
|
func (rq *request) rewrite(pr *httputil.ProxyRequest) {
|
||||||
|
upstream := rq.h.config.UpstreamURL
|
||||||
|
pr.Out.URL.Scheme = upstream.Scheme
|
||||||
|
pr.Out.URL.Host = upstream.Host
|
||||||
|
// ReverseProxy drops query parameters it cannot parse; the app gets
|
||||||
|
// the query as the client sent it.
|
||||||
|
pr.Out.URL.RawQuery = pr.In.URL.RawQuery
|
||||||
|
setForwardedHeaders(pr.In, pr.Out, rq.peer, rq.peerTrusted)
|
||||||
|
pr.Out.Header.Set(requestIDHeader, rq.line.RequestID)
|
||||||
|
pr.Out.Header.Del(asnHeader)
|
||||||
|
pr.Out.Header.Del(countryHeader)
|
||||||
|
|
||||||
|
if rq.h.config.AddLookupHeaders {
|
||||||
|
setLookupHeaders(pr.Out.Header, rq.line.ASN, rq.line.Country)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// modifyResponse looks at the app's answer before ReverseProxy passes it
|
||||||
|
// on.
|
||||||
|
func (rq *request) modifyResponse(res *http.Response) error {
|
||||||
|
rq.line.UpstreamStatus = res.StatusCode
|
||||||
|
|
||||||
|
if res.StatusCode == http.StatusSwitchingProtocols {
|
||||||
|
// An upgraded connection, such as a WebSocket, is not cut by the
|
||||||
|
// timeouts. ReverseProxy writes this answer straight to the
|
||||||
|
// connection it takes over, not through rq.out, and then copies
|
||||||
|
// what passes each way through res.Body, the connection to the app.
|
||||||
|
rq.stopTimers()
|
||||||
|
rq.out.status = res.StatusCode
|
||||||
|
rq.line.Websocket = true
|
||||||
|
|
||||||
|
conn, ok := res.Body.(io.ReadWriteCloser)
|
||||||
|
if ok {
|
||||||
|
rq.upgraded = &upgradedConn{ReadWriteCloser: conn}
|
||||||
|
res.Body = rq.upgraded
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
maxBytes := rq.h.config.ResponseMaxBytes
|
||||||
|
if maxBytes > 0 && res.Body != http.NoBody && res.ContentLength > maxBytes {
|
||||||
|
rq.refuse(refusal{
|
||||||
|
status: http.StatusBadGateway,
|
||||||
|
action: requestlog.ActionTooLarge,
|
||||||
|
limit: "SWWAF_RESPONSE_MAX_BYTES",
|
||||||
|
})
|
||||||
|
|
||||||
|
return errResponseTooLarge
|
||||||
|
}
|
||||||
|
|
||||||
|
res.Body = &responseBody{body: limitBody(res.Body, maxBytes), rq: rq}
|
||||||
|
rq.startClientResponseTimeout()
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// answerError is ReverseProxy's ErrorHandler: the request could not be
|
||||||
|
// passed to the app, or the app's answer cannot be passed on.
|
||||||
|
func (rq *request) answerError(_ http.ResponseWriter, _ *http.Request, err error) {
|
||||||
|
refused := rq.refused.Load()
|
||||||
|
if refused == nil {
|
||||||
|
if rq.in.Context().Err() != nil {
|
||||||
|
return // the client has gone, and there is no one to answer
|
||||||
|
}
|
||||||
|
|
||||||
|
rq.h.processLog.Warn("request to the app failed", "error", err.Error())
|
||||||
|
|
||||||
|
refused = &refusal{
|
||||||
|
status: http.StatusBadGateway,
|
||||||
|
action: requestlog.ActionUpstreamError,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
rq.answer(*refused)
|
||||||
|
}
|
||||||
|
|
||||||
|
// answer sends smallwebwaf's own answer, unless the response has already
|
||||||
|
// started, and records the refusal for the log line.
|
||||||
|
func (rq *request) answer(r refusal) {
|
||||||
|
rq.refused.CompareAndSwap(nil, &r)
|
||||||
|
|
||||||
|
if rq.out.status != 0 {
|
||||||
|
return // too late to answer: the connection can only be cut
|
||||||
|
}
|
||||||
|
|
||||||
|
if r.status == 0 {
|
||||||
|
// SWWAF_BAN_RESPONSE is close. This panic has Go's server close
|
||||||
|
// the connection without an answer, and log nothing; the log line
|
||||||
|
// is still written as the handler returns.
|
||||||
|
panic(http.ErrAbortHandler)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A client found too slow is read no more; any other may go on
|
||||||
|
// sending until its time is up, so that Go's server can read the
|
||||||
|
// rest of the body and end the request cleanly.
|
||||||
|
deadline := rq.clientRequestDeadline()
|
||||||
|
if r.status == http.StatusRequestTimeout {
|
||||||
|
deadline = time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
rq.stopReadingBody(deadline)
|
||||||
|
|
||||||
|
timeout := rq.h.config.ClientResponseTimeout
|
||||||
|
if timeout > 0 {
|
||||||
|
_ = rq.rc.SetWriteDeadline(time.Now().Add(timeout))
|
||||||
|
}
|
||||||
|
|
||||||
|
http.Error(rq.out, http.StatusText(r.status), r.status)
|
||||||
|
}
|
||||||
|
|
||||||
|
// refuse records r, unless an earlier refusal was, and ends the request
|
||||||
|
// to the app, if one was made: smallwebwaf reads the body of a request
|
||||||
|
// it answers itself too.
|
||||||
|
func (rq *request) refuse(r refusal) {
|
||||||
|
rq.refused.CompareAndSwap(nil, &r)
|
||||||
|
|
||||||
|
if rq.cancel != nil {
|
||||||
|
rq.cancel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// finish ends the request's timeouts, counts it in the metrics and writes
|
||||||
|
// its log line.
|
||||||
func (rq *request) finish() {
|
func (rq *request) finish() {
|
||||||
|
rq.stopTimers()
|
||||||
|
|
||||||
|
refused := rq.refused.Load()
|
||||||
|
if refused == nil {
|
||||||
|
rq.stopReadingBody(rq.clientRequestDeadline())
|
||||||
|
}
|
||||||
|
|
||||||
line := &rq.line
|
line := &rq.line
|
||||||
line.Status = rq.out.status
|
line.Status = rq.out.status
|
||||||
line.ResponseBytes = rq.out.bytes
|
line.ResponseBytes = rq.out.bytes
|
||||||
line.ResponseContentType = rq.out.Header().Get("Content-Type")
|
header := rq.out.Header()
|
||||||
line.RequestBytes = rq.requestBytes
|
line.ResponseContentType = header.Get("Content-Type")
|
||||||
|
line.CacheControl = header.Get("Cache-Control")
|
||||||
|
line.Location = header.Get("Location")
|
||||||
|
line.RequestBytes = rq.requestBytes()
|
||||||
|
|
||||||
|
// limit is the setting whose size or time limit the request passed.
|
||||||
|
var limit string
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case rq.refused != nil:
|
case refused != nil:
|
||||||
line.Action = rq.refused.action
|
line.Action = refused.action
|
||||||
case rq.out.err != nil || rq.in.Context().Err() != nil:
|
limit = refused.limit
|
||||||
|
case errors.Is(rq.out.err, os.ErrDeadlineExceeded):
|
||||||
|
// The client took longer than SWWAF_CLIENT_RESPONSE_TIMEOUT to
|
||||||
|
// take the response.
|
||||||
|
line.Action = requestlog.ActionTimedOut
|
||||||
|
limit = "SWWAF_CLIENT_RESPONSE_TIMEOUT"
|
||||||
|
case !rq.complete && (rq.out.err != nil || rq.in.Context().Err() != nil):
|
||||||
line.Aborted = true
|
line.Aborted = true
|
||||||
}
|
}
|
||||||
|
|
||||||
duration := time.Since(rq.start)
|
now := time.Now()
|
||||||
|
duration := now.Sub(rq.start)
|
||||||
line.DurationTotal = requestlog.Milliseconds(duration)
|
line.DurationTotal = requestlog.Milliseconds(duration)
|
||||||
line.DurationChecks = timing(rq.start, rq.checked)
|
line.DurationChecks = timing(rq.start, rq.checked)
|
||||||
|
|
||||||
// Counted before the log line is written, so that the metrics count
|
var upstreamDuration time.Duration
|
||||||
// every request whose line is out. An allowed request is the exception:
|
|
||||||
// its report gives the status, bytes and times the metrics count.
|
if !rq.upstreamStart.IsZero() {
|
||||||
if rq.allowed {
|
upstreamDuration = now.Sub(rq.upstreamStart)
|
||||||
rq.h.metrics.RequestAllowed()
|
line.DurationUpstreamTotal = new(requestlog.Milliseconds(upstreamDuration))
|
||||||
} else {
|
|
||||||
rq.h.metrics.RequestEnded(line, duration)
|
rq.mu.Lock()
|
||||||
|
line.DurationUpstreamConnect = timing(rq.upstreamStart, rq.connected)
|
||||||
|
line.DurationUpstreamFirstByte = timing(rq.upstreamStart, rq.answerStarted)
|
||||||
|
rq.mu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.writeLine()
|
// Counted before the log line is written, so that the metrics count
|
||||||
}
|
// every request whose line is out.
|
||||||
|
rq.h.metrics.RequestEnded(line, limit, duration, upstreamDuration)
|
||||||
|
|
||||||
// writeLine writes the request's log line.
|
err := requestlog.Write(rq.h.requestLog, line)
|
||||||
func (rq *request) writeLine() {
|
|
||||||
err := requestlog.Write(rq.h.requestLog, &rq.line)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
rq.h.processLog.Error("writing the request log failed", "error", err.Error())
|
rq.h.processLog.Error("writing the request log failed", "error", err.Error())
|
||||||
}
|
}
|
||||||
@@ -355,16 +564,18 @@ func timing(start, end time.Time) *float64 {
|
|||||||
// before either was there, and one from GeoJS that comes later is added
|
// before either was there, and one from GeoJS that comes later is added
|
||||||
// when it comes.
|
// when it comes.
|
||||||
func (rq *request) addToHistory() {
|
func (rq *request) addToHistory() {
|
||||||
|
forwarded := !rq.upstreamStart.IsZero()
|
||||||
request := ratelimit.Request{
|
request := ratelimit.Request{
|
||||||
Forwarded: rq.allowed,
|
Forwarded: forwarded,
|
||||||
Refused: rq.refused != nil,
|
Refused: !forwarded && rq.refused.Load() != nil,
|
||||||
Status: rq.out.status,
|
Status: rq.out.status,
|
||||||
RequestBytes: rq.requestBytes,
|
RequestBytes: rq.requestBytes(),
|
||||||
ResponseBytes: rq.out.bytes,
|
ResponseBytes: rq.out.bytes,
|
||||||
BrokeLimit: rq.line.Offence == requestlog.OffenceLimit,
|
BrokeLimit: rq.line.Offence == requestlog.OffenceLimit,
|
||||||
Attack: rq.attack,
|
Attack: rq.attack,
|
||||||
RuleBlocked: rq.ruleBlocked,
|
RuleBlocked: rq.ruleBlocked,
|
||||||
WAFBlocked: rq.wafBlocked,
|
WAFBlocked: rq.wafBlocked,
|
||||||
|
TokenRefused: rq.tokenRefused,
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.h.limiter.AddToHistory(rq.h.clientGroup(rq.client), rq.h.now(), request)
|
rq.h.limiter.AddToHistory(rq.h.clientGroup(rq.client), rq.h.now(), request)
|
||||||
@@ -391,7 +602,6 @@ func (rq *request) countAnomalies() {
|
|||||||
answer, _ := rq.answerAtTheEnd()
|
answer, _ := rq.answerAtTheEnd()
|
||||||
|
|
||||||
rq.h.anomalies.Count(rq.h.now(), anomaly.Request{
|
rq.h.anomalies.Count(rq.h.now(), anomaly.Request{
|
||||||
Host: rq.site,
|
|
||||||
Client: rq.client,
|
Client: rq.client,
|
||||||
ClientGroup: rq.h.clientGroup(rq.client),
|
ClientGroup: rq.h.clientGroup(rq.client),
|
||||||
ASN: answer.ASN,
|
ASN: answer.ASN,
|
||||||
@@ -425,38 +635,208 @@ func (rq *request) answerAtTheEnd() (lookup.Answer, bool) {
|
|||||||
return rq.h.geojs.Kept(rq.h.clientGroup(rq.client))
|
return rq.h.geojs.Kept(rq.h.clientGroup(rq.client))
|
||||||
}
|
}
|
||||||
|
|
||||||
// responseWriter is servewaf's answer to a request. It notes the status and
|
// requestBytes is how many bytes of the request's body have been read.
|
||||||
// size for the log line, and the first error writing the answer.
|
func (rq *request) requestBytes() int64 {
|
||||||
type responseWriter struct {
|
if rq.body == nil {
|
||||||
http.ResponseWriter
|
return 0
|
||||||
|
|
||||||
// status is the status sent, or zero before one is.
|
|
||||||
status int
|
|
||||||
bytes int64
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteHeader sends the status and headers.
|
|
||||||
func (w *responseWriter) WriteHeader(status int) {
|
|
||||||
if w.status == 0 {
|
|
||||||
w.status = status
|
|
||||||
}
|
}
|
||||||
|
|
||||||
w.ResponseWriter.WriteHeader(status)
|
return rq.body.bytes.Load()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write sends part of the body.
|
// clientRequestDeadline is when the client must have sent its whole
|
||||||
func (w *responseWriter) Write(p []byte) (int, error) {
|
// request, or zero when SWWAF_CLIENT_REQUEST_TIMEOUT is off.
|
||||||
if w.status == 0 {
|
func (rq *request) clientRequestDeadline() time.Time {
|
||||||
w.status = http.StatusOK
|
timeout := rq.h.config.ClientRequestTimeout
|
||||||
|
if timeout == 0 {
|
||||||
|
return time.Time{}
|
||||||
}
|
}
|
||||||
|
|
||||||
n, err := w.ResponseWriter.Write(p)
|
return rq.start.Add(timeout)
|
||||||
w.bytes += int64(n)
|
}
|
||||||
|
|
||||||
if w.err == nil {
|
// stopReadingBody ends, at deadline, the reading of a client body that has
|
||||||
w.err = err
|
// not arrived whole: Go's server then reads no more of it, and closes the
|
||||||
|
// connection after the answer.
|
||||||
|
func (rq *request) stopReadingBody(deadline time.Time) {
|
||||||
|
if rq.body == nil || rq.body.received.Load() {
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
return n, err
|
_ = rq.rc.SetReadDeadline(deadline)
|
||||||
|
}
|
||||||
|
|
||||||
|
// startRequestTimers starts the timeouts that run while the request goes
|
||||||
|
// to the app: SWWAF_CLIENT_REQUEST_TIMEOUT until the client has sent its
|
||||||
|
// whole body, and SWWAF_UPSTREAM_REQUEST_TIMEOUT until the app has been
|
||||||
|
// sent the whole request.
|
||||||
|
func (rq *request) startRequestTimers() {
|
||||||
|
rq.mu.Lock()
|
||||||
|
defer rq.mu.Unlock()
|
||||||
|
|
||||||
|
if rq.body != nil && rq.h.config.ClientRequestTimeout > 0 {
|
||||||
|
rq.clientRequestTimer = time.AfterFunc(
|
||||||
|
time.Until(rq.clientRequestDeadline()), func() {
|
||||||
|
rq.requestTimedOut("SWWAF_CLIENT_REQUEST_TIMEOUT")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
timeout := rq.h.config.UpstreamRequestTimeout
|
||||||
|
if timeout > 0 {
|
||||||
|
rq.upstreamRequestTimer = time.AfterFunc(timeout, func() {
|
||||||
|
rq.requestTimedOut("SWWAF_UPSTREAM_REQUEST_TIMEOUT")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// requestTimedOut is called when limit, SWWAF_CLIENT_REQUEST_TIMEOUT or
|
||||||
|
// SWWAF_UPSTREAM_REQUEST_TIMEOUT, runs out while the request is still on
|
||||||
|
// its way to the app. The answer names the side smallwebwaf was waiting
|
||||||
|
// on at that moment: 408 when it was waiting for the client to send more
|
||||||
|
// of its body, 504 when it was waiting for the app to be reached or to
|
||||||
|
// take what it had.
|
||||||
|
func (rq *request) requestTimedOut(limit string) {
|
||||||
|
rq.mu.Lock()
|
||||||
|
defer rq.mu.Unlock()
|
||||||
|
|
||||||
|
if rq.timersStopped {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if rq.body == nil || !rq.body.waiting.Load() {
|
||||||
|
rq.refuse(refusal{
|
||||||
|
status: http.StatusGatewayTimeout,
|
||||||
|
action: requestlog.ActionTimedOut,
|
||||||
|
limit: limit,
|
||||||
|
})
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
rq.refuse(refusal{
|
||||||
|
status: http.StatusRequestTimeout,
|
||||||
|
action: requestlog.ActionTimedOut,
|
||||||
|
limit: limit,
|
||||||
|
})
|
||||||
|
// The transport gives up on the app only once its Read of the
|
||||||
|
// client's body returns, so that Read is ended now. The lock keeps
|
||||||
|
// this from reaching the connection after the request is handled.
|
||||||
|
_ = rq.rc.SetReadDeadline(time.Now())
|
||||||
|
}
|
||||||
|
|
||||||
|
// bodyReceived is called once the client has sent its whole body.
|
||||||
|
func (rq *request) bodyReceived() {
|
||||||
|
rq.mu.Lock()
|
||||||
|
defer rq.mu.Unlock()
|
||||||
|
|
||||||
|
stopTimer(rq.clientRequestTimer)
|
||||||
|
}
|
||||||
|
|
||||||
|
// gotConn is called once there is a connection to the app, a new one or
|
||||||
|
// one kept open from an earlier request.
|
||||||
|
func (rq *request) gotConn(httptrace.GotConnInfo) {
|
||||||
|
rq.mu.Lock()
|
||||||
|
defer rq.mu.Unlock()
|
||||||
|
|
||||||
|
rq.connected = time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
// gotFirstResponseByte is called once the first byte of the app's answer
|
||||||
|
// has arrived.
|
||||||
|
func (rq *request) gotFirstResponseByte() {
|
||||||
|
rq.mu.Lock()
|
||||||
|
defer rq.mu.Unlock()
|
||||||
|
|
||||||
|
rq.answerStarted = time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
// wroteRequest is called once the app has been sent the whole request:
|
||||||
|
// the request timeouts end and SWWAF_UPSTREAM_RESPONSE_TIMEOUT starts.
|
||||||
|
func (rq *request) wroteRequest(info httptrace.WroteRequestInfo) {
|
||||||
|
if info.Err != nil {
|
||||||
|
return // the transport gives up, or tries again
|
||||||
|
}
|
||||||
|
|
||||||
|
rq.mu.Lock()
|
||||||
|
defer rq.mu.Unlock()
|
||||||
|
|
||||||
|
if rq.timersStopped {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
stopTimer(rq.clientRequestTimer)
|
||||||
|
stopTimer(rq.upstreamRequestTimer)
|
||||||
|
rq.requestSent = time.Now()
|
||||||
|
|
||||||
|
timeout := rq.h.config.UpstreamResponseTimeout
|
||||||
|
if timeout > 0 {
|
||||||
|
rq.upstreamResponseTimer = time.AfterFunc(timeout, rq.responseTimedOut)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// responseTimedOut is called when SWWAF_UPSTREAM_RESPONSE_TIMEOUT runs out
|
||||||
|
// before the app has sent its whole answer.
|
||||||
|
func (rq *request) responseTimedOut() {
|
||||||
|
rq.mu.Lock()
|
||||||
|
defer rq.mu.Unlock()
|
||||||
|
|
||||||
|
if !rq.timersStopped {
|
||||||
|
rq.refuse(refusal{
|
||||||
|
status: http.StatusGatewayTimeout,
|
||||||
|
action: requestlog.ActionTimedOut,
|
||||||
|
limit: "SWWAF_UPSTREAM_RESPONSE_TIMEOUT",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// responseReceived is called once the app has sent its whole answer.
|
||||||
|
func (rq *request) responseReceived() {
|
||||||
|
rq.complete = true
|
||||||
|
rq.stopTimers()
|
||||||
|
}
|
||||||
|
|
||||||
|
// startClientResponseTimeout sets SWWAF_CLIENT_RESPONSE_TIMEOUT on the
|
||||||
|
// connection to the client: the response must reach the client within it
|
||||||
|
// of the end of the request, or of now if the app answers before it has
|
||||||
|
// the whole request.
|
||||||
|
func (rq *request) startClientResponseTimeout() {
|
||||||
|
timeout := rq.h.config.ClientResponseTimeout
|
||||||
|
if timeout == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
from := rq.sentAt()
|
||||||
|
if from.IsZero() {
|
||||||
|
from = time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = rq.rc.SetWriteDeadline(from.Add(timeout))
|
||||||
|
}
|
||||||
|
|
||||||
|
// sentAt is when the app had been sent the whole request, or zero.
|
||||||
|
func (rq *request) sentAt() time.Time {
|
||||||
|
rq.mu.Lock()
|
||||||
|
defer rq.mu.Unlock()
|
||||||
|
|
||||||
|
return rq.requestSent
|
||||||
|
}
|
||||||
|
|
||||||
|
// stopTimers stops the request's timeouts and keeps any from starting
|
||||||
|
// later: the app's answer is complete, the connection upgraded, or the
|
||||||
|
// request handled.
|
||||||
|
func (rq *request) stopTimers() {
|
||||||
|
rq.mu.Lock()
|
||||||
|
defer rq.mu.Unlock()
|
||||||
|
|
||||||
|
rq.timersStopped = true
|
||||||
|
stopTimer(rq.clientRequestTimer)
|
||||||
|
stopTimer(rq.upstreamRequestTimer)
|
||||||
|
stopTimer(rq.upstreamResponseTimer)
|
||||||
|
}
|
||||||
|
|
||||||
|
// stopTimer stops t, which is nil when its timeout is off.
|
||||||
|
func stopTimer(t *time.Timer) {
|
||||||
|
if t != nil {
|
||||||
|
t.Stop()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package proxy_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"io"
|
"io"
|
||||||
|
"maps"
|
||||||
"math"
|
"math"
|
||||||
"net/http"
|
"net/http"
|
||||||
"reflect"
|
"reflect"
|
||||||
@@ -41,6 +42,9 @@ func TestLogLineHasEachFieldWhereItApplies(t *testing.T) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
w.Header().Set("Content-Type", "text/html")
|
||||||
|
w.Header().Set("Cache-Control", "no-store")
|
||||||
|
w.Header().Set("Location", "/elsewhere")
|
||||||
w.WriteHeader(http.StatusFound)
|
w.WriteHeader(http.StatusFound)
|
||||||
_, _ = io.WriteString(w, "moved")
|
_, _ = io.WriteString(w, "moved")
|
||||||
})
|
})
|
||||||
@@ -66,7 +70,7 @@ func TestLogLineHasEachFieldWhereItApplies(t *testing.T) {
|
|||||||
|
|
||||||
// A request's log line can come after its answer: each is waited for
|
// A request's log line can come after its answer: each is waited for
|
||||||
// before the next request, so that the lines are in order.
|
// before the next request, so that the lines are in order.
|
||||||
full := out.endedLines(t, 1)[0]
|
full := out.requestLines(t, 1)[0]
|
||||||
|
|
||||||
// This one comes from 127.0.0.1, which the rate limits do not count,
|
// This one comes from 127.0.0.1, which the rate limits do not count,
|
||||||
// with a body of 4 bytes whose length it does not announce, so that its
|
// with a body of 4 bytes whose length it does not announce, so that its
|
||||||
@@ -77,7 +81,7 @@ func TestLogLineHasEachFieldWhereItApplies(t *testing.T) {
|
|||||||
"Transfer-Encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\n\r\n")
|
"Transfer-Encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\n\r\n")
|
||||||
wantStatus(t, readResponse(t, conn), http.StatusNoContent)
|
wantStatus(t, readResponse(t, conn), http.StatusNoContent)
|
||||||
|
|
||||||
bare := out.endedLines(t, 2)[1]
|
bare := out.requestLines(t, 2)[1]
|
||||||
|
|
||||||
wantFullLine(t, full)
|
wantFullLine(t, full)
|
||||||
wantBareLine(t, bare)
|
wantBareLine(t, bare)
|
||||||
@@ -104,23 +108,24 @@ func wantFullLine(t *testing.T, line logLine) {
|
|||||||
headers := map[string]string{"accept": "text/html", "x-custom": "one, two"}
|
headers := map[string]string{"accept": "text/html", "x-custom": "one, two"}
|
||||||
|
|
||||||
want := withTimings(line, requestlog.Line{
|
want := withTimings(line, requestlog.Line{
|
||||||
Type: requestType, ServeWAFLine: requestlog.ServeWAFLineReport,
|
Type: requestType, Time: line.Time, Instance: instance,
|
||||||
Time: line.Time, Instance: instance,
|
|
||||||
ClientIP: ipv6Client, Method: http.MethodPost, Scheme: secure,
|
ClientIP: ipv6Client, Method: http.MethodPost, Scheme: secure,
|
||||||
Host: appHost, Path: "/full", Protocol: protocol,
|
Host: appHost, Path: "/full", Protocol: protocol,
|
||||||
Status: http.StatusFound, RequestBytes: 3, ResponseBytes: 5,
|
Status: http.StatusFound, RequestBytes: 3, ResponseBytes: 5,
|
||||||
RequestID: line.RequestID, PeerIP: localhost,
|
RequestID: "from-traefik", PeerIP: localhost,
|
||||||
ForwardedFor: "198.51.100.7, " + ipv6Client, ClientGroup: ipv6Group,
|
ForwardedFor: "198.51.100.7, " + ipv6Client, ClientGroup: ipv6Group,
|
||||||
ContentType: "application/x-www-form-urlencoded", ContentLength: 3,
|
ContentType: "application/x-www-form-urlencoded", ContentLength: 3,
|
||||||
RequestHeaders: headers, HasAuthorization: true, HasCookie: true,
|
RequestHeaders: headers, HasAuthorization: true, HasCookie: true,
|
||||||
|
ResponseContentType: "text/html", UpstreamStatus: http.StatusFound,
|
||||||
|
CacheControl: "no-store", Location: "/elsewhere",
|
||||||
Action: requestlog.ActionForward,
|
Action: requestlog.ActionForward,
|
||||||
// Its 3 bytes in and 5 out, each way counted by default.
|
// Its 3 bytes in and 5 out, each way counted by default.
|
||||||
Counts: ratelimit.Counts{
|
Counts: ratelimit.Counts{
|
||||||
Minute: 1, Hour: 1, Day: 1, MinuteBytes: 8, HourBytes: 8, DayBytes: 8,
|
Minute: 1, Hour: 1, Day: 1, MinuteBytes: 8, HourBytes: 8, DayBytes: 8,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if !reflect.DeepEqual(line.Line, want) || line.RequestID == "from-traefik" {
|
if !reflect.DeepEqual(line.Line, want) {
|
||||||
t.Errorf("log line\n%+v\nwant\n%+v, with a new request id", line.Line, want)
|
t.Errorf("log line\n%+v\nwant\n%+v", line.Line, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -130,13 +135,12 @@ func wantBareLine(t *testing.T, line logLine) {
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
want := withTimings(line, requestlog.Line{
|
want := withTimings(line, requestlog.Line{
|
||||||
Type: requestType, ServeWAFLine: requestlog.ServeWAFLineReport,
|
Type: requestType, Time: line.Time, Instance: instance,
|
||||||
Time: line.Time, Instance: instance,
|
|
||||||
ClientIP: localhost, Method: http.MethodPost, Scheme: plain,
|
ClientIP: localhost, Method: http.MethodPost, Scheme: plain,
|
||||||
Host: appHost, Path: "/bare", Protocol: protocol,
|
Host: appHost, Path: "/bare", Protocol: protocol,
|
||||||
Status: http.StatusNoContent, RequestBytes: 4, RequestID: line.RequestID,
|
Status: http.StatusNoContent, RequestBytes: 4, RequestID: line.RequestID,
|
||||||
PeerIP: localhost, ClientGroup: localhost + "/32",
|
PeerIP: localhost, ClientGroup: localhost + "/32",
|
||||||
Action: requestlog.ActionForward,
|
UpstreamStatus: http.StatusNoContent, Action: requestlog.ActionForward,
|
||||||
})
|
})
|
||||||
if !reflect.DeepEqual(line.Line, want) || line.RequestID == "" {
|
if !reflect.DeepEqual(line.Line, want) || line.RequestID == "" {
|
||||||
t.Errorf("log line\n%+v\nwant\n%+v, with a request id", line.Line, want)
|
t.Errorf("log line\n%+v\nwant\n%+v, with a request id", line.Line, want)
|
||||||
@@ -144,7 +148,8 @@ func wantBareLine(t *testing.T, line logLine) {
|
|||||||
|
|
||||||
for _, name := range []string{
|
for _, name := range []string{
|
||||||
"forwarded_for", "content_type", "content_length", "request_headers",
|
"forwarded_for", "content_type", "content_length", "request_headers",
|
||||||
"has_authorization", "has_cookie", "response_content_type", "counts",
|
"has_authorization", "has_cookie", "websocket", "response_content_type",
|
||||||
|
"cache_control", "location", "counts",
|
||||||
} {
|
} {
|
||||||
_, present := line.fields[name]
|
_, present := line.fields[name]
|
||||||
if present {
|
if present {
|
||||||
@@ -157,6 +162,8 @@ func wantBareLine(t *testing.T, line logLine) {
|
|||||||
func withTimings(line logLine, want requestlog.Line) requestlog.Line {
|
func withTimings(line logLine, want requestlog.Line) requestlog.Line {
|
||||||
want.DurationTotal = line.DurationTotal
|
want.DurationTotal = line.DurationTotal
|
||||||
want.DurationChecks = line.DurationChecks
|
want.DurationChecks = line.DurationChecks
|
||||||
|
want.DurationUpstreamConnect = line.DurationUpstreamConnect
|
||||||
|
want.DurationUpstreamFirstByte = line.DurationUpstreamFirstByte
|
||||||
want.DurationUpstreamTotal = line.DurationUpstreamTotal
|
want.DurationUpstreamTotal = line.DurationUpstreamTotal
|
||||||
|
|
||||||
return want
|
return want
|
||||||
@@ -181,7 +188,7 @@ func TestHasAuthorizationAndHasCookieEachComeFromTheirOwnHeader(t *testing.T) {
|
|||||||
req.Header.Set(tc.header, "secret")
|
req.Header.Set(tc.header, "secret")
|
||||||
wantStatus(t, do(t, req), http.StatusOK)
|
wantStatus(t, do(t, req), http.StatusOK)
|
||||||
|
|
||||||
line := out.endedLine(t)
|
line := out.requestLine(t)
|
||||||
|
|
||||||
_, otherPresent := line.fields[tc.other]
|
_, otherPresent := line.fields[tc.other]
|
||||||
if line.fields[tc.field] != true || otherPresent {
|
if line.fields[tc.field] != true || otherPresent {
|
||||||
@@ -192,20 +199,81 @@ func TestHasAuthorizationAndHasCookieEachComeFromTheirOwnHeader(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRequestIDAndSchemeComeOnlyFromATrustedProxy(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const sentID = "from-traefik"
|
||||||
|
|
||||||
|
sent := http.Header{requestIDHeader: {sentID}, forwardedProto: {secure}}
|
||||||
|
trusted := map[string]string{trustedProxies: trustLocalhost}
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
env map[string]string
|
||||||
|
header http.Header
|
||||||
|
// wantID is the request id logged, "" for a new one.
|
||||||
|
wantID, wantScheme string
|
||||||
|
}{
|
||||||
|
{"a trusted proxy's are kept", trusted, sent, sentID, secure},
|
||||||
|
{"without them, the id is new and the scheme http", trusted, nil, "", plain},
|
||||||
|
{"another peer's are replaced", nil, sent, "", plain},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
received := make(chan string, 2)
|
||||||
|
app := startApp(t, func(_ http.ResponseWriter, r *http.Request) {
|
||||||
|
received <- r.Header.Get(requestIDHeader)
|
||||||
|
})
|
||||||
|
addr, out := startProxy(t, app.URL, tc.env)
|
||||||
|
|
||||||
|
// Two requests, so that two new ids can be told apart.
|
||||||
|
ids := make([]string, 0, 2)
|
||||||
|
|
||||||
|
for i := range 2 {
|
||||||
|
req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
|
||||||
|
maps.Copy(req.Header, tc.header)
|
||||||
|
wantStatus(t, do(t, req), http.StatusOK)
|
||||||
|
|
||||||
|
line := out.requestLines(t, i+1)[i]
|
||||||
|
ids = append(ids, line.RequestID)
|
||||||
|
|
||||||
|
got := <-received
|
||||||
|
if line.RequestID != got || line.Scheme != tc.wantScheme {
|
||||||
|
t.Errorf("log line has request_id %q and scheme %q, and the "+
|
||||||
|
"app received id %q; want the same id and scheme %q",
|
||||||
|
line.RequestID, line.Scheme, got, tc.wantScheme)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case tc.wantID != "" && (ids[0] != tc.wantID || ids[1] != tc.wantID):
|
||||||
|
t.Errorf("request ids %q, want %q", ids, tc.wantID)
|
||||||
|
case tc.wantID == "" && (slices.Contains(ids, sentID) ||
|
||||||
|
slices.Contains(ids, "") || ids[0] == ids[1]):
|
||||||
|
t.Errorf("request ids %q, want two new ones", ids)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestTimingsAreInOrder(t *testing.T) {
|
func TestTimingsAreInOrder(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const denied = "192.0.2.50" // in SWWAF_DENY_NETS
|
const denied = "192.0.2.50" // in SWWAF_DENY_NETS
|
||||||
|
|
||||||
app := startApp(t, func(w http.ResponseWriter, _ *http.Request) {
|
app := startApp(t, func(w http.ResponseWriter, _ *http.Request) {
|
||||||
// The pause sets the times apart; a hold-up of the test only lengthens
|
// The pauses set the times apart; a hold-up of the test only
|
||||||
// it.
|
// lengthens them.
|
||||||
|
time.Sleep(time.Millisecond)
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
_ = http.NewResponseController(w).Flush()
|
||||||
|
|
||||||
time.Sleep(time.Millisecond)
|
time.Sleep(time.Millisecond)
|
||||||
|
|
||||||
_, _ = io.WriteString(w, "done")
|
_, _ = io.WriteString(w, "done")
|
||||||
})
|
})
|
||||||
|
addr, out := startProxy(t, app.URL, map[string]string{
|
||||||
addr, out, server := startProxyWithClock(t, app.URL, "", time.Now, map[string]string{
|
|
||||||
trustedProxies: trustLocalhost,
|
trustedProxies: trustLocalhost,
|
||||||
denyNets: denied,
|
denyNets: denied,
|
||||||
})
|
})
|
||||||
@@ -213,19 +281,20 @@ func TestTimingsAreInOrder(t *testing.T) {
|
|||||||
// Each log line is waited for before the next request, so that the
|
// Each log line is waited for before the next request, so that the
|
||||||
// lines are in order.
|
// lines are in order.
|
||||||
wantStatus(t, get(t, addr, "/"), http.StatusOK)
|
wantStatus(t, get(t, addr, "/"), http.StatusOK)
|
||||||
forwarded := out.endedLines(t, 1)[0]
|
forwarded := out.requestLines(t, 1)[0]
|
||||||
|
|
||||||
req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
|
req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
|
||||||
req.Header.Set(forwardedFor, denied)
|
req.Header.Set(forwardedFor, denied)
|
||||||
wantStatus(t, do(t, req), http.StatusForbidden)
|
wantStatus(t, do(t, req), http.StatusForbidden)
|
||||||
refused := out.endedLines(t, 2)[1]
|
refused := out.requestLines(t, 2)[1]
|
||||||
|
|
||||||
wantStatus(t, get(t, server.Addr, proxy.HealthPath), http.StatusOK)
|
wantStatus(t, get(t, addr, proxy.HealthPath), http.StatusOK)
|
||||||
health := out.endedLines(t, 3)[2]
|
health := out.requestLines(t, 3)[2]
|
||||||
|
|
||||||
// A request passed to the app has every timing; one refused, none of
|
// A request passed to the app has every timing; one refused, none of
|
||||||
// the app's; the health check, which runs no check, only the total.
|
// the app's; the health check, which runs no check, only the total.
|
||||||
wantTimings(t, forwarded, "duration_total", "duration_checks",
|
wantTimings(t, forwarded, "duration_total", "duration_checks",
|
||||||
|
"duration_upstream_connect", "duration_upstream_first_byte",
|
||||||
"duration_upstream_total")
|
"duration_upstream_total")
|
||||||
wantTimings(t, refused, "duration_total", "duration_checks")
|
wantTimings(t, refused, "duration_total", "duration_checks")
|
||||||
wantTimings(t, health, "duration_total")
|
wantTimings(t, health, "duration_total")
|
||||||
@@ -238,12 +307,17 @@ func TestTimingsAreInOrder(t *testing.T) {
|
|||||||
// exact.
|
// exact.
|
||||||
total := microseconds(forwarded.DurationTotal)
|
total := microseconds(forwarded.DurationTotal)
|
||||||
checks := microseconds(*forwarded.DurationChecks)
|
checks := microseconds(*forwarded.DurationChecks)
|
||||||
|
connect := microseconds(*forwarded.DurationUpstreamConnect)
|
||||||
|
firstByte := microseconds(*forwarded.DurationUpstreamFirstByte)
|
||||||
upstream := microseconds(*forwarded.DurationUpstreamTotal)
|
upstream := microseconds(*forwarded.DurationUpstreamTotal)
|
||||||
|
|
||||||
// The checks end before the request is handed to the app.
|
// The checks end before the request is handed to the app, and the
|
||||||
if checks+upstream > total {
|
// connection comes before the answer, which the app ends after a
|
||||||
t.Errorf("timings in microseconds: total %d, checks %d, upstream total %d",
|
// pause.
|
||||||
total, checks, upstream)
|
if checks+upstream > total || connect >= firstByte || firstByte >= upstream {
|
||||||
|
t.Errorf("timings in microseconds: total %d, checks %d, connect %d, "+
|
||||||
|
"first byte %d, upstream total %d", total, checks, connect, firstByte,
|
||||||
|
upstream)
|
||||||
}
|
}
|
||||||
|
|
||||||
if *refused.DurationChecks > refused.DurationTotal {
|
if *refused.DurationChecks > refused.DurationTotal {
|
||||||
@@ -276,3 +350,22 @@ func wantTimings(t *testing.T, line logLine, want ...string) {
|
|||||||
func microseconds(milliseconds float64) int64 {
|
func microseconds(milliseconds float64) int64 {
|
||||||
return int64(math.Round(milliseconds * 1000))
|
return int64(math.Round(milliseconds * 1000))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLogsAnUpgradedConnection(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
app := startApp(t, echoAfterUpgrade)
|
||||||
|
addr, out := startProxy(t, app.URL, nil)
|
||||||
|
|
||||||
|
conn := dial(t, addr)
|
||||||
|
send(t, conn, "GET /socket HTTP/1.1\r\nHost: app\r\n"+
|
||||||
|
"Connection: Upgrade\r\nUpgrade: websocket\r\n\r\n")
|
||||||
|
wantStatus(t, readResponse(t, conn), http.StatusSwitchingProtocols)
|
||||||
|
|
||||||
|
_ = conn.Close()
|
||||||
|
|
||||||
|
line := out.requestLine(t)
|
||||||
|
if line.fields["websocket"] != true {
|
||||||
|
t.Errorf("log line has websocket %v, want true", line.fields["websocket"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -185,6 +185,8 @@ func TestObserveModeLogsWhatTheRulesWouldDo(t *testing.T) {
|
|||||||
func TestMetricsCountRuleMatchesAndBansForAnAttack(t *testing.T) {
|
func TestMetricsCountRuleMatchesAndBansForAnAttack(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
const scraper = "192.0.2.200"
|
||||||
|
|
||||||
s, _, _ := startWithClock(t, "", map[string]string{
|
s, _, _ := startWithClock(t, "", map[string]string{
|
||||||
rulesDir: writeRules(t, testRules),
|
rulesDir: writeRules(t, testRules),
|
||||||
metricsToken: token,
|
metricsToken: token,
|
||||||
@@ -194,7 +196,7 @@ func TestMetricsCountRuleMatchesAndBansForAnAttack(t *testing.T) {
|
|||||||
s.request(client, "/.env", http.StatusForbidden, requestlog.ActionBanned)
|
s.request(client, "/.env", http.StatusForbidden, requestlog.ActionBanned)
|
||||||
s.get(client, http.StatusForbidden, requestlog.ActionBanned)
|
s.get(client, http.StatusForbidden, requestlog.ActionBanned)
|
||||||
|
|
||||||
metrics := s.scrape()
|
metrics := s.scrape(scraper)
|
||||||
wantMetric(t, metrics,
|
wantMetric(t, metrics,
|
||||||
`smallwebwaf_rule_matches_total{action="block",instance="app",rule_id="blocked"}`, 1)
|
`smallwebwaf_rule_matches_total{action="block",instance="app",rule_id="blocked"}`, 1)
|
||||||
wantMetric(t, metrics,
|
wantMetric(t, metrics,
|
||||||
|
|||||||
@@ -10,15 +10,13 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// trapPath reports whether the request asks for a path in
|
// trapPath reports whether the request asks for a path in
|
||||||
// SWWAF_TRAP_PATHS: its path as a path rule sees it, without the query,
|
// SWWAF_TRAP_PATHS: its path as a path rule sees it, before any decoding
|
||||||
// as sent or once percent-decoded, is one of them. Such a request is a
|
// and without the query, is one of them. Such a request is a clear sign of
|
||||||
// clear sign of attack, as a ban rule's match is: it bans the client's
|
// attack, as a ban rule's match is: it bans the client's netblock, or in
|
||||||
// netblock, or in observe mode raises the alert for the ban it would have
|
// observe mode raises the alert for the ban it would have made.
|
||||||
// made. The ban's notes give the path as sent.
|
|
||||||
func (rq *request) trapPath(now time.Time) bool {
|
func (rq *request) trapPath(now time.Time) bool {
|
||||||
path := rules.Path(rq.in)
|
path := rules.Path(rq.in)
|
||||||
if !slices.Contains(rq.h.config.TrapPaths, path) &&
|
if !slices.Contains(rq.h.config.TrapPaths, path) {
|
||||||
!slices.Contains(rq.h.config.TrapPaths, rules.DecodeOnce(path)) {
|
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,509 +0,0 @@
|
|||||||
package proxy
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"log/slog"
|
|
||||||
"net/http"
|
|
||||||
"net/netip"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
"unicode/utf8"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The headers of rproxy's question and of servewaf's answer, as
|
|
||||||
// "servewaf" in SPEC.md gives them.
|
|
||||||
const (
|
|
||||||
// ClientHeader marks a request as rproxy's question, and gives the
|
|
||||||
// visitor's address.
|
|
||||||
ClientHeader = "X-Smallwebwaf-Client"
|
|
||||||
// PeerHeader gives the address of rproxy's own TCP peer.
|
|
||||||
PeerHeader = "X-Smallwebwaf-Peer"
|
|
||||||
// ContentLengthHeader gives the body length the visitor announced.
|
|
||||||
ContentLengthHeader = "X-Smallwebwaf-Content-Length"
|
|
||||||
// DispositionHeader gives servewaf's answer.
|
|
||||||
DispositionHeader = "X-Smallwebwaf-Disposition"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The values of DispositionHeader.
|
|
||||||
const (
|
|
||||||
// DispositionAllow tells rproxy to pass the request to the app.
|
|
||||||
DispositionAllow = "allow"
|
|
||||||
// DispositionDeny tells rproxy to give the visitor the answer's status,
|
|
||||||
// headers and body.
|
|
||||||
DispositionDeny = "deny"
|
|
||||||
// DispositionClose tells rproxy to close the visitor's connection
|
|
||||||
// without an answer, which SWWAF_BAN_RESPONSE asks for when it is close.
|
|
||||||
// servewaf closing the question's connection instead would read as
|
|
||||||
// servewaf being down, and rproxy would then let the request through.
|
|
||||||
DispositionClose = "close"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ReportPath is where rproxy sends the report of each request servewaf
|
|
||||||
// answered.
|
|
||||||
const ReportPath = "/_smallwebwaf/report"
|
|
||||||
|
|
||||||
// QuestionBodyMaxBytes is the most of a visitor's body a question carries.
|
|
||||||
const QuestionBodyMaxBytes = 64 << 10
|
|
||||||
|
|
||||||
// The header limits: the most cookies and header fields a request may have
|
|
||||||
// when the checks reach the Core Rule Set. Real clients never reach them.
|
|
||||||
const (
|
|
||||||
maxCookies = 50
|
|
||||||
maxHeaderFields = 100
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// maxAwaitingReport is the most answered requests held until their
|
|
||||||
// reports come. Past it, the request answered longest ago is dropped.
|
|
||||||
maxAwaitingReport = 10000
|
|
||||||
// maxRequestIDBytes is the longest id a question may give.
|
|
||||||
maxRequestIDBytes = 128
|
|
||||||
// reportBodyMaxBytes is the most of a report's body that is read. It is
|
|
||||||
// far above any report's.
|
|
||||||
reportBodyMaxBytes = 1 << 20
|
|
||||||
// maxHTTPStatus is the highest status a report may give.
|
|
||||||
maxHTTPStatus = 599
|
|
||||||
// maxReportedBytes is the most bytes a report may give for each way, 1
|
|
||||||
// PiB, so that adding them up cannot overflow.
|
|
||||||
maxReportedBytes = 1 << 50
|
|
||||||
// maxReportedMilliseconds is the longest duration a report may give,
|
|
||||||
// about 31 years, so that it fits a time.Duration.
|
|
||||||
maxReportedMilliseconds = 1e12
|
|
||||||
// serveWAFTimeout bounds how long a request to servewaf takes to arrive,
|
|
||||||
// and its answer to leave, and serveWAFIdleTimeout how long a kept-open
|
|
||||||
// connection waits for its next request.
|
|
||||||
serveWAFTimeout = 60 * time.Second
|
|
||||||
serveWAFIdleTimeout = 120 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
errReportFieldMissing = errors.New("a field is missing")
|
|
||||||
errReportValue = errors.New("a value is out of range")
|
|
||||||
)
|
|
||||||
|
|
||||||
// Report is the JSON object rproxy posts to ReportPath once the visitor's
|
|
||||||
// answer has ended, its fields named as the request log's of the same
|
|
||||||
// meaning. RequestID is the question's id. Status is what the visitor got, 0
|
|
||||||
// if nothing. The durations are in milliseconds.
|
|
||||||
//
|
|
||||||
//nolint:tagliatelle // docs/request-log.md names the fields in snake_case
|
|
||||||
type Report struct {
|
|
||||||
RequestID string `json:"request_id"`
|
|
||||||
Status int `json:"status"`
|
|
||||||
RequestBytes int64 `json:"request_bytes"`
|
|
||||||
ResponseBytes int64 `json:"response_bytes"`
|
|
||||||
DurationTotal float64 `json:"duration_total"`
|
|
||||||
DurationUpstreamTotal float64 `json:"duration_upstream_total"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewServeWAF returns the server `smallwebwaf servewaf` runs. It answers
|
|
||||||
// rproxy's questions, takes rproxy's reports and answers its own endpoints. It
|
|
||||||
// passes nothing to an app. rproxy applies the size and time limits to the
|
|
||||||
// visitor. Go's default limit on a request's line and headers, 1 MiB, stays: it
|
|
||||||
// is far above rproxy's, and a question refused for its size would have rproxy
|
|
||||||
// let the request through.
|
|
||||||
func NewServeWAF(params Params) *Server {
|
|
||||||
h := newHandler(params)
|
|
||||||
|
|
||||||
return &Server{
|
|
||||||
Server: &http.Server{
|
|
||||||
Addr: params.Config.ListenAddr,
|
|
||||||
Handler: h,
|
|
||||||
ErrorLog: slog.NewLogLogger(params.ProcessLog.Handler(), slog.LevelWarn),
|
|
||||||
ReadTimeout: serveWAFTimeout,
|
|
||||||
WriteTimeout: serveWAFTimeout,
|
|
||||||
IdleTimeout: serveWAFIdleTimeout,
|
|
||||||
},
|
|
||||||
Ledger: h.ledger,
|
|
||||||
Limiter: h.limiter,
|
|
||||||
GeoJS: h.geojs,
|
|
||||||
Anomalies: h.anomalies,
|
|
||||||
LookupFile: h.lookupFile,
|
|
||||||
Lists: h.lists,
|
|
||||||
DNSBL: h.dnsbl,
|
|
||||||
AbuseIPDB: h.abuseIPDB,
|
|
||||||
Metrics: h.metrics,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ServeHTTP handles one request to servewaf: a request carrying ClientHeader is
|
|
||||||
// rproxy's question, and any other is for servewaf itself.
|
|
||||||
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if len(r.Header.Values(ClientHeader)) > 0 {
|
|
||||||
h.answerQuestion(w, r)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
h.answerOwnEndpoint(w, r)
|
|
||||||
}
|
|
||||||
|
|
||||||
// refusedWhateverTheAnswer reports whether the request is one that does not wait
|
|
||||||
// for GeoJS, since the checks will refuse it at now whatever GeoJS answers about
|
|
||||||
// its client: it is over the header limits, a blocklist lists its client while
|
|
||||||
// SWWAF_BLOCKLIST_ACTION is deny, the CrowdSec decision list lists it, or it is
|
|
||||||
// over a whole rate limit. In observe mode, which refuses nothing, none is.
|
|
||||||
// rproxy lets a request through when servewaf answers too late.
|
|
||||||
func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
|
|
||||||
if rq.h.config.Observe {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
lists := rq.h.lists
|
|
||||||
_, decided := lists.CrowdSecDecision(rq.client, now)
|
|
||||||
|
|
||||||
return decided || rq.overHeaderLimits() ||
|
|
||||||
(rq.h.config.BlocklistAction == deny && len(lists.ListedBy(rq.client)) > 0) ||
|
|
||||||
(rq.countedByRateLimits() &&
|
|
||||||
rq.h.limiter.OverARateLimit(rq.h.clientGroup(rq.client), now))
|
|
||||||
}
|
|
||||||
|
|
||||||
// overHeaderLimits reports whether servewaf refuses the request for its
|
|
||||||
// headers: it has more than maxHeaderFields fields, each value of a header sent
|
|
||||||
// twice being one, or more than maxCookies cookies, each part of a Cookie header
|
|
||||||
// between semicolons being one unless it is empty or only spaces and tabs. The
|
|
||||||
// Core Rule Set runs each of its rules on each cookie, so that thousands of small
|
|
||||||
// cookies keep it busy for over a second. It trims only spaces and control
|
|
||||||
// characters, of which a header value can hold only tabs, so a part of Unicode
|
|
||||||
// spaces is a cookie to it.
|
|
||||||
func (rq *request) overHeaderLimits() bool {
|
|
||||||
fields, cookies := 0, 0
|
|
||||||
|
|
||||||
for _, values := range rq.in.Header {
|
|
||||||
fields += len(values)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, value := range rq.in.Header.Values("Cookie") {
|
|
||||||
for part := range strings.SplitSeq(value, ";") {
|
|
||||||
if strings.Trim(part, " \t") != "" {
|
|
||||||
cookies++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return fields > maxHeaderFields || cookies > maxCookies
|
|
||||||
}
|
|
||||||
|
|
||||||
// answerQuestion runs the checks on the visitor's request a question
|
|
||||||
// carries, and answers rproxy what to do with it. A denied request's counting
|
|
||||||
// is finished here, and an allowed one's by its report.
|
|
||||||
func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
|
|
||||||
client, body, ok := h.readQuestion(w, r)
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// One that cannot be read leaves the log line's peer_ip invalid.
|
|
||||||
peer, _ := netip.ParseAddr(r.Header.Get(PeerHeader))
|
|
||||||
|
|
||||||
rq := h.startRequest(w, asTheVisitorSentIt(r, body), client.Unmap(), peer.Unmap(),
|
|
||||||
true)
|
|
||||||
// The last deferred runs first, so that what is held has the finished log
|
|
||||||
// line. Go's server sends the answer only once this function has
|
|
||||||
// returned, so the report cannot come before it is held.
|
|
||||||
defer rq.holdForReport()
|
|
||||||
defer rq.finish()
|
|
||||||
|
|
||||||
rq.site = r.Host
|
|
||||||
rq.line.ServeWAFLine = requestlog.ServeWAFLineDecision
|
|
||||||
refused := rq.check(r.Context())
|
|
||||||
rq.checked = time.Now()
|
|
||||||
underOwnPrefix := strings.HasPrefix(r.URL.Path, adminPrefix)
|
|
||||||
|
|
||||||
if refused == nil && !underOwnPrefix {
|
|
||||||
// In observe mode, what enforce mode would have refused counts here.
|
|
||||||
defer rq.countRefusal()
|
|
||||||
|
|
||||||
rq.allow()
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// The last deferred runs first: countRefusal before addToHistory, so that a
|
|
||||||
// broken error burst is in the client's history.
|
|
||||||
defer rq.addToHistory()
|
|
||||||
defer rq.countAnomalies()
|
|
||||||
defer rq.countRefusal()
|
|
||||||
|
|
||||||
if refused != nil {
|
|
||||||
rq.deny(*refused)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// The app never sees this prefix. servewaf's own endpoints are at its
|
|
||||||
// own address, so this answers as a path with no endpoint does.
|
|
||||||
rq.line.Action = requestlog.ActionAdmin
|
|
||||||
rq.out.Header().Set(DispositionHeader, DispositionDeny)
|
|
||||||
http.Error(rq.out, http.StatusText(http.StatusNotFound), http.StatusNotFound)
|
|
||||||
}
|
|
||||||
|
|
||||||
// readQuestion returns the visitor's address and the body a question carries,
|
|
||||||
// and whether servewaf takes the question. It answers a question it does not
|
|
||||||
// take with 400 and no disposition, which rproxy treats as it treats no
|
|
||||||
// answer. The id of a question it takes is held from here on, without a
|
|
||||||
// decision yet, so that another question with that id is refused while this
|
|
||||||
// one is being handled.
|
|
||||||
//
|
|
||||||
// The body is read before any check, since Go's server does not send an answer
|
|
||||||
// until it has read the body. A question whose body stops arriving gets no
|
|
||||||
// answer, and is not looked at.
|
|
||||||
func (h *handler) readQuestion(
|
|
||||||
w http.ResponseWriter, r *http.Request,
|
|
||||||
) (netip.Addr, []byte, bool) {
|
|
||||||
client, err := netip.ParseAddr(r.Header.Get(ClientHeader))
|
|
||||||
if err != nil || client.Zone() != "" {
|
|
||||||
h.refuseQuestion(w, ClientHeader+" is not an address without a zone")
|
|
||||||
|
|
||||||
return netip.Addr{}, nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
id := r.Header.Get(requestIDHeader)
|
|
||||||
if id == "" || len(id) > maxRequestIDBytes || !utf8.ValidString(id) {
|
|
||||||
h.refuseQuestion(w, requestIDHeader+" is not a valid id")
|
|
||||||
|
|
||||||
return netip.Addr{}, nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, QuestionBodyMaxBytes))
|
|
||||||
|
|
||||||
if _, tooLarge := errors.AsType[*http.MaxBytesError](err); tooLarge {
|
|
||||||
h.refuseQuestion(w, "the body is over 64 KiB")
|
|
||||||
|
|
||||||
return netip.Addr{}, nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
h.processLog.Warn("a question's body stopped arriving", "error", err.Error())
|
|
||||||
// This panic has Go's server close the connection without an answer,
|
|
||||||
// and log nothing.
|
|
||||||
panic(http.ErrAbortHandler)
|
|
||||||
}
|
|
||||||
|
|
||||||
held, _ := h.awaitingReport.ContainsOrAdd(id, nil)
|
|
||||||
if held {
|
|
||||||
h.refuseQuestion(w, requestIDHeader+" is the id of a request servewaf holds")
|
|
||||||
|
|
||||||
return netip.Addr{}, nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
return client, body, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// refuseQuestion answers a question servewaf does not take, for reason.
|
|
||||||
func (h *handler) refuseQuestion(w http.ResponseWriter, reason string) {
|
|
||||||
h.processLog.Warn("a question is refused", "reason", reason)
|
|
||||||
http.Error(w, reason, http.StatusBadRequest)
|
|
||||||
}
|
|
||||||
|
|
||||||
// asTheVisitorSentIt returns a copy of r, a question, as the checks and the
|
|
||||||
// log line are to see it: without the headers rproxy adds for servewaf, with
|
|
||||||
// body, which has been read of r, and with the length the visitor announced,
|
|
||||||
// as -1 when it announced none. The question's own length covers only the part
|
|
||||||
// of the body it carries.
|
|
||||||
func asTheVisitorSentIt(r *http.Request, body []byte) *http.Request {
|
|
||||||
sent := r.Clone(r.Context())
|
|
||||||
announced := sent.Header.Get(ContentLengthHeader)
|
|
||||||
|
|
||||||
sent.Header.Del(ClientHeader)
|
|
||||||
sent.Header.Del(PeerHeader)
|
|
||||||
sent.Header.Del(ContentLengthHeader)
|
|
||||||
sent.Header.Del("Content-Length")
|
|
||||||
|
|
||||||
sent.ContentLength = -1
|
|
||||||
|
|
||||||
length, err := strconv.ParseInt(announced, 10, 64)
|
|
||||||
if err == nil && length >= 0 {
|
|
||||||
sent.ContentLength = length
|
|
||||||
sent.Header.Set("Content-Length", announced)
|
|
||||||
}
|
|
||||||
|
|
||||||
sent.Body = http.NoBody
|
|
||||||
if len(body) > 0 {
|
|
||||||
sent.Body = io.NopCloser(bytes.NewReader(body))
|
|
||||||
}
|
|
||||||
|
|
||||||
return sent
|
|
||||||
}
|
|
||||||
|
|
||||||
// allow answers a question with DispositionAllow and, while
|
|
||||||
// SWWAF_ADD_LOOKUP_HEADERS is set, the client's AS number and country.
|
|
||||||
func (rq *request) allow() {
|
|
||||||
rq.allowed = true
|
|
||||||
|
|
||||||
header := rq.out.Header()
|
|
||||||
header.Set(DispositionHeader, DispositionAllow)
|
|
||||||
|
|
||||||
if rq.h.config.AddLookupHeaders {
|
|
||||||
setLookupHeaders(header, rq.line.ASN, rq.line.Country)
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.out.WriteHeader(http.StatusOK)
|
|
||||||
}
|
|
||||||
|
|
||||||
// holdForReport holds a copy of what the report's steps need, by the request's
|
|
||||||
// id. A report only ever reaches that copy, never the request being handled.
|
|
||||||
func (rq *request) holdForReport() {
|
|
||||||
held := rq.decision
|
|
||||||
rq.h.awaitingReport.Add(rq.line.RequestID, &held)
|
|
||||||
}
|
|
||||||
|
|
||||||
// deny answers a question with what the visitor is to get for r.
|
|
||||||
func (rq *request) deny(r refusal) {
|
|
||||||
rq.refused = &r
|
|
||||||
|
|
||||||
if r.status == 0 {
|
|
||||||
// SWWAF_BAN_RESPONSE is close. Written past rq.out, since the log line's
|
|
||||||
// status stays 0: the visitor is sent nothing.
|
|
||||||
rq.out.Header().Set(DispositionHeader, DispositionClose)
|
|
||||||
rq.out.ResponseWriter.WriteHeader(http.StatusOK)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.out.Header().Set(DispositionHeader, DispositionDeny)
|
|
||||||
http.Error(rq.out, http.StatusText(r.status), r.status)
|
|
||||||
}
|
|
||||||
|
|
||||||
// answerOwnEndpoint answers a request to servewaf's own address. No check
|
|
||||||
// runs on it: the address is on the private network, where the peers are
|
|
||||||
// rproxy and the admin's tools, not visitors.
|
|
||||||
func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.Method == http.MethodPost && r.URL.Path == ReportPath {
|
|
||||||
h.takeReport(w, r)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
peer := peerAddress(r)
|
|
||||||
|
|
||||||
rq := h.startRequest(w, r, peer, peer, false)
|
|
||||||
defer rq.finish()
|
|
||||||
|
|
||||||
if r.Method == http.MethodGet && r.URL.Path == HealthPath {
|
|
||||||
rq.answerHealth()
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.answerAdmin()
|
|
||||||
}
|
|
||||||
|
|
||||||
// takeReport answers POST ReportPath: it finishes the request the report
|
|
||||||
// names, if it holds that request's decision, and ignores the report
|
|
||||||
// otherwise.
|
|
||||||
func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) {
|
|
||||||
report, err := readReport(w, r)
|
|
||||||
if err != nil {
|
|
||||||
h.processLog.Warn("a report is refused", "error", err.Error())
|
|
||||||
http.Error(w, "the body is not a report", http.StatusBadRequest)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
held, found := h.awaitingReport.Peek(report.RequestID)
|
|
||||||
// Remove reports true to one report only, of several that name the id. A
|
|
||||||
// question still being handled is held as nil.
|
|
||||||
if found && held != nil && h.awaitingReport.Remove(report.RequestID) {
|
|
||||||
h.finishWith(held, report)
|
|
||||||
}
|
|
||||||
|
|
||||||
w.WriteHeader(http.StatusNoContent)
|
|
||||||
}
|
|
||||||
|
|
||||||
// readReport reads a report from r's body, which must be one JSON object with
|
|
||||||
// all of Report's fields, in the ranges SPEC.md gives.
|
|
||||||
func readReport(w http.ResponseWriter, r *http.Request) (Report, error) {
|
|
||||||
var report Report
|
|
||||||
|
|
||||||
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, reportBodyMaxBytes))
|
|
||||||
if err != nil {
|
|
||||||
return report, fmt.Errorf("read the body: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var fields map[string]json.RawMessage
|
|
||||||
|
|
||||||
err = json.Unmarshal(body, &fields)
|
|
||||||
if err != nil {
|
|
||||||
return report, fmt.Errorf("decode the body: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, name := range []string{
|
|
||||||
"request_id", "status", "request_bytes", "response_bytes",
|
|
||||||
"duration_total", "duration_upstream_total",
|
|
||||||
} {
|
|
||||||
value, there := fields[name]
|
|
||||||
if !there || string(value) == "null" {
|
|
||||||
return report, fmt.Errorf("%w: %s", errReportFieldMissing, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
err = json.Unmarshal(body, &report)
|
|
||||||
if err != nil {
|
|
||||||
return report, fmt.Errorf("decode the body: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return report, report.checkRanges()
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkRanges returns an error for a value of the report that cannot be true.
|
|
||||||
func (report *Report) checkRanges() error {
|
|
||||||
switch {
|
|
||||||
case report.Status != 0 &&
|
|
||||||
(report.Status < http.StatusContinue || report.Status > maxHTTPStatus):
|
|
||||||
return fmt.Errorf("%w: status", errReportValue)
|
|
||||||
case report.RequestBytes < 0 || report.RequestBytes > maxReportedBytes:
|
|
||||||
return fmt.Errorf("%w: request_bytes", errReportValue)
|
|
||||||
case report.ResponseBytes < 0 || report.ResponseBytes > maxReportedBytes:
|
|
||||||
return fmt.Errorf("%w: response_bytes", errReportValue)
|
|
||||||
case report.DurationTotal < 0 || report.DurationTotal > maxReportedMilliseconds:
|
|
||||||
return fmt.Errorf("%w: duration_total", errReportValue)
|
|
||||||
case report.DurationUpstreamTotal < 0 ||
|
|
||||||
report.DurationUpstreamTotal > maxReportedMilliseconds:
|
|
||||||
return fmt.Errorf("%w: duration_upstream_total", errReportValue)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// finishWith finishes a request with what its report says, and writes the
|
|
||||||
// request's last log line, which holds its decision's fields and the report's.
|
|
||||||
// For an allowed request it first counts the request's bytes, and adds it to its
|
|
||||||
// client's history, the anomaly counters and the metrics, with the status and
|
|
||||||
// bytes the report gives. A denied request's counting was finished at its
|
|
||||||
// decision.
|
|
||||||
func (h *handler) finishWith(held *decision, report Report) {
|
|
||||||
rq := &request{
|
|
||||||
decision: *held,
|
|
||||||
h: h,
|
|
||||||
out: &responseWriter{status: report.Status, bytes: report.ResponseBytes},
|
|
||||||
requestBytes: report.RequestBytes,
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.line.ServeWAFLine = requestlog.ServeWAFLineReport
|
|
||||||
rq.line.Status = report.Status
|
|
||||||
rq.line.RequestBytes = report.RequestBytes
|
|
||||||
rq.line.ResponseBytes = report.ResponseBytes
|
|
||||||
rq.line.DurationTotal = report.DurationTotal
|
|
||||||
|
|
||||||
if rq.allowed {
|
|
||||||
rq.line.DurationUpstreamTotal = &report.DurationUpstreamTotal
|
|
||||||
|
|
||||||
rq.countBytes()
|
|
||||||
rq.countAnomalies()
|
|
||||||
rq.addToHistory()
|
|
||||||
|
|
||||||
rq.h.metrics.RequestReported(&rq.line,
|
|
||||||
time.Duration(report.DurationTotal*float64(time.Millisecond)),
|
|
||||||
time.Duration(report.DurationUpstreamTotal*float64(time.Millisecond)))
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.writeLine()
|
|
||||||
}
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -17,7 +17,7 @@ const (
|
|||||||
unlistedAddr = "2001:db8::2"
|
unlistedAddr = "2001:db8::2"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestAllowNetsSkipEveryCheck(t *testing.T) {
|
func TestAllowNetsSkipEveryCheckButTheSizeLimit(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
var calls atomic.Int32
|
var calls atomic.Int32
|
||||||
@@ -34,6 +34,7 @@ func TestAllowNetsSkipEveryCheck(t *testing.T) {
|
|||||||
denyNets: fromKP,
|
denyNets: fromKP,
|
||||||
deniedCountries: "kp",
|
deniedCountries: "kp",
|
||||||
rateLimitPerMinute: "1",
|
rateLimitPerMinute: "1",
|
||||||
|
requestMaxBytes: "1K",
|
||||||
})
|
})
|
||||||
|
|
||||||
// Neither SWWAF_DENY_NETS, the country lists nor the limit of one
|
// Neither SWWAF_DENY_NETS, the country lists nor the limit of one
|
||||||
@@ -48,6 +49,14 @@ func TestAllowNetsSkipEveryCheck(t *testing.T) {
|
|||||||
t.Errorf("GeoJS was asked about %v, want nothing", asked())
|
t.Errorf("GeoJS was asked about %v, want nothing", asked())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The size limit still applies.
|
||||||
|
body := strings.NewReader(strings.Repeat("a", 2<<10))
|
||||||
|
req := newRequest(t, http.MethodPost, addr, "/", body)
|
||||||
|
req.Header.Set(forwardedFor, fromKP)
|
||||||
|
wantStatus(t, do(t, req), http.StatusRequestEntityTooLarge)
|
||||||
|
wantLine(t, out.requestLines(t, 3)[2],
|
||||||
|
http.StatusRequestEntityTooLarge, requestlog.ActionTooLarge)
|
||||||
|
|
||||||
if calls.Load() != 2 {
|
if calls.Load() != 2 {
|
||||||
t.Errorf("the app was called %d times, want 2", calls.Load())
|
t.Errorf("the app was called %d times, want 2", calls.Load())
|
||||||
}
|
}
|
||||||
@@ -73,7 +82,7 @@ func TestRequestFromAllowNetsIsNotCounted(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDenyNetsRefuseBeforeTheLookup(t *testing.T) {
|
func TestDenyNetsRefuseBeforeTheLookupAndTheBody(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
var calls atomic.Int32
|
var calls atomic.Int32
|
||||||
@@ -92,9 +101,13 @@ func TestDenyNetsRefuseBeforeTheLookup(t *testing.T) {
|
|||||||
req.Header.Set(forwardedFor, fromDE)
|
req.Header.Set(forwardedFor, fromDE)
|
||||||
wantStatus(t, do(t, req), http.StatusForbidden)
|
wantStatus(t, do(t, req), http.StatusForbidden)
|
||||||
|
|
||||||
line := out.endedLine(t)
|
line := out.requestLine(t)
|
||||||
wantLine(t, line, http.StatusForbidden, requestlog.ActionDenied)
|
wantLine(t, line, http.StatusForbidden, requestlog.ActionDenied)
|
||||||
|
|
||||||
|
if line.RequestBytes != 0 {
|
||||||
|
t.Errorf("log line has request_bytes %d, want 0", line.RequestBytes)
|
||||||
|
}
|
||||||
|
|
||||||
if len(asked()) != 0 {
|
if len(asked()) != 0 {
|
||||||
t.Errorf("GeoJS was asked about %v, want nothing", asked())
|
t.Errorf("GeoJS was asked about %v, want nothing", asked())
|
||||||
}
|
}
|
||||||
@@ -167,6 +180,6 @@ func wantAnswers(t *testing.T, addr string, out *output, requests []sentRequest)
|
|||||||
req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
|
req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
|
||||||
req.Header.Set(forwardedFor, sent.client)
|
req.Header.Set(forwardedFor, sent.client)
|
||||||
wantStatus(t, do(t, req), sent.status)
|
wantStatus(t, do(t, req), sent.status)
|
||||||
wantLine(t, out.endedLines(t, i+1)[i], sent.status, sent.action)
|
wantLine(t, out.requestLines(t, i+1)[i], sent.status, sent.action)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,307 @@
|
|||||||
|
package proxy_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strconv"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
|
)
|
||||||
|
|
||||||
|
// largeBodySize is more than the connections between the client,
|
||||||
|
// smallwebwaf and the app can hold while nobody reads, so that a sender
|
||||||
|
// soon waits.
|
||||||
|
const largeBodySize = 64 << 20
|
||||||
|
|
||||||
|
// writeSize is how much a test sender writes at a time.
|
||||||
|
const writeSize = 32 << 10
|
||||||
|
|
||||||
|
func TestRequestTimeouts(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
// limit is the setting set to shortTimeout, which runs out; long
|
||||||
|
// is one set to longTimeoutSetting, which does not, or "".
|
||||||
|
limit, long string
|
||||||
|
// appTakesNothing has the app never read, while the client sends
|
||||||
|
// as fast as it can; otherwise the app reads, and the client
|
||||||
|
// stops sending halfway.
|
||||||
|
appTakesNothing bool
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "client request timeout, waiting on the client",
|
||||||
|
limit: clientRequestTimeout,
|
||||||
|
want: http.StatusRequestTimeout,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "upstream request timeout, waiting on the client",
|
||||||
|
limit: upstreamRequestTimeout,
|
||||||
|
long: clientRequestTimeout,
|
||||||
|
want: http.StatusRequestTimeout,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "upstream request timeout, waiting on the app",
|
||||||
|
limit: upstreamRequestTimeout,
|
||||||
|
appTakesNothing: true,
|
||||||
|
want: http.StatusGatewayTimeout,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "client request timeout, waiting on the app",
|
||||||
|
limit: clientRequestTimeout,
|
||||||
|
long: upstreamRequestTimeout,
|
||||||
|
appTakesNothing: true,
|
||||||
|
want: http.StatusGatewayTimeout,
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var (
|
||||||
|
app *httptest.Server
|
||||||
|
appURL string
|
||||||
|
sendRequest func(*testing.T, string) net.Conn
|
||||||
|
appGotBody atomic.Bool
|
||||||
|
)
|
||||||
|
|
||||||
|
if tc.appTakesNothing {
|
||||||
|
appURL, sendRequest = startAppThatTakesNothing(t), sendLargeBody
|
||||||
|
} else {
|
||||||
|
app = startApp(t, func(_ http.ResponseWriter, r *http.Request) {
|
||||||
|
n, _ := io.Copy(io.Discard, r.Body)
|
||||||
|
appGotBody.Store(n > 0)
|
||||||
|
})
|
||||||
|
appURL, sendRequest = app.URL, sendPartOfBody
|
||||||
|
}
|
||||||
|
|
||||||
|
env := map[string]string{tc.limit: shortTimeoutSetting, metricsToken: token}
|
||||||
|
if tc.long != "" {
|
||||||
|
env[tc.long] = longTimeoutSetting
|
||||||
|
}
|
||||||
|
|
||||||
|
addr, out := startProxy(t, appURL, env)
|
||||||
|
start := time.Now()
|
||||||
|
got := readResponse(t, sendRequest(t, addr))
|
||||||
|
wantTimedOut(t, start)
|
||||||
|
|
||||||
|
want := tc.want
|
||||||
|
|
||||||
|
if app != nil {
|
||||||
|
// Close returns once the app has finished with the request.
|
||||||
|
app.Close()
|
||||||
|
|
||||||
|
// Until some of the body has reached the app, smallwebwaf
|
||||||
|
// waits on the app, and SPEC.md asks for 504; the timeout
|
||||||
|
// runs out then only if the test process is held up.
|
||||||
|
if !appGotBody.Load() {
|
||||||
|
want = http.StatusGatewayTimeout
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
wantStatus(t, got, want)
|
||||||
|
wantLine(t, out.requestLine(t), want, requestlog.ActionTimedOut)
|
||||||
|
wantLimitHits(t, addr, tc.limit, 1)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// startAppThatTakesNothing starts an app that accepts connections and
|
||||||
|
// never reads from them, and returns its URL.
|
||||||
|
func startAppThatTakesNothing(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("listen: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
mu sync.Mutex
|
||||||
|
held []net.Conn
|
||||||
|
)
|
||||||
|
|
||||||
|
hold := func(conn net.Conn) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
|
||||||
|
held = append(held, conn)
|
||||||
|
}
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
conn, err := listener.Accept()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
hold(conn)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
t.Cleanup(func() {
|
||||||
|
_ = listener.Close()
|
||||||
|
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
|
||||||
|
for _, conn := range held {
|
||||||
|
_ = conn.Close()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
return "http://" + listener.Addr().String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// sendPartOfBody sends a request that announces a large body, and only
|
||||||
|
// the first bytes of it.
|
||||||
|
func sendPartOfBody(t *testing.T, addr string) net.Conn {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
conn := dial(t, addr)
|
||||||
|
send(t, conn, "POST /upload HTTP/1.1\r\nHost: app\r\nContent-Length: "+
|
||||||
|
strconv.Itoa(largeBodySize)+"\r\n\r\nthe first bytes")
|
||||||
|
|
||||||
|
return conn
|
||||||
|
}
|
||||||
|
|
||||||
|
// sendLargeBody sends a request with a large body, as fast as smallwebwaf
|
||||||
|
// takes it, from a goroutine of its own.
|
||||||
|
func sendLargeBody(t *testing.T, addr string) net.Conn {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
conn := dial(t, addr)
|
||||||
|
send(t, conn, "POST /upload HTTP/1.1\r\nHost: app\r\nContent-Length: "+
|
||||||
|
strconv.Itoa(largeBodySize)+"\r\n\r\n")
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
chunk := make([]byte, writeSize)
|
||||||
|
for range largeBodySize / writeSize {
|
||||||
|
_, err := conn.Write(chunk)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
return conn
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAppTooSlowToAnswer(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
app := startApp(t, func(_ http.ResponseWriter, r *http.Request) {
|
||||||
|
<-r.Context().Done()
|
||||||
|
})
|
||||||
|
addr, out := startProxy(t, app.URL, map[string]string{
|
||||||
|
upstreamResponseTimeout: shortTimeoutSetting,
|
||||||
|
metricsToken: token,
|
||||||
|
})
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
|
||||||
|
wantStatus(t, get(t, addr, "/slow"), http.StatusGatewayTimeout)
|
||||||
|
wantTimedOut(t, start)
|
||||||
|
|
||||||
|
line := out.requestLine(t)
|
||||||
|
wantLine(t, line, http.StatusGatewayTimeout, requestlog.ActionTimedOut)
|
||||||
|
|
||||||
|
_, answered := line.fields["upstream_status"]
|
||||||
|
if answered {
|
||||||
|
t.Errorf("log line has upstream_status %v for an app that never answered",
|
||||||
|
line.fields["upstream_status"])
|
||||||
|
}
|
||||||
|
|
||||||
|
wantLimitHits(t, addr, upstreamResponseTimeout, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAppTooSlowToFinishItsAnswer(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_, _ = io.WriteString(w, "the first part")
|
||||||
|
_ = http.NewResponseController(w).Flush()
|
||||||
|
|
||||||
|
<-r.Context().Done()
|
||||||
|
})
|
||||||
|
addr, out := startProxy(t, app.URL, map[string]string{
|
||||||
|
upstreamResponseTimeout: shortTimeoutSetting,
|
||||||
|
})
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
got := get(t, addr, "/slow")
|
||||||
|
wantStatus(t, got, http.StatusOK)
|
||||||
|
|
||||||
|
if string(got.body) != "the first part" || !errors.Is(got.err, io.ErrUnexpectedEOF) {
|
||||||
|
t.Errorf("client read %q (%v), want the first part cut off", got.body, got.err)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantTimedOut(t, start)
|
||||||
|
|
||||||
|
line := out.requestLine(t)
|
||||||
|
wantLine(t, line, http.StatusOK, requestlog.ActionTimedOut)
|
||||||
|
|
||||||
|
if line.UpstreamStatus != http.StatusOK {
|
||||||
|
t.Errorf("log line has upstream_status %d, want %d",
|
||||||
|
line.UpstreamStatus, http.StatusOK)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClientTooSlowToTakeTheAnswer(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
app := startApp(t, func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
chunk := make([]byte, writeSize)
|
||||||
|
for range largeBodySize / writeSize {
|
||||||
|
_, err := w.Write(chunk)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
addr, out := startProxy(t, app.URL, map[string]string{
|
||||||
|
clientResponseTimeout: shortTimeoutSetting,
|
||||||
|
metricsToken: token,
|
||||||
|
})
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
|
||||||
|
// The client asks, and never reads the answer.
|
||||||
|
conn := dial(t, addr)
|
||||||
|
send(t, conn, "GET /large HTTP/1.1\r\nHost: app\r\n\r\n")
|
||||||
|
|
||||||
|
line := out.requestLine(t)
|
||||||
|
wantTimedOut(t, start)
|
||||||
|
wantLine(t, line, http.StatusOK, requestlog.ActionTimedOut)
|
||||||
|
wantLimitHits(t, addr, clientResponseTimeout, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClosesAnIdleConnection(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
app := startApp(t, func(http.ResponseWriter, *http.Request) {})
|
||||||
|
addr, _ := startProxy(t, app.URL, map[string]string{
|
||||||
|
clientIdleTimeout: shortTimeoutSetting,
|
||||||
|
})
|
||||||
|
|
||||||
|
// The idle time starts once the answer is sent, so after start.
|
||||||
|
start := time.Now()
|
||||||
|
conn := dial(t, addr)
|
||||||
|
send(t, conn, "GET / HTTP/1.1\r\nHost: app\r\n\r\n")
|
||||||
|
wantStatus(t, readResponse(t, conn), http.StatusOK)
|
||||||
|
|
||||||
|
// The read deadline readResponse set still bounds this read.
|
||||||
|
_, err := conn.Read(make([]byte, 1))
|
||||||
|
if !errors.Is(err, io.EOF) {
|
||||||
|
t.Fatalf("read on the idle connection: %v, want it closed", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantTimedOut(t, start)
|
||||||
|
}
|
||||||
@@ -32,9 +32,10 @@ func TestTrapPathBansAsABanRuleDoes(t *testing.T) {
|
|||||||
})
|
})
|
||||||
start := clk.Now()
|
start := clk.Now()
|
||||||
|
|
||||||
// Only the path itself, the whole of it in the same case, is a trap
|
// Only the path itself, as the client sent it, is a trap path.
|
||||||
// path.
|
for _, path := range []string{
|
||||||
for _, path := range []string{"/wp-login.php/", "/WP-LOGIN.PHP", "/blog/xmlrpc.php"} {
|
"/wp-login.php/", "/WP-LOGIN.PHP", "/blog/xmlrpc.php", "/%77p-login.php",
|
||||||
|
} {
|
||||||
s.request(otherClient, path, http.StatusOK, requestlog.ActionForward)
|
s.request(otherClient, path, http.StatusOK, requestlog.ActionForward)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -83,22 +84,6 @@ func TestTrapPathBansAsABanRuleDoes(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTrapPathMatchesThePathDecodedOnce(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s, _, server := startWithClock(t, "", map[string]string{trapPaths: trapPathList})
|
|
||||||
|
|
||||||
// %2577 decodes once to %77, not to w.
|
|
||||||
s.request(otherClient, "/%2577p-login.php", http.StatusOK, requestlog.ActionForward)
|
|
||||||
s.request(client, "/%77p-login.php", http.StatusForbidden, requestlog.ActionBanned)
|
|
||||||
|
|
||||||
// The ban's notes give the path as the client sent it.
|
|
||||||
got := server.Ledger.Bans(netip.MustParsePrefix(client + "/32"))
|
|
||||||
if len(got) != 1 || got[0].Notes.TrapPath != "/%77p-login.php" {
|
|
||||||
t.Fatalf("bans %+v, want one whose notes give the trap path /%%77p-login.php", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTrapPathsNeedNoRuleFiles(t *testing.T) {
|
func TestTrapPathsNeedNoRuleFiles(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -23,8 +23,8 @@ func TestHistoryKeepsEveryRequest(t *testing.T) {
|
|||||||
{Forwarded: true, Status: 502, ResponseBytes: 12},
|
{Forwarded: true, Status: 502, ResponseBytes: 12},
|
||||||
// Closed without an answer: refused, and no response.
|
// Closed without an answer: refused, and no response.
|
||||||
{Refused: true, Status: 0},
|
{Refused: true, Status: 0},
|
||||||
// Answered 404 for a path under /_smallwebwaf/: neither forwarded nor
|
// Answered 404 at smallwebwaf's own endpoints: neither forwarded
|
||||||
// refused.
|
// nor refused.
|
||||||
{Status: 404},
|
{Status: 404},
|
||||||
} {
|
} {
|
||||||
limiter.AddToHistory(client, start.Add(time.Duration(i)*time.Minute), r)
|
limiter.AddToHistory(client, start.Add(time.Duration(i)*time.Minute), r)
|
||||||
|
|||||||
@@ -92,9 +92,10 @@ type History struct {
|
|||||||
ASName string `json:"as_name,omitempty"`
|
ASName string `json:"as_name,omitempty"`
|
||||||
Country string `json:"country,omitempty"`
|
Country string `json:"country,omitempty"`
|
||||||
LookedUp time.Time `json:"looked_up,omitzero"`
|
LookedUp time.Time `json:"looked_up,omitzero"`
|
||||||
// Requests are all the client's requests: Forwarded those servewaf allowed,
|
// Requests are all the client's requests: Forwarded those passed to
|
||||||
// Refused those it refused, and neither those for a path under
|
// the app, Refused those refused before anything reached it, a 401 at
|
||||||
// /_smallwebwaf/.
|
// smallwebwaf's own endpoints included, and neither the others
|
||||||
|
// smallwebwaf answered there.
|
||||||
Requests int64 `json:"requests"`
|
Requests int64 `json:"requests"`
|
||||||
Forwarded int64 `json:"forwarded"`
|
Forwarded int64 `json:"forwarded"`
|
||||||
Refused int64 `json:"refused"`
|
Refused int64 `json:"refused"`
|
||||||
@@ -123,20 +124,21 @@ type Offences struct {
|
|||||||
// Limit is its requests that broke a rate limit, a byte limit or the
|
// Limit is its requests that broke a rate limit, a byte limit or the
|
||||||
// error burst, Attack those that were a clear sign of attack, a match
|
// error burst, Attack those that were a clear sign of attack, a match
|
||||||
// of a ban rule or a request for a trap path, RuleBlocked those a block
|
// of a ban rule or a request for a trap path, RuleBlocked those a block
|
||||||
// rule refused, and WAFBlocked those the Core Rule Set refused.
|
// rule refused, WAFBlocked those the Core Rule Set refused, and
|
||||||
Limit int64 `json:"limit"`
|
// TokenRefused those refused for a missing or wrong token.
|
||||||
Attack int64 `json:"attack"`
|
Limit int64 `json:"limit"`
|
||||||
RuleBlocked int64 `json:"rule_blocked"`
|
Attack int64 `json:"attack"`
|
||||||
WAFBlocked int64 `json:"waf_blocked"`
|
RuleBlocked int64 `json:"rule_blocked"`
|
||||||
// TokenRefused is what a clients.json written by an earlier version gives,
|
WAFBlocked int64 `json:"waf_blocked"`
|
||||||
// which must still load. Nothing adds to it.
|
TokenRefused int64 `json:"token_refused"`
|
||||||
TokenRefused int64 `json:"token_refused,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Request is what a client's history keeps of one of its requests.
|
// Request is what a client's history keeps of one of its requests.
|
||||||
type Request struct {
|
type Request struct {
|
||||||
// Forwarded is true for a request servewaf allowed, and Refused for one it
|
// Forwarded is true for a request passed to the app, Refused for one
|
||||||
// refused. Both are false for a request for a path under /_smallwebwaf/.
|
// refused before anything reached it, a 401 at smallwebwaf's own
|
||||||
|
// endpoints included. Both are false for any other request smallwebwaf
|
||||||
|
// answered there.
|
||||||
Forwarded bool
|
Forwarded bool
|
||||||
Refused bool
|
Refused bool
|
||||||
// Status is what the client was sent, 0 if nothing was.
|
// Status is what the client was sent, 0 if nothing was.
|
||||||
@@ -147,12 +149,14 @@ type Request struct {
|
|||||||
ResponseBytes int64
|
ResponseBytes int64
|
||||||
// BrokeLimit is true for a request that broke a rate limit, a byte
|
// BrokeLimit is true for a request that broke a rate limit, a byte
|
||||||
// limit or the error burst, Attack for one that matched a ban rule or
|
// limit or the error burst, Attack for one that matched a ban rule or
|
||||||
// asked for a trap path, RuleBlocked for one a block rule refused, and
|
// asked for a trap path, RuleBlocked for one a block rule refused,
|
||||||
// WAFBlocked for one the Core Rule Set refused.
|
// WAFBlocked for one the Core Rule Set refused, and TokenRefused for
|
||||||
BrokeLimit bool
|
// one refused for a missing or wrong token.
|
||||||
Attack bool
|
BrokeLimit bool
|
||||||
RuleBlocked bool
|
Attack bool
|
||||||
WAFBlocked bool
|
RuleBlocked bool
|
||||||
|
WAFBlocked bool
|
||||||
|
TokenRefused bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// New returns a Limiter for limits, with no client counted yet, whose
|
// New returns a Limiter for limits, with no client counted yet, whose
|
||||||
@@ -202,7 +206,7 @@ type Hit struct {
|
|||||||
// Counts are a client's requests and bytes in the minute, the hour and
|
// Counts are a client's requests and bytes in the minute, the hour and
|
||||||
// the day that end at a request, that request's included.
|
// the day that end at a request, that request's included.
|
||||||
//
|
//
|
||||||
//nolint:tagliatelle // docs/request-log.md names the fields in snake_case
|
//nolint:tagliatelle // SPEC.md's request log names its fields in snake_case
|
||||||
type Counts struct {
|
type Counts struct {
|
||||||
Minute float64 `json:"minute"`
|
Minute float64 `json:"minute"`
|
||||||
Hour float64 `json:"hour"`
|
Hour float64 `json:"hour"`
|
||||||
@@ -224,30 +228,6 @@ func (l *Limiter) Count(
|
|||||||
return l.count(client, now, 1, 0, percent)
|
return l.count(client, now, 1, 0, percent)
|
||||||
}
|
}
|
||||||
|
|
||||||
// OverARateLimit reports whether one more request from client at now would
|
|
||||||
// take it over a whole rate limit. A client's percentage can only lower a
|
|
||||||
// limit, so that request is over the limit at any percentage. It counts
|
|
||||||
// nothing, and does not make the client the most recently seen.
|
|
||||||
func (l *Limiter) OverARateLimit(client netip.Prefix, now time.Time) bool {
|
|
||||||
l.mu.Lock()
|
|
||||||
defer l.mu.Unlock()
|
|
||||||
|
|
||||||
c, seen := l.clients.Peek(client)
|
|
||||||
if !seen {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
buckets := c.buckets()
|
|
||||||
|
|
||||||
for i, w := range l.windows {
|
|
||||||
if w.limit > 0 && buckets[i].Add(now, w.length, 0)+1 > float64(w.limit) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// CountBytes counts bytes, those of a request from client that has ended,
|
// CountBytes counts bytes, those of a request from client that has ended,
|
||||||
// at now, in every window, and returns the client's counts in each window.
|
// at now, in every window, and returns the client's counts in each window.
|
||||||
// It reports whether the bytes take the client over a byte limit, of which
|
// It reports whether the bytes take the client over a byte limit, of which
|
||||||
@@ -330,6 +310,10 @@ func (l *Limiter) AddToHistory(client netip.Prefix, now time.Time, r Request) {
|
|||||||
if r.WAFBlocked {
|
if r.WAFBlocked {
|
||||||
h.Offences.WAFBlocked++
|
h.Offences.WAFBlocked++
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if r.TokenRefused {
|
||||||
|
h.Offences.TokenRefused++
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// AddLookup gives client's history its AS number, AS name and country, as
|
// AddLookup gives client's history its AS number, AS name and country, as
|
||||||
|
|||||||
@@ -26,50 +26,6 @@ const (
|
|||||||
hour = "hour"
|
hour = "hour"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
|
||||||
now := midnight()
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
|
|
||||||
tableSize)
|
|
||||||
|
|
||||||
if limiter.OverARateLimit(client, now) {
|
|
||||||
t.Error("a client never seen is over a rate limit")
|
|
||||||
}
|
|
||||||
|
|
||||||
if limiter.Len() != 0 {
|
|
||||||
t.Error("asking about a client never seen added it to the table")
|
|
||||||
}
|
|
||||||
|
|
||||||
for range limit {
|
|
||||||
if limiter.OverARateLimit(client, now) {
|
|
||||||
t.Error("a client with room for another request is over a rate limit")
|
|
||||||
}
|
|
||||||
|
|
||||||
wantCount(t, limiter, client, now, "")
|
|
||||||
}
|
|
||||||
|
|
||||||
if !limiter.OverARateLimit(client, now) {
|
|
||||||
t.Error("a client at its limit is not over a rate limit")
|
|
||||||
}
|
|
||||||
|
|
||||||
held, _ := limiter.Client(client)
|
|
||||||
if held.Minute.Current != limit {
|
|
||||||
t.Errorf("the client's minute holds %d requests, want %d", held.Minute.Current, limit)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantCount(t, limiter, client, now, minute)
|
|
||||||
|
|
||||||
// A limit that is off is never gone over.
|
|
||||||
off := ratelimit.New(ratelimit.Limits{}, tableSize)
|
|
||||||
off.Count(client, now, whole)
|
|
||||||
|
|
||||||
if off.OverARateLimit(client, now) {
|
|
||||||
t.Error("a client is over a rate limit that is off")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
|
func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
// Package requestlog writes the lines smallwebwaf prints on stdout: one
|
// Package requestlog writes the lines smallwebwaf prints on stdout: one
|
||||||
// JSON object per request, two from servewaf and a short one from rproxy, marked
|
// JSON object per request, marked "type":"request", and the process's own
|
||||||
// "type":"request", and the process's own messages as JSON lines marked
|
// messages as JSON lines marked "type":"process".
|
||||||
// "type":"process".
|
|
||||||
package requestlog
|
package requestlog
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -14,13 +13,18 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The action a request line names: what servewaf did with the request.
|
// The action a request line names: what smallwebwaf did with the
|
||||||
|
// request.
|
||||||
const (
|
const (
|
||||||
// ActionForward is a request servewaf allowed, which rproxy passes to the
|
// ActionForward is a request passed to the app.
|
||||||
// app.
|
|
||||||
ActionForward = "forward"
|
ActionForward = "forward"
|
||||||
// ActionTooLarge is a request over the header limits.
|
// ActionTooLarge is a request or response over its size limit.
|
||||||
ActionTooLarge = "too_large"
|
ActionTooLarge = "too_large"
|
||||||
|
// ActionTimedOut is a request or response that ran out of time.
|
||||||
|
ActionTimedOut = "timed_out"
|
||||||
|
// ActionUpstreamError is a request the app could not be reached
|
||||||
|
// for, or whose answer could not be passed on.
|
||||||
|
ActionUpstreamError = "upstream_error"
|
||||||
// ActionRateLimited is a request refused because it took its client
|
// ActionRateLimited is a request refused because it took its client
|
||||||
// over a rate limit, which bans the client.
|
// over a rate limit, which bans the client.
|
||||||
ActionRateLimited = "rate_limited"
|
ActionRateLimited = "rate_limited"
|
||||||
@@ -42,8 +46,8 @@ const (
|
|||||||
ActionDenied = "denied"
|
ActionDenied = "denied"
|
||||||
// ActionCountryDenied is a request refused for its client's country.
|
// ActionCountryDenied is a request refused for its client's country.
|
||||||
ActionCountryDenied = "country_denied"
|
ActionCountryDenied = "country_denied"
|
||||||
// ActionAdmin is a request servewaf answered at one of its own endpoints,
|
// ActionAdmin is a request smallwebwaf answered at one of its own
|
||||||
// or a question about a path under /_smallwebwaf/.
|
// endpoints, under /_smallwebwaf/.
|
||||||
ActionAdmin = "admin"
|
ActionAdmin = "admin"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -56,32 +60,20 @@ const OffenceLimit = "limit"
|
|||||||
// that broke the error burst.
|
// that broke the error burst.
|
||||||
const LimitHitErrorBurst = "error_burst"
|
const LimitHitErrorBurst = "error_burst"
|
||||||
|
|
||||||
// The servewaf_line a request line names, which says which of servewaf's two
|
|
||||||
// lines about a question it is.
|
|
||||||
const (
|
|
||||||
// ServeWAFLineDecision is servewaf's line at its decision.
|
|
||||||
ServeWAFLineDecision = "decision"
|
|
||||||
// ServeWAFLineReport is servewaf's line when rproxy's report comes, with
|
|
||||||
// the decision's fields and the report's together.
|
|
||||||
ServeWAFLineReport = "report"
|
|
||||||
)
|
|
||||||
|
|
||||||
// timeLayout is RFC 3339 with milliseconds.
|
// timeLayout is RFC 3339 with milliseconds.
|
||||||
const timeLayout = "2006-01-02T15:04:05.000Z07:00"
|
const timeLayout = "2006-01-02T15:04:05.000Z07:00"
|
||||||
|
|
||||||
// Line is one request's line in the request log. The field names, and
|
// Line is one request's line in the request log. The field names, and
|
||||||
// their order, are those of docs/request-log.md. A field that may not
|
// their order, are those of the "Request log" section of SPEC.md. A field
|
||||||
// apply to a request is left out of its line when it does not.
|
// that may not apply to a request is left out of its line when it does
|
||||||
|
// not.
|
||||||
//
|
//
|
||||||
//nolint:tagliatelle // docs/request-log.md names the fields in snake_case
|
//nolint:tagliatelle // SPEC.md's request log names its fields in snake_case
|
||||||
type Line struct {
|
type Line struct {
|
||||||
Type string `json:"type"`
|
Type string `json:"type"`
|
||||||
// ServeWAFLine is ServeWAFLineDecision or ServeWAFLineReport on servewaf's
|
|
||||||
// two lines about a question.
|
|
||||||
ServeWAFLine string `json:"servewaf_line,omitempty"`
|
|
||||||
|
|
||||||
// The standard web log fields. Scheme is how the client reached rproxy, or
|
// The standard web log fields. Scheme is how the client reached
|
||||||
// the trusted proxy in front of it.
|
// smallwebwaf, or the trusted proxy in front of it.
|
||||||
Time string `json:"time"`
|
Time string `json:"time"`
|
||||||
Instance string `json:"instance"`
|
Instance string `json:"instance"`
|
||||||
ClientIP string `json:"client_ip"`
|
ClientIP string `json:"client_ip"`
|
||||||
@@ -97,8 +89,8 @@ type Line struct {
|
|||||||
Referer string `json:"referer"`
|
Referer string `json:"referer"`
|
||||||
UserAgent string `json:"user_agent"`
|
UserAgent string `json:"user_agent"`
|
||||||
|
|
||||||
// Request detail. RequestID is the id rproxy made for the request, or for a
|
// Request detail. RequestID is the X-Request-ID a trusted proxy sent,
|
||||||
// request to servewaf's own endpoints a new one. ForwardedFor is the
|
// or a new one, and is sent on to the app. ForwardedFor is the
|
||||||
// X-Forwarded-For header as received. ClientGroup is the netblock the
|
// X-Forwarded-For header as received. ClientGroup is the netblock the
|
||||||
// client is counted as. ASN, ASName and Country are the client's AS
|
// client is counted as. ASN, ASName and Country are the client's AS
|
||||||
// number, AS name and country, as looked up.
|
// number, AS name and country, as looked up.
|
||||||
@@ -117,10 +109,17 @@ type Line struct {
|
|||||||
RequestHeaders map[string]string `json:"request_headers,omitempty"`
|
RequestHeaders map[string]string `json:"request_headers,omitempty"`
|
||||||
HasAuthorization bool `json:"has_authorization,omitempty"`
|
HasAuthorization bool `json:"has_authorization,omitempty"`
|
||||||
HasCookie bool `json:"has_cookie,omitempty"`
|
HasCookie bool `json:"has_cookie,omitempty"`
|
||||||
|
// Websocket is true when the connection was upgraded, as for a
|
||||||
|
// WebSocket.
|
||||||
|
Websocket bool `json:"websocket,omitempty"`
|
||||||
|
|
||||||
// Response detail, of servewaf's own answer. Aborted is true when rproxy, or
|
// Response detail, from the headers of the answer: the app's, as
|
||||||
// the admin's tool, went away before the answer.
|
// passed on, or those of smallwebwaf's own. Aborted is true when the
|
||||||
|
// client went away early.
|
||||||
ResponseContentType string `json:"response_content_type,omitempty"`
|
ResponseContentType string `json:"response_content_type,omitempty"`
|
||||||
|
UpstreamStatus int `json:"upstream_status,omitempty"`
|
||||||
|
CacheControl string `json:"cache_control,omitempty"`
|
||||||
|
Location string `json:"location,omitempty"`
|
||||||
Aborted bool `json:"aborted,omitempty"`
|
Aborted bool `json:"aborted,omitempty"`
|
||||||
|
|
||||||
// The decision.
|
// The decision.
|
||||||
@@ -128,8 +127,7 @@ type Line struct {
|
|||||||
// WouldAction is, in observe mode, the action enforce mode would have
|
// WouldAction is, in observe mode, the action enforce mode would have
|
||||||
// taken with a request it would have refused: ActionDenied,
|
// taken with a request it would have refused: ActionDenied,
|
||||||
// ActionBanned, ActionCountryDenied, ActionRateLimited,
|
// ActionBanned, ActionCountryDenied, ActionRateLimited,
|
||||||
// ActionRuleBlocked, ActionWAFBlocked, or, in servewaf, ActionTooLarge
|
// ActionRuleBlocked or ActionWAFBlocked.
|
||||||
// for the header limits.
|
|
||||||
WouldAction string `json:"would_action,omitempty"`
|
WouldAction string `json:"would_action,omitempty"`
|
||||||
// LimitPercent and LimitPercentSetting are, for a request the rate
|
// LimitPercent and LimitPercentSetting are, for a request the rate
|
||||||
// limits counted whose client a biased threshold gives a percentage of
|
// limits counted whose client a biased threshold gives a percentage of
|
||||||
@@ -170,60 +168,23 @@ type Line struct {
|
|||||||
|
|
||||||
// The timings, in milliseconds. DurationChecks is the time until the
|
// The timings, in milliseconds. DurationChecks is the time until the
|
||||||
// checks were done, and DurationWAF the part of it the Core Rule Set
|
// checks were done, and DurationWAF the part of it the Core Rule Set
|
||||||
// took. DurationUpstreamTotal is what the report of an allowed request gives,
|
// took. DurationUpstreamConnect, DurationUpstreamFirstByte and
|
||||||
// from the hand-over to the app until the end of its answer. Each but
|
// DurationUpstreamTotal run from when the request was handed to the
|
||||||
// DurationTotal is nil for a request that did not get that far.
|
// app: until there was a connection to it, until the first byte of its
|
||||||
DurationTotal float64 `json:"duration_total"`
|
// answer arrived, and until the end. Each but DurationTotal is nil for
|
||||||
DurationChecks *float64 `json:"duration_checks,omitempty"`
|
// a request that did not get that far.
|
||||||
DurationWAF *float64 `json:"duration_waf,omitempty"`
|
DurationTotal float64 `json:"duration_total"`
|
||||||
DurationUpstreamTotal *float64 `json:"duration_upstream_total,omitempty"`
|
DurationChecks *float64 `json:"duration_checks,omitempty"`
|
||||||
}
|
DurationWAF *float64 `json:"duration_waf,omitempty"`
|
||||||
|
DurationUpstreamConnect *float64 `json:"duration_upstream_connect,omitempty"`
|
||||||
// RProxyLine is rproxy's short line in the request log about one request. The
|
DurationUpstreamFirstByte *float64 `json:"duration_upstream_first_byte,omitempty"`
|
||||||
// field names, and their order, are those of docs/request-log.md. A field that
|
DurationUpstreamTotal *float64 `json:"duration_upstream_total,omitempty"`
|
||||||
// does not apply to a request is left out of its line.
|
|
||||||
//
|
|
||||||
//nolint:tagliatelle // docs/request-log.md names the fields in snake_case
|
|
||||||
type RProxyLine struct {
|
|
||||||
Type string `json:"type"`
|
|
||||||
Time string `json:"time"`
|
|
||||||
RequestID string `json:"request_id"`
|
|
||||||
ClientIP string `json:"client_ip"`
|
|
||||||
Host string `json:"host"`
|
|
||||||
Method string `json:"method"`
|
|
||||||
Path string `json:"path"`
|
|
||||||
Query string `json:"query"`
|
|
||||||
Status int `json:"status"`
|
|
||||||
RequestBytes int64 `json:"request_bytes"`
|
|
||||||
ResponseBytes int64 `json:"response_bytes"`
|
|
||||||
// DurationRequest runs in milliseconds from the request's headers being read
|
|
||||||
// until the app has been sent the whole request, or the answer has started
|
|
||||||
// if that is sooner. DurationResponse runs from then until the answer ends.
|
|
||||||
DurationRequest float64 `json:"duration_request"`
|
|
||||||
DurationResponse float64 `json:"duration_response"`
|
|
||||||
// Disposition is the valid disposition servewaf answered or, while
|
|
||||||
// DefaultDisposition is true, the value of SWRP_DEFAULT_DISPOSITION. It is ""
|
|
||||||
// when neither gave one.
|
|
||||||
Disposition string `json:"disposition,omitempty"`
|
|
||||||
DefaultDisposition bool `json:"default_disposition,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write writes line to w as one JSON line marked "type":"request".
|
// Write writes line to w as one JSON line marked "type":"request".
|
||||||
func Write(w io.Writer, line *Line) error {
|
func Write(w io.Writer, line *Line) error {
|
||||||
line.Type = "request"
|
line.Type = "request"
|
||||||
|
|
||||||
return writeJSONLine(w, line)
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteRProxyLine writes line to w as one JSON line marked "type":"request".
|
|
||||||
func WriteRProxyLine(w io.Writer, line *RProxyLine) error {
|
|
||||||
line.Type = "request"
|
|
||||||
|
|
||||||
return writeJSONLine(w, line)
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeJSONLine writes line to w as one JSON line.
|
|
||||||
func writeJSONLine(w io.Writer, line any) error {
|
|
||||||
encoded, err := json.Marshal(line)
|
encoded, err := json.Marshal(line)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("encode the request log line: %w", err)
|
return fmt.Errorf("encode the request log line: %w", err)
|
||||||
|
|||||||
@@ -67,28 +67,6 @@ func TestWriteWritesOneJSONLineMarkedRequest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWriteRProxyLineWritesOneJSONLineWithOnlyItsFields(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
err := requestlog.WriteRProxyLine(&out, &requestlog.RProxyLine{
|
|
||||||
RequestID: "an-id",
|
|
||||||
Disposition: "allow",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("write: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
const want = `{"type":"request","time":"","request_id":"an-id","client_ip":"",` +
|
|
||||||
`"host":"","method":"","path":"","query":"","status":0,"request_bytes":0,` +
|
|
||||||
`"response_bytes":0,"duration_request":0,"duration_response":0,` +
|
|
||||||
`"disposition":"allow"}` + "\n"
|
|
||||||
if out.String() != want {
|
|
||||||
t.Errorf("wrote %s, want %s", out.String(), want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
|
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -1,122 +0,0 @@
|
|||||||
package rproxy
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/netip"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The headers rproxy sets on the requests it passes to the app. Go writes every
|
|
||||||
// header name in this form, as it sends it and as it receives it, so
|
|
||||||
// X-Client-ASN arrives as X-Client-Asn, and Del removes a visitor's own whatever
|
|
||||||
// their case; header names are not case-sensitive.
|
|
||||||
const (
|
|
||||||
requestIDHeader = "X-Request-ID"
|
|
||||||
asnHeader = "X-Client-Asn"
|
|
||||||
countryHeader = "X-Client-Country"
|
|
||||||
)
|
|
||||||
|
|
||||||
// peerAddress is the address of the request's TCP peer, normally traefik, without
|
|
||||||
// any IPv6 zone, which servewaf refuses.
|
|
||||||
func peerAddress(r *http.Request) netip.Addr {
|
|
||||||
addrPort, err := netip.ParseAddrPort(r.RemoteAddr)
|
|
||||||
if err != nil {
|
|
||||||
return netip.Addr{}
|
|
||||||
}
|
|
||||||
|
|
||||||
return addrPort.Addr().Unmap().WithZone("")
|
|
||||||
}
|
|
||||||
|
|
||||||
// clientAddress works out who the visitor is. A peer outside the trusted
|
|
||||||
// proxies is the visitor, and what it says in X-Forwarded-For is ignored. For a
|
|
||||||
// peer inside them, X-Forwarded-For is read from the right, and the first
|
|
||||||
// address outside them is the visitor; if every address in it is inside, the
|
|
||||||
// leftmost is, and with no header, the peer. An entry that is not an address
|
|
||||||
// ends the reading, since nothing to its left can be believed. The visitor's
|
|
||||||
// address never has an IPv6 zone, which servewaf refuses.
|
|
||||||
func clientAddress(
|
|
||||||
peer netip.Addr, forwardedFor []string, trusted []netip.Prefix,
|
|
||||||
) netip.Addr {
|
|
||||||
client := peer
|
|
||||||
if !isInside(peer, trusted) {
|
|
||||||
return client
|
|
||||||
}
|
|
||||||
|
|
||||||
entries := strings.Split(strings.Join(forwardedFor, ","), ",")
|
|
||||||
for _, entry := range slices.Backward(entries) {
|
|
||||||
addr, err := netip.ParseAddr(strings.TrimSpace(entry))
|
|
||||||
if err != nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
client = addr.Unmap().WithZone("")
|
|
||||||
if !isInside(client, trusted) {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return client
|
|
||||||
}
|
|
||||||
|
|
||||||
// scheme is how the visitor reached traefik, as a trusted proxy says in
|
|
||||||
// X-Forwarded-Proto, or otherwise http, the only scheme rproxy serves.
|
|
||||||
func scheme(r *http.Request, peerTrusted bool) string {
|
|
||||||
proto := r.Header.Get("X-Forwarded-Proto")
|
|
||||||
if !peerTrusted || proto == "" {
|
|
||||||
return "http"
|
|
||||||
}
|
|
||||||
|
|
||||||
return proto
|
|
||||||
}
|
|
||||||
|
|
||||||
// isInside reports whether addr is in one of the netblocks.
|
|
||||||
func isInside(addr netip.Addr, netblocks []netip.Prefix) bool {
|
|
||||||
return slices.ContainsFunc(netblocks, func(netblock netip.Prefix) bool {
|
|
||||||
return netblock.Contains(addr)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// setForwardedHeaders sets the headers in which the app learns about the
|
|
||||||
// visitor, so that it sees what it would see from traefik directly. A trusted
|
|
||||||
// proxy's forwarded headers pass on, with the proxy's own address added to
|
|
||||||
// X-Forwarded-For. Those of any other peer are its own claims and are replaced:
|
|
||||||
// X-Forwarded-For names the peer, X-Forwarded-Host the host it asked for, and
|
|
||||||
// X-Forwarded-Proto plain http, which is how it reached rproxy.
|
|
||||||
func setForwardedHeaders(in, out *http.Request, peer netip.Addr, trusted bool) {
|
|
||||||
forwardedFor := peer.String()
|
|
||||||
|
|
||||||
if trusted {
|
|
||||||
// ReverseProxy removes these from out before Rewrite.
|
|
||||||
for _, name := range []string{"Forwarded", "X-Forwarded-Host", "X-Forwarded-Proto"} {
|
|
||||||
values, ok := in.Header[name]
|
|
||||||
if ok {
|
|
||||||
out.Header[name] = values
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
prior := in.Header.Values("X-Forwarded-For")
|
|
||||||
if len(prior) > 0 {
|
|
||||||
forwardedFor = strings.Join(prior, ", ") + ", " + forwardedFor
|
|
||||||
}
|
|
||||||
|
|
||||||
out.Header.Set("X-Forwarded-For", forwardedFor)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// ReverseProxy has removed Forwarded and the three set below; these are the
|
|
||||||
// other headers in which traefik tells the app about the visitor and its
|
|
||||||
// request.
|
|
||||||
for _, name := range []string{
|
|
||||||
"X-Forwarded-Port", "X-Forwarded-Server", "X-Forwarded-Uri",
|
|
||||||
"X-Forwarded-Method", "X-Forwarded-Prefix", "X-Forwarded-Tls-Client-Cert",
|
|
||||||
"X-Forwarded-Tls-Client-Cert-Info", "X-Real-Ip",
|
|
||||||
} {
|
|
||||||
out.Header.Del(name)
|
|
||||||
}
|
|
||||||
|
|
||||||
out.Header.Set("X-Forwarded-For", forwardedFor)
|
|
||||||
out.Header.Set("X-Forwarded-Host", in.Host)
|
|
||||||
out.Header.Set("X-Forwarded-Proto", "http")
|
|
||||||
}
|
|
||||||
@@ -1,811 +0,0 @@
|
|||||||
package rproxy_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"io"
|
|
||||||
"log/slog"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/proxy"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
|
||||||
)
|
|
||||||
|
|
||||||
// post is the start of a POST whose headers end with what follows it.
|
|
||||||
const post = "POST / HTTP/1.1\r\nHost: site.example\r\n"
|
|
||||||
|
|
||||||
// announcedLength is the header that announces a body of size bytes, and the
|
|
||||||
// end of the headers.
|
|
||||||
func announcedLength(size int) string {
|
|
||||||
return "Content-Length: " + strconv.Itoa(size) + "\r\n\r\n"
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestServerHasTheFixedLimits(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cfg, err := config.RProxyFromEnvironment(func(name string) (string, bool) {
|
|
||||||
if name == "SWRP_WAF_BACKEND" {
|
|
||||||
return nowhere, true
|
|
||||||
}
|
|
||||||
|
|
||||||
return "", false
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("settings: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
server := rproxy.New(rproxy.Params{
|
|
||||||
Config: cfg,
|
|
||||||
RequestLog: io.Discard,
|
|
||||||
ProcessLog: requestlog.NewProcessLogger(io.Discard, "test", slog.LevelInfo),
|
|
||||||
})
|
|
||||||
|
|
||||||
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
|
|
||||||
server.IdleTimeout != 2*time.Minute || server.ReadHeaderTimeout != time.Minute {
|
|
||||||
t.Errorf("server listens on %q with header limit %d, idle time %s and "+
|
|
||||||
"header timeout %s", server.Addr, server.MaxHeaderBytes,
|
|
||||||
server.IdleTimeout, server.ReadHeaderTimeout)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestABodyAnnouncedOverItsLimitIsRefusedOnceServeWAFAllowsIt(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name, limit string
|
|
||||||
size, asked int
|
|
||||||
}{
|
|
||||||
{"with the limit within the first part", "1K", 1025, 1024},
|
|
||||||
{"with the limit past the first part", "128K", 129 << 10, questionBodyMaxBytes},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, func(http.ResponseWriter, *http.Request) {
|
|
||||||
t.Error("the app was asked")
|
|
||||||
})
|
|
||||||
addr, _ := startRProxy(t, servewaf.URL, app.URL, map[string]string{
|
|
||||||
maxRequestSize: tc.limit,
|
|
||||||
})
|
|
||||||
|
|
||||||
got := visit(t, newRequest(t, http.MethodPost, addr, "/",
|
|
||||||
bytes.NewReader(make([]byte, tc.size))))
|
|
||||||
if got.status != http.StatusRequestEntityTooLarge {
|
|
||||||
t.Errorf("answered %+v, want 413", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
if asked := servewaf.question(t); len(asked.body) != tc.asked {
|
|
||||||
t.Errorf("the question carries %d bytes, want %d", len(asked.body), tc.asked)
|
|
||||||
}
|
|
||||||
|
|
||||||
report := servewaf.report(t)
|
|
||||||
if report.Status != http.StatusRequestEntityTooLarge {
|
|
||||||
t.Errorf("report %+v, want status 413", report)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestADenyOutranksTheBodySizeLimit(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, deny)
|
|
||||||
app := startApp(t, greet)
|
|
||||||
addr, _ := startRProxy(t, servewaf.URL, app.URL, map[string]string{
|
|
||||||
maxRequestSize: "1K",
|
|
||||||
})
|
|
||||||
|
|
||||||
got := visit(t, newRequest(t, http.MethodPost, addr, "/",
|
|
||||||
bytes.NewReader(make([]byte, 1025))))
|
|
||||||
if got.status != http.StatusForbidden {
|
|
||||||
t.Errorf("answered %+v, want servewaf's 403", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestABodyNotAnnouncedOverItsLimitIsRefused(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name, limit string
|
|
||||||
size int
|
|
||||||
// mayReachApp is true when the part of the body the limit lets through
|
|
||||||
// goes on to the app before the limit is met. The app may still not have
|
|
||||||
// started on the request when it is cut off.
|
|
||||||
mayReachApp bool
|
|
||||||
}{
|
|
||||||
{"within the first part", "1K", 2 << 10, false},
|
|
||||||
{"past the first part", "128K", 129 << 10, true},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var asked atomic.Bool
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
asked.Store(true)
|
|
||||||
greet(w, r)
|
|
||||||
})
|
|
||||||
addr, _ := startRProxy(t, servewaf.URL, app.URL, map[string]string{
|
|
||||||
maxRequestSize: tc.limit,
|
|
||||||
})
|
|
||||||
|
|
||||||
conn := dial(t, addr)
|
|
||||||
send(t, conn, post+"Transfer-Encoding: chunked\r\n\r\n"+
|
|
||||||
strconv.FormatInt(int64(tc.size), 16)+"\r\n"+strings.Repeat("x", tc.size)+
|
|
||||||
"\r\n0\r\n\r\n")
|
|
||||||
|
|
||||||
if status := readStatus(t, conn); status != http.StatusRequestEntityTooLarge {
|
|
||||||
t.Errorf("status %d, want 413", status)
|
|
||||||
}
|
|
||||||
|
|
||||||
if asked.Load() && !tc.mayReachApp {
|
|
||||||
t.Error("the app was asked")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAResponseOverItsLimit(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
large := strings.Repeat("x", 2<<10)
|
|
||||||
|
|
||||||
t.Run("announced, is refused", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.Header().Set(contentLengthHeader, strconv.Itoa(len(large)))
|
|
||||||
_, _ = io.WriteString(w, large)
|
|
||||||
})
|
|
||||||
addr, _ := startRProxy(t, servewaf.URL, app.URL, map[string]string{
|
|
||||||
maxResponseSize: "1K",
|
|
||||||
})
|
|
||||||
|
|
||||||
got := get(t, addr, "/")
|
|
||||||
if got.status != http.StatusBadGateway {
|
|
||||||
t.Errorf("answered %d, want 502", got.status)
|
|
||||||
}
|
|
||||||
|
|
||||||
if report := servewaf.report(t); report.Status != http.StatusBadGateway {
|
|
||||||
t.Errorf("report %+v, want status 502", report)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("not announced, is cut off", func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
_, _ = io.WriteString(w, large[:1])
|
|
||||||
_ = http.NewResponseController(w).Flush()
|
|
||||||
_, _ = io.WriteString(w, large[1:])
|
|
||||||
})
|
|
||||||
addr, _ := startRProxy(t, servewaf.URL, app.URL, map[string]string{
|
|
||||||
maxResponseSize: "1K",
|
|
||||||
})
|
|
||||||
|
|
||||||
conn := dial(t, addr)
|
|
||||||
send(t, conn, "GET / HTTP/1.1\r\nHost: site.example\r\n\r\n")
|
|
||||||
|
|
||||||
text, err := io.ReadAll(conn)
|
|
||||||
if err != nil || bytes.Contains(text, []byte(large)) {
|
|
||||||
t.Errorf("read %d bytes (%v), want the answer cut off", len(text), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
report := servewaf.report(t)
|
|
||||||
if report.Status != http.StatusOK || report.ResponseBytes > 1<<10 {
|
|
||||||
t.Errorf("report %+v, want status 200 and at most 1K of the answer", report)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAHeadWhoseAnswerAnnouncesMoreThanTheResponseLimitIsPassedOn(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const announced = "2048"
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.Header().Set(contentLengthHeader, announced)
|
|
||||||
})
|
|
||||||
addr, _ := startRProxy(t, servewaf.URL, app.URL, map[string]string{
|
|
||||||
maxResponseSize: "1K",
|
|
||||||
})
|
|
||||||
|
|
||||||
got := visit(t, newRequest(t, http.MethodHead, addr, "/", http.NoBody))
|
|
||||||
if got.status != http.StatusOK || got.header.Get(contentLengthHeader) != announced {
|
|
||||||
t.Errorf("answered %+v, want the app's 200 announcing %s bytes", got, announced)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestASettingOfOffSetsNoLimit(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, setting := range []string{
|
|
||||||
maxRequestSize, maxResponseSize, maxRequestTime, maxResponseTime,
|
|
||||||
} {
|
|
||||||
t.Run(setting, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, greet)
|
|
||||||
addr, _ := startRProxy(t, servewaf.URL, app.URL, map[string]string{
|
|
||||||
setting: "off",
|
|
||||||
})
|
|
||||||
|
|
||||||
// A body and an answer that a limit of zero would cut off, as it would the
|
|
||||||
// next answer, which rproxy gives itself.
|
|
||||||
wantGreeting(t, visit(t, newRequest(t, http.MethodPost, addr, "/",
|
|
||||||
strings.NewReader("a body"))))
|
|
||||||
|
|
||||||
if got := get(t, addr, proxy.HealthPath); got.status != http.StatusOK {
|
|
||||||
t.Errorf("answered %+v, want rproxy's own 200", got)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAVisitorTooSlowToSendTheFirstPartIsAnswered408(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name, rest string
|
|
||||||
}{
|
|
||||||
{"with its length announced", announcedLength(100) + "some of it"},
|
|
||||||
{"with no length announced", "Transfer-Encoding: chunked\r\n\r\na\r\nsome of it\r\n"},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, greet)
|
|
||||||
addr, out := startRProxy(t, servewaf.URL, app.URL, map[string]string{
|
|
||||||
maxRequestTime: "500",
|
|
||||||
})
|
|
||||||
|
|
||||||
conn := dial(t, addr)
|
|
||||||
send(t, conn, post+tc.rest)
|
|
||||||
|
|
||||||
if status := readStatus(t, conn); status != http.StatusRequestTimeout {
|
|
||||||
t.Errorf("status %d, want 408", status)
|
|
||||||
}
|
|
||||||
|
|
||||||
line := out.requestLine(t)
|
|
||||||
if line["disposition"] != nil || line["request_bytes"] != 10.0 {
|
|
||||||
t.Errorf("line %v, want no disposition and 10 bytes", line)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(servewaf.questions) != 0 {
|
|
||||||
t.Error("servewaf was asked")
|
|
||||||
}
|
|
||||||
|
|
||||||
servewaf.wantNoReport(t)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAVisitorThatGoesAwayBeforeTheFirstPartIsNotAsked(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, greet)
|
|
||||||
addr, out := startRProxy(t, servewaf.URL, app.URL, nil)
|
|
||||||
|
|
||||||
conn := dial(t, addr)
|
|
||||||
send(t, conn, post+announcedLength(100)+"some of it")
|
|
||||||
_ = conn.Close()
|
|
||||||
|
|
||||||
line := out.requestLine(t)
|
|
||||||
if line["status"] != 0.0 || line["disposition"] != nil {
|
|
||||||
t.Errorf("line %v, want no answer and no disposition", line)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(servewaf.questions) != 0 {
|
|
||||||
t.Error("servewaf was asked")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestABodyThatCannotBeReadClosesTheConnectionWithoutAnAnswer(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name, request string
|
|
||||||
stopsSending bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
"a malformed chunk", post + "Transfer-Encoding: chunked\r\n\r\nnot a chunk\r\n",
|
|
||||||
false,
|
|
||||||
},
|
|
||||||
{"a visitor that stops sending", post + announcedLength(100) + "some of it", true},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, greet)
|
|
||||||
addr, out := startRProxy(t, servewaf.URL, app.URL, nil)
|
|
||||||
|
|
||||||
conn := dial(t, addr)
|
|
||||||
send(t, conn, tc.request)
|
|
||||||
|
|
||||||
if tc.stopsSending {
|
|
||||||
tcpConn, ok := conn.(*net.TCPConn)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("the connection is a %T", conn)
|
|
||||||
}
|
|
||||||
|
|
||||||
err := tcpConn.CloseWrite()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("stop sending: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
text, err := io.ReadAll(conn)
|
|
||||||
if err != nil || len(text) != 0 {
|
|
||||||
t.Errorf("read %q (%v), want the connection closed without an answer",
|
|
||||||
text, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
line := out.requestLine(t)
|
|
||||||
if line["status"] != 0.0 || line["disposition"] != nil {
|
|
||||||
t.Errorf("line %v, want no answer and no disposition", line)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(servewaf.questions) != 0 {
|
|
||||||
t.Error("servewaf was asked")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// deadlineRecorder notes the time limits rproxy sets on reading the visitor's body
|
|
||||||
// and on sending its answer.
|
|
||||||
type deadlineRecorder struct {
|
|
||||||
*httptest.ResponseRecorder
|
|
||||||
|
|
||||||
readDeadline time.Time
|
|
||||||
writeDeadline time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetReadDeadline notes deadline.
|
|
||||||
func (w *deadlineRecorder) SetReadDeadline(deadline time.Time) error {
|
|
||||||
w.readDeadline = deadline
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetWriteDeadline notes deadline.
|
|
||||||
func (w *deadlineRecorder) SetWriteDeadline(deadline time.Time) error {
|
|
||||||
w.writeDeadline = deadline
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestABodyLeftUnreadIsReadOnlyUntilTheRequestTimeLimit(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
limit = 5 * time.Minute
|
|
||||||
// announced is over what Go's server reads of a body left unread, so that
|
|
||||||
// the app's own server answers without waiting for the rest.
|
|
||||||
announced = 100 * questionBodyMaxBytes
|
|
||||||
)
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusNoContent)
|
|
||||||
})
|
|
||||||
_, _, server := startServer(t, servewaf.URL, app.URL, map[string]string{
|
|
||||||
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
|
||||||
})
|
|
||||||
|
|
||||||
// The visitor sends the first part and a little more, and then nothing.
|
|
||||||
stalled, stop := io.Pipe()
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = stop.Close() })
|
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/",
|
|
||||||
io.MultiReader(bytes.NewReader(make([]byte, questionBodyMaxBytes+1)), stalled))
|
|
||||||
req.ContentLength = announced
|
|
||||||
req.Header.Set(contentLengthHeader, strconv.Itoa(announced))
|
|
||||||
|
|
||||||
w := &deadlineRecorder{ResponseRecorder: httptest.NewRecorder()}
|
|
||||||
|
|
||||||
before := time.Now()
|
|
||||||
|
|
||||||
server.Handler.ServeHTTP(w, req)
|
|
||||||
|
|
||||||
if w.Code != http.StatusNoContent {
|
|
||||||
t.Errorf("status %d, want the app's 204", w.Code)
|
|
||||||
}
|
|
||||||
|
|
||||||
if w.readDeadline.Before(before.Add(limit)) ||
|
|
||||||
w.readDeadline.After(time.Now().Add(limit)) {
|
|
||||||
t.Errorf("the reading of the body ends at %s, want %s from the request",
|
|
||||||
w.readDeadline, limit)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnAnswerRProxyGivesItselfHasTheResponseTimeToBeSent(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const limit = 5 * time.Minute
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name, target string
|
|
||||||
status int
|
|
||||||
}{
|
|
||||||
{"servewaf's deny", "/", http.StatusForbidden},
|
|
||||||
{"rproxy's own 400", "x:admin", http.StatusBadRequest},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, deny)
|
|
||||||
app := startApp(t, greet)
|
|
||||||
_, _, server := startServer(t, servewaf.URL, app.URL, map[string]string{
|
|
||||||
maxResponseTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
|
||||||
})
|
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, tc.target,
|
|
||||||
http.NoBody)
|
|
||||||
w := &deadlineRecorder{ResponseRecorder: httptest.NewRecorder()}
|
|
||||||
|
|
||||||
before := time.Now()
|
|
||||||
|
|
||||||
server.Handler.ServeHTTP(w, req)
|
|
||||||
|
|
||||||
if w.Code != tc.status {
|
|
||||||
t.Errorf("status %d, want %d", w.Code, tc.status)
|
|
||||||
}
|
|
||||||
|
|
||||||
if w.writeDeadline.Before(before.Add(limit)) ||
|
|
||||||
w.writeDeadline.After(time.Now().Add(limit)) {
|
|
||||||
t.Errorf("the time limit on sending ends at %s, want %s from the answer",
|
|
||||||
w.writeDeadline, limit)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAVisitorTooSlowToSendTheRestIsAnswered408(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, greet)
|
|
||||||
addr, _ := startRProxy(t, servewaf.URL, app.URL, map[string]string{
|
|
||||||
maxRequestTime: "3000",
|
|
||||||
})
|
|
||||||
|
|
||||||
conn := dial(t, addr)
|
|
||||||
send(t, conn, post+announcedLength(2*questionBodyMaxBytes)+
|
|
||||||
strings.Repeat("x", questionBodyMaxBytes+1))
|
|
||||||
|
|
||||||
if status := readStatus(t, conn); status != http.StatusRequestTimeout {
|
|
||||||
t.Errorf("status %d, want 408", status)
|
|
||||||
}
|
|
||||||
|
|
||||||
if asked := servewaf.question(t); len(asked.body) != questionBodyMaxBytes {
|
|
||||||
t.Errorf("the question carries %d bytes", len(asked.body))
|
|
||||||
}
|
|
||||||
|
|
||||||
if report := servewaf.report(t); report.Status != http.StatusRequestTimeout {
|
|
||||||
t.Errorf("report %+v, want status 408", report)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// startAppThatTakesNothing starts an app that accepts connections and never
|
|
||||||
// reads from them, and returns its URL.
|
|
||||||
func startAppThatTakesNothing(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("listen: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
stopped := make(chan struct{})
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
close(stopped)
|
|
||||||
|
|
||||||
_ = listener.Close()
|
|
||||||
})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
for {
|
|
||||||
conn, err := listener.Accept()
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
<-stopped
|
|
||||||
|
|
||||||
_ = conn.Close()
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
return "http://" + listener.Addr().String()
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnAppTooSlowToTakeTheRequestIsAnswered504(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// More than the connections hold while nobody reads, written a little at a
|
|
||||||
// time, so that rproxy soon waits for the app.
|
|
||||||
const (
|
|
||||||
bodySize = 64 << 20
|
|
||||||
writeSize = 32 << 10
|
|
||||||
)
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
addr, _ := startRProxy(t, servewaf.URL, startAppThatTakesNothing(t),
|
|
||||||
map[string]string{maxRequestTime: "3000"})
|
|
||||||
|
|
||||||
conn := dial(t, addr)
|
|
||||||
send(t, conn, post+announcedLength(bodySize))
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
part := make([]byte, writeSize)
|
|
||||||
|
|
||||||
for range bodySize / writeSize {
|
|
||||||
_, err := conn.Write(part)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
if status := readStatus(t, conn); status != http.StatusGatewayTimeout {
|
|
||||||
t.Errorf("status %d, want 504", status)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnAppTooSlowToAnswerIsAnswered504(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
release := make(chan struct{})
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, func(http.ResponseWriter, *http.Request) {
|
|
||||||
<-release
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Cleanup(func() { close(release) })
|
|
||||||
|
|
||||||
addr, _ := startRProxy(t, servewaf.URL, app.URL, map[string]string{
|
|
||||||
maxResponseTime: "300",
|
|
||||||
})
|
|
||||||
|
|
||||||
got := get(t, addr, "/")
|
|
||||||
if got.status != http.StatusGatewayTimeout {
|
|
||||||
t.Errorf("answered %d, want 504", got.status)
|
|
||||||
}
|
|
||||||
|
|
||||||
if report := servewaf.report(t); report.Status != http.StatusGatewayTimeout {
|
|
||||||
t.Errorf("report %+v, want status 504", report)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnAppTooSlowToFinishItsAnswerIsCutOff(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
release := make(chan struct{})
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
_, _ = io.WriteString(w, "the start")
|
|
||||||
_ = http.NewResponseController(w).Flush()
|
|
||||||
|
|
||||||
<-release
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Cleanup(func() { close(release) })
|
|
||||||
|
|
||||||
addr, _ := startRProxy(t, servewaf.URL, app.URL, map[string]string{
|
|
||||||
maxResponseTime: "300",
|
|
||||||
})
|
|
||||||
|
|
||||||
conn := dial(t, addr)
|
|
||||||
send(t, conn, "GET / HTTP/1.1\r\nHost: site.example\r\n\r\n")
|
|
||||||
|
|
||||||
// dial's deadline ends this read with an error if the connection stays open.
|
|
||||||
_, err := io.ReadAll(conn)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("read: %v, want the connection closed", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if report := servewaf.report(t); report.Status != http.StatusOK {
|
|
||||||
t.Errorf("report %+v, want the status the visitor got", report)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAVisitorTooSlowToTakeTheAnswerIsCutOff(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// More than the connections hold while nobody reads.
|
|
||||||
const (
|
|
||||||
bodySize = 64 << 20
|
|
||||||
writeSize = 32 << 10
|
|
||||||
)
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
part := make([]byte, writeSize)
|
|
||||||
|
|
||||||
for range bodySize / writeSize {
|
|
||||||
_, err := w.Write(part)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
addr, out := startRProxy(t, servewaf.URL, app.URL, map[string]string{
|
|
||||||
maxResponseTime: "500",
|
|
||||||
})
|
|
||||||
|
|
||||||
// The visitor asks, and never reads the answer.
|
|
||||||
conn := dial(t, addr)
|
|
||||||
send(t, conn, "GET / HTTP/1.1\r\nHost: site.example\r\n\r\n")
|
|
||||||
|
|
||||||
line := out.requestLine(t)
|
|
||||||
|
|
||||||
responseBytes, _ := line["response_bytes"].(float64)
|
|
||||||
if line["status"] != float64(http.StatusOK) || responseBytes >= bodySize {
|
|
||||||
t.Errorf("line %v, want status 200 and the answer cut off", line)
|
|
||||||
}
|
|
||||||
|
|
||||||
report := servewaf.report(t)
|
|
||||||
if report.Status != http.StatusOK || report.ResponseBytes >= bodySize {
|
|
||||||
t.Errorf("report %+v, want status 200 and the answer cut off", report)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnAppThatCannotBeReachedIsAnswered502WithAReport(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
addr, _ := startRProxy(t, servewaf.URL, nowhere, nil)
|
|
||||||
|
|
||||||
got := get(t, addr, "/")
|
|
||||||
if got.status != http.StatusBadGateway {
|
|
||||||
t.Errorf("answered %d, want 502", got.status)
|
|
||||||
}
|
|
||||||
|
|
||||||
if report := servewaf.report(t); report.Status != http.StatusBadGateway {
|
|
||||||
t.Errorf("report %+v, want status 502", report)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAVisitorThatStopsSendingAfterTheAppHasAnsweredGetsTheAnswer(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
_, _ = io.WriteString(w, greeting)
|
|
||||||
})
|
|
||||||
addr, _ := startRProxy(t, servewaf.URL, app.URL, map[string]string{
|
|
||||||
maxRequestTime: "3000",
|
|
||||||
})
|
|
||||||
|
|
||||||
// Far more announced than sent, so that Go's server answers for the app
|
|
||||||
// without waiting for the rest.
|
|
||||||
conn := dial(t, addr)
|
|
||||||
send(t, conn, post+announcedLength(100*questionBodyMaxBytes)+
|
|
||||||
strings.Repeat("x", questionBodyMaxBytes))
|
|
||||||
|
|
||||||
if status := readStatus(t, conn); status != http.StatusOK {
|
|
||||||
t.Errorf("status %d, want the app's 200", status)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAVisitorThatStopsSendingWhileTheAppWorksGetsNoAnswer(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
working := make(chan struct{})
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, func(_ http.ResponseWriter, r *http.Request) {
|
|
||||||
close(working)
|
|
||||||
<-r.Context().Done()
|
|
||||||
})
|
|
||||||
addr, out := startRProxy(t, servewaf.URL, app.URL, nil)
|
|
||||||
|
|
||||||
conn := dial(t, addr)
|
|
||||||
send(t, conn, "GET / HTTP/1.1\r\nHost: site.example\r\n\r\n")
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-working:
|
|
||||||
case <-time.After(waitLimit):
|
|
||||||
t.Fatal("the app was not asked")
|
|
||||||
}
|
|
||||||
|
|
||||||
tcpConn, ok := conn.(*net.TCPConn)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("the connection is a %T", conn)
|
|
||||||
}
|
|
||||||
|
|
||||||
err := tcpConn.CloseWrite()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("stop sending: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
text, err := io.ReadAll(conn)
|
|
||||||
if err != nil || len(text) != 0 {
|
|
||||||
t.Errorf("read %q (%v), want the connection closed without an answer", text, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if line := out.requestLine(t); line["status"] != 0.0 {
|
|
||||||
t.Errorf("line %v, want status 0", line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTheRequestTimeLimitDoesNotCutAnAnswerOnceTheAppHasTheRequest(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const answer = "the end of a long answer"
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
_, _ = io.Copy(io.Discard, r.Body)
|
|
||||||
_ = http.NewResponseController(w).Flush()
|
|
||||||
|
|
||||||
time.Sleep(2100 * time.Millisecond)
|
|
||||||
|
|
||||||
_, _ = io.WriteString(w, answer)
|
|
||||||
})
|
|
||||||
addr, _ := startRProxy(t, servewaf.URL, app.URL, map[string]string{
|
|
||||||
maxRequestTime: "2000",
|
|
||||||
})
|
|
||||||
|
|
||||||
got := visit(t, newRequest(t, http.MethodPost, addr, "/", strings.NewReader("a body")))
|
|
||||||
if got.status != http.StatusOK || got.body != answer {
|
|
||||||
t.Errorf("answered %d %q, want the app's whole answer", got.status, got.body)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAVisitorThatStopsSendingGetsAnAnswerRProxyGivesByTheDeadline(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name, start string
|
|
||||||
status int
|
|
||||||
}{
|
|
||||||
{"a deny", post, http.StatusForbidden},
|
|
||||||
{
|
|
||||||
"the health check",
|
|
||||||
"GET " + proxy.HealthPath + " HTTP/1.1\r\nHost: site.example\r\n",
|
|
||||||
http.StatusOK,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, deny)
|
|
||||||
app := startApp(t, greet)
|
|
||||||
addr, _ := startRProxy(t, servewaf.URL, app.URL, map[string]string{
|
|
||||||
maxRequestTime: "500",
|
|
||||||
})
|
|
||||||
|
|
||||||
// Go's server waits for a little of the body left before it sends an answer,
|
|
||||||
// and the time for sending the request ends that wait.
|
|
||||||
conn := dial(t, addr)
|
|
||||||
send(t, conn, tc.start+announcedLength(questionBodyMaxBytes+100)+
|
|
||||||
strings.Repeat("x", questionBodyMaxBytes))
|
|
||||||
|
|
||||||
if status := readStatus(t, conn); status != tc.status {
|
|
||||||
t.Errorf("status %d, want %d", status, tc.status)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,523 +0,0 @@
|
|||||||
package rproxy_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"crypto/rand"
|
|
||||||
"io"
|
|
||||||
"maps"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"slices"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/proxy"
|
|
||||||
)
|
|
||||||
|
|
||||||
// questionBodyMaxBytes is the most of a body a question carries.
|
|
||||||
const questionBodyMaxBytes = proxy.QuestionBodyMaxBytes
|
|
||||||
|
|
||||||
// chunked is the encoding of a body whose length the visitor did not announce.
|
|
||||||
const chunked = "chunked"
|
|
||||||
|
|
||||||
// pathGoReencodes is a path Go's client would decode and escape again, %2F turned
|
|
||||||
// into / and | into %7C, unless told to send it as it is. A test sets it as the
|
|
||||||
// visitor's URL.Opaque, so that the visitor sends it unchanged too.
|
|
||||||
const pathGoReencodes = "/some%2Fpath/with%20space|pipe"
|
|
||||||
|
|
||||||
func TestQuestionIsTheVisitorsRequestWithRProxysHeaders(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, greet)
|
|
||||||
addr, _ := startRProxy(t, servewaf.URL, app.URL, nil)
|
|
||||||
|
|
||||||
const target = pathGoReencodes + "?b=2&a=1&bad=%zz;x"
|
|
||||||
|
|
||||||
req := newRequest(t, http.MethodDelete, addr, target, http.NoBody)
|
|
||||||
req.URL.Opaque = pathGoReencodes
|
|
||||||
req.Host = site
|
|
||||||
req.Header.Set("X-Test", "from the visitor")
|
|
||||||
req.Header.Set(forwardedForHeader, "198.51.100.1")
|
|
||||||
req.Header.Set(forwardedProtoHeader, "gopher")
|
|
||||||
req.Header.Set(requestIDHeader, "the-visitor's")
|
|
||||||
req.Header.Set(proxy.ClientHeader, "198.51.100.2")
|
|
||||||
req.Header.Set(proxy.PeerHeader, "198.51.100.3")
|
|
||||||
req.Header.Set(proxy.ContentLengthHeader, "12345")
|
|
||||||
// Go's client then sends no User-Agent, so that one in the question would be
|
|
||||||
// Go's own.
|
|
||||||
req.Header["User-Agent"] = nil
|
|
||||||
|
|
||||||
wantGreeting(t, visit(t, req))
|
|
||||||
|
|
||||||
asked := servewaf.question(t)
|
|
||||||
if asked.method != http.MethodDelete || asked.target != target ||
|
|
||||||
asked.host != site {
|
|
||||||
t.Errorf("asked about %s %s at %s", asked.method, asked.target, asked.host)
|
|
||||||
}
|
|
||||||
|
|
||||||
if asked.header.Get(requestIDHeader) == "the-visitor's" {
|
|
||||||
t.Error("the question carries the visitor's id")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Exactly these: visit's connection has the visitor send Connection: close.
|
|
||||||
want := http.Header{}
|
|
||||||
want.Set("Connection", "close")
|
|
||||||
want.Set("X-Test", "from the visitor")
|
|
||||||
want.Set(forwardedForHeader, "198.51.100.1")
|
|
||||||
want.Set(forwardedProtoHeader, "http")
|
|
||||||
want.Set(requestIDHeader, asked.header.Get(requestIDHeader))
|
|
||||||
want.Set(proxy.ClientHeader, localhost)
|
|
||||||
want.Set(proxy.PeerHeader, localhost)
|
|
||||||
|
|
||||||
if !maps.EqualFunc(asked.header, want, slices.Equal) {
|
|
||||||
t.Errorf("the question's headers are %v, want %v", asked.header, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestQuestionNamesTheVisitorATrustedProxySpeaksFor(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const visitor = "203.0.113.9"
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name, trusted, client, proto, forwarded string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
"from a trusted proxy", localhost + "/32", visitor, "https",
|
|
||||||
visitor + ", " + localhost,
|
|
||||||
},
|
|
||||||
{"from a peer not trusted", "10.0.0.0/8", localhost, "http", localhost},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
forwarded := make(chan http.Header, 1)
|
|
||||||
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
forwarded <- r.Header.Clone()
|
|
||||||
|
|
||||||
greet(w, r)
|
|
||||||
})
|
|
||||||
addr, out := startRProxy(t, servewaf.URL, app.URL, map[string]string{
|
|
||||||
trustedProxies: tc.trusted,
|
|
||||||
})
|
|
||||||
|
|
||||||
req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
|
|
||||||
req.Header.Set(forwardedForHeader, visitor)
|
|
||||||
req.Header.Set(forwardedProtoHeader, "https")
|
|
||||||
wantGreeting(t, visit(t, req))
|
|
||||||
|
|
||||||
asked := servewaf.question(t)
|
|
||||||
if asked.header.Get(proxy.ClientHeader) != tc.client ||
|
|
||||||
asked.header.Get(proxy.PeerHeader) != localhost ||
|
|
||||||
asked.header.Get(forwardedForHeader) != visitor ||
|
|
||||||
asked.header.Get(forwardedProtoHeader) != tc.proto {
|
|
||||||
t.Errorf("the question's headers are %v", asked.header)
|
|
||||||
}
|
|
||||||
|
|
||||||
if line := out.requestLine(t); line["client_ip"] != tc.client {
|
|
||||||
t.Errorf("line %v, want client_ip %s", line, tc.client)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := (<-forwarded).Get(forwardedForHeader); got != tc.forwarded {
|
|
||||||
t.Errorf("the app saw %s %q, want %q", forwardedForHeader, got, tc.forwarded)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestQuestionNamesTheClientAsTheForwardedAddressesSay(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
header []string
|
|
||||||
client string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
"the first address outside the trusted proxies, from the right",
|
|
||||||
[]string{"198.51.100.7, 203.0.113.9, 10.0.0.2"}, "203.0.113.9",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"the leftmost, when every address is a trusted proxy",
|
|
||||||
[]string{"10.0.0.5, 10.0.0.2"}, "10.0.0.5",
|
|
||||||
},
|
|
||||||
{"the peer, with no header", nil, localhost},
|
|
||||||
{
|
|
||||||
"the last address read, before an entry that is not an address",
|
|
||||||
[]string{"203.0.113.9, unknown, 10.0.0.2"}, "10.0.0.2",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"read as one list, with several header lines",
|
|
||||||
[]string{"2001:db8::7", "10.0.0.2"}, "2001:db8::7",
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, greet)
|
|
||||||
addr, _ := startRProxy(t, servewaf.URL, app.URL, map[string]string{
|
|
||||||
trustedProxies: localhost + "/32,10.0.0.0/8",
|
|
||||||
})
|
|
||||||
|
|
||||||
req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
|
|
||||||
req.Header[forwardedForHeader] = tc.header
|
|
||||||
wantGreeting(t, visit(t, req))
|
|
||||||
|
|
||||||
if got := servewaf.question(t).header.Get(proxy.ClientHeader); got != tc.client {
|
|
||||||
t.Errorf("the question names %s, want %s", got, tc.client)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// randomBytes returns n random bytes.
|
|
||||||
func randomBytes(n int) []byte {
|
|
||||||
body := make([]byte, n)
|
|
||||||
_, _ = rand.Read(body) // it never returns an error
|
|
||||||
|
|
||||||
return body
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFirst64KiBReachServeWAFAndTheWholeBodyTheApp(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, size := range []int{
|
|
||||||
1, questionBodyMaxBytes - 1, questionBodyMaxBytes, questionBodyMaxBytes + 1,
|
|
||||||
5 * questionBodyMaxBytes,
|
|
||||||
} {
|
|
||||||
for _, announced := range []bool{true, false} {
|
|
||||||
name := strconv.Itoa(size) + " bytes, length announced: " +
|
|
||||||
strconv.FormatBool(announced)
|
|
||||||
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
body := randomBytes(size)
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
received := make(chan []byte, 1)
|
|
||||||
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
whole, _ := io.ReadAll(r.Body)
|
|
||||||
received <- whole
|
|
||||||
|
|
||||||
greet(w, r)
|
|
||||||
})
|
|
||||||
addr, _ := startRProxy(t, servewaf.URL, app.URL, nil)
|
|
||||||
|
|
||||||
req := newRequest(t, http.MethodPost, addr, "/", bytes.NewReader(body))
|
|
||||||
if !announced {
|
|
||||||
// Go's client sends a body of unknown length in chunks.
|
|
||||||
req.ContentLength = -1
|
|
||||||
}
|
|
||||||
|
|
||||||
wantGreeting(t, visit(t, req))
|
|
||||||
|
|
||||||
asked := servewaf.question(t)
|
|
||||||
want := body[:min(size, questionBodyMaxBytes)]
|
|
||||||
wantQuestionBody(t, asked, want, size, announced)
|
|
||||||
|
|
||||||
if !bytes.Equal(<-received, body) {
|
|
||||||
t.Error("the app did not get the whole body")
|
|
||||||
}
|
|
||||||
|
|
||||||
if report := servewaf.report(t); report.RequestBytes != int64(size) {
|
|
||||||
t.Errorf("report %+v, of %d bytes", report, size)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// wantQuestionBody checks the body a question carries, want, and how it gives
|
|
||||||
// the length of the visitor's body, of size bytes.
|
|
||||||
func wantQuestionBody(
|
|
||||||
t *testing.T, asked question, want []byte, size int, announced bool,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
if !bytes.Equal(asked.body, want) {
|
|
||||||
t.Errorf("the question carries %d bytes, want the first %d", len(asked.body),
|
|
||||||
len(want))
|
|
||||||
}
|
|
||||||
|
|
||||||
values := asked.header.Values(proxy.ContentLengthHeader)
|
|
||||||
|
|
||||||
if announced {
|
|
||||||
if !slices.Equal(values, []string{strconv.Itoa(size)}) ||
|
|
||||||
asked.contentLength != int64(len(want)) {
|
|
||||||
t.Errorf("announced %v and length %d, want %d and %d", values,
|
|
||||||
asked.contentLength, size, len(want))
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(values) != 0 || !slices.Equal(asked.transferEncoding, []string{chunked}) {
|
|
||||||
t.Errorf("announced %v and encoded %v, want none and chunked", values,
|
|
||||||
asked.transferEncoding)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDenyNeedsOnlyTheFirstPartOfALargeBody(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, deny)
|
|
||||||
app := startApp(t, greet)
|
|
||||||
addr, _ := startRProxy(t, servewaf.URL, app.URL, nil)
|
|
||||||
|
|
||||||
// Far more announced than sent. Go's server does not wait for that much, so
|
|
||||||
// the answer comes at once.
|
|
||||||
conn := dial(t, addr)
|
|
||||||
send(t, conn, post+announcedLength(100*questionBodyMaxBytes)+
|
|
||||||
strings.Repeat("x", questionBodyMaxBytes))
|
|
||||||
|
|
||||||
if status := readStatus(t, conn); status != http.StatusForbidden {
|
|
||||||
t.Errorf("status %d, want servewaf's 403", status)
|
|
||||||
}
|
|
||||||
|
|
||||||
if report := servewaf.report(t); report.RequestBytes != questionBodyMaxBytes {
|
|
||||||
t.Errorf("report %+v, want the first part's bytes", report)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAQuestionCarriesItsBodyWhateverTheMethod(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name, framing, body string
|
|
||||||
encoding []string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
chunked, "Transfer-Encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\n\r\n", "body",
|
|
||||||
[]string{chunked},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"chunked and empty", "Transfer-Encoding: chunked\r\n\r\n0\r\n\r\n", "",
|
|
||||||
[]string{chunked},
|
|
||||||
},
|
|
||||||
{"with its length", announcedLength(4) + "body", "body", nil},
|
|
||||||
} {
|
|
||||||
for _, method := range []string{
|
|
||||||
http.MethodGet, http.MethodDelete, http.MethodPost,
|
|
||||||
} {
|
|
||||||
t.Run(method+" "+tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, deny)
|
|
||||||
addr, _ := startRProxy(t, servewaf.URL, nowhere, nil)
|
|
||||||
|
|
||||||
conn := dial(t, addr)
|
|
||||||
send(t, conn, method+" /x HTTP/1.1\r\nHost: site.example\r\n"+tc.framing)
|
|
||||||
|
|
||||||
if status := readStatus(t, conn); status != http.StatusForbidden {
|
|
||||||
t.Errorf("status %d, want servewaf's 403", status)
|
|
||||||
}
|
|
||||||
|
|
||||||
asked := servewaf.question(t)
|
|
||||||
if string(asked.body) != tc.body ||
|
|
||||||
!slices.Equal(asked.transferEncoding, tc.encoding) {
|
|
||||||
t.Errorf("the question carries %q, encoded %v, want %q, encoded %v",
|
|
||||||
asked.body, asked.transferEncoding, tc.body, tc.encoding)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAMethodNotPassedOnIsAnswered405(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, method := range []string{http.MethodTrace, "PROPFIND", "LOCK", "BREW"} {
|
|
||||||
t.Run(method, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, func(http.ResponseWriter, *http.Request) {
|
|
||||||
t.Error("the app was asked")
|
|
||||||
})
|
|
||||||
addr, out := startRProxy(t, servewaf.URL, app.URL, nil)
|
|
||||||
|
|
||||||
got := visit(t, newRequest(t, method, addr, "/", http.NoBody))
|
|
||||||
if got.status != http.StatusMethodNotAllowed ||
|
|
||||||
got.header.Get("Allow") != "GET, HEAD, POST, OPTIONS, PUT, DELETE, PATCH, QUERY" {
|
|
||||||
t.Errorf("answered %+v, want 405 naming the methods passed on", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
line := out.requestLine(t)
|
|
||||||
if line["status"] != float64(http.StatusMethodNotAllowed) ||
|
|
||||||
line["disposition"] != nil || line["default_disposition"] != nil {
|
|
||||||
t.Errorf("line %v, want status 405 and no disposition", line)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(servewaf.questions) != 0 {
|
|
||||||
t.Error("servewaf was asked")
|
|
||||||
}
|
|
||||||
|
|
||||||
servewaf.wantNoReport(t)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEachMethodPassedOnReachesServeWAFAndTheApp(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, method := range []string{
|
|
||||||
http.MethodGet, http.MethodHead, http.MethodPost, http.MethodOptions,
|
|
||||||
http.MethodPut, http.MethodDelete, http.MethodPatch, "QUERY",
|
|
||||||
} {
|
|
||||||
t.Run(method, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
reached := make(chan string, 1)
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, func(_ http.ResponseWriter, r *http.Request) {
|
|
||||||
reached <- r.Method
|
|
||||||
})
|
|
||||||
addr, _ := startRProxy(t, servewaf.URL, app.URL, nil)
|
|
||||||
|
|
||||||
got := visit(t, newRequest(t, method, addr, "/", http.NoBody))
|
|
||||||
if got.status != http.StatusOK {
|
|
||||||
t.Errorf("answered %+v, want the app's 200", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
if asked := servewaf.question(t); asked.method != method {
|
|
||||||
t.Errorf("servewaf was asked about %s", asked.method)
|
|
||||||
}
|
|
||||||
|
|
||||||
if atApp := <-reached; atApp != method {
|
|
||||||
t.Errorf("the app was asked %s", atApp)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestARequestThatCannotGoToServeWAFAsSentIsRefused(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const host = "Host: site.example\r\n\r\n"
|
|
||||||
|
|
||||||
for name, request := range map[string]string{
|
|
||||||
"a target of a scheme": "GET x:admin HTTP/1.1\r\n" + host,
|
|
||||||
"a target of a URL": "GET http://site.example/ HTTP/1.1\r\n" + host,
|
|
||||||
"a host Go will not send": "GET http://a<b/ HTTP/1.1\r\n" + host,
|
|
||||||
"a target of a host": "GET site.example:443 HTTP/1.1\r\n" + host,
|
|
||||||
"a target of *": "GET * HTTP/1.1\r\n" + host,
|
|
||||||
"a target of //": "GET //x/y HTTP/1.1\r\n" + host,
|
|
||||||
"a target of // escaped": "GET //x%2Fadmin| HTTP/1.1\r\n" + host,
|
|
||||||
"an empty Host": "GET / HTTP/1.1\r\nHost:\r\n\r\n",
|
|
||||||
"HTTP/1.0 with no Host": "GET / HTTP/1.0\r\n\r\n",
|
|
||||||
"a Host with a zone": "GET / HTTP/1.1\r\nHost: [fe80::1%25eth0]:80\r\n\r\n",
|
|
||||||
"two User-Agents": "GET / HTTP/1.1\r\nUser-Agent: a\r\nUser-Agent: b\r\n" +
|
|
||||||
host,
|
|
||||||
} {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, greet)
|
|
||||||
addr, out := startRProxy(t, servewaf.URL, app.URL, nil)
|
|
||||||
|
|
||||||
conn := dial(t, addr)
|
|
||||||
send(t, conn, request)
|
|
||||||
|
|
||||||
if status := readStatus(t, conn); status != http.StatusBadRequest {
|
|
||||||
t.Errorf("status %d, want 400", status)
|
|
||||||
}
|
|
||||||
|
|
||||||
line := out.requestLine(t)
|
|
||||||
if line["status"] != float64(http.StatusBadRequest) ||
|
|
||||||
line["disposition"] != nil || line["default_disposition"] != nil {
|
|
||||||
t.Errorf("line %v, want status 400 and no disposition", line)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(servewaf.questions) != 0 {
|
|
||||||
t.Error("servewaf was asked")
|
|
||||||
}
|
|
||||||
|
|
||||||
servewaf.wantNoReport(t)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAHostWithAPercentSignButNoBracketGoesToServeWAF(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const host = "a%b"
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, greet)
|
|
||||||
addr, _ := startRProxy(t, servewaf.URL, app.URL, nil)
|
|
||||||
|
|
||||||
conn := dial(t, addr)
|
|
||||||
send(t, conn, "GET / HTTP/1.1\r\nHost: "+host+"\r\n\r\n")
|
|
||||||
|
|
||||||
if status := readStatus(t, conn); status != http.StatusOK {
|
|
||||||
t.Errorf("status %d, want the app's 200", status)
|
|
||||||
}
|
|
||||||
|
|
||||||
if asked := servewaf.question(t); asked.host != host {
|
|
||||||
t.Errorf("asked about the host %q, want %q", asked.host, host)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestATrustedPeerWithNoForwardedProtoIsAskedAboutAsHTTP(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const want = "http"
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, allow)
|
|
||||||
app := startApp(t, greet)
|
|
||||||
addr, _ := startRProxy(t, servewaf.URL, app.URL, map[string]string{
|
|
||||||
trustedProxies: localhost + "/32",
|
|
||||||
})
|
|
||||||
|
|
||||||
wantGreeting(t, get(t, addr, "/"))
|
|
||||||
|
|
||||||
asked := servewaf.question(t)
|
|
||||||
if got := asked.header.Get(forwardedProtoHeader); got != want {
|
|
||||||
t.Errorf("the question's %s is %q, want %s", forwardedProtoHeader, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAQuestionNamesNoZone(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name, remoteAddr, forwardedFor, client, peer string
|
|
||||||
}{
|
|
||||||
{"from a link-local peer", "[fe80::1%eth0]:1234", "", "fe80::1", "fe80::1"},
|
|
||||||
{
|
|
||||||
"from X-Forwarded-For through a trusted peer", localhost + ":1234",
|
|
||||||
"2001:db8::1%eth0", "2001:db8::1", localhost,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
servewaf := startServeWAF(t, deny)
|
|
||||||
app := startApp(t, greet)
|
|
||||||
_, out, server := startServer(t, servewaf.URL, app.URL, map[string]string{
|
|
||||||
trustedProxies: localhost + "/32",
|
|
||||||
})
|
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", http.NoBody)
|
|
||||||
req.RemoteAddr = tc.remoteAddr
|
|
||||||
req.Header.Set(forwardedForHeader, tc.forwardedFor)
|
|
||||||
|
|
||||||
server.Handler.ServeHTTP(httptest.NewRecorder(), req)
|
|
||||||
|
|
||||||
asked := servewaf.question(t)
|
|
||||||
if asked.header.Get(proxy.ClientHeader) != tc.client ||
|
|
||||||
asked.header.Get(proxy.PeerHeader) != tc.peer {
|
|
||||||
t.Errorf("the question's headers are %v", asked.header)
|
|
||||||
}
|
|
||||||
|
|
||||||
if line := out.requestLine(t); line["client_ip"] != tc.client {
|
|
||||||
t.Errorf("line %v, want client_ip %s", line, tc.client)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,593 +0,0 @@
|
|||||||
package rproxy
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/rand"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptrace"
|
|
||||||
"net/http/httputil"
|
|
||||||
"net/netip"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"strconv"
|
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/proxy"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
|
||||||
)
|
|
||||||
|
|
||||||
// flushAfterEachWrite has ReverseProxy pass on each part of the app's answer as
|
|
||||||
// soon as it arrives.
|
|
||||||
const flushAfterEachWrite time.Duration = -1
|
|
||||||
|
|
||||||
// reportStatusMax is the highest status servewaf takes in a report. An app can
|
|
||||||
// send a higher one, up to 999, and the visitor gets that as it is.
|
|
||||||
const reportStatusMax = 599
|
|
||||||
|
|
||||||
// plainText is the Content-Type of the answers rproxy gives itself.
|
|
||||||
const plainText = "text/plain; charset=utf-8"
|
|
||||||
|
|
||||||
// request is one request on its way through rproxy, from the moment its headers
|
|
||||||
// have been read to its log line and report.
|
|
||||||
type request struct {
|
|
||||||
h *handler
|
|
||||||
in *http.Request
|
|
||||||
// rc sets the deadlines of the connection to the visitor.
|
|
||||||
rc *http.ResponseController
|
|
||||||
out *responseWriter
|
|
||||||
body *requestBody // nil for a request without a body
|
|
||||||
|
|
||||||
// id is the fresh id made for the request when it arrived.
|
|
||||||
id string
|
|
||||||
client netip.Addr
|
|
||||||
peer netip.Addr
|
|
||||||
peerTrusted bool
|
|
||||||
start time.Time
|
|
||||||
|
|
||||||
// disposition is what is done with the request: servewaf's valid disposition,
|
|
||||||
// or SWRP_DEFAULT_DISPOSITION. It is "" when neither gave one.
|
|
||||||
disposition string
|
|
||||||
// answered is true when servewaf gave disposition, which makes a report due,
|
|
||||||
// and byDefault when SWRP_DEFAULT_DISPOSITION did.
|
|
||||||
answered bool
|
|
||||||
byDefault bool
|
|
||||||
// asn and country are servewaf's X-Client-ASN and X-Client-Country, for the
|
|
||||||
// app.
|
|
||||||
asn, country string
|
|
||||||
|
|
||||||
// upstreamStart is when the request was handed to the app.
|
|
||||||
upstreamStart time.Time
|
|
||||||
// cancel ends the request to the app.
|
|
||||||
cancel context.CancelFunc
|
|
||||||
// refused is the status of the first refusal, from whichever goroutine meets
|
|
||||||
// it: what the visitor is answered if the response has not started yet.
|
|
||||||
refused atomic.Int64
|
|
||||||
// upgraded is the connection to the app once the app has switched protocols,
|
|
||||||
// as for a WebSocket, and nil otherwise.
|
|
||||||
upgraded *upgradedConn
|
|
||||||
|
|
||||||
// mu guards what follows. The time limits run on goroutines of their own,
|
|
||||||
// and the transport starts and stops them, and notes the times below, from
|
|
||||||
// its own; once timersStopped is set, neither time limit acts any more.
|
|
||||||
mu sync.Mutex
|
|
||||||
timersStopped bool
|
|
||||||
requestTimer *time.Timer
|
|
||||||
responseTimer *time.Timer
|
|
||||||
// requestEnd is when the app had been sent the whole request, or the answer
|
|
||||||
// started, if that was sooner.
|
|
||||||
requestEnd time.Time
|
|
||||||
// answerStarted is when the first byte of the app's answer arrived.
|
|
||||||
answerStarted time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// newRequest starts handling r: it notes the time, works out the visitor from
|
|
||||||
// the TCP peer and X-Forwarded-For, and makes the request's id.
|
|
||||||
func (h *handler) newRequest(w http.ResponseWriter, r *http.Request) *request {
|
|
||||||
peer := peerAddress(r)
|
|
||||||
trusted := h.config.TrustedProxies
|
|
||||||
|
|
||||||
rq := &request{
|
|
||||||
h: h,
|
|
||||||
in: r,
|
|
||||||
rc: http.NewResponseController(w),
|
|
||||||
out: &responseWriter{ResponseWriter: w},
|
|
||||||
id: rand.Text(),
|
|
||||||
client: clientAddress(peer, r.Header.Values("X-Forwarded-For"), trusted),
|
|
||||||
peer: peer,
|
|
||||||
peerTrusted: isInside(peer, trusted),
|
|
||||||
start: time.Now(),
|
|
||||||
}
|
|
||||||
|
|
||||||
if r.Body != http.NoBody {
|
|
||||||
rq.body = &requestBody{body: limitBody(r.Body, h.config.MaxRequestSize), rq: rq}
|
|
||||||
}
|
|
||||||
|
|
||||||
return rq
|
|
||||||
}
|
|
||||||
|
|
||||||
// answerHealth answers the health check.
|
|
||||||
func (rq *request) answerHealth() {
|
|
||||||
rq.prepareOwnAnswer(rq.requestDeadline())
|
|
||||||
rq.out.Header().Set("Content-Type", plainText)
|
|
||||||
rq.send(http.StatusOK, []byte("ok\n"))
|
|
||||||
}
|
|
||||||
|
|
||||||
// readFirstPartOfBody reads up to the first 64 KiB of the body into memory,
|
|
||||||
// within SWRP_MAX_REQUEST_TIME_MS, and reports whether the request goes on. A
|
|
||||||
// visitor too slow to send it is answered 408. For a malformed chunk, or a
|
|
||||||
// visitor that went away or stopped its sending side, the connection is closed
|
|
||||||
// without an answer. A body over SWRP_MAX_REQUEST_SIZE goes on with the part
|
|
||||||
// read, so that servewaf is asked first; passToApp then refuses it.
|
|
||||||
func (rq *request) readFirstPartOfBody() bool {
|
|
||||||
if rq.body == nil {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
_ = rq.rc.SetReadDeadline(rq.requestDeadline())
|
|
||||||
held, err := io.ReadAll(io.LimitReader(rq.body, proxy.QuestionBodyMaxBytes))
|
|
||||||
// From here on, the request timer ends a read of the rest of the body, so that
|
|
||||||
// the answer can name the side rproxy was waiting on.
|
|
||||||
_ = rq.rc.SetReadDeadline(time.Time{})
|
|
||||||
|
|
||||||
rq.body.held = held
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case err == nil || rq.refused.Load() != 0:
|
|
||||||
return true
|
|
||||||
case errors.Is(err, os.ErrDeadlineExceeded):
|
|
||||||
rq.answer(http.StatusRequestTimeout)
|
|
||||||
|
|
||||||
return false
|
|
||||||
default:
|
|
||||||
// This panic has Go's server close the connection without an answer, and
|
|
||||||
// log nothing; the deferred finish still runs.
|
|
||||||
panic(http.ErrAbortHandler)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// setPathAsSent has out, the URL of a request made from in, send the path as the
|
|
||||||
// visitor sent it. Go sends the path URL.EscapedPath gives, which, for a path
|
|
||||||
// holding a character Go escapes, such as |, is the whole path decoded and
|
|
||||||
// escaped again, %2F turned into /. Opaque is sent as it is.
|
|
||||||
func setPathAsSent(out *url.URL, in *http.Request) {
|
|
||||||
out.Opaque = rules.Path(in)
|
|
||||||
}
|
|
||||||
|
|
||||||
// forward passes the request to the app, the part of the body held and then the
|
|
||||||
// rest, and the app's answer back. ctx is the request's own context.
|
|
||||||
func (rq *request) forward(ctx context.Context) {
|
|
||||||
ctx, cancel := context.WithCancel(ctx)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
rq.cancel = cancel
|
|
||||||
ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{
|
|
||||||
WroteRequest: rq.wroteRequest,
|
|
||||||
GotFirstResponseByte: rq.gotFirstResponseByte,
|
|
||||||
})
|
|
||||||
|
|
||||||
out := rq.in.WithContext(ctx)
|
|
||||||
if rq.body != nil {
|
|
||||||
out.Body = rq.body
|
|
||||||
}
|
|
||||||
|
|
||||||
reverseProxy := &httputil.ReverseProxy{
|
|
||||||
Rewrite: rq.rewrite,
|
|
||||||
Transport: rq.h.transport,
|
|
||||||
FlushInterval: flushAfterEachWrite,
|
|
||||||
ErrorLog: rq.h.errorLog,
|
|
||||||
ModifyResponse: rq.modifyResponse,
|
|
||||||
ErrorHandler: rq.answerError,
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.startRequestTimer()
|
|
||||||
rq.upstreamStart = time.Now()
|
|
||||||
reverseProxy.ServeHTTP(rq.out, out)
|
|
||||||
}
|
|
||||||
|
|
||||||
// rewrite makes the request the app receives: the visitor's request, unchanged,
|
|
||||||
// sent to SWRP_UPSTREAM_URL, with the forwarded headers and the request's id set,
|
|
||||||
// without any X-Client-ASN or X-Client-Country the visitor sent, and with
|
|
||||||
// servewaf's.
|
|
||||||
func (rq *request) rewrite(pr *httputil.ProxyRequest) {
|
|
||||||
upstream := rq.h.config.UpstreamURL
|
|
||||||
pr.Out.URL.Scheme = upstream.Scheme
|
|
||||||
pr.Out.URL.Host = upstream.Host
|
|
||||||
// ReverseProxy drops query parameters it cannot parse; the app gets the
|
|
||||||
// query as the visitor sent it.
|
|
||||||
pr.Out.URL.RawQuery = pr.In.URL.RawQuery
|
|
||||||
setPathAsSent(pr.Out.URL, pr.In)
|
|
||||||
|
|
||||||
setForwardedHeaders(pr.In, pr.Out, rq.peer, rq.peerTrusted)
|
|
||||||
pr.Out.Header.Set(requestIDHeader, rq.id)
|
|
||||||
pr.Out.Header.Del(asnHeader)
|
|
||||||
pr.Out.Header.Del(countryHeader)
|
|
||||||
|
|
||||||
if rq.asn != "" {
|
|
||||||
pr.Out.Header.Set(asnHeader, rq.asn)
|
|
||||||
}
|
|
||||||
|
|
||||||
if rq.country != "" {
|
|
||||||
pr.Out.Header.Set(countryHeader, rq.country)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// modifyResponse looks at the app's answer before ReverseProxy passes it on.
|
|
||||||
func (rq *request) modifyResponse(res *http.Response) error {
|
|
||||||
if res.StatusCode == http.StatusSwitchingProtocols {
|
|
||||||
// An upgraded connection, such as a WebSocket, is not cut by the time
|
|
||||||
// limits. ReverseProxy writes this answer straight to the connection it
|
|
||||||
// takes over, not through rq.out, and then copies what passes each way
|
|
||||||
// through res.Body, the connection to the app. It may still refuse the
|
|
||||||
// switch, as when the app's Upgrade is not the visitor's, so the status
|
|
||||||
// is noted only once it has taken over the connection, by rq.out's
|
|
||||||
// Hijack.
|
|
||||||
rq.stopTimers()
|
|
||||||
|
|
||||||
conn, ok := res.Body.(io.ReadWriteCloser)
|
|
||||||
if ok {
|
|
||||||
rq.upgraded = &upgradedConn{ReadWriteCloser: conn}
|
|
||||||
res.Body = rq.upgraded
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
maxBytes := rq.h.config.MaxResponseSize
|
|
||||||
if maxBytes > 0 && res.Body != http.NoBody && res.ContentLength > maxBytes {
|
|
||||||
rq.refuse(http.StatusBadGateway)
|
|
||||||
|
|
||||||
return errResponseTooLarge
|
|
||||||
}
|
|
||||||
|
|
||||||
// Go's server adds a Content-Type it guesses from the body to a response
|
|
||||||
// whose headers have no Content-Type key. The key with no value sends none,
|
|
||||||
// as the app sent none.
|
|
||||||
if res.Header.Get("Content-Type") == "" {
|
|
||||||
rq.out.Header()["Content-Type"] = nil
|
|
||||||
}
|
|
||||||
|
|
||||||
res.Body = &responseBody{body: limitBody(res.Body, maxBytes), rq: rq}
|
|
||||||
rq.setResponseDeadline()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// answerError is ReverseProxy's ErrorHandler: the request could not be passed to
|
|
||||||
// the app, or the app's answer cannot be passed on.
|
|
||||||
func (rq *request) answerError(_ http.ResponseWriter, _ *http.Request, err error) {
|
|
||||||
if rq.upgraded != nil {
|
|
||||||
// The app switched protocols, but the switch was not passed on, and
|
|
||||||
// ReverseProxy leaves the app's connection open when it refuses it.
|
|
||||||
_ = rq.upgraded.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
status := int(rq.refused.Load())
|
|
||||||
if status == 0 {
|
|
||||||
if rq.in.Context().Err() != nil {
|
|
||||||
// The visitor went away or stopped its sending side. This panic has Go's
|
|
||||||
// server close the connection without an answer, and log nothing; the
|
|
||||||
// deferred finish still runs.
|
|
||||||
panic(http.ErrAbortHandler)
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.h.processLog.Warn("request to the app failed", "error", err.Error())
|
|
||||||
|
|
||||||
status = http.StatusBadGateway
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.answer(status)
|
|
||||||
}
|
|
||||||
|
|
||||||
// answer sends rproxy's own answer, unless the response has already started.
|
|
||||||
func (rq *request) answer(status int) {
|
|
||||||
rq.refused.CompareAndSwap(0, int64(status))
|
|
||||||
|
|
||||||
if rq.out.status != 0 {
|
|
||||||
return // too late to answer: the connection can only be cut
|
|
||||||
}
|
|
||||||
|
|
||||||
// A visitor found too slow is read no more; any other may go on sending until
|
|
||||||
// its time is up, so that Go's server can read the rest of the body and end
|
|
||||||
// the request cleanly.
|
|
||||||
deadline := rq.requestDeadline()
|
|
||||||
if status == http.StatusRequestTimeout {
|
|
||||||
deadline = time.Now()
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.prepareOwnAnswer(deadline)
|
|
||||||
|
|
||||||
header := rq.out.Header()
|
|
||||||
header.Set("Content-Type", plainText)
|
|
||||||
header.Set("X-Content-Type-Options", "nosniff")
|
|
||||||
rq.send(status, []byte(http.StatusText(status)+"\n"))
|
|
||||||
}
|
|
||||||
|
|
||||||
// send gives the visitor an answer rproxy holds whole: status, the headers set so
|
|
||||||
// far and body. Go's server would otherwise send so short an answer only once the
|
|
||||||
// handler has returned, after finish has written the log line and sent the report.
|
|
||||||
// Without Content-Length, the flush would send the body in chunks, the last of
|
|
||||||
// which would still wait for the handler. A HEAD's answer has no body, so any
|
|
||||||
// Content-Length it has stays as servewaf gave it.
|
|
||||||
func (rq *request) send(status int, body []byte) {
|
|
||||||
if rq.in.Method != http.MethodHead {
|
|
||||||
rq.out.Header().Set("Content-Length", strconv.Itoa(len(body)))
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.out.WriteHeader(status)
|
|
||||||
_, _ = rq.out.Write(body)
|
|
||||||
_ = rq.rc.Flush()
|
|
||||||
}
|
|
||||||
|
|
||||||
// prepareOwnAnswer readies the connection for an answer rproxy gives itself: the
|
|
||||||
// reading of a body that has not arrived whole ends at deadline, and the answer
|
|
||||||
// has SWRP_MAX_RESPONSE_TIME_MS from now to reach the visitor, even when the app
|
|
||||||
// used that time up first.
|
|
||||||
func (rq *request) prepareOwnAnswer(deadline time.Time) {
|
|
||||||
rq.stopReadingBody(deadline)
|
|
||||||
rq.endRequest()
|
|
||||||
|
|
||||||
limit := rq.h.config.MaxResponseTime
|
|
||||||
if limit > 0 {
|
|
||||||
_ = rq.rc.SetWriteDeadline(time.Now().Add(limit))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// refuse records status, unless an earlier refusal was, and ends the request to
|
|
||||||
// the app, if one was made.
|
|
||||||
func (rq *request) refuse(status int) {
|
|
||||||
rq.refused.CompareAndSwap(0, int64(status))
|
|
||||||
|
|
||||||
if rq.cancel != nil {
|
|
||||||
rq.cancel()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// finish ends the request's time limits, writes its log line and, if servewaf
|
|
||||||
// answered the question, sends the report. ctx is the request's own context.
|
|
||||||
func (rq *request) finish(ctx context.Context) {
|
|
||||||
rq.stopTimers()
|
|
||||||
|
|
||||||
if rq.refused.Load() == 0 {
|
|
||||||
rq.stopReadingBody(rq.requestDeadline())
|
|
||||||
}
|
|
||||||
|
|
||||||
requestEnd := rq.endRequest()
|
|
||||||
end := time.Now()
|
|
||||||
requestBytes, responseBytes := rq.bytesEachWay()
|
|
||||||
|
|
||||||
err := requestlog.WriteRProxyLine(rq.h.requestLog, &requestlog.RProxyLine{
|
|
||||||
Time: requestlog.FormatTime(rq.start),
|
|
||||||
RequestID: rq.id,
|
|
||||||
ClientIP: rq.client.String(),
|
|
||||||
Host: rq.in.Host,
|
|
||||||
Method: rq.in.Method,
|
|
||||||
Path: rules.Path(rq.in),
|
|
||||||
Query: rq.in.URL.RawQuery,
|
|
||||||
Status: rq.out.status,
|
|
||||||
RequestBytes: requestBytes,
|
|
||||||
ResponseBytes: responseBytes,
|
|
||||||
DurationRequest: requestlog.Milliseconds(requestEnd.Sub(rq.start)),
|
|
||||||
DurationResponse: requestlog.Milliseconds(end.Sub(requestEnd)),
|
|
||||||
Disposition: rq.disposition,
|
|
||||||
DefaultDisposition: rq.byDefault,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
rq.h.processLog.Error("writing the request log failed", "error", err.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
if !rq.answered {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
report := &proxy.Report{
|
|
||||||
RequestID: rq.id,
|
|
||||||
Status: min(rq.out.status, reportStatusMax),
|
|
||||||
RequestBytes: requestBytes,
|
|
||||||
ResponseBytes: responseBytes,
|
|
||||||
DurationTotal: requestlog.Milliseconds(end.Sub(rq.start)),
|
|
||||||
}
|
|
||||||
if !rq.upstreamStart.IsZero() {
|
|
||||||
report.DurationUpstreamTotal = requestlog.Milliseconds(end.Sub(rq.upstreamStart))
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.h.sendReport(ctx, report)
|
|
||||||
}
|
|
||||||
|
|
||||||
// bytesEachWay returns the body bytes of the request and of the answer. For an
|
|
||||||
// upgraded connection, such as a WebSocket, which has closed by then, what it
|
|
||||||
// carried from the visitor counts with the request's and what it carried from the
|
|
||||||
// app with the answer's.
|
|
||||||
func (rq *request) bytesEachWay() (int64, int64) {
|
|
||||||
var requestBytes int64
|
|
||||||
if rq.body != nil {
|
|
||||||
requestBytes = rq.body.bytes.Load()
|
|
||||||
}
|
|
||||||
|
|
||||||
responseBytes := rq.out.bytes
|
|
||||||
|
|
||||||
if rq.upgraded != nil {
|
|
||||||
requestBytes += rq.upgraded.toApp.Load()
|
|
||||||
responseBytes += rq.upgraded.fromApp.Load()
|
|
||||||
}
|
|
||||||
|
|
||||||
return requestBytes, responseBytes
|
|
||||||
}
|
|
||||||
|
|
||||||
// requestDeadline is when the visitor must have sent its whole request, or zero
|
|
||||||
// when SWRP_MAX_REQUEST_TIME_MS is off.
|
|
||||||
func (rq *request) requestDeadline() time.Time {
|
|
||||||
limit := rq.h.config.MaxRequestTime
|
|
||||||
if limit == 0 {
|
|
||||||
return time.Time{}
|
|
||||||
}
|
|
||||||
|
|
||||||
return rq.start.Add(limit)
|
|
||||||
}
|
|
||||||
|
|
||||||
// stopReadingBody ends, at deadline, the reading of a visitor's body that has not
|
|
||||||
// arrived whole: Go's server then reads no more of it, and closes the connection
|
|
||||||
// after the answer.
|
|
||||||
func (rq *request) stopReadingBody(deadline time.Time) {
|
|
||||||
if rq.body == nil || rq.body.received.Load() {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
_ = rq.rc.SetReadDeadline(deadline)
|
|
||||||
}
|
|
||||||
|
|
||||||
// startRequestTimer starts SWRP_MAX_REQUEST_TIME_MS, which runs from the
|
|
||||||
// request's headers being read until the app has been sent the whole request.
|
|
||||||
func (rq *request) startRequestTimer() {
|
|
||||||
deadline := rq.requestDeadline()
|
|
||||||
if deadline.IsZero() {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.mu.Lock()
|
|
||||||
defer rq.mu.Unlock()
|
|
||||||
|
|
||||||
rq.requestTimer = time.AfterFunc(time.Until(deadline), rq.requestTimedOut)
|
|
||||||
}
|
|
||||||
|
|
||||||
// requestTimedOut is called when SWRP_MAX_REQUEST_TIME_MS runs out while the
|
|
||||||
// request is still on its way to the app. The answer names the side rproxy was
|
|
||||||
// waiting on at that moment: 408 when it was waiting for the visitor to send more
|
|
||||||
// of its body, 504 when it was waiting for the app to be reached or to take what
|
|
||||||
// it had.
|
|
||||||
func (rq *request) requestTimedOut() {
|
|
||||||
rq.mu.Lock()
|
|
||||||
defer rq.mu.Unlock()
|
|
||||||
|
|
||||||
if rq.timersStopped {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if rq.body == nil || !rq.body.waiting.Load() {
|
|
||||||
// Once the app has answered, the transport, ReverseProxy and Go's server
|
|
||||||
// each close the visitor's body, and closing it reads what is left of a
|
|
||||||
// body under 256 KiB without requestBody.Read, so waiting is false while
|
|
||||||
// that read waits for the visitor. That read ends now, before refuse:
|
|
||||||
// should rproxy still answer itself, answer then sets the deadline it
|
|
||||||
// gives its own answers.
|
|
||||||
if !rq.answerStarted.IsZero() {
|
|
||||||
rq.stopReadingBody(time.Now())
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.refuse(http.StatusGatewayTimeout)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.refuse(http.StatusRequestTimeout)
|
|
||||||
// The transport gives up on the app only once its Read of the visitor's body
|
|
||||||
// returns, so that Read is ended now. The lock keeps this from reaching the
|
|
||||||
// connection after the request is handled.
|
|
||||||
_ = rq.rc.SetReadDeadline(time.Now())
|
|
||||||
}
|
|
||||||
|
|
||||||
// gotFirstResponseByte is called once the first byte of the app's answer has
|
|
||||||
// arrived.
|
|
||||||
func (rq *request) gotFirstResponseByte() {
|
|
||||||
rq.mu.Lock()
|
|
||||||
defer rq.mu.Unlock()
|
|
||||||
|
|
||||||
rq.answerStarted = time.Now()
|
|
||||||
}
|
|
||||||
|
|
||||||
// wroteRequest is called once the app has been sent the whole request:
|
|
||||||
// SWRP_MAX_REQUEST_TIME_MS ends and SWRP_MAX_RESPONSE_TIME_MS starts.
|
|
||||||
func (rq *request) wroteRequest(info httptrace.WroteRequestInfo) {
|
|
||||||
if info.Err != nil {
|
|
||||||
return // the transport gives up, or tries again
|
|
||||||
}
|
|
||||||
|
|
||||||
requestEnd := rq.endRequest()
|
|
||||||
|
|
||||||
rq.mu.Lock()
|
|
||||||
defer rq.mu.Unlock()
|
|
||||||
|
|
||||||
if rq.timersStopped {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
stopTimer(rq.requestTimer)
|
|
||||||
|
|
||||||
limit := rq.h.config.MaxResponseTime
|
|
||||||
if limit > 0 {
|
|
||||||
rq.responseTimer = time.AfterFunc(time.Until(requestEnd.Add(limit)),
|
|
||||||
rq.responseTimedOut)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// responseTimedOut is called when SWRP_MAX_RESPONSE_TIME_MS runs out before the
|
|
||||||
// app has sent its whole answer.
|
|
||||||
func (rq *request) responseTimedOut() {
|
|
||||||
rq.mu.Lock()
|
|
||||||
defer rq.mu.Unlock()
|
|
||||||
|
|
||||||
if !rq.timersStopped {
|
|
||||||
rq.refuse(http.StatusGatewayTimeout)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// responseReceived is called once the app has sent its whole answer. An app may
|
|
||||||
// answer before the visitor has sent its whole body, as with a 404 it sends
|
|
||||||
// without reading it. The time limits then run on: until the transport's read of
|
|
||||||
// the body returns, Go's server cannot send the answer, and
|
|
||||||
// SWRP_MAX_REQUEST_TIME_MS is what ends that read for a visitor that stops
|
|
||||||
// sending.
|
|
||||||
func (rq *request) responseReceived() {
|
|
||||||
if rq.body != nil && !rq.body.received.Load() {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.stopTimers()
|
|
||||||
}
|
|
||||||
|
|
||||||
// setResponseDeadline sets SWRP_MAX_RESPONSE_TIME_MS on the connection to the
|
|
||||||
// visitor: the app's answer must reach the visitor within it of the end of the
|
|
||||||
// request.
|
|
||||||
func (rq *request) setResponseDeadline() {
|
|
||||||
requestEnd := rq.endRequest()
|
|
||||||
|
|
||||||
limit := rq.h.config.MaxResponseTime
|
|
||||||
if limit > 0 {
|
|
||||||
_ = rq.rc.SetWriteDeadline(requestEnd.Add(limit))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// endRequest notes now as the end of the request, unless an end is noted, and
|
|
||||||
// returns the end noted.
|
|
||||||
func (rq *request) endRequest() time.Time {
|
|
||||||
rq.mu.Lock()
|
|
||||||
defer rq.mu.Unlock()
|
|
||||||
|
|
||||||
if rq.requestEnd.IsZero() {
|
|
||||||
rq.requestEnd = time.Now()
|
|
||||||
}
|
|
||||||
|
|
||||||
return rq.requestEnd
|
|
||||||
}
|
|
||||||
|
|
||||||
// stopTimers stops the request's time limits and keeps any from starting later:
|
|
||||||
// the app's answer is complete, the connection upgraded, or the request handled.
|
|
||||||
func (rq *request) stopTimers() {
|
|
||||||
rq.mu.Lock()
|
|
||||||
defer rq.mu.Unlock()
|
|
||||||
|
|
||||||
rq.timersStopped = true
|
|
||||||
stopTimer(rq.requestTimer)
|
|
||||||
stopTimer(rq.responseTimer)
|
|
||||||
}
|
|
||||||
|
|
||||||
// stopTimer stops t, which is nil when its time limit is off.
|
|
||||||
func stopTimer(t *time.Timer) {
|
|
||||||
if t != nil {
|
|
||||||
t.Stop()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,228 +0,0 @@
|
|||||||
// Package rproxy is the reverse proxy in front of one app that asks servewaf
|
|
||||||
// about each request. It passes a request servewaf allows to the app and the
|
|
||||||
// app's answer back, within the size and time limits. Any other answer from
|
|
||||||
// servewaf goes to the visitor as servewaf gave it. SWRP_DEFAULT_DISPOSITION
|
|
||||||
// decides only when servewaf cannot be connected to, does not answer at all, or
|
|
||||||
// does not answer within SWRP_TIMEOUT_MS. rproxy itself refuses a request with a
|
|
||||||
// method it does not pass on, or one it cannot put to servewaf as the visitor
|
|
||||||
// sent it. It keeps nothing between requests and never touches disk. "rproxy" in
|
|
||||||
// SPEC.md says what it does.
|
|
||||||
package rproxy
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"io"
|
|
||||||
"log"
|
|
||||||
"log/slog"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/proxy"
|
|
||||||
)
|
|
||||||
|
|
||||||
// How rproxy keeps connections to the app and to servewaf open between
|
|
||||||
// requests.
|
|
||||||
const (
|
|
||||||
idleConns = 100
|
|
||||||
idleConnTimeout = 90 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
// The limits on a connection from a visitor that have no setting.
|
|
||||||
const (
|
|
||||||
// requestHeaderMaxBytes is the largest request line and headers a visitor
|
|
||||||
// may send.
|
|
||||||
requestHeaderMaxBytes = 32 << 10
|
|
||||||
// goServerExtraHeaderBytes is how far past MaxHeaderBytes Go's server reads
|
|
||||||
// before it refuses.
|
|
||||||
goServerExtraHeaderBytes = 4 << 10
|
|
||||||
// idleTimeout is how long a kept-open connection may wait for its next
|
|
||||||
// request.
|
|
||||||
idleTimeout = 120 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
// adminPrefix starts the path of every request for smallwebwaf itself, which
|
|
||||||
// never reaches the app.
|
|
||||||
const adminPrefix = "/_smallwebwaf/"
|
|
||||||
|
|
||||||
// allowedMethods lists the methods rproxy passes on, as an Allow header gives
|
|
||||||
// them. It answers any other method 405 itself.
|
|
||||||
const allowedMethods = "GET, HEAD, POST, OPTIONS, PUT, DELETE, PATCH, QUERY"
|
|
||||||
|
|
||||||
// Params are what New needs.
|
|
||||||
type Params struct {
|
|
||||||
Config *config.RProxy
|
|
||||||
// RequestLog receives one short JSON line per request.
|
|
||||||
RequestLog io.Writer
|
|
||||||
// ProcessLog receives the process's own messages.
|
|
||||||
ProcessLog *slog.Logger
|
|
||||||
}
|
|
||||||
|
|
||||||
// Server is the server `smallwebwaf rproxy` runs.
|
|
||||||
type Server struct {
|
|
||||||
*http.Server
|
|
||||||
|
|
||||||
h *handler
|
|
||||||
}
|
|
||||||
|
|
||||||
// New returns the server `smallwebwaf rproxy` runs. Go's server itself refuses
|
|
||||||
// a request line and headers over requestHeaderMaxBytes, with 431, closes a
|
|
||||||
// connection idle for idleTimeout, and applies SWRP_MAX_REQUEST_TIME_MS while
|
|
||||||
// the headers arrive. The handler applies the time and size limits from then
|
|
||||||
// on.
|
|
||||||
func New(params Params) *Server {
|
|
||||||
backend := params.Config.WAFBackend
|
|
||||||
errorLog := slog.NewLogLogger(params.ProcessLog.Handler(), slog.LevelWarn)
|
|
||||||
h := &handler{
|
|
||||||
config: params.Config,
|
|
||||||
requestLog: params.RequestLog,
|
|
||||||
processLog: params.ProcessLog,
|
|
||||||
errorLog: errorLog,
|
|
||||||
transport: newTransport(),
|
|
||||||
serveWAFTransport: newServeWAFTransport(),
|
|
||||||
reportURL: (&url.URL{
|
|
||||||
Scheme: backend.Scheme, Host: backend.Host, Path: proxy.ReportPath,
|
|
||||||
}).String(),
|
|
||||||
}
|
|
||||||
|
|
||||||
return &Server{
|
|
||||||
Server: &http.Server{
|
|
||||||
Addr: params.Config.ListenAddr,
|
|
||||||
Handler: h,
|
|
||||||
ErrorLog: errorLog,
|
|
||||||
ReadHeaderTimeout: params.Config.MaxRequestTime,
|
|
||||||
IdleTimeout: idleTimeout,
|
|
||||||
MaxHeaderBytes: requestHeaderMaxBytes - goServerExtraHeaderBytes,
|
|
||||||
},
|
|
||||||
h: h,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// WaitForReports returns once the reports being sent have gone or failed,
|
|
||||||
// which SWRP_TIMEOUT_MS bounds. Any report after this call is dropped: that of
|
|
||||||
// a WebSocket that closes while rproxy stops, for one.
|
|
||||||
func (s *Server) WaitForReports() {
|
|
||||||
s.h.reportsMu.Lock()
|
|
||||||
s.h.reportsStopped = true
|
|
||||||
s.h.reportsMu.Unlock()
|
|
||||||
|
|
||||||
s.h.reports.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
// handler is the proxy. It holds what every request shares; what belongs to
|
|
||||||
// one request is in a request.
|
|
||||||
type handler struct {
|
|
||||||
config *config.RProxy
|
|
||||||
requestLog io.Writer
|
|
||||||
processLog *slog.Logger
|
|
||||||
errorLog *log.Logger
|
|
||||||
// transport carries requests to the app, and serveWAFTransport questions and
|
|
||||||
// reports to servewaf.
|
|
||||||
transport http.RoundTripper
|
|
||||||
serveWAFTransport http.RoundTripper
|
|
||||||
// reportURL is where the reports go.
|
|
||||||
reportURL string
|
|
||||||
// reports waits for the reports being sent. reportsMu guards starting one
|
|
||||||
// against reportsStopped, which is set once nothing is to wait for another.
|
|
||||||
reports sync.WaitGroup
|
|
||||||
reportsMu sync.Mutex
|
|
||||||
reportsStopped bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// newTransport returns what carries requests to the app. It never goes through
|
|
||||||
// a proxy named in the environment, and leaves answers compressed or not as they
|
|
||||||
// were sent.
|
|
||||||
func newTransport() *http.Transport {
|
|
||||||
return &http.Transport{
|
|
||||||
MaxIdleConns: idleConns,
|
|
||||||
MaxIdleConnsPerHost: idleConns,
|
|
||||||
IdleConnTimeout: idleConnTimeout,
|
|
||||||
DisableCompression: true,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// newServeWAFTransport returns what carries questions and reports to servewaf.
|
|
||||||
// It speaks HTTP/1.1 only. Go's HTTP/2 client refuses to send a visitor's Upgrade
|
|
||||||
// header, or a Connection header other than close or keep-alive, and that would
|
|
||||||
// read as servewaf giving no answer.
|
|
||||||
func newServeWAFTransport() *http.Transport {
|
|
||||||
transport := newTransport()
|
|
||||||
transport.Protocols = new(http.Protocols)
|
|
||||||
transport.Protocols.SetHTTP1(true)
|
|
||||||
|
|
||||||
return transport
|
|
||||||
}
|
|
||||||
|
|
||||||
// ServeHTTP handles one request: it answers the health check itself, refuses a
|
|
||||||
// request with a method it does not pass on or one it cannot put to servewaf as
|
|
||||||
// the visitor sent it, and for any other request reads the first part of the
|
|
||||||
// body, asks servewaf, and does what the answer says. Once the answer has ended,
|
|
||||||
// it writes the request's log line and sends its report.
|
|
||||||
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
||||||
rq := h.newRequest(w, r)
|
|
||||||
defer rq.finish(r.Context())
|
|
||||||
|
|
||||||
if r.Method == http.MethodGet && r.URL.Path == proxy.HealthPath {
|
|
||||||
rq.answerHealth()
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Contains(strings.Split(allowedMethods, ", "), r.Method) {
|
|
||||||
rq.out.Header().Set("Allow", allowedMethods)
|
|
||||||
rq.answer(http.StatusMethodNotAllowed)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if !canBeAsked(r) {
|
|
||||||
rq.answer(http.StatusBadRequest)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if !rq.readFirstPartOfBody() {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
answer, err := rq.ask(r.Context())
|
|
||||||
if err != nil {
|
|
||||||
rq.handleNoUsableAnswer(r.Context(), err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.noteAnswer(answer)
|
|
||||||
|
|
||||||
switch rq.disposition {
|
|
||||||
case proxy.DispositionAllow:
|
|
||||||
rq.passToApp(r.Context())
|
|
||||||
case proxy.DispositionClose:
|
|
||||||
// This panic has Go's server close the connection without an answer,
|
|
||||||
// and log nothing; the deferred finish still runs.
|
|
||||||
panic(http.ErrAbortHandler)
|
|
||||||
default: // a deny, an error, or an answer without a valid disposition
|
|
||||||
rq.answerWith(answer)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// passToApp passes the request on to the app, unless a limit or its path keeps
|
|
||||||
// it from the app. servewaf answers a path under adminPrefix itself, so such a
|
|
||||||
// request gets here only when SWRP_DEFAULT_DISPOSITION allowed it. ctx is the
|
|
||||||
// request's own context.
|
|
||||||
func (rq *request) passToApp(ctx context.Context) {
|
|
||||||
maxBytes := rq.h.config.MaxRequestSize
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case rq.refused.Load() != 0 || (maxBytes > 0 && rq.in.ContentLength > maxBytes):
|
|
||||||
rq.answer(http.StatusRequestEntityTooLarge)
|
|
||||||
case strings.HasPrefix(rq.in.URL.Path, adminPrefix):
|
|
||||||
rq.answer(http.StatusNotFound)
|
|
||||||
default:
|
|
||||||
rq.forward(ctx)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,266 +0,0 @@
|
|||||||
package rproxy
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"maps"
|
|
||||||
"net/http"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/proxy"
|
|
||||||
)
|
|
||||||
|
|
||||||
// answerBodyMaxBytes is the most of the body of servewaf's answer that is read.
|
|
||||||
// It is far above any answer's.
|
|
||||||
const answerBodyMaxBytes = 1 << 20
|
|
||||||
|
|
||||||
// reportAnswerMaxBytes is the most of the answer to a report that is read,
|
|
||||||
// which has no body when servewaf takes the report.
|
|
||||||
const reportAnswerMaxBytes = 4 << 10
|
|
||||||
|
|
||||||
var (
|
|
||||||
errCannotPassOn = errors.New("the answer cannot be passed to the visitor")
|
|
||||||
errReportRefused = errors.New("the report was answered")
|
|
||||||
)
|
|
||||||
|
|
||||||
// servewafAnswer is what servewaf answered to a question.
|
|
||||||
type servewafAnswer struct {
|
|
||||||
status int
|
|
||||||
header http.Header
|
|
||||||
body []byte
|
|
||||||
}
|
|
||||||
|
|
||||||
// noteAnswer notes what servewaf answered. An answer without a valid disposition
|
|
||||||
// notes nothing, and no report is due for it: servewaf holds no decision to report
|
|
||||||
// on.
|
|
||||||
func (rq *request) noteAnswer(answer *servewafAnswer) {
|
|
||||||
disposition := answer.header.Get(proxy.DispositionHeader)
|
|
||||||
|
|
||||||
switch disposition {
|
|
||||||
case proxy.DispositionAllow, proxy.DispositionDeny, proxy.DispositionClose:
|
|
||||||
rq.disposition = disposition
|
|
||||||
rq.answered = true
|
|
||||||
rq.asn = answer.header.Get(asnHeader)
|
|
||||||
rq.country = answer.header.Get(countryHeader)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// handleNoUsableAnswer deals with a request whose question ended in err, which
|
|
||||||
// leaves no answer to act on. ctx is the request's own context.
|
|
||||||
func (rq *request) handleNoUsableAnswer(ctx context.Context, err error) {
|
|
||||||
switch {
|
|
||||||
case ctx.Err() != nil:
|
|
||||||
// The visitor went away while servewaf was being asked. servewaf has not
|
|
||||||
// failed, and nothing is noted. This panic has Go's server close the
|
|
||||||
// connection without an answer, and log nothing; the deferred finish still
|
|
||||||
// runs.
|
|
||||||
panic(http.ErrAbortHandler)
|
|
||||||
case errors.Is(err, errCannotPassOn):
|
|
||||||
rq.h.processLog.Warn("servewaf's answer is dropped", "error", err.Error())
|
|
||||||
rq.answer(http.StatusBadGateway)
|
|
||||||
default:
|
|
||||||
// servewaf cannot be connected to, does not answer at all, or does not answer
|
|
||||||
// within SWRP_TIMEOUT_MS. A request that keeps servewaf busy past
|
|
||||||
// SWRP_TIMEOUT_MS, like any request while servewaf is overloaded, gets
|
|
||||||
// SWRP_DEFAULT_DISPOSITION here. deny closes this, at the cost of answering 503
|
|
||||||
// whenever servewaf is slow. Making such requests hard to send is
|
|
||||||
// https://git.eeqj.de/sneak/smallwebwaf/issues/182.
|
|
||||||
rq.h.processLog.Warn("servewaf gave no answer", "error", err.Error())
|
|
||||||
|
|
||||||
rq.disposition = rq.h.config.DefaultDisposition
|
|
||||||
rq.byDefault = true
|
|
||||||
|
|
||||||
if rq.disposition == proxy.DispositionAllow {
|
|
||||||
rq.passToApp(ctx)
|
|
||||||
} else {
|
|
||||||
rq.answer(http.StatusServiceUnavailable)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ask sends servewaf the question, and returns its answer, which must have come
|
|
||||||
// whole within SWRP_TIMEOUT_MS. It ends when ctx does, as when the visitor goes
|
|
||||||
// away.
|
|
||||||
func (rq *request) ask(ctx context.Context) (*servewafAnswer, error) {
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, rq.h.config.Timeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
question, err := rq.newQuestion(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
res, err := rq.h.serveWAFTransport.RoundTrip(question)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("send the question: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
_ = res.Body.Close()
|
|
||||||
}()
|
|
||||||
|
|
||||||
// The body of a 101 is the connection itself, and the time limit does not end
|
|
||||||
// a read of that. Go's client also takes a status below 100, which Go's server
|
|
||||||
// panics when asked to send.
|
|
||||||
if res.StatusCode < http.StatusOK {
|
|
||||||
return nil, fmt.Errorf("%w: its status is %d", errCannotPassOn, res.StatusCode)
|
|
||||||
}
|
|
||||||
|
|
||||||
body, err := io.ReadAll(io.LimitReader(res.Body, answerBodyMaxBytes+1))
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("read the answer: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(body) > answerBodyMaxBytes {
|
|
||||||
return nil, fmt.Errorf("%w: its body is over 1 MiB", errCannotPassOn)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &servewafAnswer{res.StatusCode, res.Header, body}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// canBeAsked reports whether Go's client can send servewaf r as the visitor sent
|
|
||||||
// it. It cannot send these as they came:
|
|
||||||
// - a target that does not start with /, or that starts with //;
|
|
||||||
// - an empty Host, for which it sends servewaf's own address;
|
|
||||||
// - a Host holding an IPv6 zone, which it removes;
|
|
||||||
// - more than one User-Agent, of which it sends the first;
|
|
||||||
// - an empty User-Agent, which it leaves out, the one case that still goes to
|
|
||||||
// servewaf.
|
|
||||||
func canBeAsked(r *http.Request) bool {
|
|
||||||
target, host := r.RequestURI, r.Host
|
|
||||||
hostHasZone := strings.HasPrefix(host, "[") && strings.Contains(host, "%")
|
|
||||||
|
|
||||||
return strings.HasPrefix(target, "/") && !strings.HasPrefix(target, "//") &&
|
|
||||||
host != "" && !hostHasZone && len(r.Header.Values("User-Agent")) <= 1
|
|
||||||
}
|
|
||||||
|
|
||||||
// newQuestion returns the question about the request, as "The question" in
|
|
||||||
// SPEC.md gives it: the visitor's request as the visitor sent it, with the part
|
|
||||||
// of the body it holds and rproxy's headers.
|
|
||||||
func (rq *request) newQuestion(ctx context.Context) (*http.Request, error) {
|
|
||||||
in := rq.in
|
|
||||||
|
|
||||||
var held []byte
|
|
||||||
if rq.body != nil {
|
|
||||||
held = rq.body.held
|
|
||||||
}
|
|
||||||
|
|
||||||
question, err := http.NewRequestWithContext( //nolint:gosec // a setting's URL
|
|
||||||
ctx, in.Method, rq.h.config.WAFBackend.String(), bytes.NewReader(held))
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("make the question: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
backend := question.URL
|
|
||||||
target := *in.URL
|
|
||||||
target.Scheme = backend.Scheme
|
|
||||||
target.Host = backend.Host
|
|
||||||
setPathAsSent(&target, in)
|
|
||||||
question.URL = &target
|
|
||||||
question.Host = in.Host
|
|
||||||
|
|
||||||
// The length NewRequestWithContext set is that of what is held. This line makes
|
|
||||||
// a body whose length the visitor did not announce go chunked, as "The question"
|
|
||||||
// in SPEC.md requires.
|
|
||||||
question.TransferEncoding = in.TransferEncoding
|
|
||||||
|
|
||||||
question.Header = in.Header.Clone()
|
|
||||||
question.Header.Set(proxy.ClientHeader, rq.client.String())
|
|
||||||
question.Header.Set(proxy.PeerHeader, rq.peer.String())
|
|
||||||
question.Header.Del(proxy.ContentLengthHeader)
|
|
||||||
|
|
||||||
announced := in.Header.Get("Content-Length")
|
|
||||||
if announced != "" {
|
|
||||||
question.Header.Set(proxy.ContentLengthHeader, announced)
|
|
||||||
}
|
|
||||||
|
|
||||||
question.Header.Set(requestIDHeader, rq.id)
|
|
||||||
question.Header.Set("X-Forwarded-Proto", scheme(in, rq.peerTrusted))
|
|
||||||
|
|
||||||
// Go's client adds a User-Agent of its own to a request without the key.
|
|
||||||
_, sent := question.Header["User-Agent"]
|
|
||||||
if !sent {
|
|
||||||
question.Header["User-Agent"] = nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return question, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// answerWith gives the visitor servewaf's answer, less its disposition.
|
|
||||||
func (rq *request) answerWith(answer *servewafAnswer) {
|
|
||||||
rq.prepareOwnAnswer(rq.requestDeadline())
|
|
||||||
|
|
||||||
header := rq.out.Header()
|
|
||||||
maps.Copy(header, answer.header)
|
|
||||||
header.Del(proxy.DispositionHeader)
|
|
||||||
|
|
||||||
// Go's server adds a Content-Type it guesses from the body to a response whose
|
|
||||||
// headers have no Content-Type key. The key with no value sends none.
|
|
||||||
if header.Get("Content-Type") == "" {
|
|
||||||
header["Content-Type"] = nil
|
|
||||||
}
|
|
||||||
|
|
||||||
rq.send(answer.status, answer.body)
|
|
||||||
}
|
|
||||||
|
|
||||||
// sendReport sends servewaf report on a goroutine of its own, so that the
|
|
||||||
// visitor does not wait for it. The report goes on after ctx, the request's own
|
|
||||||
// context, has ended.
|
|
||||||
func (h *handler) sendReport(ctx context.Context, report *proxy.Report) {
|
|
||||||
h.reportsMu.Lock()
|
|
||||||
defer h.reportsMu.Unlock()
|
|
||||||
|
|
||||||
if h.reportsStopped {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
h.reports.Go(func() {
|
|
||||||
err := h.postReport(context.WithoutCancel(ctx), report)
|
|
||||||
if err != nil {
|
|
||||||
h.processLog.Warn("a report is dropped", "error", err.Error())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// postReport posts report to servewaf, and waits for its answer for up to
|
|
||||||
// SWRP_TIMEOUT_MS.
|
|
||||||
func (h *handler) postReport(ctx context.Context, report *proxy.Report) error {
|
|
||||||
body, err := json.Marshal(report)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("encode the report: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, h.config.Timeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext( //nolint:gosec // a setting's URL
|
|
||||||
ctx, http.MethodPost, h.reportURL, bytes.NewReader(body))
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("make the request: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
|
|
||||||
res, err := h.serveWAFTransport.RoundTrip(req)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("send the report: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The body of a 101 is the connection itself, and the time limit does not end
|
|
||||||
// a read of that.
|
|
||||||
if res.StatusCode >= http.StatusOK {
|
|
||||||
_, _ = io.Copy(io.Discard, io.LimitReader(res.Body, reportAnswerMaxBytes))
|
|
||||||
}
|
|
||||||
|
|
||||||
_ = res.Body.Close()
|
|
||||||
|
|
||||||
if res.StatusCode != http.StatusNoContent {
|
|
||||||
return fmt.Errorf("%w %s", errReportRefused, res.Status)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
+16
-26
@@ -50,13 +50,6 @@ const quietTime = 2 * time.Second
|
|||||||
// header:<Name>.
|
// header:<Name>.
|
||||||
const headerTarget = "header:"
|
const headerTarget = "header:"
|
||||||
|
|
||||||
// pathTarget and uriTarget are the targets matched both as received and
|
|
||||||
// once percent-decoded: the path, and the path and query together.
|
|
||||||
const (
|
|
||||||
pathTarget = "path"
|
|
||||||
uriTarget = "uri"
|
|
||||||
)
|
|
||||||
|
|
||||||
// escapeLength is the length of a percent escape, such as %2e.
|
// escapeLength is the length of a percent escape, such as %2e.
|
||||||
const escapeLength = 3
|
const escapeLength = 3
|
||||||
|
|
||||||
@@ -381,7 +374,7 @@ func parse(line string) (Rule, bool, error) {
|
|||||||
// isTarget reports whether target is one a rule may have.
|
// isTarget reports whether target is one a rule may have.
|
||||||
func isTarget(target string) bool {
|
func isTarget(target string) bool {
|
||||||
switch target {
|
switch target {
|
||||||
case pathTarget, "query", uriTarget, "method", "host", "user_agent", "referer":
|
case "path", "query", "uri", "method", "host", "user_agent", "referer":
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -391,38 +384,35 @@ func isTarget(target string) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// matches reports whether the rule's regex matches its target in r. For
|
// matches reports whether the rule's regex matches its target in r. For
|
||||||
// path and uri it is matched against the target as received, and against
|
// uri it is matched against the path and query as received, and against
|
||||||
// it once percent-decoded, so that an encoded probe cannot slip past.
|
// them once percent-decoded, so that an encoded probe cannot slip past.
|
||||||
func (rule Rule) matches(r *http.Request) bool {
|
func (rule Rule) matches(r *http.Request) bool {
|
||||||
target := value(rule.Target, r)
|
if rule.Target == "uri" {
|
||||||
|
uri := pathAndQuery(r)
|
||||||
|
|
||||||
if rule.Target == pathTarget || rule.Target == uriTarget {
|
return rule.regex.MatchString(uri) || rule.regex.MatchString(decodeOnce(uri))
|
||||||
return rule.regex.MatchString(target) || rule.regex.MatchString(DecodeOnce(target))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return rule.regex.MatchString(target)
|
return rule.regex.MatchString(value(rule.Target, r))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Path returns r's path as the client sent it, before any decoding or
|
// Path returns r's path as the client sent it, before any decoding or
|
||||||
// re-encoding, up to the first ?: what a path rule is matched against, as
|
// re-encoding, up to the first ?: what a path rule is matched against.
|
||||||
// it is and once percent-decoded.
|
|
||||||
func Path(r *http.Request) string {
|
func Path(r *http.Request) string {
|
||||||
path, _, _ := strings.Cut(pathAndQuery(r), "?")
|
path, _, _ := strings.Cut(pathAndQuery(r), "?")
|
||||||
|
|
||||||
return path
|
return path
|
||||||
}
|
}
|
||||||
|
|
||||||
// value returns what a rule with target is matched against in r: the
|
// value returns what a rule with target, other than uri, is matched
|
||||||
// path, as Path gives it, the path and query, as pathAndQuery gives
|
// against in r: the path, as Path gives it, and the query as the client
|
||||||
// them, and the query as the client sent it, before any decoding or
|
// sent it, before any decoding or re-encoding, after the first ?, and a
|
||||||
// re-encoding, after the first ?, and a header's values joined by ", ",
|
// header's values joined by ", ", as HTTP joins those of a header sent
|
||||||
// as HTTP joins those of a header sent more than once.
|
// more than once.
|
||||||
func value(target string, r *http.Request) string {
|
func value(target string, r *http.Request) string {
|
||||||
switch target {
|
switch target {
|
||||||
case pathTarget:
|
case "path":
|
||||||
return Path(r)
|
return Path(r)
|
||||||
case uriTarget:
|
|
||||||
return pathAndQuery(r)
|
|
||||||
case "query":
|
case "query":
|
||||||
_, query, _ := strings.Cut(pathAndQuery(r), "?")
|
_, query, _ := strings.Cut(pathAndQuery(r), "?")
|
||||||
|
|
||||||
@@ -474,11 +464,11 @@ func header(r *http.Request, name string) string {
|
|||||||
return strings.Join(r.Header.Values(name), ", ")
|
return strings.Join(r.Header.Values(name), ", ")
|
||||||
}
|
}
|
||||||
|
|
||||||
// DecodeOnce returns s with each percent escape, such as %2e, replaced by
|
// decodeOnce returns s with each percent escape, such as %2e, replaced by
|
||||||
// the byte it stands for. A % that is not followed by two hex digits is
|
// the byte it stands for. A % that is not followed by two hex digits is
|
||||||
// left as it is, so that a malformed escape cannot keep the rest of s
|
// left as it is, so that a malformed escape cannot keep the rest of s
|
||||||
// from being decoded.
|
// from being decoded.
|
||||||
func DecodeOnce(s string) string {
|
func decodeOnce(s string) string {
|
||||||
var decoded strings.Builder
|
var decoded strings.Builder
|
||||||
|
|
||||||
for i := 0; i < len(s); i++ {
|
for i := 0; i < len(s); i++ {
|
||||||
|
|||||||
@@ -49,9 +49,7 @@ func TestEachTargetMatchesWhatItNames(t *testing.T) {
|
|||||||
want bool
|
want bool
|
||||||
}{
|
}{
|
||||||
{"path as received", `path log ^/%2eenv$`, "/%2eenv", nil, true},
|
{"path as received", `path log ^/%2eenv$`, "/%2eenv", nil, true},
|
||||||
{"path decoded", `path log ^/\.env$`, "/%2Eenv", nil, true},
|
{"path not decoded", `path log ^/\.env$`, "/%2eenv", nil, false},
|
||||||
{"path decoded only once", `path log ^/\.env$`, "/%252eenv", nil, false},
|
|
||||||
{"path decoded without the query", `path log ^/a\?b$`, "/a%3fb?c", nil, true},
|
|
||||||
{"path without the query", `path log ^/a$`, "/a?b=c", nil, true},
|
{"path without the query", `path log ^/a$`, "/a?b=c", nil, true},
|
||||||
{"query as received", `query log ^b=%2e$`, "/a?b=%2e", nil, true},
|
{"query as received", `query log ^b=%2e$`, "/a?b=%2e", nil, true},
|
||||||
{"uri as received", `uri log ^/a\?b=%2e$`, "/a?b=%2e", nil, true},
|
{"uri as received", `uri log ^/a\?b=%2e$`, "/a?b=%2e", nil, true},
|
||||||
@@ -420,7 +418,7 @@ func TestDefaultFileBansProbesAtTheSiteRootAlone(t *testing.T) {
|
|||||||
|
|
||||||
// Probes sent by a browser, by the rule that refuses them.
|
// Probes sent by a browser, by the rule that refuses them.
|
||||||
for rule, targets := range map[string][]string{
|
for rule, targets := range map[string][]string{
|
||||||
"env-file": {"/.env", "/.env.production", "/.ENV", "/%2eenv"},
|
"env-file": {"/.env", "/.env.production", "/.ENV"},
|
||||||
"vcs-dir": {"/.git/config", "/.git", "/.svn/entries"},
|
"vcs-dir": {"/.git/config", "/.git", "/.svn/entries"},
|
||||||
"secrets-dir": {"/.aws/credentials", "/.ssh/id_rsa"},
|
"secrets-dir": {"/.aws/credentials", "/.ssh/id_rsa"},
|
||||||
"secret-file": {"/.htpasswd", "/.DS_Store", "/.git-credentials"},
|
"secret-file": {"/.htpasswd", "/.DS_Store", "/.git-credentials"},
|
||||||
|
|||||||
@@ -17,16 +17,13 @@ import (
|
|||||||
// healthCheckTimeout bounds the whole health check.
|
// healthCheckTimeout bounds the whole health check.
|
||||||
const healthCheckTimeout = 5 * time.Second
|
const healthCheckTimeout = 5 * time.Second
|
||||||
|
|
||||||
var errHealthEndpoint = errors.New("the health endpoint answered")
|
var errHealthEndpoint = errors.New("smallwebwaf's health endpoint answered")
|
||||||
|
|
||||||
// HealthCheck is the container's health check. In servewaf's container, where
|
// HealthCheck is the container's health check. It returns 0 while
|
||||||
// SMALLWEBWAF_COMMAND is servewaf, it returns 0 while servewaf answers its health
|
// smallwebwaf answers its health endpoint on 127.0.0.1, at the port in
|
||||||
// endpoint on 127.0.0.1, at the port in SWWAF_LISTEN_ADDR. In rproxy's container,
|
// SWWAF_LISTEN_ADDR, and the app accepts connections at the address in
|
||||||
// where SMALLWEBWAF_COMMAND is rproxy, unset or empty, it returns 0 while rproxy
|
// SWWAF_UPSTREAM_URL. Otherwise it writes why to stderr and returns 1.
|
||||||
// answers its health endpoint on 127.0.0.1, at the port in SWRP_LISTEN_ADDR, and
|
// It reads no other setting, nor a file that another names, so neither
|
||||||
// the app accepts connections at the address in SWRP_UPSTREAM_URL. Otherwise,
|
|
||||||
// any other value of SMALLWEBWAF_COMMAND included, it writes why to stderr and
|
|
||||||
// returns 1. It reads no other setting, nor a file that another names, so neither
|
|
||||||
// can fail it.
|
// can fail it.
|
||||||
// args are the arguments after `healthcheck`; it takes none, and given
|
// args are the arguments after `healthcheck`; it takes none, and given
|
||||||
// one it names it on stderr and returns 1 without checking anything.
|
// one it names it on stderr and returns 1 without checking anything.
|
||||||
@@ -41,20 +38,7 @@ func HealthCheck(
|
|||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, healthCheckTimeout)
|
err := healthCheck(ctx, lookupEnv)
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
command, err := config.Command(lookupEnv)
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case err != nil:
|
|
||||||
err = fmt.Errorf("invalid setting: %w", err)
|
|
||||||
case command == "servewaf":
|
|
||||||
err = checkServeWAF(ctx, lookupEnv)
|
|
||||||
default:
|
|
||||||
err = checkRProxyAndApp(ctx, lookupEnv)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_, _ = fmt.Fprintln(stderr, "unhealthy:", err)
|
_, _ = fmt.Fprintln(stderr, "unhealthy:", err)
|
||||||
|
|
||||||
@@ -64,48 +48,16 @@ func HealthCheck(
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkServeWAF asks servewaf's health endpoint.
|
func healthCheck(ctx context.Context, lookupEnv func(string) (string, bool)) error {
|
||||||
func checkServeWAF(ctx context.Context, lookupEnv func(string) (string, bool)) error {
|
ctx, cancel := context.WithTimeout(ctx, healthCheckTimeout)
|
||||||
listenAddr, err := config.ServeWAFListenAddr(lookupEnv)
|
defer cancel()
|
||||||
|
|
||||||
|
listenAddr, upstreamURL, err := config.ListenAddrAndUpstreamURL(lookupEnv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("invalid setting: %w", err)
|
return fmt.Errorf("invalid setting: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = askHealthEndpoint(ctx, listenAddr)
|
// The settings have checked that the address has a port.
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("ask servewaf: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkRProxyAndApp asks rproxy's health endpoint, and connects to the app.
|
|
||||||
func checkRProxyAndApp(
|
|
||||||
ctx context.Context, lookupEnv func(string) (string, bool),
|
|
||||||
) error {
|
|
||||||
listenAddr, upstreamURL, err := config.RProxyListenAddrAndUpstreamURL(lookupEnv)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("invalid setting: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = askHealthEndpoint(ctx, listenAddr)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("ask rproxy: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", appAddress(upstreamURL))
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("connect to the app: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_ = conn.Close()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// askHealthEndpoint asks the health endpoint on 127.0.0.1 at the port of
|
|
||||||
// listenAddr, which the settings have checked has one.
|
|
||||||
func askHealthEndpoint(ctx context.Context, listenAddr string) error {
|
|
||||||
_, port, _ := net.SplitHostPort(listenAddr)
|
_, port, _ := net.SplitHostPort(listenAddr)
|
||||||
health := "http://" + net.JoinHostPort("127.0.0.1", port) + proxy.HealthPath
|
health := "http://" + net.JoinHostPort("127.0.0.1", port) + proxy.HealthPath
|
||||||
|
|
||||||
@@ -116,7 +68,7 @@ func askHealthEndpoint(ctx context.Context, listenAddr string) error {
|
|||||||
|
|
||||||
res, err := http.DefaultClient.Do(req)
|
res, err := http.DefaultClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("send the request: %w", err)
|
return fmt.Errorf("ask smallwebwaf: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
_ = res.Body.Close()
|
_ = res.Body.Close()
|
||||||
@@ -125,6 +77,13 @@ func askHealthEndpoint(ctx context.Context, listenAddr string) error {
|
|||||||
return fmt.Errorf("%w %s", errHealthEndpoint, res.Status)
|
return fmt.Errorf("%w %s", errHealthEndpoint, res.Status)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", appAddress(upstreamURL))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("connect to the app: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = conn.Close()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -15,26 +15,22 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/smallwebwaf"
|
"sneak.berlin/go/smallwebwaf/internal/smallwebwaf"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
func TestHealthCheck(t *testing.T) {
|
||||||
// commandSetting says which command a container runs, and which the health
|
|
||||||
// check checks.
|
|
||||||
commandSetting = "SMALLWEBWAF_COMMAND"
|
|
||||||
// serveWAFCommand is the command that runs servewaf.
|
|
||||||
serveWAFCommand = "servewaf"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestHealthCheckInServeWAFsContainer(t *testing.T) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
app := httptest.NewServer(http.NotFoundHandler())
|
||||||
|
defer app.Close()
|
||||||
|
|
||||||
ctx, stop := context.WithCancel(t.Context())
|
ctx, stop := context.WithCancel(t.Context())
|
||||||
defer stop()
|
defer stop()
|
||||||
|
|
||||||
out := &output{}
|
out := &output{}
|
||||||
exited := make(chan int, 1)
|
exited := make(chan int, 1)
|
||||||
settings := map[string]string{
|
settings := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: t.TempDir(),
|
upstreamURL: app.URL,
|
||||||
rulesDir: t.TempDir(),
|
stateDir: t.TempDir(),
|
||||||
|
rulesDir: t.TempDir(),
|
||||||
}
|
}
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
@@ -43,36 +39,29 @@ func TestHealthCheckInServeWAFsContainer(t *testing.T) {
|
|||||||
|
|
||||||
addr, _ := out.line(t, "msg", "starting")["address"].(string)
|
addr, _ := out.line(t, "msg", "starting")["address"].(string)
|
||||||
_, port, _ := net.SplitHostPort(addr)
|
_, port, _ := net.SplitHostPort(addr)
|
||||||
|
|
||||||
// The container's settings: an address to listen on with an empty
|
// The container's settings: an address to listen on with an empty
|
||||||
// host part, which the health check asks at 127.0.0.1.
|
// host part, which the health check asks at 127.0.0.1.
|
||||||
wantHealthCheck(t, map[string]string{
|
env := map[string]string{listenAddr: ":" + port, upstreamURL: app.URL}
|
||||||
commandSetting: serveWAFCommand, listenAddr: ":" + port,
|
|
||||||
}, 0, "")
|
|
||||||
|
|
||||||
// The health check reads that setting alone, here given as a file: a removed
|
wantHealthCheck(t, env, 0, "")
|
||||||
// or invalid token file, or an invalid value of another setting, does not
|
|
||||||
// fail it.
|
// The health check reads those two settings alone, here given as
|
||||||
|
// files: a removed or invalid token file, or an invalid value of
|
||||||
|
// another setting, does not fail it.
|
||||||
for _, other := range []struct{ name, value string }{
|
for _, other := range []struct{ name, value string }{
|
||||||
{"SWWAF_METRICS_TOKEN_FILE", filepath.Join(t.TempDir(), "removed")},
|
{"SWWAF_METRICS_TOKEN_FILE", filepath.Join(t.TempDir(), "removed")},
|
||||||
{"SWWAF_METRICS_TOKEN_FILE", writeFile(t, "too short\n")},
|
{"SWWAF_METRICS_TOKEN_FILE", writeFile(t, "too short\n")},
|
||||||
{"SWWAF_MODE", "neither"},
|
{"SWWAF_MODE", "neither"},
|
||||||
} {
|
} {
|
||||||
wantHealthCheck(t, map[string]string{
|
wantHealthCheck(t, map[string]string{
|
||||||
commandSetting: serveWAFCommand,
|
listenAddr + "_FILE": writeFile(t, ":"+port+"\n"),
|
||||||
listenAddr + "_FILE": writeFile(t, ":"+port+"\n"),
|
upstreamURL + "_FILE": writeFile(t, app.URL+"\n"),
|
||||||
other.name: other.value,
|
other.name: other.value,
|
||||||
}, 0, "")
|
}, 0, "")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Listening on port 0 takes any free port, so nothing listens on port 0
|
app.Close()
|
||||||
// itself.
|
wantHealthCheck(t, env, 1, "unhealthy: connect to the app: ")
|
||||||
wantHealthCheck(t, map[string]string{
|
|
||||||
commandSetting: serveWAFCommand, listenAddr: ":0",
|
|
||||||
}, 1, "unhealthy: ask servewaf: ")
|
|
||||||
wantHealthCheck(t, map[string]string{
|
|
||||||
commandSetting: serveWAFCommand, listenAddr: "8080",
|
|
||||||
}, 1, "unhealthy: invalid setting: SWWAF_LISTEN_ADDR: ")
|
|
||||||
|
|
||||||
stop()
|
stop()
|
||||||
|
|
||||||
@@ -81,63 +70,10 @@ func TestHealthCheckInServeWAFsContainer(t *testing.T) {
|
|||||||
case <-time.After(waitLimit):
|
case <-time.After(waitLimit):
|
||||||
t.Fatal("still running after being told to stop")
|
t.Fatal("still running after being told to stop")
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
func TestHealthCheckInRProxysContainer(t *testing.T) {
|
wantHealthCheck(t, env, 1, "unhealthy: ask smallwebwaf: ")
|
||||||
t.Parallel()
|
wantHealthCheck(t, map[string]string{listenAddr: "8080"}, 1,
|
||||||
|
"unhealthy: invalid setting: SWWAF_LISTEN_ADDR: ")
|
||||||
app := httptest.NewServer(http.NotFoundHandler())
|
|
||||||
defer app.Close()
|
|
||||||
|
|
||||||
// Nothing listens at the backend. The health check asks rproxy and the app,
|
|
||||||
// whether or not servewaf answers.
|
|
||||||
site, _ := startRProxy(t, "http://127.0.0.1:9", app.URL)
|
|
||||||
_, port, _ := net.SplitHostPort(strings.TrimPrefix(site, "http://"))
|
|
||||||
settings := func(more ...string) map[string]string {
|
|
||||||
env := map[string]string{rproxyListenAddr: ":" + port, rproxyUpstreamURL: app.URL}
|
|
||||||
for i := 0; i < len(more); i += 2 {
|
|
||||||
env[more[i]] = more[i+1]
|
|
||||||
}
|
|
||||||
|
|
||||||
return env
|
|
||||||
}
|
|
||||||
|
|
||||||
wantHealthCheck(t, settings(), 0, "")
|
|
||||||
wantHealthCheck(t, settings(commandSetting, ""), 0, "")
|
|
||||||
wantHealthCheck(t, settings(commandSetting, "rproxy"), 0, "")
|
|
||||||
|
|
||||||
// SMALLWEBWAF_COMMAND cannot be given as a file.
|
|
||||||
wantHealthCheck(t, settings(commandSetting+"_FILE", writeFile(t, "servewaf\n")),
|
|
||||||
0, "")
|
|
||||||
|
|
||||||
// Another setting does not fail it, even an invalid one, and neither does
|
|
||||||
// SWRP_WAF_BACKEND being unset.
|
|
||||||
wantHealthCheck(t, settings("SWRP_TIMEOUT_MS", "soon"), 0, "")
|
|
||||||
|
|
||||||
// Another test can take a closed server's port at once, so the app's
|
|
||||||
// address here is a multicast address, to which TCP cannot connect.
|
|
||||||
wantHealthCheck(t, settings(rproxyUpstreamURL, "http://224.0.0.1"), 1,
|
|
||||||
"unhealthy: connect to the app: ")
|
|
||||||
|
|
||||||
// Listening on port 0 takes any free port, so nothing listens on port 0
|
|
||||||
// itself.
|
|
||||||
wantHealthCheck(t, settings(rproxyListenAddr, ":0"), 1, "unhealthy: ask rproxy: ")
|
|
||||||
wantHealthCheck(t, settings(rproxyListenAddr, "8080"), 1,
|
|
||||||
"unhealthy: invalid setting: SWRP_LISTEN_ADDR: ")
|
|
||||||
|
|
||||||
// No setting of rproxy's has a form given as a file.
|
|
||||||
wantHealthCheck(t, map[string]string{
|
|
||||||
rproxyListenAddr + "_FILE": writeFile(t, ":"+port+"\n"),
|
|
||||||
rproxyUpstreamURL: app.URL,
|
|
||||||
}, 1, "unhealthy: ask rproxy: ")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHealthCheckFailsWithAnyOtherCommand(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
wantHealthCheck(t, map[string]string{commandSetting: "healthcheck"}, 1,
|
|
||||||
"unhealthy: invalid setting: SMALLWEBWAF_COMMAND: "+
|
|
||||||
"\"healthcheck\" is not rproxy or servewaf\n")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHealthCheckRefusesAnArgument(t *testing.T) {
|
func TestHealthCheckRefusesAnArgument(t *testing.T) {
|
||||||
|
|||||||
@@ -1,99 +0,0 @@
|
|||||||
package smallwebwaf
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"log/slog"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
|
||||||
)
|
|
||||||
|
|
||||||
// RunRProxy is `smallwebwaf rproxy`: it reads rproxy's settings, then serves
|
|
||||||
// requests until ctx is done. Then it gives the requests in progress
|
|
||||||
// shutdownTimeout to finish, and the reports being sent SWRP_TIMEOUT_MS. It
|
|
||||||
// returns the process's exit status, 1 when rproxy cannot start. Its process
|
|
||||||
// messages name the host's name as instance, and it writes those at info or
|
|
||||||
// more severe: rproxy has no setting for either.
|
|
||||||
func RunRProxy(ctx context.Context, params Params) int {
|
|
||||||
hostname, _ := os.Hostname() // "" when the host has no name to give
|
|
||||||
processLog := requestlog.NewProcessLogger(params.Stdout, hostname, slog.LevelInfo)
|
|
||||||
|
|
||||||
cfg, err := config.RProxyFromEnvironment(params.LookupEnv)
|
|
||||||
if err != nil {
|
|
||||||
processLog.Error("invalid setting", "error", err.Error())
|
|
||||||
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
server := rproxy.New(rproxy.Params{
|
|
||||||
Config: cfg,
|
|
||||||
RequestLog: params.Stdout,
|
|
||||||
ProcessLog: processLog,
|
|
||||||
})
|
|
||||||
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", cfg.ListenAddr)
|
|
||||||
if err != nil {
|
|
||||||
processLog.Error("cannot listen on SWRP_LISTEN_ADDR", "error", err.Error())
|
|
||||||
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
processLog.Info("starting",
|
|
||||||
"version", params.Version,
|
|
||||||
"address", listener.Addr().String(),
|
|
||||||
"settings", cfg)
|
|
||||||
|
|
||||||
return serveRProxy(ctx, server, listener, processLog)
|
|
||||||
}
|
|
||||||
|
|
||||||
// serveRProxy serves requests on listener until ctx is done, and then stops as
|
|
||||||
// RunRProxy says.
|
|
||||||
func serveRProxy(
|
|
||||||
ctx context.Context, server *rproxy.Server, listener net.Listener,
|
|
||||||
processLog *slog.Logger,
|
|
||||||
) int {
|
|
||||||
served := make(chan error, 1)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
served <- server.Serve(listener)
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case err := <-served:
|
|
||||||
processLog.Error("serving failed", "error", err.Error())
|
|
||||||
|
|
||||||
return 1
|
|
||||||
case <-ctx.Done():
|
|
||||||
}
|
|
||||||
|
|
||||||
processLog.Info("stopping")
|
|
||||||
|
|
||||||
shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx),
|
|
||||||
shutdownTimeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
err := server.Shutdown(shutdownCtx)
|
|
||||||
if err != nil {
|
|
||||||
processLog.Warn("requests still in progress were cut off",
|
|
||||||
"error", err.Error())
|
|
||||||
|
|
||||||
_ = server.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
err = <-served
|
|
||||||
if !errors.Is(err, http.ErrServerClosed) {
|
|
||||||
processLog.Error("serving failed", "error", err.Error())
|
|
||||||
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
server.WaitForReports()
|
|
||||||
processLog.Info("stopped")
|
|
||||||
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
@@ -1,789 +0,0 @@
|
|||||||
package smallwebwaf_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"context"
|
|
||||||
"encoding/pem"
|
|
||||||
"io"
|
|
||||||
"maps"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"sync/atomic"
|
|
||||||
"syscall"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/proxy"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/smallwebwaf"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// wafBackend is the setting that names servewaf to rproxy.
|
|
||||||
wafBackend = "SWRP_WAF_BACKEND"
|
|
||||||
// rproxyListenAddr is the setting that says where rproxy listens.
|
|
||||||
rproxyListenAddr = "SWRP_LISTEN_ADDR"
|
|
||||||
// rproxyUpstreamURL is the setting that says where the app is.
|
|
||||||
rproxyUpstreamURL = "SWRP_UPSTREAM_URL"
|
|
||||||
// rproxyTimeout is the setting that says how long rproxy waits for servewaf.
|
|
||||||
rproxyTimeout = "SWRP_TIMEOUT_MS"
|
|
||||||
// bannedProbe is a path the default rule file bans a client for.
|
|
||||||
bannedProbe = "/.env"
|
|
||||||
// otherClient is a client the tests do not ban.
|
|
||||||
otherClient = "198.51.100.7"
|
|
||||||
)
|
|
||||||
|
|
||||||
// runRProxy runs `smallwebwaf rproxy` with the settings in env until ctx is
|
|
||||||
// done, and returns its exit status.
|
|
||||||
func runRProxy(ctx context.Context, env map[string]string, out *output) int {
|
|
||||||
return smallwebwaf.RunRProxy(ctx, smallwebwaf.Params{
|
|
||||||
Version: testVersion,
|
|
||||||
LookupEnv: func(name string) (string, bool) {
|
|
||||||
value, ok := env[name]
|
|
||||||
|
|
||||||
return value, ok
|
|
||||||
},
|
|
||||||
Stdout: out,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// startRProxy runs rproxy with servewaf at backend and the app at appURL, and
|
|
||||||
// returns its URL and what it writes. It believes X-Forwarded-For from the
|
|
||||||
// tests. It stops as the test ends, and that it stopped in order is checked.
|
|
||||||
func startRProxy(t *testing.T, backend, appURL string) (string, *output) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
ctx, stop := context.WithCancel(t.Context())
|
|
||||||
out := &output{}
|
|
||||||
exited := make(chan int, 1)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
exited <- runRProxy(ctx, map[string]string{
|
|
||||||
rproxyListenAddr: localhost + ":0",
|
|
||||||
"SWRP_UPSTREAM_URL": appURL,
|
|
||||||
"SWRP_TRUSTED_PROXIES": localhost + "/32",
|
|
||||||
wafBackend: backend,
|
|
||||||
}, out)
|
|
||||||
}()
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
stop()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case status := <-exited:
|
|
||||||
if status != 0 {
|
|
||||||
t.Errorf("rproxy's exit status %d, want 0; output:\n%s", status, out.text())
|
|
||||||
}
|
|
||||||
case <-time.After(waitLimit):
|
|
||||||
t.Error("rproxy still running after being told to stop")
|
|
||||||
}
|
|
||||||
|
|
||||||
out.line(t, "msg", "stopped")
|
|
||||||
})
|
|
||||||
|
|
||||||
addr, _ := out.line(t, "msg", "starting")["address"].(string)
|
|
||||||
|
|
||||||
return "http://" + addr, out
|
|
||||||
}
|
|
||||||
|
|
||||||
// startServeWAF runs servewaf with the settings in env, and returns its address
|
|
||||||
// and URL, what it writes, and what stops it, which checks that it stopped in
|
|
||||||
// order.
|
|
||||||
func startServeWAF(t *testing.T, env map[string]string) (string, *output, func()) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
ctx, stop := context.WithCancel(t.Context())
|
|
||||||
out := &output{}
|
|
||||||
exited := make(chan int, 1)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
exited <- run(ctx, env, out)
|
|
||||||
}()
|
|
||||||
|
|
||||||
addr, _ := out.line(t, "msg", "starting")["address"].(string)
|
|
||||||
|
|
||||||
return addr, out, func() {
|
|
||||||
stop()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case status := <-exited:
|
|
||||||
if status != 0 {
|
|
||||||
t.Fatalf("servewaf's exit status %d, want 0; output:\n%s", status, out.text())
|
|
||||||
}
|
|
||||||
case <-time.After(waitLimit):
|
|
||||||
t.Fatal("servewaf still running after being told to stop")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// serveWAFSettings are servewaf's settings with the default rule file, to listen
|
|
||||||
// at any free port.
|
|
||||||
func serveWAFSettings(t *testing.T) map[string]string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
return map[string]string{
|
|
||||||
listenAddr: localhost + ":0",
|
|
||||||
stateDir: t.TempDir(),
|
|
||||||
rulesDir: filepath.Join("..", "..", "share", "rules.d"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestABanThroughOneSiteRefusesTheClientOnTheOther(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
addr, serveWAFOut, stop := startServeWAF(t, serveWAFSettings(t))
|
|
||||||
defer stop()
|
|
||||||
|
|
||||||
backend := "http://" + addr
|
|
||||||
first, firstOut := startRProxy(t, backend, startApp(t))
|
|
||||||
second, _ := startRProxy(t, backend, startApp(t))
|
|
||||||
|
|
||||||
wantStatus(t, first+"/", placed, http.StatusOK)
|
|
||||||
wantStatus(t, second+"/", placed, http.StatusOK)
|
|
||||||
|
|
||||||
// The default rule file bans a probe for this.
|
|
||||||
wantStatus(t, first+bannedProbe, placed, http.StatusForbidden)
|
|
||||||
wantStatus(t, second+"/", placed, http.StatusForbidden)
|
|
||||||
wantStatus(t, second+"/", otherClient, http.StatusOK)
|
|
||||||
|
|
||||||
// One request's three lines carry one id, rproxy's with servewaf's answer.
|
|
||||||
line := firstOut.line(t, "path", bannedProbe)
|
|
||||||
if line["disposition"] != proxy.DispositionDeny || line["client_ip"] != placed {
|
|
||||||
t.Errorf("rproxy's line is %v", line)
|
|
||||||
}
|
|
||||||
|
|
||||||
id, _ := line["request_id"].(string)
|
|
||||||
decision := serveWAFLine(t, serveWAFOut, id, "decision")
|
|
||||||
report := serveWAFLine(t, serveWAFOut, id, "report")
|
|
||||||
|
|
||||||
if decision["host"] != line["host"] || decision["client_ip"] != placed ||
|
|
||||||
report["status"] != float64(http.StatusForbidden) {
|
|
||||||
t.Errorf("servewaf's lines are %v and %v", decision, report)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// serveWAFLine waits for servewaf's line with request id and servewaf_line kind.
|
|
||||||
func serveWAFLine(t *testing.T, out *output, id, kind string) map[string]any {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
deadline := time.Now().Add(waitLimit)
|
|
||||||
for time.Now().Before(deadline) {
|
|
||||||
line := out.findLine(t, func(fields map[string]any) bool {
|
|
||||||
return fields["request_id"] == id && fields["servewaf_line"] == kind
|
|
||||||
})
|
|
||||||
if line != nil {
|
|
||||||
return line
|
|
||||||
}
|
|
||||||
|
|
||||||
time.Sleep(pollInterval)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Fatalf("no %s line for %s in:\n%s", kind, id, out.text())
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTheSiteAnswersWhileServeWAFIsStoppedAndStartedAgain(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := serveWAFSettings(t)
|
|
||||||
addr, _, stop := startServeWAF(t, env)
|
|
||||||
site, out := startRProxy(t, "http://"+addr, startApp(t))
|
|
||||||
|
|
||||||
// A visitor who keeps asking throughout, and is never refused.
|
|
||||||
visitorDone := make(chan struct{})
|
|
||||||
visitorStop := make(chan struct{})
|
|
||||||
|
|
||||||
var refused atomic.Int64
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
defer close(visitorDone)
|
|
||||||
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-visitorStop:
|
|
||||||
return
|
|
||||||
case <-time.After(pollInterval):
|
|
||||||
}
|
|
||||||
|
|
||||||
if !answersNormally(t.Context(), site+"/", otherClient) {
|
|
||||||
refused.Add(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
wantStatus(t, site+bannedProbe, placed, http.StatusForbidden)
|
|
||||||
wantStatus(t, site+"/", placed, http.StatusForbidden)
|
|
||||||
stop()
|
|
||||||
|
|
||||||
// With servewaf down, the site answers, and rproxy says the default did it.
|
|
||||||
wantStatus(t, site+"/after-stop", placed, http.StatusOK)
|
|
||||||
|
|
||||||
line := out.line(t, "path", "/after-stop")
|
|
||||||
if line["disposition"] != proxy.DispositionAllow ||
|
|
||||||
line["default_disposition"] != true {
|
|
||||||
t.Errorf("rproxy's line is %v", line)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Back at its address, with its state files, it applies the ban again.
|
|
||||||
restarted := maps.Clone(env)
|
|
||||||
restarted[listenAddr] = addr
|
|
||||||
_, _, stop = startServeWAF(t, restarted)
|
|
||||||
|
|
||||||
defer stop()
|
|
||||||
|
|
||||||
wantStatus(t, site+"/", placed, http.StatusForbidden)
|
|
||||||
|
|
||||||
close(visitorStop)
|
|
||||||
<-visitorDone
|
|
||||||
|
|
||||||
if refused.Load() != 0 {
|
|
||||||
t.Errorf("the site refused its visitor %d times", refused.Load())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// answersNormally reports whether a request to url from the client at from, as
|
|
||||||
// X-Forwarded-For names it, gets the app's answer.
|
|
||||||
func answersNormally(ctx context.Context, url, from string) bool {
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody)
|
|
||||||
if err != nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
req.Header.Set("X-Forwarded-For", from)
|
|
||||||
|
|
||||||
transport := &http.Transport{}
|
|
||||||
defer transport.CloseIdleConnections()
|
|
||||||
|
|
||||||
res, err := transport.RoundTrip(req)
|
|
||||||
if err != nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
body, err := io.ReadAll(res.Body)
|
|
||||||
_ = res.Body.Close()
|
|
||||||
|
|
||||||
return err == nil && res.StatusCode == http.StatusOK && string(body) == greeting
|
|
||||||
}
|
|
||||||
|
|
||||||
// sendRaw sends request, as it is, to rproxy at site, and returns the status of
|
|
||||||
// the answer.
|
|
||||||
func sendRaw(t *testing.T, site, request string) int {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
conn, err := (&net.Dialer{}).DialContext(t.Context(), "tcp",
|
|
||||||
strings.TrimPrefix(site, "http://"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("dial: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
_ = conn.Close()
|
|
||||||
}()
|
|
||||||
|
|
||||||
err = conn.SetDeadline(time.Now().Add(waitLimit))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("set the deadline: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = io.WriteString(conn, request)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("send: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
res, err := http.ReadResponse(bufio.NewReader(conn), nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read the answer: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_ = res.Body.Close()
|
|
||||||
|
|
||||||
return res.StatusCode
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAChunkedBodyStaysABodyAtServeWAF(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
addr, serveWAFOut, stop := startServeWAF(t, serveWAFSettings(t))
|
|
||||||
defer stop()
|
|
||||||
|
|
||||||
site, _ := startRProxy(t, "http://"+addr, startApp(t))
|
|
||||||
|
|
||||||
// Read as a further request, this body would ban otherClient on every site.
|
|
||||||
const smuggledID = "smuggled-request"
|
|
||||||
|
|
||||||
smuggled := "GET " + bannedProbe + " HTTP/1.1\r\nHost: site.example\r\n" +
|
|
||||||
proxy.ClientHeader + ": " + otherClient + "\r\n" +
|
|
||||||
"X-Request-ID: " + smuggledID + "\r\n\r\n"
|
|
||||||
|
|
||||||
status := sendRaw(t, site, "POST /upload HTTP/1.1\r\nHost: site.example\r\n"+
|
|
||||||
"X-Forwarded-For: "+placed+"\r\nTransfer-Encoding: chunked\r\n\r\n"+
|
|
||||||
strconv.FormatInt(int64(len(smuggled)), 16)+"\r\n"+smuggled+"\r\n0\r\n\r\n")
|
|
||||||
if status != http.StatusOK {
|
|
||||||
t.Errorf("status %d, want the app's 200", status)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Read as a further request, it would act straight after the answer.
|
|
||||||
for range 20 {
|
|
||||||
wantStatus(t, site+"/", otherClient, http.StatusOK)
|
|
||||||
time.Sleep(pollInterval)
|
|
||||||
}
|
|
||||||
|
|
||||||
line := serveWAFOut.findLine(t, func(fields map[string]any) bool {
|
|
||||||
return fields["request_id"] == smuggledID
|
|
||||||
})
|
|
||||||
if line != nil {
|
|
||||||
t.Errorf("servewaf read the body as a request: %v", line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestATargetThatCannotGoToServeWAFIsRefusedForABannedClient(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
addr, _, stop := startServeWAF(t, serveWAFSettings(t))
|
|
||||||
defer stop()
|
|
||||||
|
|
||||||
site, out := startRProxy(t, "http://"+addr, startApp(t))
|
|
||||||
|
|
||||||
wantStatus(t, site+bannedProbe, placed, http.StatusForbidden)
|
|
||||||
|
|
||||||
status := sendRaw(t, site, "GET x:admin HTTP/1.1\r\nHost: site.example\r\n"+
|
|
||||||
"X-Forwarded-For: "+placed+"\r\n\r\n")
|
|
||||||
if status != http.StatusBadRequest {
|
|
||||||
t.Errorf("status %d, want 400", status)
|
|
||||||
}
|
|
||||||
|
|
||||||
line := out.line(t, "path", "admin")
|
|
||||||
if line["disposition"] != nil || line["default_disposition"] != nil {
|
|
||||||
t.Errorf("rproxy's line is %v, want no disposition", line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAClientWithAZoneIsTheClientWithout(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const zoned = "2001:db8::1%eth0"
|
|
||||||
|
|
||||||
addr, _, stop := startServeWAF(t, serveWAFSettings(t))
|
|
||||||
defer stop()
|
|
||||||
|
|
||||||
site, out := startRProxy(t, "http://"+addr, startApp(t))
|
|
||||||
|
|
||||||
wantStatus(t, site+bannedProbe, zoned, http.StatusForbidden)
|
|
||||||
wantStatus(t, site+"/", zoned, http.StatusForbidden)
|
|
||||||
wantStatus(t, site+"/", "2001:db8::1", http.StatusForbidden)
|
|
||||||
|
|
||||||
line := out.line(t, "path", bannedProbe)
|
|
||||||
if line["client_ip"] != "2001:db8::1" || line["default_disposition"] != nil {
|
|
||||||
t.Errorf("rproxy's line is %v", line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAStatusServeWAFDoesNotTakeIsReportedAs599(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const status = 799
|
|
||||||
|
|
||||||
app := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(status)
|
|
||||||
}))
|
|
||||||
t.Cleanup(app.Close)
|
|
||||||
|
|
||||||
addr, serveWAFOut, stop := startServeWAF(t, serveWAFSettings(t))
|
|
||||||
defer stop()
|
|
||||||
|
|
||||||
site, out := startRProxy(t, "http://"+addr, app.URL)
|
|
||||||
|
|
||||||
wantStatus(t, site+"/", placed, status)
|
|
||||||
|
|
||||||
line := out.line(t, "path", "/")
|
|
||||||
if line["status"] != float64(status) {
|
|
||||||
t.Errorf("rproxy's line is %v, want the status the visitor got", line)
|
|
||||||
}
|
|
||||||
|
|
||||||
id, _ := line["request_id"].(string)
|
|
||||||
|
|
||||||
report := serveWAFLine(t, serveWAFOut, id, "report")
|
|
||||||
if report["status"] != 599.0 {
|
|
||||||
t.Errorf("servewaf's line is %v, want status 599", report)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// startRProxyProcess runs `smallwebwaf rproxy` in a process of its own, with the
|
|
||||||
// settings in env, each as NAME=value. It returns where rproxy listens, the
|
|
||||||
// process, and what rproxy writes.
|
|
||||||
func startRProxyProcess(t *testing.T, env ...string) (string, *exec.Cmd, *output) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
binary, err := os.Executable()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("find the test binary: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), 3*waitLimit)
|
|
||||||
t.Cleanup(cancel)
|
|
||||||
|
|
||||||
out := &output{}
|
|
||||||
cmd := exec.CommandContext(ctx, binary) //nolint:gosec // this test binary
|
|
||||||
cmd.Stdout = out
|
|
||||||
|
|
||||||
cmd.Env = append(os.Environ(),
|
|
||||||
commandForMain+"=rproxy",
|
|
||||||
"SWRP_LISTEN_ADDR="+localhost+":0")
|
|
||||||
cmd.Env = append(cmd.Env, env...)
|
|
||||||
|
|
||||||
err = cmd.Start()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("start: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
_ = cmd.Process.Kill()
|
|
||||||
_ = cmd.Wait()
|
|
||||||
})
|
|
||||||
|
|
||||||
addr, _ := out.line(t, "msg", "starting")["address"].(string)
|
|
||||||
|
|
||||||
return addr, cmd, out
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRProxyAsksAServeWAFThatSpeaksHTTP2OverHTTP1(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
protocols := make(chan string, 10)
|
|
||||||
backend := httptest.NewUnstartedServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.Header.Get(proxy.ClientHeader) != "" {
|
|
||||||
protocols <- r.Proto
|
|
||||||
|
|
||||||
w.Header().Set(proxy.DispositionHeader, proxy.DispositionDeny)
|
|
||||||
w.WriteHeader(http.StatusForbidden)
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
backend.EnableHTTP2 = true
|
|
||||||
backend.StartTLS()
|
|
||||||
t.Cleanup(backend.Close)
|
|
||||||
|
|
||||||
// rproxy has no setting for the certificates it trusts, so it is told of this
|
|
||||||
// one as Go's client is told of any, in a process of its own.
|
|
||||||
certificates := filepath.Join(t.TempDir(), "servewaf.pem")
|
|
||||||
|
|
||||||
err := os.WriteFile(certificates, pem.EncodeToMemory(&pem.Block{
|
|
||||||
Type: "CERTIFICATE", Bytes: backend.Certificate().Raw,
|
|
||||||
}), 0o600)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("write the certificate: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
addr, _, _ := startRProxyProcess(t,
|
|
||||||
wafBackend+"="+backend.URL,
|
|
||||||
"SWRP_UPSTREAM_URL="+startApp(t),
|
|
||||||
"SSL_CERT_FILE="+certificates)
|
|
||||||
|
|
||||||
for name, headers := range map[string]string{
|
|
||||||
"no such header": "",
|
|
||||||
"Upgrade": "Upgrade: anything\r\n",
|
|
||||||
"Connection": "Connection: anything\r\n",
|
|
||||||
"a WebSocket upgrade": "Connection: Upgrade\r\nUpgrade: websocket\r\n",
|
|
||||||
} {
|
|
||||||
status := sendRaw(t, addr, "GET / HTTP/1.1\r\nHost: site.example\r\n"+headers+"\r\n")
|
|
||||||
if status != http.StatusForbidden {
|
|
||||||
t.Errorf("with %s: status %d, want servewaf's 403", name, status)
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
|
||||||
case protocol := <-protocols:
|
|
||||||
if protocol != "HTTP/1.1" {
|
|
||||||
t.Errorf("with %s: asked over %s, want HTTP/1.1", name, protocol)
|
|
||||||
}
|
|
||||||
case <-time.After(waitLimit):
|
|
||||||
t.Errorf("with %s: servewaf was not asked", name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRProxyProcessLinesNameTheHostAndComeAtInfo(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
hostname, err := os.Hostname()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("the host's name: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Nothing listens at the backend's address; no request asks it.
|
|
||||||
_, out := startRProxy(t, "http://127.0.0.1:9", startApp(t))
|
|
||||||
|
|
||||||
line := out.line(t, "msg", "starting")
|
|
||||||
if line["instance"] != hostname || line["level"] != "INFO" {
|
|
||||||
t.Errorf("rproxy's line is %v, want instance %s at INFO", line, hostname)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
reportGotThere := make(chan struct{})
|
|
||||||
release := make(chan struct{})
|
|
||||||
backend := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.Header.Get(proxy.ClientHeader) != "" {
|
|
||||||
w.Header().Set(proxy.DispositionHeader, proxy.DispositionAllow)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
close(reportGotThere)
|
|
||||||
<-release
|
|
||||||
w.WriteHeader(http.StatusNoContent)
|
|
||||||
}))
|
|
||||||
t.Cleanup(backend.Close)
|
|
||||||
|
|
||||||
ctx, stop := context.WithCancel(t.Context())
|
|
||||||
defer stop()
|
|
||||||
|
|
||||||
env := map[string]string{
|
|
||||||
rproxyListenAddr: localhost + ":0",
|
|
||||||
"SWRP_UPSTREAM_URL": startApp(t),
|
|
||||||
rproxyTimeout: "60000",
|
|
||||||
wafBackend: backend.URL,
|
|
||||||
}
|
|
||||||
out := &output{}
|
|
||||||
exited := make(chan int, 1)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
exited <- runRProxy(ctx, env, out)
|
|
||||||
}()
|
|
||||||
|
|
||||||
addr, _ := out.line(t, "msg", "starting")["address"].(string)
|
|
||||||
|
|
||||||
wantStatus(t, "http://"+addr+"/", placed, http.StatusOK)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-reportGotThere:
|
|
||||||
case <-time.After(waitLimit):
|
|
||||||
t.Fatal("no report")
|
|
||||||
}
|
|
||||||
|
|
||||||
stop()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-exited:
|
|
||||||
t.Error("rproxy stopped with a report on its way")
|
|
||||||
close(release)
|
|
||||||
|
|
||||||
return
|
|
||||||
case <-time.After(10 * pollInterval):
|
|
||||||
}
|
|
||||||
|
|
||||||
close(release)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case status := <-exited:
|
|
||||||
if status != 0 {
|
|
||||||
t.Errorf("rproxy's exit status %d, want 0; output:\n%s", status, out.text())
|
|
||||||
}
|
|
||||||
case <-time.After(waitLimit):
|
|
||||||
t.Error("rproxy still running, the report having been answered")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection. A
|
|
||||||
// read from it ends with an error once waitLimit is up.
|
|
||||||
func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
conn, err := (&net.Dialer{}).DialContext(t.Context(), "tcp", addr)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("dial: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
_ = conn.Close()
|
|
||||||
})
|
|
||||||
|
|
||||||
err = conn.SetDeadline(time.Now().Add(waitLimit))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("set the deadline: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = io.WriteString(conn, request)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("send: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return conn
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
cutOffAfter = 5 * time.Second
|
|
||||||
upstreamSetting = "SWRP_UPSTREAM_URL"
|
|
||||||
)
|
|
||||||
|
|
||||||
asked := make(chan struct{})
|
|
||||||
release := make(chan struct{})
|
|
||||||
app := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(http.ResponseWriter, *http.Request) {
|
|
||||||
close(asked)
|
|
||||||
<-release
|
|
||||||
}))
|
|
||||||
t.Cleanup(app.Close)
|
|
||||||
t.Cleanup(func() { close(release) })
|
|
||||||
|
|
||||||
backend := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.Header().Set(proxy.DispositionHeader, proxy.DispositionAllow)
|
|
||||||
}))
|
|
||||||
t.Cleanup(backend.Close)
|
|
||||||
|
|
||||||
ctx, stop := context.WithCancel(t.Context())
|
|
||||||
defer stop()
|
|
||||||
|
|
||||||
env := map[string]string{
|
|
||||||
rproxyListenAddr: localhost + ":0",
|
|
||||||
upstreamSetting: app.URL,
|
|
||||||
wafBackend: backend.URL,
|
|
||||||
}
|
|
||||||
out := &output{}
|
|
||||||
exited := make(chan int, 1)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
exited <- runRProxy(ctx, env, out)
|
|
||||||
}()
|
|
||||||
|
|
||||||
addr, _ := out.line(t, "msg", "starting")["address"].(string)
|
|
||||||
conn := sendAndKeepOpen(t, addr, "GET / HTTP/1.1\r\nHost: site.example\r\n\r\n")
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-asked:
|
|
||||||
case <-time.After(waitLimit):
|
|
||||||
t.Fatal("the app was not asked")
|
|
||||||
}
|
|
||||||
|
|
||||||
stopped := time.Now()
|
|
||||||
|
|
||||||
stop()
|
|
||||||
|
|
||||||
text, err := io.ReadAll(conn)
|
|
||||||
if err != nil || len(text) != 0 {
|
|
||||||
t.Errorf("read %q (%v), want the connection cut off", text, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if elapsed := time.Since(stopped); elapsed < cutOffAfter {
|
|
||||||
t.Errorf("cut off after %s, want %s", elapsed, cutOffAfter)
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
|
||||||
case status := <-exited:
|
|
||||||
if status != 0 {
|
|
||||||
t.Errorf("rproxy's exit status %d, want 0; output:\n%s", status, out.text())
|
|
||||||
}
|
|
||||||
case <-time.After(waitLimit):
|
|
||||||
t.Error("rproxy still running after the cut-off")
|
|
||||||
}
|
|
||||||
|
|
||||||
out.line(t, "msg", "requests still in progress were cut off")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRProxyInvalidSettingStopsTheStart(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name, setting string
|
|
||||||
env map[string]string
|
|
||||||
}{
|
|
||||||
{"no backend", wafBackend, nil},
|
|
||||||
{
|
|
||||||
"another invalid", rproxyTimeout,
|
|
||||||
map[string]string{wafBackend: "http://servewaf:8080", rproxyTimeout: "soon"},
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
out := &output{}
|
|
||||||
|
|
||||||
if status := runRProxy(t.Context(), tc.env, out); status != 1 {
|
|
||||||
t.Errorf("exit status %d, want 1", status)
|
|
||||||
}
|
|
||||||
|
|
||||||
line := out.line(t, "msg", "invalid setting")
|
|
||||||
message, _ := line["error"].(string)
|
|
||||||
|
|
||||||
if !strings.HasPrefix(message, tc.setting+": ") {
|
|
||||||
t.Errorf("start refused with %v, want it to name %s", line, tc.setting)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRProxyAddressInUseStopsTheStart(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
taken, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("listen: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
_ = taken.Close()
|
|
||||||
}()
|
|
||||||
|
|
||||||
out := &output{}
|
|
||||||
|
|
||||||
status := runRProxy(t.Context(), map[string]string{
|
|
||||||
wafBackend: "http://servewaf:8080",
|
|
||||||
rproxyListenAddr: taken.Addr().String(),
|
|
||||||
}, out)
|
|
||||||
if status != 1 {
|
|
||||||
t.Errorf("exit status %d, want 1", status)
|
|
||||||
}
|
|
||||||
|
|
||||||
out.line(t, "msg", "cannot listen on SWRP_LISTEN_ADDR")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMainRunsRProxy(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Nothing listens there; the health check does not ask.
|
|
||||||
addr, cmd, out := startRProxyProcess(t, wafBackend+"=http://127.0.0.1:9")
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet,
|
|
||||||
"http://"+addr+proxy.HealthPath, http.NoBody)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("new request: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
transport := &http.Transport{}
|
|
||||||
defer transport.CloseIdleConnections()
|
|
||||||
|
|
||||||
res, err := transport.RoundTrip(req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("health check: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
body, err := io.ReadAll(res.Body)
|
|
||||||
_ = res.Body.Close()
|
|
||||||
|
|
||||||
if err != nil || string(body) != "ok\n" {
|
|
||||||
t.Errorf("health check answered %q (%v)", body, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = cmd.Process.Signal(syscall.SIGTERM)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("signal: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = cmd.Wait()
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("exit: %v; output:\n%s", err, out.text())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,204 +0,0 @@
|
|||||||
package smallwebwaf_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"crypto/rand"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"syscall"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/proxy"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/smallwebwaf"
|
|
||||||
)
|
|
||||||
|
|
||||||
// commandForMain, while set in the environment, has this test binary run Main
|
|
||||||
// with the command it gives, in place of the tests, so that a test can run
|
|
||||||
// Main in a process of its own.
|
|
||||||
const commandForMain = "SMALLWEBWAF_TEST_COMMAND"
|
|
||||||
|
|
||||||
func TestMain(m *testing.M) {
|
|
||||||
command, set := os.LookupEnv(commandForMain)
|
|
||||||
if !set {
|
|
||||||
os.Exit(m.Run())
|
|
||||||
}
|
|
||||||
|
|
||||||
os.Args = strings.Fields("smallwebwaf " + command)
|
|
||||||
|
|
||||||
os.Exit(smallwebwaf.Main(testVersion))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMainRunsServeWAF(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
binary, err := os.Executable()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("find the test binary: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), 3*waitLimit)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
out := &output{}
|
|
||||||
cmd := exec.CommandContext(ctx, binary) //nolint:gosec // this test binary
|
|
||||||
cmd.Stdout = out
|
|
||||||
|
|
||||||
cmd.Env = append(os.Environ(),
|
|
||||||
commandForMain+"="+serveWAFCommand,
|
|
||||||
listenAddr+"="+localhost+":0",
|
|
||||||
stateDir+"="+t.TempDir(),
|
|
||||||
rulesDir+"="+t.TempDir(),
|
|
||||||
lookupSource+"=off")
|
|
||||||
|
|
||||||
err = cmd.Start()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("start: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
_ = cmd.Process.Kill()
|
|
||||||
_ = cmd.Wait()
|
|
||||||
})
|
|
||||||
|
|
||||||
addr, _ := out.line(t, "msg", "starting")["address"].(string)
|
|
||||||
ask(t, addr, "/", http.StatusOK, proxy.DispositionAllow)
|
|
||||||
|
|
||||||
err = cmd.Process.Signal(syscall.SIGTERM)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("signal: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = cmd.Wait()
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("exit: %v; output:\n%s", err, out.text())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMainWithoutACommandItKnowsPrintsTheCommands(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
binary, err := os.Executable()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("find the test binary: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, command := range []string{"", "serve"} {
|
|
||||||
t.Run("command "+command, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), waitLimit)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
|
|
||||||
cmd := exec.CommandContext(ctx, binary)
|
|
||||||
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
|
||||||
|
|
||||||
cmd.Env = append(os.Environ(), commandForMain+"="+command)
|
|
||||||
|
|
||||||
err := cmd.Run()
|
|
||||||
|
|
||||||
if code := cmd.ProcessState.ExitCode(); code != 2 {
|
|
||||||
t.Errorf("exit status %d (%v), want 2", code, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, name := range []string{"rproxy", serveWAFCommand, "healthcheck"} {
|
|
||||||
if !strings.Contains(stderr.String(), name) {
|
|
||||||
t.Errorf("stderr %q does not name %s", stderr.String(), name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if stdout.Len() != 0 {
|
|
||||||
t.Errorf("stdout %q, want nothing", stdout.String())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ask sends servewaf at addr rproxy's question about a request for path from
|
|
||||||
// placed, checks the answer's status and disposition, and closes it.
|
|
||||||
func ask(t *testing.T, addr, path string, status int, disposition string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet,
|
|
||||||
"http://"+addr+path, http.NoBody)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("new request: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
req.Header.Set(proxy.ClientHeader, placed)
|
|
||||||
req.Header.Set("X-Request-ID", rand.Text())
|
|
||||||
|
|
||||||
transport := &http.Transport{}
|
|
||||||
defer transport.CloseIdleConnections()
|
|
||||||
|
|
||||||
res, err := (&http.Client{Transport: transport}).Do(req)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("question: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, _ = io.Copy(io.Discard, res.Body)
|
|
||||||
_ = res.Body.Close()
|
|
||||||
|
|
||||||
if res.StatusCode != status ||
|
|
||||||
res.Header.Get(proxy.DispositionHeader) != disposition {
|
|
||||||
t.Errorf("answered %d with disposition %q, want %d with %q", res.StatusCode,
|
|
||||||
res.Header.Get(proxy.DispositionHeader), status, disposition)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// runServeWAF runs `smallwebwaf servewaf` with the settings in env, has use
|
|
||||||
// send it requests at the address it listens on, stops it as SIGTERM does, and
|
|
||||||
// checks that it stopped in order.
|
|
||||||
func runServeWAF(t *testing.T, env map[string]string, use func(addr string)) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
ctx, stop := context.WithCancel(t.Context())
|
|
||||||
out := &output{}
|
|
||||||
exited := make(chan int, 1)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
exited <- run(ctx, env, out)
|
|
||||||
}()
|
|
||||||
|
|
||||||
addr, _ := out.line(t, "msg", "starting")["address"].(string)
|
|
||||||
use(addr)
|
|
||||||
stop()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case status := <-exited:
|
|
||||||
if status != 0 {
|
|
||||||
t.Fatalf("exit status %d, want 0; output:\n%s", status, out.text())
|
|
||||||
}
|
|
||||||
case <-time.After(waitLimit):
|
|
||||||
t.Fatal("still running after being told to stop")
|
|
||||||
}
|
|
||||||
|
|
||||||
out.line(t, "msg", "stopped")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestServeWAFKeepsItsBansAcrossARestartAndNeedsNoApp(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := map[string]string{
|
|
||||||
listenAddr: localhost + ":0",
|
|
||||||
stateDir: t.TempDir(),
|
|
||||||
rulesDir: filepath.Join("..", "..", "share", "rules.d"),
|
|
||||||
}
|
|
||||||
|
|
||||||
runServeWAF(t, env, func(addr string) {
|
|
||||||
ask(t, addr, "/", http.StatusOK, proxy.DispositionAllow)
|
|
||||||
// The default rule file bans a probe for this.
|
|
||||||
ask(t, addr, "/.env", http.StatusForbidden, proxy.DispositionDeny)
|
|
||||||
})
|
|
||||||
|
|
||||||
runServeWAF(t, env, func(addr string) {
|
|
||||||
ask(t, addr, "/", http.StatusForbidden, proxy.DispositionDeny)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -1,16 +1,12 @@
|
|||||||
// Package smallwebwaf runs the smallwebwaf process, as the command it is given.
|
// Package smallwebwaf runs the smallwebwaf process: it reads the settings,
|
||||||
// `smallwebwaf servewaf` reads the settings, the rule files, the lookup database
|
// the rule files, the lookup database and the state files, serves requests
|
||||||
// and the state files, answers rproxy's questions until it is told to stop, and
|
// until it is told to stop, and then stops in an orderly way, writing the
|
||||||
// then stops in an orderly way, writing the state files. `smallwebwaf rproxy`
|
// state files.
|
||||||
// reads rproxy's settings alone, and runs the reverse proxy in front of one app
|
|
||||||
// that asks servewaf about each request. `smallwebwaf healthcheck` is the
|
|
||||||
// container's health check.
|
|
||||||
package smallwebwaf
|
package smallwebwaf
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net"
|
"net"
|
||||||
@@ -36,25 +32,12 @@ import (
|
|||||||
// runit and docker wait a little longer before they kill the process.
|
// runit and docker wait a little longer before they kill the process.
|
||||||
const shutdownTimeout = 5 * time.Second
|
const shutdownTimeout = 5 * time.Second
|
||||||
|
|
||||||
// remoteLogStopTimeout is how long, as servewaf stops, the log lines still
|
// remoteLogStopTimeout is how long, as smallwebwaf stops, the log lines
|
||||||
// waiting are sent to SWWAF_LOG_REMOTE_URL before they are given up. stdout has
|
// still waiting are sent to SWWAF_LOG_REMOTE_URL before they are given
|
||||||
// carried them.
|
// up. stdout has carried them.
|
||||||
const remoteLogStopTimeout = 2 * time.Second
|
const remoteLogStopTimeout = 2 * time.Second
|
||||||
|
|
||||||
// usageStatus is the exit status with no command, or one smallwebwaf does not
|
// Params are what Run needs from the process.
|
||||||
// know.
|
|
||||||
const usageStatus = 2
|
|
||||||
|
|
||||||
// usage is what smallwebwaf prints then.
|
|
||||||
const usage = `usage: smallwebwaf <command>
|
|
||||||
|
|
||||||
commands:
|
|
||||||
rproxy the reverse proxy in front of one app
|
|
||||||
servewaf decides for rproxy, and keeps bans, counters and history
|
|
||||||
healthcheck the container's health check
|
|
||||||
`
|
|
||||||
|
|
||||||
// Params are what RunServeWAF and RunRProxy need from the process.
|
|
||||||
type Params struct {
|
type Params struct {
|
||||||
// Version is the version of the binary, set when it is built.
|
// Version is the version of the binary, set when it is built.
|
||||||
Version string
|
Version string
|
||||||
@@ -64,45 +47,29 @@ type Params struct {
|
|||||||
Stdout io.Writer
|
Stdout io.Writer
|
||||||
}
|
}
|
||||||
|
|
||||||
// Main runs the command os.Args names, and returns the process's exit status.
|
// Main runs smallwebwaf until SIGTERM or SIGINT, and returns the
|
||||||
// `smallwebwaf rproxy` and `smallwebwaf servewaf` run until SIGTERM or SIGINT.
|
// process's exit status. Run as `smallwebwaf healthcheck`, it is the
|
||||||
// With no command, or one it does not know, it prints the commands and returns
|
// container's health check instead.
|
||||||
// usageStatus.
|
|
||||||
func Main(version string) int {
|
func Main(version string) int {
|
||||||
args := os.Args[1:]
|
if len(os.Args) > 1 && os.Args[1] == "healthcheck" {
|
||||||
if len(args) == 0 {
|
return HealthCheck(context.Background(), os.Args[2:], os.LookupEnv, os.Stderr)
|
||||||
return printUsage(os.Stderr)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx, stop := signal.NotifyContext(context.Background(),
|
ctx, stop := signal.NotifyContext(context.Background(),
|
||||||
syscall.SIGTERM, os.Interrupt)
|
syscall.SIGTERM, os.Interrupt)
|
||||||
defer stop()
|
defer stop()
|
||||||
|
|
||||||
params := Params{Version: version, LookupEnv: os.LookupEnv, Stdout: os.Stdout}
|
return Run(ctx, Params{
|
||||||
|
Version: version,
|
||||||
switch args[0] {
|
LookupEnv: os.LookupEnv,
|
||||||
case "rproxy":
|
Stdout: os.Stdout,
|
||||||
return RunRProxy(ctx, params)
|
})
|
||||||
case "servewaf":
|
|
||||||
return RunServeWAF(ctx, params)
|
|
||||||
case "healthcheck":
|
|
||||||
return HealthCheck(ctx, args[1:], os.LookupEnv, os.Stderr)
|
|
||||||
default:
|
|
||||||
return printUsage(os.Stderr)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// printUsage writes the commands to w, and returns usageStatus.
|
// Run reads the settings, the rule files, the lookup database and the
|
||||||
func printUsage(w io.Writer) int {
|
// state files, then serves requests until ctx is done. It returns the
|
||||||
_, _ = fmt.Fprint(w, usage)
|
// process's exit status, 1 when smallwebwaf cannot start.
|
||||||
|
func Run(ctx context.Context, params Params) int {
|
||||||
return usageStatus
|
|
||||||
}
|
|
||||||
|
|
||||||
// RunServeWAF is `smallwebwaf servewaf`: it reads the settings, the rule files,
|
|
||||||
// the lookup database and the state files, then serves requests until ctx is
|
|
||||||
// done. It returns the process's exit status, 1 when servewaf cannot start.
|
|
||||||
func RunServeWAF(ctx context.Context, params Params) int {
|
|
||||||
// Until the settings are read, the one message is an invalid setting's
|
// Until the settings are read, the one message is an invalid setting's
|
||||||
// error, which every SWWAF_LOG_LEVEL lets through.
|
// error, which every SWWAF_LOG_LEVEL lets through.
|
||||||
processLog := requestlog.NewProcessLogger(params.Stdout,
|
processLog := requestlog.NewProcessLogger(params.Stdout,
|
||||||
@@ -184,9 +151,9 @@ func RunServeWAF(ctx context.Context, params Params) int {
|
|||||||
return serve(ctx, server, listener, files, ruleFiles, alertQueue, processLog)
|
return serve(ctx, server, listener, files, ruleFiles, alertQueue, processLog)
|
||||||
}
|
}
|
||||||
|
|
||||||
// newServer returns the server servewaf runs, with the metrics of the alerts,
|
// newServer returns the server smallwebwaf runs, with the metrics of the
|
||||||
// after reading the lookup database while SWWAF_LOOKUP_SOURCE is file. A lookup
|
// alerts, after reading the lookup database while SWWAF_LOOKUP_SOURCE is
|
||||||
// database that cannot be read is an error.
|
// file. A lookup database that cannot be read is an error.
|
||||||
func newServer(
|
func newServer(
|
||||||
cfg *config.Config, stdout io.Writer, processLog *slog.Logger,
|
cfg *config.Config, stdout io.Writer, processLog *slog.Logger,
|
||||||
now func() time.Time, ruleFiles *rules.Files, alertQueue *alerts.Queue,
|
now func() time.Time, ruleFiles *rules.Files, alertQueue *alerts.Queue,
|
||||||
@@ -207,7 +174,7 @@ func newServer(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
server := proxy.NewServeWAF(proxy.Params{
|
server := proxy.New(proxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: stdout,
|
RequestLog: stdout,
|
||||||
ProcessLog: processLog,
|
ProcessLog: processLog,
|
||||||
@@ -371,10 +338,12 @@ func serve(
|
|||||||
|
|
||||||
// Run and Watch have ended, so nothing else reads or writes the
|
// Run and Watch have ended, so nothing else reads or writes the
|
||||||
// files, and no alert is being sent, so that alerts.json keeps every
|
// files, and no alert is being sent, so that alerts.json keeps every
|
||||||
// alert not yet sent. Every request has ended too, but for one cut off
|
// alert not yet sent. Every request has ended too, but for two kinds
|
||||||
// because Shutdown timed out, which adds to its client's history only as
|
// that Go's server does not wait for: one cut off because Shutdown
|
||||||
// it ends. That can be after this write, and then that request is missing
|
// timed out, and one whose connection switched protocols, such as a
|
||||||
// from clients.json.
|
// WebSocket. Such a request adds to its client's history only as it
|
||||||
|
// ends, which can be after this write, and then that request is
|
||||||
|
// missing from clients.json.
|
||||||
<-written
|
<-written
|
||||||
<-watched
|
<-watched
|
||||||
<-rulesWatched
|
<-rulesWatched
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package smallwebwaf_test
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/rand"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"io"
|
"io"
|
||||||
"net"
|
"net"
|
||||||
@@ -20,7 +19,6 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
|
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/proxy"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/smallwebwaf"
|
"sneak.berlin/go/smallwebwaf/internal/smallwebwaf"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -34,6 +32,8 @@ const (
|
|||||||
// localhost is where the tests listen.
|
// localhost is where the tests listen.
|
||||||
localhost = "127.0.0.1"
|
localhost = "127.0.0.1"
|
||||||
listenAddr = "SWWAF_LISTEN_ADDR"
|
listenAddr = "SWWAF_LISTEN_ADDR"
|
||||||
|
upstreamURL = "SWWAF_UPSTREAM_URL"
|
||||||
|
trustedProxies = "SWWAF_TRUSTED_PROXIES"
|
||||||
stateDir = "SWWAF_STATE_DIR"
|
stateDir = "SWWAF_STATE_DIR"
|
||||||
stateWriteDelay = "SWWAF_STATE_WRITE_DELAY"
|
stateWriteDelay = "SWWAF_STATE_WRITE_DELAY"
|
||||||
stateCounterInterval = "SWWAF_STATE_COUNTER_INTERVAL"
|
stateCounterInterval = "SWWAF_STATE_COUNTER_INTERVAL"
|
||||||
@@ -76,11 +76,21 @@ func (o *output) line(t *testing.T, key, value string) map[string]any {
|
|||||||
|
|
||||||
deadline := time.Now().Add(waitLimit)
|
deadline := time.Now().Add(waitLimit)
|
||||||
for time.Now().Before(deadline) {
|
for time.Now().Before(deadline) {
|
||||||
fields := o.findLine(t, func(fields map[string]any) bool {
|
o.mu.Lock()
|
||||||
return fields[key] == value
|
text := o.buf.String()
|
||||||
})
|
o.mu.Unlock()
|
||||||
if fields != nil {
|
|
||||||
return fields
|
for line := range strings.Lines(text) {
|
||||||
|
var fields map[string]any
|
||||||
|
|
||||||
|
err := json.Unmarshal([]byte(line), &fields)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("output line %q is not JSON: %v", line, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if fields[key] == value {
|
||||||
|
return fields
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
time.Sleep(pollInterval)
|
time.Sleep(pollInterval)
|
||||||
@@ -91,29 +101,6 @@ func (o *output) line(t *testing.T, key, value string) map[string]any {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// findLine returns the first line written so far for which matches is true, or
|
|
||||||
// nil.
|
|
||||||
func (o *output) findLine(
|
|
||||||
t *testing.T, matches func(map[string]any) bool,
|
|
||||||
) map[string]any {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
for line := range strings.Lines(o.text()) {
|
|
||||||
var fields map[string]any
|
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(line), &fields)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("output line %q is not JSON: %v", line, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if matches(fields) {
|
|
||||||
return fields
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// text returns everything written so far.
|
// text returns everything written so far.
|
||||||
func (o *output) text() string {
|
func (o *output) text() string {
|
||||||
o.mu.Lock()
|
o.mu.Lock()
|
||||||
@@ -122,11 +109,11 @@ func (o *output) text() string {
|
|||||||
return o.buf.String()
|
return o.buf.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
// run runs `smallwebwaf servewaf` with the settings in env until ctx is done,
|
// run runs smallwebwaf with the settings in env until ctx is done, and
|
||||||
// and returns its exit status. SWWAF_LOOKUP_SOURCE is off unless env sets it,
|
// returns its exit status. SWWAF_LOOKUP_SOURCE is off unless env sets it,
|
||||||
// so that no test sends GeoJS its clients' addresses.
|
// so that no test sends GeoJS its clients' addresses.
|
||||||
func run(ctx context.Context, env map[string]string, out *output) int {
|
func run(ctx context.Context, env map[string]string, out *output) int {
|
||||||
return smallwebwaf.RunServeWAF(ctx, smallwebwaf.Params{
|
return smallwebwaf.Run(ctx, smallwebwaf.Params{
|
||||||
Version: testVersion,
|
Version: testVersion,
|
||||||
LookupEnv: func(name string) (string, bool) {
|
LookupEnv: func(name string) (string, bool) {
|
||||||
value, ok := env[name]
|
value, ok := env[name]
|
||||||
@@ -146,8 +133,8 @@ func TestInvalidSettingStopsTheStart(t *testing.T) {
|
|||||||
out := &output{}
|
out := &output{}
|
||||||
|
|
||||||
status := run(t.Context(), map[string]string{
|
status := run(t.Context(), map[string]string{
|
||||||
"SWWAF_BYTES_LIMIT_PER_MINUTE": "lots",
|
"SWWAF_REQUEST_MAX_BYTES": "lots",
|
||||||
instanceName: instance,
|
instanceName: instance,
|
||||||
}, out)
|
}, out)
|
||||||
if status != 1 {
|
if status != 1 {
|
||||||
t.Errorf("exit status %d, want 1", status)
|
t.Errorf("exit status %d, want 1", status)
|
||||||
@@ -158,7 +145,7 @@ func TestInvalidSettingStopsTheStart(t *testing.T) {
|
|||||||
|
|
||||||
if line["type"] != "process" || line["instance"] != instance ||
|
if line["type"] != "process" || line["instance"] != instance ||
|
||||||
line["level"] != "ERROR" ||
|
line["level"] != "ERROR" ||
|
||||||
!strings.HasPrefix(message, "SWWAF_BYTES_LIMIT_PER_MINUTE: ") {
|
!strings.HasPrefix(message, "SWWAF_REQUEST_MAX_BYTES: ") {
|
||||||
t.Errorf("start refused with %v", line)
|
t.Errorf("start refused with %v", line)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -220,6 +207,7 @@ func TestAddressInUseStopsTheStart(t *testing.T) {
|
|||||||
func TestServesUntilToldToStop(t *testing.T) {
|
func TestServesUntilToldToStop(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
appURL := startApp(t)
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
|
|
||||||
ctx, stop := context.WithCancel(t.Context())
|
ctx, stop := context.WithCancel(t.Context())
|
||||||
@@ -228,9 +216,10 @@ func TestServesUntilToldToStop(t *testing.T) {
|
|||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
exited <- run(ctx, map[string]string{
|
exited <- run(ctx, map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: dir,
|
upstreamURL: appURL,
|
||||||
rulesDir: filepath.Join("..", "..", "share", "rules.d"),
|
stateDir: dir,
|
||||||
|
rulesDir: filepath.Join("..", "..", "share", "rules.d"),
|
||||||
}, out)
|
}, out)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
@@ -240,10 +229,10 @@ func TestServesUntilToldToStop(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
starting := out.line(t, "msg", "starting")
|
starting := out.line(t, "msg", "starting")
|
||||||
wantStartingLine(t, starting, dir)
|
wantStartingLine(t, starting, appURL, dir)
|
||||||
|
|
||||||
addr, _ := starting["address"].(string)
|
addr, _ := starting["address"].(string)
|
||||||
wantAllowed(t, "http://"+addr+"/")
|
wantGreeting(t, "http://"+addr+"/")
|
||||||
out.line(t, "type", "request")
|
out.line(t, "type", "request")
|
||||||
|
|
||||||
stop()
|
stop()
|
||||||
@@ -315,6 +304,7 @@ func TestEveryLogLineAndMetricCarriesTheInstanceName(t *testing.T) {
|
|||||||
|
|
||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
|
upstreamURL: startApp(t),
|
||||||
stateDir: t.TempDir(),
|
stateDir: t.TempDir(),
|
||||||
rulesDir: t.TempDir(),
|
rulesDir: t.TempDir(),
|
||||||
metricsToken: token,
|
metricsToken: token,
|
||||||
@@ -352,6 +342,7 @@ func TestStateKeptAcrossRestarts(t *testing.T) {
|
|||||||
|
|
||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
|
upstreamURL: startApp(t),
|
||||||
stateDir: t.TempDir(),
|
stateDir: t.TempDir(),
|
||||||
rulesDir: t.TempDir(),
|
rulesDir: t.TempDir(),
|
||||||
rateLimitPerDay: "2",
|
rateLimitPerDay: "2",
|
||||||
@@ -363,8 +354,8 @@ func TestStateKeptAcrossRestarts(t *testing.T) {
|
|||||||
|
|
||||||
// The two requests a day allows, and a stop.
|
// The two requests a day allows, and a stop.
|
||||||
runUntilStopped(t, env, func(url string) {
|
runUntilStopped(t, env, func(url string) {
|
||||||
wantAllowed(t, url)
|
wantGreeting(t, url)
|
||||||
wantAllowed(t, url)
|
wantGreeting(t, url)
|
||||||
})
|
})
|
||||||
|
|
||||||
// After a restart the client has no fresh allowance: its third
|
// After a restart the client has no fresh allowance: its third
|
||||||
@@ -388,8 +379,10 @@ func TestBanRefusesItsNetblockAfterARestartWithAnotherScope(t *testing.T) {
|
|||||||
|
|
||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
|
upstreamURL: startApp(t),
|
||||||
stateDir: t.TempDir(),
|
stateDir: t.TempDir(),
|
||||||
rulesDir: t.TempDir(),
|
rulesDir: t.TempDir(),
|
||||||
|
trustedProxies: localhost + "/32",
|
||||||
rateLimitPerDay: "1",
|
rateLimitPerDay: "1",
|
||||||
scope: "24",
|
scope: "24",
|
||||||
}
|
}
|
||||||
@@ -433,9 +426,11 @@ func TestBanAddedAndLiftedByEditingBansJSON(t *testing.T) {
|
|||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: dir,
|
upstreamURL: startApp(t),
|
||||||
rulesDir: t.TempDir(),
|
stateDir: dir,
|
||||||
|
rulesDir: t.TempDir(),
|
||||||
|
trustedProxies: localhost + "/32",
|
||||||
// No write comes due in the test, so only the watch on the
|
// No write comes due in the test, so only the watch on the
|
||||||
// directory can take the edits in.
|
// directory can take the edits in.
|
||||||
stateWriteDelay: "1h",
|
stateWriteDelay: "1h",
|
||||||
@@ -456,10 +451,12 @@ func TestBanAddedAndLiftedThroughTheEndpointsKeptInBansJSON(t *testing.T) {
|
|||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: dir,
|
upstreamURL: startApp(t),
|
||||||
rulesDir: t.TempDir(),
|
stateDir: dir,
|
||||||
adminToken: adminSecret,
|
rulesDir: t.TempDir(),
|
||||||
|
trustedProxies: localhost + "/32",
|
||||||
|
adminToken: adminSecret,
|
||||||
// Neither comes due in the test: the files are written as
|
// Neither comes due in the test: the files are written as
|
||||||
// smallwebwaf stops.
|
// smallwebwaf stops.
|
||||||
stateWriteDelay: "1h",
|
stateWriteDelay: "1h",
|
||||||
@@ -499,16 +496,17 @@ func TestRuleFileAddedWhileRunningTakesEffect(t *testing.T) {
|
|||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: t.TempDir(),
|
upstreamURL: startApp(t),
|
||||||
rulesDir: dir,
|
stateDir: t.TempDir(),
|
||||||
|
rulesDir: dir,
|
||||||
// The requests sent until the rule takes effect must not break a
|
// The requests sent until the rule takes effect must not break a
|
||||||
// rate limit, whose ban would refuse them too.
|
// rate limit, whose ban would refuse them too.
|
||||||
"SWWAF_RATE_LIMIT_PER_MINUTE": "off",
|
"SWWAF_RATE_LIMIT_PER_MINUTE": "off",
|
||||||
}
|
}
|
||||||
|
|
||||||
out := runUntilStopped(t, env, func(url string) {
|
out := runUntilStopped(t, env, func(url string) {
|
||||||
wantAllowed(t, url)
|
wantGreeting(t, url)
|
||||||
|
|
||||||
// Written once: each change would start the rule files' wait
|
// Written once: each change would start the rule files' wait
|
||||||
// again. A file written before smallwebwaf watches the directory is
|
// again. A file written before smallwebwaf watches the directory is
|
||||||
@@ -561,8 +559,10 @@ func TestLookupDatabaseReplacedWhileRunningTakesEffect(t *testing.T) {
|
|||||||
|
|
||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
|
upstreamURL: startApp(t),
|
||||||
stateDir: t.TempDir(),
|
stateDir: t.TempDir(),
|
||||||
rulesDir: t.TempDir(),
|
rulesDir: t.TempDir(),
|
||||||
|
trustedProxies: localhost + "/32",
|
||||||
metricsToken: token,
|
metricsToken: token,
|
||||||
instanceName: instance,
|
instanceName: instance,
|
||||||
lookupSource: "file",
|
lookupSource: "file",
|
||||||
@@ -658,8 +658,10 @@ func TestBlocklistTriesAndCopiesKeptInReputationJSONAcrossRestarts(t *testing.T)
|
|||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
|
upstreamURL: startApp(t),
|
||||||
stateDir: dir,
|
stateDir: dir,
|
||||||
rulesDir: t.TempDir(),
|
rulesDir: t.TempDir(),
|
||||||
|
trustedProxies: localhost + "/32",
|
||||||
metricsToken: token,
|
metricsToken: token,
|
||||||
instanceName: instance,
|
instanceName: instance,
|
||||||
"SWWAF_BLOCKLIST_URLS": torURL,
|
"SWWAF_BLOCKLIST_URLS": torURL,
|
||||||
@@ -740,8 +742,10 @@ func TestCrowdSecDecisionListKeptInReputationJSONAcrossARestart(t *testing.T) {
|
|||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
|
upstreamURL: startApp(t),
|
||||||
stateDir: dir,
|
stateDir: dir,
|
||||||
rulesDir: t.TempDir(),
|
rulesDir: t.TempDir(),
|
||||||
|
trustedProxies: localhost + "/32",
|
||||||
"SWWAF_CROWDSEC_LAPI_URL": engine.URL,
|
"SWWAF_CROWDSEC_LAPI_URL": engine.URL,
|
||||||
"SWWAF_CROWDSEC_LAPI_KEY": bouncerKey,
|
"SWWAF_CROWDSEC_LAPI_KEY": bouncerKey,
|
||||||
}
|
}
|
||||||
@@ -839,13 +843,14 @@ func TestEveryLineIsAlsoSentToTheRemoteLogEndpoint(t *testing.T) {
|
|||||||
|
|
||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
|
upstreamURL: startApp(t),
|
||||||
stateDir: t.TempDir(),
|
stateDir: t.TempDir(),
|
||||||
rulesDir: t.TempDir(),
|
rulesDir: t.TempDir(),
|
||||||
"SWWAF_LOG_REMOTE_URL": "syslog+tcp://" + endpoint.Addr().String(),
|
"SWWAF_LOG_REMOTE_URL": "syslog+tcp://" + endpoint.Addr().String(),
|
||||||
}
|
}
|
||||||
|
|
||||||
out := runUntilStopped(t, env, func(url string) {
|
out := runUntilStopped(t, env, func(url string) {
|
||||||
wantAllowed(t, url)
|
wantGreeting(t, url)
|
||||||
})
|
})
|
||||||
out.line(t, "type", "request")
|
out.line(t, "type", "request")
|
||||||
|
|
||||||
@@ -894,6 +899,7 @@ func TestStalledRemoteLogEndpointHoldsUpNoRequest(t *testing.T) {
|
|||||||
|
|
||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
|
upstreamURL: startApp(t),
|
||||||
stateDir: t.TempDir(),
|
stateDir: t.TempDir(),
|
||||||
rulesDir: t.TempDir(),
|
rulesDir: t.TempDir(),
|
||||||
"SWWAF_LOG_REMOTE_URL": "syslog+tls://" + endpoint.Addr().String(),
|
"SWWAF_LOG_REMOTE_URL": "syslog+tls://" + endpoint.Addr().String(),
|
||||||
@@ -903,7 +909,7 @@ func TestStalledRemoteLogEndpointHoldsUpNoRequest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
out := runUntilStopped(t, env, func(url string) {
|
out := runUntilStopped(t, env, func(url string) {
|
||||||
wantAllowed(t, url)
|
wantGreeting(t, url)
|
||||||
|
|
||||||
// More than one line has been written, and the buffer holds the
|
// More than one line has been written, and the buffer holds the
|
||||||
// last.
|
// last.
|
||||||
@@ -953,6 +959,7 @@ func TestBanIsAlertedAndAnAlertNotSentIsKeptAcrossARestart(t *testing.T) {
|
|||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
|
upstreamURL: startApp(t),
|
||||||
stateDir: dir,
|
stateDir: dir,
|
||||||
rulesDir: rules,
|
rulesDir: rules,
|
||||||
"SWWAF_ALERT_WEBHOOK_URL": webhook.url,
|
"SWWAF_ALERT_WEBHOOK_URL": webhook.url,
|
||||||
@@ -1037,8 +1044,10 @@ func TestBanIsAlertedToSlackAndNtfy(t *testing.T) {
|
|||||||
slack, ntfy := startDestination(t), startDestination(t)
|
slack, ntfy := startDestination(t), startDestination(t)
|
||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
|
upstreamURL: startApp(t),
|
||||||
stateDir: t.TempDir(),
|
stateDir: t.TempDir(),
|
||||||
rulesDir: t.TempDir(),
|
rulesDir: t.TempDir(),
|
||||||
|
trustedProxies: localhost + "/32",
|
||||||
rateLimitPerDay: "1",
|
rateLimitPerDay: "1",
|
||||||
// The metrics are read from 127.0.0.1, which no limit counts.
|
// The metrics are read from 127.0.0.1, which no limit counts.
|
||||||
"SWWAF_ALLOW_NETS": localhost + "/32",
|
"SWWAF_ALLOW_NETS": localhost + "/32",
|
||||||
@@ -1178,8 +1187,8 @@ func startApp(t *testing.T) string {
|
|||||||
return app.URL
|
return app.URL
|
||||||
}
|
}
|
||||||
|
|
||||||
// runUntilStopped runs `smallwebwaf servewaf` with the settings in env, has use
|
// runUntilStopped runs smallwebwaf with the settings in env, has use send
|
||||||
// send it requests at url, then stops it as SIGTERM does, checks that it
|
// it requests at url, then stops it as SIGTERM does, checks that it
|
||||||
// stopped in order, and returns its output.
|
// stopped in order, and returns its output.
|
||||||
func runUntilStopped(
|
func runUntilStopped(
|
||||||
t *testing.T, env map[string]string, use func(url string),
|
t *testing.T, env map[string]string, use func(url string),
|
||||||
@@ -1212,33 +1221,43 @@ func runUntilStopped(
|
|||||||
|
|
||||||
// wantStartingLine checks that the line at start gives the version and
|
// wantStartingLine checks that the line at start gives the version and
|
||||||
// every setting's value.
|
// every setting's value.
|
||||||
func wantStartingLine(t *testing.T, line map[string]any, dir string) {
|
func wantStartingLine(t *testing.T, line map[string]any, appURL, dir string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
settings, _ := line["settings"].(map[string]any)
|
settings, _ := line["settings"].(map[string]any)
|
||||||
want := map[string]any{
|
want := map[string]any{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: dir,
|
upstreamURL: appURL,
|
||||||
"SWWAF_MODE": "enforce",
|
stateDir: dir,
|
||||||
stateWriteDelay: "10s",
|
"SWWAF_MODE": "enforce",
|
||||||
stateCounterInterval: "15m",
|
stateWriteDelay: "10s",
|
||||||
"SWWAF_ALLOW_NETS": "",
|
stateCounterInterval: "15m",
|
||||||
rateLimitExemptNets: "",
|
trustedProxies: "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16",
|
||||||
"SWWAF_DENY_NETS": "",
|
"SWWAF_CLIENT_REQUEST_TIMEOUT": "60s",
|
||||||
"SWWAF_RATE_LIMIT_PER_MINUTE": "500",
|
"SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES": "32K",
|
||||||
"SWWAF_RATE_LIMIT_PER_HOUR": "20000",
|
"SWWAF_CLIENT_IDLE_TIMEOUT": "120s",
|
||||||
rateLimitPerDay: "300000",
|
"SWWAF_CLIENT_RESPONSE_TIMEOUT": "30m",
|
||||||
"SWWAF_RATE_LIMIT_EXEMPT_PATHS": "",
|
"SWWAF_UPSTREAM_REQUEST_TIMEOUT": "60s",
|
||||||
"SWWAF_DENIED_COUNTRIES": "",
|
"SWWAF_UPSTREAM_RESPONSE_TIMEOUT": "30m",
|
||||||
"SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES": "",
|
"SWWAF_REQUEST_MAX_BYTES": "100M",
|
||||||
"SWWAF_BAN_RESPONSE": "403",
|
"SWWAF_RESPONSE_MAX_BYTES": "5G",
|
||||||
"SWWAF_LIMIT_BAN_DURATION": "1h",
|
"SWWAF_ALLOW_NETS": "",
|
||||||
"SWWAF_LIMIT_BAN_REPEAT_WINDOW": "24h",
|
rateLimitExemptNets: "",
|
||||||
"SWWAF_MAX_BAN_DURATION": "7d",
|
"SWWAF_DENY_NETS": "",
|
||||||
"SWWAF_ATTACK_BAN_DURATION": "7d",
|
"SWWAF_RATE_LIMIT_PER_MINUTE": "1000",
|
||||||
"SWWAF_MAX_BANS": "5000",
|
"SWWAF_RATE_LIMIT_PER_HOUR": "10000",
|
||||||
"SWWAF_BAN_SCOPE_V4_PREFIX": "32",
|
rateLimitPerDay: "50000",
|
||||||
"SWWAF_RULES_ENABLED": "true",
|
"SWWAF_RATE_LIMIT_EXEMPT_PATHS": "",
|
||||||
|
"SWWAF_DENIED_COUNTRIES": "",
|
||||||
|
"SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES": "",
|
||||||
|
"SWWAF_BAN_RESPONSE": "403",
|
||||||
|
"SWWAF_LIMIT_BAN_DURATION": "1h",
|
||||||
|
"SWWAF_LIMIT_BAN_REPEAT_WINDOW": "24h",
|
||||||
|
"SWWAF_MAX_BAN_DURATION": "7d",
|
||||||
|
"SWWAF_ATTACK_BAN_DURATION": "7d",
|
||||||
|
"SWWAF_MAX_BANS": "5000",
|
||||||
|
"SWWAF_BAN_SCOPE_V4_PREFIX": "32",
|
||||||
|
"SWWAF_RULES_ENABLED": "true",
|
||||||
}
|
}
|
||||||
|
|
||||||
for name, value := range want {
|
for name, value := range want {
|
||||||
@@ -1252,11 +1271,30 @@ func wantStartingLine(t *testing.T, line map[string]any, dir string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// wantAllowed checks that servewaf at url allows a request from localhost.
|
// wantGreeting checks that a request to url gets the app's answer.
|
||||||
func wantAllowed(t *testing.T, url string) {
|
func wantGreeting(t *testing.T, url string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
wantStatus(t, url, localhost, http.StatusOK)
|
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, url,
|
||||||
|
http.NoBody)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("new request: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
transport := &http.Transport{}
|
||||||
|
defer transport.CloseIdleConnections()
|
||||||
|
|
||||||
|
res, err := (&http.Client{Transport: transport}).Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("request: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := io.ReadAll(res.Body)
|
||||||
|
_ = res.Body.Close()
|
||||||
|
|
||||||
|
if err != nil || string(body) != greeting {
|
||||||
|
t.Errorf("got %q (%v), want the app's answer", body, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// messages returns the message of each record in received, octet-counted
|
// messages returns the message of each record in received, octet-counted
|
||||||
@@ -1437,16 +1475,34 @@ func onlyBan(t *testing.T, dir string) map[string]any {
|
|||||||
return file.Bans[0]
|
return file.Bans[0]
|
||||||
}
|
}
|
||||||
|
|
||||||
// wantRefused checks that servewaf at url refuses a request from localhost with
|
// wantRefused checks that a request to url is refused with 403, the
|
||||||
// 403, the default SWWAF_BAN_RESPONSE.
|
// default SWWAF_BAN_RESPONSE.
|
||||||
func wantRefused(t *testing.T, url string) {
|
func wantRefused(t *testing.T, url string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
wantStatus(t, url, localhost, http.StatusForbidden)
|
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, url,
|
||||||
|
http.NoBody)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("new request: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
transport := &http.Transport{}
|
||||||
|
defer transport.CloseIdleConnections()
|
||||||
|
|
||||||
|
res, err := (&http.Client{Transport: transport}).Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("request: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = res.Body.Close()
|
||||||
|
|
||||||
|
if res.StatusCode != http.StatusForbidden {
|
||||||
|
t.Errorf("status %d, want %d", res.StatusCode, http.StatusForbidden)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// wantStatus checks that a request to url from the client at from, as statusFrom
|
// wantStatus checks that a request to url from the client at from, as
|
||||||
// sends it, is answered with status.
|
// X-Forwarded-For names it, is answered with status.
|
||||||
func wantStatus(t *testing.T, url, from string, status int) {
|
func wantStatus(t *testing.T, url, from string, status int) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -1613,10 +1669,8 @@ func (w *webhook) waitFor(t *testing.T, event string, answered bool) webhookPost
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// statusFrom returns the status a request to url from the client at from is
|
// statusFrom returns the status a request to url from the client at
|
||||||
// answered with. The request serves rproxy and servewaf alike. rproxy reads the
|
// from, as X-Forwarded-For names it, is answered with.
|
||||||
// client from X-Forwarded-For, and replaces the other two headers. To servewaf,
|
|
||||||
// they make it a question about that client.
|
|
||||||
func statusFrom(t *testing.T, url, from string) int {
|
func statusFrom(t *testing.T, url, from string) int {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -1627,8 +1681,6 @@ func statusFrom(t *testing.T, url, from string) int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
req.Header.Set("X-Forwarded-For", from)
|
req.Header.Set("X-Forwarded-For", from)
|
||||||
req.Header.Set(proxy.ClientHeader, from)
|
|
||||||
req.Header.Set("X-Request-ID", rand.Text())
|
|
||||||
|
|
||||||
transport := &http.Transport{}
|
transport := &http.Transport{}
|
||||||
defer transport.CloseIdleConnections()
|
defer transport.CloseIdleConnections()
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
// Package state keeps smallwebwaf's state in JSON files in
|
// Package state keeps smallwebwaf's state in JSON files in
|
||||||
// SWWAF_STATE_DIR, as the "Persistent state" section of SPEC.md requires
|
// SWWAF_STATE_DIR, as the "Persistent state" section of SPEC.md describes:
|
||||||
// and docs/state-files.md lists them:
|
|
||||||
// bans.json holds the bans, clients.json each client's counters and
|
// bans.json holds the bans, clients.json each client's counters and
|
||||||
// history, lookups.json GeoJS's answers, reputation.json the last try and
|
// history, lookups.json GeoJS's answers, reputation.json the last try and
|
||||||
// last good copy of each list fetched from a URL, the DNSBL zones'
|
// last good copy of each list fetched from a URL, the DNSBL zones'
|
||||||
@@ -62,7 +61,6 @@ var (
|
|||||||
// errMissing is for an entry without a field it needs.
|
// errMissing is for an entry without a field it needs.
|
||||||
errMissing = errors.New("has no")
|
errMissing = errors.New("has no")
|
||||||
errCause = errors.New("is not limit, attack, admin or crowdsec")
|
errCause = errors.New("is not limit, attack, admin or crowdsec")
|
||||||
errExpires = errors.New("is not after its start")
|
|
||||||
errDestination = errors.New("is not webhook, slack or ntfy")
|
errDestination = errors.New("is not webhook, slack or ntfy")
|
||||||
errScope = errors.New("is not client, net, asn, total or watch")
|
errScope = errors.New("is not client, net, asn, total or watch")
|
||||||
errWaitingList = errors.New(`waiting is a list, but now lists the alerts by ` +
|
errWaitingList = errors.New(`waiting is a list, but now lists the alerts by ` +
|
||||||
@@ -648,10 +646,8 @@ func (e BanEntry) ban() bans.Ban {
|
|||||||
// client, a start, from which the length of the netblock's next ban is
|
// client, a start, from which the length of the netblock's next ban is
|
||||||
// worked out, or an expires, which would make it permanent. A permanent
|
// worked out, or an expires, which would make it permanent. A permanent
|
||||||
// ban's expires is null, which Bans cannot tell from a missing one, so
|
// ban's expires is null, which Bans cannot tell from a missing one, so
|
||||||
// each expires is read again as written. An expires not after the start
|
// each expires is read again as written. A cause other than limit,
|
||||||
// is refused too, since the netblock's next ban for a broken limit, three
|
// attack, admin or crowdsec, most likely misspelt, is refused too.
|
||||||
// times as long as this one, would then refuse nothing; and so is a cause
|
|
||||||
// other than limit, attack, admin or crowdsec, most likely misspelt.
|
|
||||||
func (f *bansFile) check(data []byte) error {
|
func (f *bansFile) check(data []byte) error {
|
||||||
var written struct {
|
var written struct {
|
||||||
Bans []struct {
|
Bans []struct {
|
||||||
@@ -672,8 +668,6 @@ func (f *bansFile) check(data []byte) error {
|
|||||||
return missing(i, "start")
|
return missing(i, "start")
|
||||||
case written.Bans[i].Expires == nil:
|
case written.Bans[i].Expires == nil:
|
||||||
return missing(i, "expires")
|
return missing(i, "expires")
|
||||||
case entry.Expires != nil && !entry.Expires.After(entry.Start):
|
|
||||||
return fmt.Errorf("entry %d's expires %w", i+1, errExpires)
|
|
||||||
case entry.Cause != "" && entry.Cause != bans.CauseLimit &&
|
case entry.Cause != "" && entry.Cause != bans.CauseLimit &&
|
||||||
entry.Cause != bans.CauseAttack && entry.Cause != bans.CauseAdmin &&
|
entry.Cause != bans.CauseAttack && entry.Cause != bans.CauseAdmin &&
|
||||||
entry.Cause != bans.CauseCrowdSec:
|
entry.Cause != bans.CauseCrowdSec:
|
||||||
|
|||||||
@@ -648,27 +648,6 @@ func TestClientWithRefusalsInTheMinuteWithoutTheirStartStopsTheStart(t *testing.
|
|||||||
`: entry 1 has no "minute_refusals.start"`)
|
`: entry 1 has no "minute_refusals.start"`)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestClientsJSONFromAnEarlierVersionWithTokenRefusalsLoads(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
|
|
||||||
err := os.WriteFile(filepath.Join(dir, clientsJSON),
|
|
||||||
[]byte(`{"version": 1, "clients": [{"client": "203.0.113.9/32", `+
|
|
||||||
`"history": {"offences": {"token_refused": 2}}}]}`), 0o600)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("write %s: %v", clientsJSON, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
params := newParams(dir)
|
|
||||||
load(t, params)
|
|
||||||
|
|
||||||
clients := params.Limiter.Snapshot()
|
|
||||||
if len(clients) != 1 || clients[0].History.Offences.TokenRefused != 2 {
|
|
||||||
t.Errorf("clients %+v, want one with its two refusals kept", clients)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReputationJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
|
func TestReputationJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -886,21 +865,6 @@ func TestBanWithAnotherCauseStopsTheStart(t *testing.T) {
|
|||||||
`: entry 4's cause "atack" is not limit, attack, admin or crowdsec`)
|
`: entry 4's cause "atack" is not limit, attack, admin or crowdsec`)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBanThatDoesNotEndAfterItsStartStopsTheStart(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// The first ban ends a second after its start; the second ends as it
|
|
||||||
// starts, or an hour before.
|
|
||||||
for _, expires := range []string{`"2026-10-06T00:00:00Z"`, `"2026-10-05T23:00:00Z"`} {
|
|
||||||
wantRefused(t, bansJSON, `{"version": 1, "bans": [`+
|
|
||||||
`{"netblock": "203.0.113.9/32", "start": "2026-10-06T00:00:00Z", `+
|
|
||||||
`"expires": "2026-10-06T00:00:01Z"}, `+
|
|
||||||
`{"netblock": "203.0.113.10/32", "start": "2026-10-06T00:00:00Z", `+
|
|
||||||
`"expires": `+expires+`}]}`,
|
|
||||||
`: entry 2's expires is not after its start`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUnknownVersionStopsTheStart(t *testing.T) {
|
func TestUnknownVersionStopsTheStart(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -1040,8 +1004,8 @@ func TestEditJustBeforeAScheduledWriteSurvivesIt(t *testing.T) {
|
|||||||
edit(t, dir, bansJSON, permanentBansJSON)
|
edit(t, dir, bansJSON, permanentBansJSON)
|
||||||
|
|
||||||
// The write takes the edit in first, and writes it back. The second
|
// The write takes the edit in first, and writes it back. The second
|
||||||
// ban, made after the admin opened the file, is lost, as the
|
// ban, made after the admin opened the file, is lost, as "Edits
|
||||||
// "Persistent state" section of SPEC.md says.
|
// while running" in SPEC.md says.
|
||||||
time.Sleep(time.Nanosecond)
|
time.Sleep(time.Nanosecond)
|
||||||
synctest.Wait()
|
synctest.Wait()
|
||||||
|
|
||||||
|
|||||||
+14
-12
@@ -1,8 +1,8 @@
|
|||||||
// Package waf runs the OWASP Core Rule Set 4.25.0, through Coraza, on the
|
// Package waf runs the OWASP Core Rule Set 4.25.0, through Coraza, on the
|
||||||
// method, the URL with its query and the headers of a request, and on its
|
// method, the URL with its query and the headers of a request, and on its
|
||||||
// body while SWWAF_WAF_BODY_LIMIT is set, with the six changes smallwebwaf
|
// body while SWWAF_WAF_BODY_LIMIT is set, with the six changes smallwebwaf
|
||||||
// makes to it, as "Attack detection" under "Settings" in SPEC.md describes
|
// makes to it, as "Attack detection" under "Configuration surface" in
|
||||||
// them. It reads no response.
|
// SPEC.md describes them. It reads no response.
|
||||||
//
|
//
|
||||||
// smallwebwaf writes only to its state directory, so Coraza is built with
|
// smallwebwaf writes only to its state directory, so Coraza is built with
|
||||||
// its no_fs_access tag, as the Dockerfile and script/build build it: of a
|
// its no_fs_access tag, as the Dockerfile and script/build build it: of a
|
||||||
@@ -105,10 +105,9 @@ SecRuleUpdateTargetById 944110 "!REQUEST_HEADERS:Referer"
|
|||||||
// bodyDirectives have the Core Rule Set read the part of a request body
|
// bodyDirectives have the Core Rule Set read the part of a request body
|
||||||
// Inspect gives it, which is at most one byte longer than the limit, up to
|
// Inspect gives it, which is at most one byte longer than the limit, up to
|
||||||
// the limit, %d bytes, and read JSON and XML as Coraza's recommended
|
// the limit, %d bytes, and read JSON and XML as Coraza's recommended
|
||||||
// configuration has it in its rules 200000, 200001 and 200006, with
|
// configuration has it in its rules 200000, 200001 and 200006, with any
|
||||||
// text/json, and any application or text type ending in +xml or +json,
|
// type ending in +xml, and text/json, besides; form data and multipart
|
||||||
// besides; form data and multipart Coraza knows by itself. %% stands for
|
// Coraza knows by itself. %% stands for a % Coraza reads.
|
||||||
// a % Coraza reads.
|
|
||||||
const bodyDirectives = `
|
const bodyDirectives = `
|
||||||
SecRequestBodyAccess On
|
SecRequestBodyAccess On
|
||||||
SecRequestBodyLimit %d
|
SecRequestBodyLimit %d
|
||||||
@@ -133,11 +132,13 @@ SecRule REQBODY_PROCESSOR "@rx ^(?:URLENCODED|MULTIPART|JSON|XML)$" \
|
|||||||
# its strict checks (900450), each add 5 to the score, as a rule the Core
|
# its strict checks (900450), each add 5 to the score, as a rule the Core
|
||||||
# Rule Set rates critical does: no rule reads what comes after the fault,
|
# Rule Set rates critical does: no rule reads what comes after the fault,
|
||||||
# which the app may still read. Coraza's recommended configuration refuses
|
# which the app may still read. Coraza's recommended configuration refuses
|
||||||
# them in its rules 200002 and 200003. A multipart body the limit cuts
|
# them in its rules 200002 and 200003. A multipart body that reaches the
|
||||||
# before the colon of a part's header line, or between the carriage return
|
# limit is let off both (900430), since the limit can cut it inside a
|
||||||
# and the line feed that end a part's header line or the empty line after
|
# part's header, which Coraza takes for such a fault. Coraza parses any
|
||||||
# its headers, adds 5 too, since Coraza takes the line the limit cuts for a
|
# form data body.
|
||||||
# malformed header. Coraza parses any form data body.
|
SecRule INBOUND_DATA_ERROR "@eq 1" "id:900430,phase:2,pass,nolog,chain"
|
||||||
|
SecRule REQBODY_PROCESSOR "@streq MULTIPART" \
|
||||||
|
"ctl:ruleRemoveById=900440,ctl:ruleRemoveById=900450"
|
||||||
SecRule REQBODY_ERROR "!@eq 0" "id:900440,phase:2,pass,severity:'CRITICAL',\
|
SecRule REQBODY_ERROR "!@eq 0" "id:900440,phase:2,pass,severity:'CRITICAL',\
|
||||||
setvar:'tx.inbound_anomaly_score_pl1=+5'"
|
setvar:'tx.inbound_anomaly_score_pl1=+5'"
|
||||||
SecRule MULTIPART_STRICT_ERROR "!@eq 0" "id:900450,phase:2,pass,\
|
SecRule MULTIPART_STRICT_ERROR "!@eq 0" "id:900450,phase:2,pass,\
|
||||||
@@ -212,7 +213,8 @@ type Result struct {
|
|||||||
// method, its URL with the query, and its headers, the Cookie header
|
// method, its URL with the query, and its headers, the Cookie header
|
||||||
// without the cookies in cookiesNotRead, and on body, r's body as the
|
// without the cookies in cookiesNotRead, and on body, r's body as the
|
||||||
// caller has it, as readBody reads it. It returns what it found, what it
|
// caller has it, as readBody reads it. It returns what it found, what it
|
||||||
// read of body, and the error that ended the reading early, if one did.
|
// read of body, which the app is still to be sent, and the error that
|
||||||
|
// ended the reading early, if one did.
|
||||||
func (c *CoreRuleSet) Inspect(
|
func (c *CoreRuleSet) Inspect(
|
||||||
r *http.Request, client netip.Addr, body io.Reader,
|
r *http.Request, client netip.Addr, body io.Reader,
|
||||||
) (Result, []byte, error) {
|
) (Result, []byte, error) {
|
||||||
|
|||||||
+10
-38
@@ -613,57 +613,29 @@ func TestBodyCorazaCannotParseIsAMatch(t *testing.T) {
|
|||||||
|
|
||||||
// The multipart bodies Coraza cannot parse fail its strict checks too:
|
// The multipart bodies Coraza cannot parse fail its strict checks too:
|
||||||
// one whose type names its boundary twice, and one with a part header
|
// one whose type names its boundary twice, and one with a part header
|
||||||
// that has no colon, before the attack. They do so padded past the
|
// that has no colon, before the attack.
|
||||||
// limit too, which cuts them in the padding.
|
body = field("q", injection) + end
|
||||||
noColon := "--b\r\nContent-Disposition form-data; name=\"a\"\r\n\r\n1\r\n"
|
wantBody(t, crs, post(multipart+"; boundary=c"), body, matched(900440, 900450),
|
||||||
pad := field("pad", strings.Repeat("a", bodyLimit))
|
body)
|
||||||
|
|
||||||
for _, tc := range []struct{ header, head string }{
|
body = "--b\r\nContent-Disposition form-data; name=\"a\"\r\n\r\n1\r\n" +
|
||||||
{multipart + "; boundary=c", ""},
|
field("q", injection) + end
|
||||||
{multipart, noColon},
|
wantBody(t, crs, post(multipart), body, matched(900440, 900450), body)
|
||||||
} {
|
|
||||||
body = tc.head + field("q", injection) + end
|
|
||||||
wantBody(t, crs, post(tc.header), body, matched(900440, 900450), body)
|
|
||||||
|
|
||||||
body = tc.head + field("q", injection) + pad + end
|
|
||||||
wantBody(t, crs, post(tc.header), body, matched(900440, 900450),
|
|
||||||
body[:bodyLimit+1])
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMultipartBodyCutBeforeAPartHeadersColonIsAMatch(t *testing.T) {
|
func TestMultipartBodyCutInsideAPartHeaderIsNotAMatch(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// The limit falls in the middle of the name of the second part's
|
// The limit falls in the middle of the name of the second part's
|
||||||
// header, which Coraza, reading up to the limit, cannot tell from a
|
// header, which Coraza, reading up to the limit, finds without a colon.
|
||||||
// header without a colon.
|
|
||||||
cut := "--b\r\nContent-Di"
|
cut := "--b\r\nContent-Di"
|
||||||
first := field("pad", strings.Repeat("a", bodyLimit-len(field("pad", ""))-len(cut)))
|
first := field("pad", strings.Repeat("a", bodyLimit-len(field("pad", ""))-len(cut)))
|
||||||
body := first + cut + "sposition: form-data; name=\"q\"\r\n\r\n1\r\n" + end
|
body := first + cut + "sposition: form-data; name=\"q\"\r\n\r\n1\r\n" + end
|
||||||
|
|
||||||
wantBody(t, readingBodies(t), post(multipart), body, matched(900440, 900450),
|
wantBody(t, readingBodies(t), post(multipart), body, waf.Result{},
|
||||||
body[:bodyLimit+1])
|
body[:bodyLimit+1])
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMultipartBodyCutBeforeALineFeedInAPartsHeadersIsAMatch(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
crs := readingBodies(t)
|
|
||||||
headerLine := "--b\r\nContent-Disposition: form-data; name=\"q\"\r"
|
|
||||||
|
|
||||||
// The limit falls between the carriage return and the line feed that
|
|
||||||
// end the second part's header line, and then between those that end
|
|
||||||
// the empty line after it. Coraza, reading up to the limit, takes the
|
|
||||||
// line ending in a lone carriage return for a malformed header.
|
|
||||||
for _, cut := range []int{len(headerLine), len(headerLine + "\n\r")} {
|
|
||||||
first := field("pad", strings.Repeat("a", bodyLimit-len(field("pad", ""))-cut))
|
|
||||||
body := first + field("q", "1") + end
|
|
||||||
|
|
||||||
wantBody(t, crs, post(multipart), body, matched(900440, 900450),
|
|
||||||
body[:bodyLimit+1])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestCorazaWritesNoFile is not parallel, since it sets TMPDIR, the
|
// TestCorazaWritesNoFile is not parallel, since it sets TMPDIR, the
|
||||||
// system's temporary directory, for the whole test process.
|
// system's temporary directory, for the whole test process.
|
||||||
func TestCorazaWritesNoFile(t *testing.T) {
|
func TestCorazaWritesNoFile(t *testing.T) {
|
||||||
|
|||||||
+54
-163
@@ -1,27 +1,15 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/example-app: build the image, and on it the example app in
|
# script/example-app: build the image, and on it the example app in
|
||||||
# deploy/example-app, then run the app's container, with rproxy in it, and
|
# deploy/example-app, then run the app's container with a volume for the
|
||||||
# servewaf in a container of its own with a volume for the state files. It
|
# state files and check that the health check passes, that a request is
|
||||||
# checks:
|
# served through smallwebwaf, that a second one in a minute bans the
|
||||||
# - that the health checks pass;
|
# client, that a probe for /.env bans another client, which its next
|
||||||
# - that a request is served through rproxy, and a second one in a minute
|
# request bans for good, that `sv stop` stops smallwebwaf in order, that
|
||||||
# bans the client;
|
# `docker stop` stops the container without having to kill it, and that
|
||||||
# - that a probe for /.env bans another client, which its next request bans
|
# a new container on the same volume still refuses the banned client. The
|
||||||
# for good;
|
# containers run with SWWAF_LOOKUP_SOURCE=off, so that no address is sent
|
||||||
# - that only servewaf's container changes the state directory's owner;
|
# to GeoJS. The containers, the volume and both images are removed however
|
||||||
# - that the app still answers through rproxy with servewaf stopped, and
|
# the script ends. Building the app needs network access, for nixpkgs'
|
||||||
# that once servewaf has started again, bans apply again;
|
|
||||||
# - that `sv stop` stops rproxy in order, and `docker stop` stops each
|
|
||||||
# container without having to kill it;
|
|
||||||
# - that a new servewaf container on the same volume still refuses the banned
|
|
||||||
# client;
|
|
||||||
# - that a servewaf container with its state files on a new volume mounted at
|
|
||||||
# the directory SWWAF_STATE_DIR_FILE names becomes healthy;
|
|
||||||
# - that one whose SWWAF_STATE_DIR does not exist logs servewaf's message
|
|
||||||
# about it.
|
|
||||||
# servewaf runs with SWWAF_LOOKUP_SOURCE=off, so that no address is sent to
|
|
||||||
# GeoJS. The containers, the network, the volumes and both images are removed
|
|
||||||
# however the script ends. Building the app needs network access, for nixpkgs'
|
|
||||||
# binary cache. script/check does not run this.
|
# binary cache. script/check does not run this.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
@@ -33,28 +21,18 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
|||||||
NAME="$("$SCRIPT_DIR/projectname")-example-$$"
|
NAME="$("$SCRIPT_DIR/projectname")-example-$$"
|
||||||
IMAGE="$NAME-base"
|
IMAGE="$NAME-base"
|
||||||
APP_IMAGE="$NAME-app"
|
APP_IMAGE="$NAME-app"
|
||||||
NETWORK="$NAME"
|
CONTAINER="$NAME"
|
||||||
APP="$NAME-app-container"
|
|
||||||
SERVEWAF="$NAME-servewaf"
|
|
||||||
VOLUME="$NAME-state"
|
VOLUME="$NAME-state"
|
||||||
# The volumes of the container whose state directory SWWAF_STATE_DIR_FILE
|
|
||||||
# names: its state files, and the file that names their directory.
|
|
||||||
MOVED_VOLUME="$NAME-moved-state"
|
|
||||||
SETTINGS_VOLUME="$NAME-settings"
|
|
||||||
|
|
||||||
cleanup() {
|
cleanup() {
|
||||||
docker rm --force "$APP" "$SERVEWAF" >/dev/null 2>&1 || true
|
docker rm --force "$CONTAINER" >/dev/null 2>&1 || true
|
||||||
docker network rm "$NETWORK" >/dev/null 2>&1 || true
|
docker volume rm --force "$VOLUME" >/dev/null 2>&1 || true
|
||||||
docker volume rm --force "$VOLUME" "$MOVED_VOLUME" "$SETTINGS_VOLUME" \
|
|
||||||
>/dev/null 2>&1 || true
|
|
||||||
docker rmi --force "$APP_IMAGE" "$IMAGE" >/dev/null 2>&1 || true
|
docker rmi --force "$APP_IMAGE" "$IMAGE" >/dev/null 2>&1 || true
|
||||||
}
|
}
|
||||||
|
|
||||||
fail() {
|
fail() {
|
||||||
echo "example-app: $*; the app container's output:" >&2
|
echo "example-app: $*; the container's output:" >&2
|
||||||
docker logs "$APP" >&2 || true
|
docker logs "$CONTAINER" >&2 || true
|
||||||
echo "example-app: servewaf's output:" >&2
|
|
||||||
docker logs "$SERVEWAF" >&2 || true
|
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -71,73 +49,31 @@ wait_for() {
|
|||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
# healthy <container>: docker says the container's health check passes.
|
|
||||||
healthy() {
|
healthy() {
|
||||||
status="$(docker inspect --format '{{.State.Health.Status}}' "$1")"
|
status="$(docker inspect --format '{{.State.Health.Status}}' "$CONTAINER")"
|
||||||
[ "$status" = healthy ]
|
[ "$status" = healthy ]
|
||||||
}
|
}
|
||||||
|
|
||||||
# logged <container> <text>...: a line of the container's output holds every
|
# logged <text>...: a line of the container's output holds every text,
|
||||||
# text, in any order.
|
# in any order.
|
||||||
logged() {
|
logged() {
|
||||||
lines="$(docker logs "$1" 2>&1)"
|
lines="$(docker logs "$CONTAINER" 2>&1)"
|
||||||
shift
|
|
||||||
for text in "$@"; do
|
for text in "$@"; do
|
||||||
lines="$(printf '%s\n' "$lines" | grep -F "$text")" || return 1
|
lines="$(printf '%s\n' "$lines" | grep -F "$text")" || return 1
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
# start_servewaf: run servewaf from the image, with the state files on the
|
# start_container: run the app's container, with the state files on the
|
||||||
# volume, a rate limit of one request a minute and no client looked up, and wait
|
# volume, a rate limit of one request a minute and no client looked up,
|
||||||
# until it is healthy.
|
# and wait until it is healthy.
|
||||||
start_servewaf() {
|
start_container() {
|
||||||
docker run --detach --name "$SERVEWAF" --network "$NETWORK" \
|
docker run --detach --name "$CONTAINER" --publish 127.0.0.1::8080 \
|
||||||
--volume "$VOLUME:/var/lib/smallwebwaf" \
|
--volume "$VOLUME:/var/lib/smallwebwaf" \
|
||||||
--env SMALLWEBWAF_COMMAND=servewaf \
|
|
||||||
--env SWWAF_RATE_LIMIT_PER_MINUTE=1 \
|
--env SWWAF_RATE_LIMIT_PER_MINUTE=1 \
|
||||||
--env SWWAF_LOOKUP_SOURCE=off \
|
--env SWWAF_LOOKUP_SOURCE=off \
|
||||||
"$IMAGE" >/dev/null
|
|
||||||
wait_for "servewaf's health check did not pass" healthy "$SERVEWAF"
|
|
||||||
}
|
|
||||||
|
|
||||||
# start_app: run the app's container, with rproxy in it, and wait until it is
|
|
||||||
# healthy.
|
|
||||||
start_app() {
|
|
||||||
docker run --detach --name "$APP" --network "$NETWORK" --publish 127.0.0.1::8080 \
|
|
||||||
--env SWRP_WAF_BACKEND="http://$SERVEWAF:8080" \
|
|
||||||
"$APP_IMAGE" >/dev/null
|
"$APP_IMAGE" >/dev/null
|
||||||
wait_for "the app's health check did not pass" healthy "$APP"
|
wait_for "the health check did not pass" healthy
|
||||||
address="$(docker port "$APP" 8080/tcp)"
|
address="$(docker port "$CONTAINER" 8080/tcp)"
|
||||||
}
|
|
||||||
|
|
||||||
# start_moved_servewaf: run servewaf with the state files on a new volume at
|
|
||||||
# /srv/state, which SWWAF_STATE_DIR_FILE names in a file only the smallwebwaf
|
|
||||||
# user can read, as under "Settings given as files" in docs/settings.md, and
|
|
||||||
# wait until it is healthy. servewaf can write the new volume only once the run
|
|
||||||
# script has given it to that user.
|
|
||||||
start_moved_servewaf() {
|
|
||||||
docker volume create "$MOVED_VOLUME" >/dev/null
|
|
||||||
docker volume create "$SETTINGS_VOLUME" >/dev/null
|
|
||||||
docker run --rm --volume "$SETTINGS_VOLUME:/settings" \
|
|
||||||
--entrypoint bash "$IMAGE" -c 'echo /srv/state > /settings/state-dir &&
|
|
||||||
chown smallwebwaf:smallwebwaf /settings/state-dir &&
|
|
||||||
chmod 0400 /settings/state-dir'
|
|
||||||
docker run --detach --name "$SERVEWAF" --network "$NETWORK" \
|
|
||||||
--volume "$MOVED_VOLUME:/srv/state" \
|
|
||||||
--volume "$SETTINGS_VOLUME:/etc/smallwebwaf/settings:ro" \
|
|
||||||
--env SMALLWEBWAF_COMMAND=servewaf \
|
|
||||||
--env SWWAF_STATE_DIR_FILE=/etc/smallwebwaf/settings/state-dir \
|
|
||||||
--env SWWAF_LOOKUP_SOURCE=off \
|
|
||||||
"$IMAGE" >/dev/null
|
|
||||||
wait_for "servewaf's health check did not pass" healthy "$SERVEWAF"
|
|
||||||
}
|
|
||||||
|
|
||||||
# status_from <client> <path>: print the status a request for path from client,
|
|
||||||
# as X-Forwarded-For names it, gets. rproxy believes the header from docker's
|
|
||||||
# gateway, a private address.
|
|
||||||
status_from() {
|
|
||||||
curl --silent --output /dev/null --write-out '%{http_code}' \
|
|
||||||
--max-time 10 --header "X-Forwarded-For: $1" "http://$address$2" || true
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# refused: a request to the container gets 403, SWWAF_BAN_RESPONSE's
|
# refused: a request to the container gets 403, SWWAF_BAN_RESPONSE's
|
||||||
@@ -148,14 +84,13 @@ refused() {
|
|||||||
[ "$code" = 403 ]
|
[ "$code" = 403 ]
|
||||||
}
|
}
|
||||||
|
|
||||||
# refused_from <client> <path>: a request for path from client gets 403.
|
# refused_from <client> <path>: a request for path from client, as
|
||||||
|
# X-Forwarded-For names it, gets 403. smallwebwaf believes the header
|
||||||
|
# from docker's gateway, a private address.
|
||||||
refused_from() {
|
refused_from() {
|
||||||
[ "$(status_from "$1" "$2")" = 403 ]
|
code="$(curl --silent --output /dev/null --write-out '%{http_code}' \
|
||||||
}
|
--max-time 10 --header "X-Forwarded-For: $1" "http://$address$2")" || true
|
||||||
|
[ "$code" = 403 ]
|
||||||
# owner_of_state_dir <container>: print who owns /var/lib/smallwebwaf there.
|
|
||||||
owner_of_state_dir() {
|
|
||||||
docker exec "$1" stat --format '%U' /var/lib/smallwebwaf
|
|
||||||
}
|
}
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
@@ -167,88 +102,44 @@ main() {
|
|||||||
docker build --no-cache --build-arg SMALLWEBWAF_IMAGE="$IMAGE" \
|
docker build --no-cache --build-arg SMALLWEBWAF_IMAGE="$IMAGE" \
|
||||||
-t "$APP_IMAGE" deploy/example-app
|
-t "$APP_IMAGE" deploy/example-app
|
||||||
|
|
||||||
docker network create "$NETWORK" >/dev/null
|
|
||||||
docker volume create "$VOLUME" >/dev/null
|
docker volume create "$VOLUME" >/dev/null
|
||||||
start_servewaf
|
start_container
|
||||||
start_app
|
echo "example-app: the health check passes"
|
||||||
echo "example-app: the health checks pass"
|
|
||||||
|
|
||||||
[ "$(owner_of_state_dir "$SERVEWAF")" = smallwebwaf ] ||
|
|
||||||
fail "servewaf's container did not give the state directory to smallwebwaf"
|
|
||||||
[ "$(owner_of_state_dir "$APP")" = root ] ||
|
|
||||||
fail "the app's container changed the state directory's owner"
|
|
||||||
echo "example-app: only servewaf's container changes the state directory's owner"
|
|
||||||
|
|
||||||
page="$(curl --fail --silent --show-error --max-time 10 "http://$address/")" ||
|
page="$(curl --fail --silent --show-error --max-time 10 "http://$address/")" ||
|
||||||
fail "no answer on port 8080"
|
fail "no answer on port 8080"
|
||||||
[ "$page" = "hello from the example app" ] || fail "port 8080 answered $page"
|
[ "$page" = "hello from the example app" ] || fail "port 8080 answered $page"
|
||||||
wait_for "servewaf logged no request it allowed" \
|
wait_for "smallwebwaf logged no request it forwarded" logged '"action":"forward"'
|
||||||
logged "$SERVEWAF" '"action":"forward"' '"servewaf_line":"report"'
|
echo "example-app: smallwebwaf passes a request to the app and its answer back"
|
||||||
echo "example-app: rproxy passes a request to the app and its answer back"
|
|
||||||
|
|
||||||
refused || fail "a second request in a minute was not refused"
|
refused || fail "a second request in a minute was not refused"
|
||||||
wait_for "servewaf logged no ban" logged "$SERVEWAF" '"action":"rate_limited"'
|
wait_for "smallwebwaf logged no ban" logged '"action":"rate_limited"'
|
||||||
echo "example-app: a second request in a minute bans the client"
|
echo "example-app: a second request in a minute bans the client"
|
||||||
|
|
||||||
refused_from 203.0.113.9 /.env || fail "a probe for /.env was not refused"
|
refused_from 203.0.113.9 /.env || fail "a probe for /.env was not refused"
|
||||||
wait_for "servewaf logged no ban for the probe" \
|
wait_for "smallwebwaf logged no ban for the probe" \
|
||||||
logged "$SERVEWAF" '"action":"banned"' '"rule_ids":["env-file"]'
|
logged '"action":"banned"' '"rule_ids":["env-file"]'
|
||||||
refused_from 203.0.113.9 / || fail "the client of the probe was let through"
|
refused_from 203.0.113.9 / || fail "the client of the probe was let through"
|
||||||
wait_for "the client's next request did not make its ban permanent" \
|
wait_for "the client's next request did not make its ban permanent" \
|
||||||
logged "$SERVEWAF" '"ban_expires":"permanent"'
|
logged '"ban_expires":"permanent"'
|
||||||
echo "example-app: a probe for /.env bans the client, its next request for good"
|
echo "example-app: a probe for /.env bans the client, its next request for good"
|
||||||
|
|
||||||
docker exec "$SERVEWAF" sv stop smallwebwaf >/dev/null ||
|
docker exec "$CONTAINER" sv stop smallwebwaf >/dev/null ||
|
||||||
fail "sv stop smallwebwaf failed in servewaf's container"
|
fail "sv stop smallwebwaf failed"
|
||||||
wait_for "servewaf did not stop in order" logged "$SERVEWAF" '"msg":"stopped"'
|
wait_for "smallwebwaf did not stop in order" logged '"msg":"stopped"'
|
||||||
[ "$(status_from 203.0.113.9 /)" = 200 ] ||
|
echo "example-app: sv stop stops smallwebwaf in order"
|
||||||
fail "with servewaf stopped, the app did not answer through rproxy"
|
|
||||||
wait_for "rproxy logged no request SWRP_DEFAULT_DISPOSITION decided" \
|
|
||||||
logged "$APP" '"default_disposition":true'
|
|
||||||
docker exec "$APP" /usr/local/bin/smallwebwaf healthcheck ||
|
|
||||||
fail "with servewaf stopped, the app's health check failed"
|
|
||||||
echo "example-app: with servewaf stopped, the app answers through rproxy"
|
|
||||||
|
|
||||||
docker exec "$SERVEWAF" sv start smallwebwaf >/dev/null ||
|
docker stop "$CONTAINER" >/dev/null
|
||||||
fail "sv start smallwebwaf failed in servewaf's container"
|
status="$(docker inspect --format '{{.State.ExitCode}}' "$CONTAINER")"
|
||||||
wait_for "once servewaf had started again, the ban did not apply" \
|
[ "$status" = 0 ] || fail "docker stop left exit status $status"
|
||||||
refused_from 203.0.113.9 /
|
echo "example-app: docker stop stops the container in order"
|
||||||
echo "example-app: once servewaf has started again, bans apply again"
|
|
||||||
|
|
||||||
docker exec "$APP" sv stop smallwebwaf >/dev/null ||
|
docker rm "$CONTAINER" >/dev/null
|
||||||
fail "sv stop smallwebwaf failed in the app's container"
|
start_container
|
||||||
wait_for "rproxy did not stop in order" logged "$APP" '"msg":"stopped"'
|
refused || fail "the new container let the banned client through"
|
||||||
echo "example-app: sv stop stops rproxy in order"
|
wait_for "smallwebwaf logged no request refused under the ban" \
|
||||||
|
logged '"action":"banned"'
|
||||||
for container in "$APP" "$SERVEWAF"; do
|
echo "example-app: a new container on the same volume keeps the ban"
|
||||||
docker stop "$container" >/dev/null
|
|
||||||
status="$(docker inspect --format '{{.State.ExitCode}}' "$container")"
|
|
||||||
[ "$status" = 0 ] || fail "docker stop left exit status $status"
|
|
||||||
done
|
|
||||||
echo "example-app: docker stop stops each container in order"
|
|
||||||
|
|
||||||
docker rm "$APP" "$SERVEWAF" >/dev/null
|
|
||||||
start_servewaf
|
|
||||||
start_app
|
|
||||||
refused || fail "the new servewaf container let the banned client through"
|
|
||||||
wait_for "servewaf logged no request refused under the ban" \
|
|
||||||
logged "$SERVEWAF" '"action":"banned"'
|
|
||||||
echo "example-app: a new servewaf container on the same volume keeps the ban"
|
|
||||||
|
|
||||||
docker rm --force "$SERVEWAF" >/dev/null
|
|
||||||
start_moved_servewaf
|
|
||||||
echo "example-app: the run script gives the state files' directory named" \
|
|
||||||
"by SWWAF_STATE_DIR_FILE to the smallwebwaf user"
|
|
||||||
|
|
||||||
docker rm --force "$SERVEWAF" >/dev/null
|
|
||||||
docker run --detach --name "$SERVEWAF" --network "$NETWORK" \
|
|
||||||
--env SMALLWEBWAF_COMMAND=servewaf \
|
|
||||||
--env SWWAF_STATE_DIR=/srv/missing \
|
|
||||||
--env SWWAF_LOOKUP_SOURCE=off \
|
|
||||||
"$IMAGE" >/dev/null
|
|
||||||
wait_for "servewaf did not report the missing state directory" \
|
|
||||||
logged "$SERVEWAF" "SWWAF_STATE_DIR cannot be written"
|
|
||||||
echo "example-app: servewaf reports a state directory that does not exist"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
@@ -1,350 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# script/gitea-requests: build the image and run gitea 1.27.3 behind rproxy
|
|
||||||
# and servewaf, three containers on one network. Every setting is at its
|
|
||||||
# default but these: SWRP_UPSTREAM_URL names gitea's container, SWRP_WAF_BACKEND
|
|
||||||
# names servewaf's, and SWWAF_LOG_REQUEST_HEADERS adds the header that labels a
|
|
||||||
# request. Then send through rproxy the gitea requests that reading gitea's
|
|
||||||
# source found the Core Rule Set refuses, which docs/gitea.md records, and a
|
|
||||||
# short sample of gitea's ordinary use: signing in, pages of files and commits,
|
|
||||||
# a diff, the API, a push, clone and fetch over HTTP, and two container image
|
|
||||||
# pushes. Last, print each request servewaf logged, with what it did with it and
|
|
||||||
# the ids of the Core Rule Set's rules that matched it.
|
|
||||||
# Every request comes from one client, docker's gateway, so more than
|
|
||||||
# SWWAF_ERROR_BURST_THRESHOLD, 30, refusals in a minute would ban it, and
|
|
||||||
# the requests after would show as banned. The containers, the network,
|
|
||||||
# the images and the files the script makes are removed however it ends;
|
|
||||||
# gitea's own image stays, as the base images of a build do. It needs
|
|
||||||
# network access, and a docker daemon that pushes to a registry on
|
|
||||||
# localhost over plain HTTP, as docker does by default. script/check does
|
|
||||||
# not run this.
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
|
||||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
|
||||||
|
|
||||||
# gitea 1.27.3, 2026-08-29
|
|
||||||
GITEA_IMAGE="gitea/gitea@sha256:87a67ee09d3ae0d1df5fda5dcda3e2a1f9236a45b0a59025d6e00e46adc43bef"
|
|
||||||
|
|
||||||
# Named after this run, so that runs in other clones on the same host
|
|
||||||
# never touch each other's.
|
|
||||||
NAME="$("$SCRIPT_DIR/projectname")-gitea-$$"
|
|
||||||
IMAGE="$NAME-image"
|
|
||||||
NETWORK="$NAME"
|
|
||||||
RPROXY="$NAME-rproxy"
|
|
||||||
SERVEWAF="$NAME-servewaf"
|
|
||||||
GITEA="$NAME-gitea"
|
|
||||||
|
|
||||||
# gitea's one user, who owns the repository and the container images.
|
|
||||||
PASSWORD="owner-password"
|
|
||||||
|
|
||||||
BROWSER="Mozilla/5.0 (X11; Linux x86_64; rv:131.0) Gecko/20100101 Firefox/131.0"
|
|
||||||
|
|
||||||
# The header that labels a request in the report. rproxy makes every request's
|
|
||||||
# id itself, so a label cannot be the X-Request-ID.
|
|
||||||
LABEL_HEADER="X-Case"
|
|
||||||
|
|
||||||
# The default of SWWAF_LOG_REQUEST_HEADERS, and the header above.
|
|
||||||
LOGGED_HEADERS="accept,accept-language,accept-encoding,content-type,origin,range,x-case"
|
|
||||||
|
|
||||||
# The files the script makes, and rproxy's address once it runs, localhost and
|
|
||||||
# the port docker publishes it on: a host name, since the Core Rule Set takes a
|
|
||||||
# Host that is an IP address for a warning (920350).
|
|
||||||
WORK=""
|
|
||||||
address=""
|
|
||||||
|
|
||||||
cleanup() {
|
|
||||||
# So that a signal cannot stop it partway through.
|
|
||||||
trap '' HUP INT TERM
|
|
||||||
docker rm --force --volumes "$RPROXY" "$SERVEWAF" "$GITEA" >/dev/null 2>&1 || true
|
|
||||||
docker network rm "$NETWORK" >/dev/null 2>&1 || true
|
|
||||||
docker rmi --force "$IMAGE" >/dev/null 2>&1 || true
|
|
||||||
if [ -n "$address" ]; then
|
|
||||||
docker rmi --force "$address/owner/python3-app:1" "$address/owner/app:1" \
|
|
||||||
>/dev/null 2>&1 || true
|
|
||||||
fi
|
|
||||||
rm -rf "$WORK"
|
|
||||||
}
|
|
||||||
|
|
||||||
fail() {
|
|
||||||
echo "gitea-requests: $*" >&2
|
|
||||||
report
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
# wait_for <what fails> <command>...: run the command every second until
|
|
||||||
# it succeeds, for at most a minute.
|
|
||||||
wait_for() {
|
|
||||||
failure="$1"
|
|
||||||
shift
|
|
||||||
tries=0
|
|
||||||
until "$@"; do
|
|
||||||
tries=$((tries + 1))
|
|
||||||
[ "$tries" -lt 60 ] || fail "$failure"
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
# healthy <container>: docker says the container's health check passes.
|
|
||||||
healthy() {
|
|
||||||
status="$(docker inspect --format '{{.State.Health.Status}}' "$1")"
|
|
||||||
[ "$status" = healthy ]
|
|
||||||
}
|
|
||||||
|
|
||||||
# logged <text>...: a line of servewaf's output holds every text, in any order.
|
|
||||||
logged() {
|
|
||||||
lines="$(docker logs "$SERVEWAF" 2>&1)"
|
|
||||||
for text in "$@"; do
|
|
||||||
lines="$(printf '%s\n' "$lines" | grep -F "$text")" || return 1
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
# send <label> <path> <curl option>...: send a request for path to rproxy, with
|
|
||||||
# label in LABEL_HEADER, which the report gives, and keep the body of the answer
|
|
||||||
# in $WORK/answer. The Core Rule Set reads that header too, so a label names no
|
|
||||||
# file the cases name.
|
|
||||||
send() {
|
|
||||||
label="$1"
|
|
||||||
path="$2"
|
|
||||||
shift 2
|
|
||||||
curl --silent --show-error --max-time 30 --output "$WORK/answer" \
|
|
||||||
--header "$LABEL_HEADER: $label" "$@" "http://$address$path"
|
|
||||||
}
|
|
||||||
|
|
||||||
# page <label> <path> <curl option>...: send the request as a browser
|
|
||||||
# does, with the cookies gitea has set. Its Referer is the repository's
|
|
||||||
# page, which links to the others, and never a page the Core Rule Set
|
|
||||||
# refused: its address would have the next request refused too.
|
|
||||||
page() {
|
|
||||||
label="$1"
|
|
||||||
path="$2"
|
|
||||||
shift 2
|
|
||||||
send "$label" "$path" --compressed --user-agent "$BROWSER" \
|
|
||||||
--header "Accept: text/html,application/xhtml+xml,*/*;q=0.8" \
|
|
||||||
--header "Accept-Language: en-US,en;q=0.5" \
|
|
||||||
--cookie "$WORK/cookies" --cookie-jar "$WORK/cookies" \
|
|
||||||
--referer "http://$address/owner/repo" "$@"
|
|
||||||
}
|
|
||||||
|
|
||||||
# api <label> <path> <curl option>...: send the request as a client of
|
|
||||||
# gitea's API does, as the owner.
|
|
||||||
api() {
|
|
||||||
label="$1"
|
|
||||||
path="$2"
|
|
||||||
shift 2
|
|
||||||
send "$label" "$path" --user "owner:$PASSWORD" \
|
|
||||||
--header "Accept: application/json" "$@"
|
|
||||||
}
|
|
||||||
|
|
||||||
# repo_git <git argument>...: run git as the owner, without the host's
|
|
||||||
# own git settings, so that nothing it does is kept outside $WORK.
|
|
||||||
repo_git() {
|
|
||||||
GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 git \
|
|
||||||
-c user.name=owner -c user.email=owner@example.invalid "$@"
|
|
||||||
}
|
|
||||||
|
|
||||||
# lines: forty lines of text, enough for a diff to hide some of them.
|
|
||||||
lines() {
|
|
||||||
i=1
|
|
||||||
while [ "$i" -le 40 ]; do
|
|
||||||
echo "line $i"
|
|
||||||
i=$((i + 1))
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
# make_repository: create the repository owner/repo, holding a file of
|
|
||||||
# each name the cases name, and push it, which is the push over HTTP of
|
|
||||||
# the sample. The second commit adds a line at the end of Runtime.java,
|
|
||||||
# so that its diff hides the lines above.
|
|
||||||
make_repository() {
|
|
||||||
status="$(api create-repository /api/v1/user/repos \
|
|
||||||
--header "Content-Type: application/json" --data '{"name":"repo"}' \
|
|
||||||
--write-out '%{http_code}')"
|
|
||||||
[ "$status" = 201 ] || fail "creating the repository was answered $status"
|
|
||||||
|
|
||||||
dir="$WORK/repo"
|
|
||||||
repo_git init --quiet --initial-branch=main "$dir"
|
|
||||||
for file in \
|
|
||||||
src/java.base/share/classes/java/lang/Runtime.java \
|
|
||||||
java.lang.String.md \
|
|
||||||
META-INF/services/org.apache.commons.logging.LogFactory \
|
|
||||||
document.write.js \
|
|
||||||
process.env.d.ts \
|
|
||||||
test/fixtures/__proto__.json \
|
|
||||||
install-script.sh \
|
|
||||||
src/backend/catalog/information_schema.sql \
|
|
||||||
.../notes.txt \
|
|
||||||
README.md; do
|
|
||||||
mkdir -p "$dir/$(dirname "$file")"
|
|
||||||
lines >"$dir/$file"
|
|
||||||
repo_git -C "$dir" add "$file"
|
|
||||||
done
|
|
||||||
repo_git -C "$dir" commit --quiet --message "Add the files"
|
|
||||||
echo "line 41" >>"$dir/src/java.base/share/classes/java/lang/Runtime.java"
|
|
||||||
repo_git -C "$dir" commit --quiet --all --message "Add a line"
|
|
||||||
|
|
||||||
repo_git -C "$dir" -c "http.extraHeader=$LABEL_HEADER: git-push" push --quiet \
|
|
||||||
"http://owner:$PASSWORD@$address/owner/repo.git" main ||
|
|
||||||
fail "the push of the repository failed"
|
|
||||||
}
|
|
||||||
|
|
||||||
# browse: sign in, then open pages as a person does. The cases are those
|
|
||||||
# of https://git.eeqj.de/sneak/smallwebwaf/issues/30.
|
|
||||||
browse() {
|
|
||||||
# Case: gitea sends a visitor who is not signed in to the sign-in
|
|
||||||
# page, with the page asked for in redirect_to, when that page needs
|
|
||||||
# signing in, as a file's edit page does.
|
|
||||||
page issue-30-case /owner/repo/_edit/main/document.write.js --location
|
|
||||||
|
|
||||||
page sign-in /user/login
|
|
||||||
status="$(page sign-in /user/login --header "Origin: http://$address" \
|
|
||||||
--data-urlencode user_name=owner --data-urlencode "password=$PASSWORD" \
|
|
||||||
--write-out '%{http_code}')"
|
|
||||||
[ "$status" = 303 ] || fail "signing in was answered $status"
|
|
||||||
|
|
||||||
page browse /
|
|
||||||
page browse /owner/repo
|
|
||||||
page browse /owner/repo/src/branch/main/README.md
|
|
||||||
page browse /owner/repo/commits/branch/main
|
|
||||||
page browse "/owner/repo/commit/$(repo_git -C "$WORK/repo" rev-parse HEAD)"
|
|
||||||
|
|
||||||
# Case: expanding the lines the diff hides, by the link the diff's
|
|
||||||
# page gives for it.
|
|
||||||
excerpt="$(sed -n 's/.*data-fetch-url="\([^"]*blob_excerpt[^"]*\)".*/\1/p' \
|
|
||||||
"$WORK/answer" | head -n 1 | sed 's/&/\&/g')"
|
|
||||||
[ -n "$excerpt" ] || fail "the diff's page has no link to its hidden lines"
|
|
||||||
page issue-30-case "$excerpt"
|
|
||||||
|
|
||||||
# Cases: the commit graph of one file, and the pages of files whose
|
|
||||||
# path the Core Rule Set takes for an attack.
|
|
||||||
page issue-30-case "/owner/repo/graph?file=src/java.base/share/classes/java/lang/Runtime.java"
|
|
||||||
page issue-30-case /owner/repo/src/branch/main/META-INF/services/org.apache.commons.logging.LogFactory
|
|
||||||
page issue-30-case /owner/repo/src/branch/main/java.lang.String.md
|
|
||||||
page issue-30-case /owner/repo/src/branch/main/process.env.d.ts
|
|
||||||
page issue-30-case /owner/repo/src/branch/main/.../notes.txt
|
|
||||||
}
|
|
||||||
|
|
||||||
# call_api: the API's calls the cases name: the commits of a file, for a
|
|
||||||
# file of each name the Core Rule Set takes for an attack in the query
|
|
||||||
# parameter path, and the file-contents call, which takes its list of
|
|
||||||
# files as JSON in the query parameter body.
|
|
||||||
call_api() {
|
|
||||||
for file in \
|
|
||||||
src/java.base/share/classes/java/lang/Runtime.java \
|
|
||||||
java.lang.String.md \
|
|
||||||
document.write.js \
|
|
||||||
process.env.d.ts \
|
|
||||||
test/fixtures/__proto__.json \
|
|
||||||
install-script.sh \
|
|
||||||
src/backend/catalog/information_schema.sql; do
|
|
||||||
api issue-30-case /api/v1/repos/owner/repo/commits --get --data-urlencode "path=$file"
|
|
||||||
done
|
|
||||||
api issue-30-case /api/v1/repos/owner/repo/file-contents --get \
|
|
||||||
--data-urlencode 'body={"files":["src/java.base/share/classes/java/lang/Runtime.java"]}'
|
|
||||||
}
|
|
||||||
|
|
||||||
# push_images: push a container image to owner/python3-app, then the same
|
|
||||||
# image to owner/app. For the second, the case, docker asks gitea to take
|
|
||||||
# its layer from owner/python3-app, naming it in the query parameter
|
|
||||||
# from. docker sends no label, so the report gives its requests none. The image
|
|
||||||
# is one small file, imported rather than built.
|
|
||||||
push_images() {
|
|
||||||
echo "$NAME" >"$WORK/layer.txt"
|
|
||||||
tar -cf - -C "$WORK" layer.txt |
|
|
||||||
docker import - "$address/owner/python3-app:1" >/dev/null
|
|
||||||
docker tag "$address/owner/python3-app:1" "$address/owner/app:1"
|
|
||||||
|
|
||||||
# The login is written here rather than made with docker login, which
|
|
||||||
# would keep it in the host's credential helper when it has one.
|
|
||||||
mkdir "$WORK/docker"
|
|
||||||
printf '{"auths":{"%s":{"auth":"%s"}}}\n' "$address" \
|
|
||||||
"$(printf 'owner:%s' "$PASSWORD" | base64)" >"$WORK/docker/config.json"
|
|
||||||
for image in "$address/owner/python3-app:1" "$address/owner/app:1"; do
|
|
||||||
DOCKER_CONFIG="$WORK/docker" docker push --quiet "$image" >/dev/null ||
|
|
||||||
echo "gitea-requests: the push of $image failed" >&2
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
# clone_and_fetch: the clone and fetch over HTTP of the sample.
|
|
||||||
clone_and_fetch() {
|
|
||||||
repo_git -c "http.extraHeader=$LABEL_HEADER: git-clone" clone --quiet \
|
|
||||||
"http://owner:$PASSWORD@$address/owner/repo.git" "$WORK/clone" ||
|
|
||||||
fail "the clone failed"
|
|
||||||
repo_git -C "$WORK/clone" -c "http.extraHeader=$LABEL_HEADER: git-fetch" fetch --quiet ||
|
|
||||||
fail "the fetch failed"
|
|
||||||
}
|
|
||||||
|
|
||||||
# field <name> <line>: the text of the field name in line, a line of the
|
|
||||||
# request log, or nothing when it has none.
|
|
||||||
field() {
|
|
||||||
printf '%s\n' "$2" | sed -n "s/.*\"$1\":\"\([^\"]*\)\".*/\1/p"
|
|
||||||
}
|
|
||||||
|
|
||||||
# report: print a line for each request rproxy reported to servewaf: what
|
|
||||||
# servewaf did with the request, the ids of the Core Rule Set's rules that
|
|
||||||
# matched it, its label, and the request. waf_blocked is a refusal by the Core
|
|
||||||
# Rule Set, and forward a request passed on to gitea, whatever gitea answered.
|
|
||||||
report() {
|
|
||||||
docker logs "$SERVEWAF" 2>/dev/null | grep '"servewaf_line":"report"' |
|
|
||||||
sed 's/\\u0026/\&/g' >"$WORK/requests"
|
|
||||||
while IFS= read -r line; do
|
|
||||||
ids="$(printf '%s\n' "$line" | sed -n 's/.*"waf_rule_ids":\[\([0-9,]*\)\].*/\1/p')"
|
|
||||||
query="$(field query "$line")"
|
|
||||||
label="$(field x-case "$line")"
|
|
||||||
printf '%-11s %-13s %-17s %s %s%s\n' "$(field action "$line")" "${ids:--}" \
|
|
||||||
"${label:--}" "$(field method "$line")" \
|
|
||||||
"$(field path "$line")" "${query:+?$query}"
|
|
||||||
done <"$WORK/requests"
|
|
||||||
echo "gitea-requests: $(grep -c . "$WORK/requests") requests," \
|
|
||||||
"$(grep -c '"action":"waf_blocked"' "$WORK/requests") refused by the Core Rule Set"
|
|
||||||
}
|
|
||||||
|
|
||||||
main() {
|
|
||||||
cd "$ROOT"
|
|
||||||
trap cleanup EXIT
|
|
||||||
trap 'exit 1' HUP INT TERM
|
|
||||||
WORK="$(mktemp -d)"
|
|
||||||
|
|
||||||
docker build --no-cache -t "$IMAGE" .
|
|
||||||
|
|
||||||
# rproxy first, so that gitea can be given the address it is reached at,
|
|
||||||
# and gitea's image pulled before, so that gitea starts within the minute
|
|
||||||
# rproxy's health check waits for it.
|
|
||||||
docker pull --quiet "$GITEA_IMAGE" >/dev/null
|
|
||||||
docker network create "$NETWORK" >/dev/null
|
|
||||||
docker run --detach --name "$SERVEWAF" --network "$NETWORK" \
|
|
||||||
--env SMALLWEBWAF_COMMAND=servewaf \
|
|
||||||
--env SWWAF_LOG_REQUEST_HEADERS="$LOGGED_HEADERS" \
|
|
||||||
"$IMAGE" >/dev/null
|
|
||||||
wait_for "servewaf's health check did not pass" healthy "$SERVEWAF"
|
|
||||||
docker run --detach --name "$RPROXY" --network "$NETWORK" \
|
|
||||||
--publish 127.0.0.1::8080 \
|
|
||||||
--env SWRP_UPSTREAM_URL="http://$GITEA:3000" \
|
|
||||||
--env SWRP_WAF_BACKEND="http://$SERVEWAF:8080" \
|
|
||||||
"$IMAGE" >/dev/null
|
|
||||||
published="$(docker port "$RPROXY" 8080/tcp)"
|
|
||||||
address="localhost:${published##*:}"
|
|
||||||
|
|
||||||
docker run --detach --name "$GITEA" --network "$NETWORK" \
|
|
||||||
--env GITEA__security__INSTALL_LOCK=true \
|
|
||||||
--env GITEA__server__ROOT_URL="http://$address/" \
|
|
||||||
"$GITEA_IMAGE" >/dev/null
|
|
||||||
# rproxy's health check passes once gitea takes connections.
|
|
||||||
wait_for "rproxy's health check did not pass" healthy "$RPROXY"
|
|
||||||
docker exec --user git "$GITEA" gitea admin user create --admin \
|
|
||||||
--username owner --password "$PASSWORD" --email owner@example.invalid \
|
|
||||||
--must-change-password=false >/dev/null
|
|
||||||
|
|
||||||
make_repository
|
|
||||||
browse
|
|
||||||
call_api
|
|
||||||
push_images
|
|
||||||
clone_and_fetch
|
|
||||||
|
|
||||||
# The fetch's last request is the last of all; once it is logged, so
|
|
||||||
# is every other.
|
|
||||||
wait_for "servewaf logged no report of the fetch" \
|
|
||||||
logged '"x-case":"git-fetch"' '"method":"POST"' '"servewaf_line":"report"'
|
|
||||||
report
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
+5
-31
@@ -1,29 +1,14 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/run: build bin/smallwebwaf with script/build and run servewaf and
|
# script/run: build bin/smallwebwaf with script/build and run it, with
|
||||||
# rproxy on the host, with the settings in the environment, until rproxy
|
# the settings in the environment. Unless SWWAF_STATE_DIR is set, the
|
||||||
# exits or the script is told to stop. rproxy listens on port 8080 unless
|
# state files go in bin/state, beside the binary, and unless
|
||||||
# SWRP_LISTEN_ADDR says otherwise, in front of the app at SWRP_UPSTREAM_URL.
|
# SWWAF_RULES_DIR is set, the rule files are those of share/rules.d,
|
||||||
# servewaf listens on port 8090 unless SWWAF_LISTEN_ADDR says otherwise, and
|
|
||||||
# rproxy asks it there unless SWRP_WAF_BACKEND says otherwise. Unless
|
|
||||||
# SWWAF_STATE_DIR is set, the state files go in bin/state, beside the binary,
|
|
||||||
# and unless SWWAF_RULES_DIR is set, the rule files are those of share/rules.d,
|
|
||||||
# which the image ships.
|
# which the image ships.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||||
|
|
||||||
SERVEWAF=""
|
|
||||||
RPROXY=""
|
|
||||||
|
|
||||||
# stop_both: stop what runs, and wait for it, so that servewaf writes its state
|
|
||||||
# files.
|
|
||||||
stop_both() {
|
|
||||||
trap '' HUP INT TERM
|
|
||||||
kill $SERVEWAF $RPROXY 2>/dev/null || true
|
|
||||||
wait || true
|
|
||||||
}
|
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
"$SCRIPT_DIR/build"
|
"$SCRIPT_DIR/build"
|
||||||
if [ -z "${SWWAF_STATE_DIR+set}" ]; then
|
if [ -z "${SWWAF_STATE_DIR+set}" ]; then
|
||||||
@@ -35,18 +20,7 @@ main() {
|
|||||||
SWWAF_RULES_DIR="$ROOT/share/rules.d"
|
SWWAF_RULES_DIR="$ROOT/share/rules.d"
|
||||||
export SWWAF_RULES_DIR
|
export SWWAF_RULES_DIR
|
||||||
fi
|
fi
|
||||||
SWWAF_LISTEN_ADDR="${SWWAF_LISTEN_ADDR:-:8090}"
|
exec "$ROOT/bin/smallwebwaf"
|
||||||
SWRP_WAF_BACKEND="${SWRP_WAF_BACKEND:-http://127.0.0.1:${SWWAF_LISTEN_ADDR##*:}}"
|
|
||||||
export SWWAF_LISTEN_ADDR SWRP_WAF_BACKEND
|
|
||||||
|
|
||||||
trap stop_both EXIT
|
|
||||||
trap 'exit 1' HUP INT TERM
|
|
||||||
|
|
||||||
"$ROOT/bin/smallwebwaf" servewaf &
|
|
||||||
SERVEWAF=$!
|
|
||||||
"$ROOT/bin/smallwebwaf" rproxy &
|
|
||||||
RPROXY=$!
|
|
||||||
wait "$RPROXY"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
+7
-43
@@ -2,51 +2,15 @@
|
|||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
# runit's run script for smallwebwaf, run again whenever smallwebwaf
|
# runit's run script for smallwebwaf, run again whenever smallwebwaf
|
||||||
# exits; the wait spaces out the restarts. SMALLWEBWAF_COMMAND names the
|
# exits; the wait spaces out the restarts. The state directory and every
|
||||||
# command to run: rproxy when it is unset or empty, or servewaf. With any
|
# file in it are given to the smallwebwaf user, so that a volume mounted
|
||||||
# other value the script starts nothing and stops with a message naming it.
|
# there needs no change of owner; chown -R changes a symbolic link itself,
|
||||||
# exec, so that the signal `sv stop` sends reaches smallwebwaf itself.
|
# never what it points to. exec, so that the signal `sv stop` sends
|
||||||
|
# reaches smallwebwaf itself.
|
||||||
main() {
|
main() {
|
||||||
sleep 1
|
sleep 1
|
||||||
local command="${SMALLWEBWAF_COMMAND:-rproxy}"
|
chown -R smallwebwaf:smallwebwaf "${SWWAF_STATE_DIR:-/var/lib/smallwebwaf}"
|
||||||
case "$command" in
|
exec chpst -u smallwebwaf:smallwebwaf /usr/local/bin/smallwebwaf
|
||||||
rproxy) ;;
|
|
||||||
servewaf) give_state_dir_to_smallwebwaf ;;
|
|
||||||
*)
|
|
||||||
printf 'SMALLWEBWAF_COMMAND: "%s" is not rproxy or servewaf\n' \
|
|
||||||
"$command" >&2
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
exec chpst -u smallwebwaf:smallwebwaf /usr/local/bin/smallwebwaf "$command"
|
|
||||||
}
|
|
||||||
|
|
||||||
# give_state_dir_to_smallwebwaf gives the state directory and every file in
|
|
||||||
# it to the smallwebwaf user, so that a volume mounted there needs no change
|
|
||||||
# of owner; chown -R changes a symbolic link itself, never what it points
|
|
||||||
# to. When SWWAF_STATE_DIR, in either form, cannot be read or is not the
|
|
||||||
# absolute path of a directory, no owner is changed, and servewaf starts and
|
|
||||||
# stops with a message naming the setting. A change of owner that fails, as
|
|
||||||
# on a volume mounted read-only, does not end the script either: servewaf
|
|
||||||
# reports a directory it cannot write.
|
|
||||||
give_state_dir_to_smallwebwaf() {
|
|
||||||
local dir
|
|
||||||
if dir="$(state_dir)" && [[ "$dir" == /* && -d "$dir" ]]; then
|
|
||||||
chown -R smallwebwaf:smallwebwaf "$dir" || true
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# state_dir prints SWWAF_STATE_DIR, /var/lib/smallwebwaf when it is unset,
|
|
||||||
# or what is in the file SWWAF_STATE_DIR_FILE names, less every newline at
|
|
||||||
# its end. It fails when both are set or the file cannot be read.
|
|
||||||
state_dir() {
|
|
||||||
if [[ ! -v SWWAF_STATE_DIR_FILE ]]; then
|
|
||||||
printf '%s\n' "${SWWAF_STATE_DIR-/var/lib/smallwebwaf}"
|
|
||||||
elif [[ ! -v SWWAF_STATE_DIR ]]; then
|
|
||||||
cat -- "$SWWAF_STATE_DIR_FILE" 2>/dev/null
|
|
||||||
else
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
Reference in New Issue
Block a user