Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 sneak
|
021e5c4325 |
+1
-6
@@ -140,14 +140,10 @@ RUN mkdir /etc/sysusers.d \
|
||||
# apt also fetches the live archive's InRelease files, which change daily
|
||||
# and which the install does not use. The snapshot service is HTTPS only
|
||||
# and this image has no CA certificates yet, so this step uses the Go
|
||||
# image's. The snapshot service answers 503 at times; apt retries a
|
||||
# failed download, waiting twice as long before each retry, and this
|
||||
# step gives it five retries, about half a minute in all, where its
|
||||
# default of three gives up after seven seconds.
|
||||
# image's.
|
||||
RUN --mount=type=bind,from=builder,source=/etc/ssl/certs/ca-certificates.crt,target=/tmp/go-image-ca.crt \
|
||||
apt-get update --snapshot 20261001T000000Z \
|
||||
-o Acquire::https::CaInfo=/tmp/go-image-ca.crt \
|
||||
-o Acquire::Retries=5 \
|
||||
&& printf '%s\n' \
|
||||
'45f95ce276cdba3e41870516a130e03c58b8b7a79e9546b0efe9e526d255740c snapshot.ubuntu.com_ubuntu_20261001T000000Z_dists_resolute_InRelease' \
|
||||
'802e675dd9de4c7f3916434a95e7c1d8eec0e82886622d7805ab19a2c6fe0365 snapshot.ubuntu.com_ubuntu_20261001T000000Z_dists_resolute-updates_InRelease' \
|
||||
@@ -157,7 +153,6 @@ RUN --mount=type=bind,from=builder,source=/etc/ssl/certs/ca-certificates.crt,tar
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \
|
||||
--snapshot 20261001T000000Z \
|
||||
-o Acquire::https::CaInfo=/tmp/go-image-ca.crt \
|
||||
-o Acquire::Retries=5 \
|
||||
ca-certificates nix-bin runit \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
.PHONY: bootstrap setup test lint fmt fmt-check tidy check docker hooks build \
|
||||
run example-app gitea-requests
|
||||
run example-app
|
||||
|
||||
# Makefile targets are thin shims; the implementations live in script/
|
||||
# per the scripts-to-rule-them-all pattern (see the Entrypoints section
|
||||
# of README.md). tidy writes go.mod and go.sum as `go mod tidy` does,
|
||||
# which test checks. build and run are for working on the code by hand;
|
||||
# example-app checks the image with an app built on it; gitea-requests
|
||||
# reports the gitea requests the Core Rule Set refuses at the defaults.
|
||||
# example-app checks the image with an app built on it.
|
||||
|
||||
bootstrap:
|
||||
@script/bootstrap
|
||||
@@ -46,6 +45,3 @@ run:
|
||||
|
||||
example-app:
|
||||
@script/example-app
|
||||
|
||||
gitea-requests:
|
||||
@script/gitea-requests
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
// Command smallwebwaf is a web application firewall for one app: it runs
|
||||
// between traefik and the app and passes requests through within its limits,
|
||||
// or, as `smallwebwaf servewaf`, answers rproxy's questions about them.
|
||||
// Everything it does is in internal/smallwebwaf.
|
||||
// between traefik and the app, and passes requests through within its
|
||||
// limits. Everything it does is in internal/smallwebwaf.
|
||||
package main
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
# Admin endpoints
|
||||
|
||||
The endpoints under `/_smallwebwaf/`, what each takes and what each answers.
|
||||
"Admin endpoints" in [`SPEC.md`](../SPEC.md) says which checks they go through
|
||||
and how their tokens work.
|
||||
|
||||
- `GET /_smallwebwaf/healthz`: `200` and `ok`, to anyone.
|
||||
- `GET /_smallwebwaf/metrics`: the metrics, for `SWWAF_METRICS_TOKEN` (see
|
||||
[`metrics.md`](metrics.md)).
|
||||
- `GET /_smallwebwaf/bans`: every ban held, under `bans`, each as `bans.json`
|
||||
lists it. Needs `SWWAF_ADMIN_TOKEN`, as do the three below.
|
||||
- `POST /_smallwebwaf/bans`: bans a netblock. The body is a JSON object of
|
||||
`netblock`, `duration` and, if you like, `reason`. `netblock` is a netblock
|
||||
such as `203.0.113.0/24`, or a client's address, which bans its IPv4 netblock
|
||||
(`SWWAF_BAN_SCOPE_V4_PREFIX`) or its IPv6 group. `duration` is a duration such
|
||||
as `1h` or `7d`, or `permanent`. A bad body is answered `400`, saying what is
|
||||
wrong:
|
||||
- one that is not such an object, or has another field;
|
||||
- one longer than 4 KiB;
|
||||
- one that names an IPv4-mapped netblock, or an address with a zone.
|
||||
- `DELETE /_smallwebwaf/bans/<client>`: lifts every active ban covering the
|
||||
address `<client>`. It answers `404` when none is active.
|
||||
- `GET /_smallwebwaf/clients/<ip>`: what `smallwebwaf` knows of the client.
|
||||
Under `client`, its counters and history as `clients.json` holds them, or
|
||||
`null` when `clients.json` does not hold it. Under `bans`, every ban covering
|
||||
the address, with its notes.
|
||||
|
||||
## Examples
|
||||
|
||||
With `SWWAF_ADMIN_TOKEN` set, and the token in `$TOKEN`:
|
||||
|
||||
```sh
|
||||
# Every ban.
|
||||
curl -H "Authorization: Bearer $TOKEN" https://app.example/_smallwebwaf/bans
|
||||
|
||||
# Ban 203.0.113.0/24 for seven days.
|
||||
curl -H "Authorization: Bearer $TOKEN" \
|
||||
--json '{"netblock": "203.0.113.0/24", "duration": "7d", "reason": "probes for logins"}' \
|
||||
https://app.example/_smallwebwaf/bans
|
||||
|
||||
# Lift the bans on 203.0.113.9.
|
||||
curl -H "Authorization: Bearer $TOKEN" -X DELETE \
|
||||
https://app.example/_smallwebwaf/bans/203.0.113.9
|
||||
|
||||
# What smallwebwaf knows of 203.0.113.9.
|
||||
curl -H "Authorization: Bearer $TOKEN" \
|
||||
https://app.example/_smallwebwaf/clients/203.0.113.9
|
||||
```
|
||||
|
||||
## `servewaf`
|
||||
|
||||
`smallwebwaf servewaf` answers these endpoints at its own address. No check runs
|
||||
on those requests (see "`servewaf`" in [`SPEC.md`](../SPEC.md)).
|
||||
-136
@@ -1,136 +0,0 @@
|
||||
# Alerts
|
||||
|
||||
The alert events, the JSON object the webhook gets, and the messages Slack and
|
||||
ntfy get. "Alerting" in [`SPEC.md`](../SPEC.md) says when an alert is sent, held
|
||||
back or dropped.
|
||||
|
||||
## Events
|
||||
|
||||
- `ban`: a ban `smallwebwaf` made.
|
||||
- `permanent_ban`: a permanent ban it made, or an existing ban made permanent by
|
||||
a request.
|
||||
- `waf_block`: the Core Rule Set scored a request at or over the threshold. The
|
||||
request was refused unless the alert's `detail` gives `mode`: `detect` or
|
||||
`observe`.
|
||||
- `anomaly`: a count over an anomaly threshold.
|
||||
- `reputation_hit`: a request from a client on a blocklist, the CrowdSec
|
||||
decision list or a DNS blocklist, or with an AbuseIPDB score that is a hit.
|
||||
One alert per source.
|
||||
- `source_failure`: one of:
|
||||
- GeoJS failing;
|
||||
- a list that cannot be fetched;
|
||||
- a DNS blocklist or AbuseIPDB failing or refusing a query;
|
||||
- the day's AbuseIPDB checks used up.
|
||||
- `file_error`: one of:
|
||||
- a rule file or state file edit with an error;
|
||||
- a state file that cannot be written;
|
||||
- a replacement lookup database that cannot be read.
|
||||
- `summary`: the hourly summary of alerts held back.
|
||||
|
||||
## The JSON object
|
||||
|
||||
This is the alert for a ban for a broken rate limit, shown indented; it is sent
|
||||
on one line:
|
||||
|
||||
```json
|
||||
{
|
||||
"instance": "fsn1app1/gitea",
|
||||
"time": "2026-10-06T12:00:50.123461Z",
|
||||
"event": "ban",
|
||||
"client": "203.0.113.9",
|
||||
"netblock": "203.0.113.9/32",
|
||||
"asn": "",
|
||||
"as_name": "",
|
||||
"country": "",
|
||||
"reason": "requests per minute over the limit of 500",
|
||||
"detail": {
|
||||
"ban_expires": "2026-10-06T13:00:50.123Z",
|
||||
"cause": "limit",
|
||||
"notes": {
|
||||
"asn": "",
|
||||
"as_name": "",
|
||||
"country": "",
|
||||
"kind": "requests",
|
||||
"limit": 500,
|
||||
"window": "minute",
|
||||
"count": 501,
|
||||
"request": {
|
||||
"time": "2026-10-06T12:00:50.123456789Z",
|
||||
"method": "GET",
|
||||
"host": "app.example",
|
||||
"path": "/owner/repo/commits/branch/main?page=501",
|
||||
"status": 403,
|
||||
"user_agent": "scraper/1.0"
|
||||
},
|
||||
"requests": 501,
|
||||
"refused": 0,
|
||||
"earlier_bans": {
|
||||
"limit": 0,
|
||||
"attack": 0,
|
||||
"admin": 0,
|
||||
"crowdsec": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"suppressed_repeats": 0
|
||||
}
|
||||
```
|
||||
|
||||
- `instance`: `SWWAF_INSTANCE_NAME`. `time`: when the alert was raised. `event`:
|
||||
one of the events above.
|
||||
- `host`: the site of the request that raised the alert. Only `servewaf` sends
|
||||
this field, since it serves many sites, and not for `source_failure`,
|
||||
`file_error` and `summary`.
|
||||
- `client`: the address whose request raised the alert. Empty for
|
||||
`source_failure`, `file_error` and `summary`.
|
||||
- `netblock`: empty for `source_failure`, `file_error` and `summary`; otherwise
|
||||
one of:
|
||||
- the ban's netblock;
|
||||
- the netblock an anomaly counted, empty in the `asn` and `total` scopes;
|
||||
- for the other events, the client as rate limiting counts it: an IPv4
|
||||
address as a /32, or an IPv6 group.
|
||||
- `asn`, `as_name` and `country`: the client's, as known when the alert was
|
||||
raised. They are empty when GeoJS had not answered yet.
|
||||
- `reason`: a short sentence, such as
|
||||
`requests per minute over the limit of 500` or `listed by a blocklist`.
|
||||
- `detail`: what is particular to the event:
|
||||
- for a ban, its `cause`, `ban_expires` and `notes` (the notes are listed
|
||||
under `bans.json` in [`state-files.md`](state-files.md));
|
||||
- for an anomaly, the `scope`, `window`, `kind`, `count` and `threshold`;
|
||||
- for a reputation hit, the `source` and, for AbuseIPDB, the `score`;
|
||||
- for a Core Rule Set match, the `rule_ids`, `score`, `method` and `path`;
|
||||
- for a source failure, the `source` and `error`;
|
||||
- for a file error, the `file` and `error`;
|
||||
- for a summary, the `hour` it covers, the `count` of alerts held back, and
|
||||
that count by event in `events`;
|
||||
- `mode`, when the request was not refused: `detect` or `observe`.
|
||||
- `suppressed_repeats`: how many repeats the cooldown held back before this
|
||||
alert.
|
||||
|
||||
## Slack and ntfy
|
||||
|
||||
Slack and ntfy get a message with a title and a text. The title is the instance
|
||||
and the event, such as `fsn1app1/gitea: ban`. The text is the `reason`, then one
|
||||
line for each of these that the alert has:
|
||||
|
||||
- the host, the client, the netblock and the country;
|
||||
- the file, the source, the error and the mode from `detail`;
|
||||
- the suppressed repeats, if any.
|
||||
|
||||
Slack gets the title in bold and the text below it, with `&`, `<` and `>`
|
||||
escaped. ntfy gets the text as the message, with these headers:
|
||||
|
||||
- `Title`: the title;
|
||||
- `Authorization`: the token, as `Bearer <token>`, while
|
||||
`SWWAF_ALERT_NTFY_TOKEN` is set;
|
||||
- `Priority` and `Tags`: by event, as listed below.
|
||||
|
||||
The priority and tag by event:
|
||||
|
||||
- `ban`: priority `default`, tag `no_entry`.
|
||||
- `permanent_ban`: priority `high`, tag `no_entry`.
|
||||
- `waf_block`: priority `default`, tag `shield`.
|
||||
- `anomaly`: priority `high`, tag `chart_with_upwards_trend`.
|
||||
- `reputation_hit`: priority `low`, tag `label`.
|
||||
- `source_failure` and `file_error`: priority `high`, tag `warning`.
|
||||
- `summary`: priority `default`, tag `bar_chart`.
|
||||
-220
@@ -1,220 +0,0 @@
|
||||
# smallwebwaf in front of gitea
|
||||
|
||||
These notes record what `smallwebwaf` does in front of gitea at the default
|
||||
settings. They are observations, not requirements; the requirements are in
|
||||
[`SPEC.md`](../SPEC.md). `script/gitea-requests` runs gitea behind the image and
|
||||
sends through it a sample of gitea's ordinary use and the requests named below.
|
||||
It reports each request the Core Rule Set refuses, so that these notes can be
|
||||
checked again when gitea or the Core Rule Set changes version.
|
||||
|
||||
## What the defaults cover
|
||||
|
||||
- SSH access to gitea does not pass through `smallwebwaf` and is not protected
|
||||
by it.
|
||||
- A clone is a response. It fits the defaults of 30 minutes and 1 GiB unless the
|
||||
repository is large or the link slow. A repository whose clone is larger than
|
||||
1 GiB needs `SWWAF_RESPONSE_MAX_BYTES` raised, and the byte limits with it
|
||||
("Byte limits, per client" in `SPEC.md`).
|
||||
- A push is a request, and so is every other upload:
|
||||
- a git push;
|
||||
- an LFS object;
|
||||
- a container image layer;
|
||||
- a package file;
|
||||
- a release attachment.
|
||||
|
||||
At the defaults, an upload larger than 100 MiB or taking more than 60
|
||||
seconds is cut off ("Size and time limits" in `SPEC.md`). A gitea that takes
|
||||
large uploads needs `SWWAF_REQUEST_MAX_BYTES`,
|
||||
`SWWAF_CLIENT_REQUEST_TIMEOUT` and `SWWAF_UPSTREAM_REQUEST_TIMEOUT` raised
|
||||
to fit.
|
||||
|
||||
- Archive download and blame or history pages are what scrapers hammer. The
|
||||
request limits do most of the work there.
|
||||
- An Actions runner that reaches gitea through `smallwebwaf` sends requests all
|
||||
day. One older than version 0.4 (April 2026) asks for work every 2 seconds,
|
||||
43,200 requests a day. While it runs a job, it reports the job's log and state
|
||||
every second instead, two requests a second. So one runner busy all day sends
|
||||
120 requests a minute and 172,800 a day, under the request limits. Two such
|
||||
runners on one address, each busy all day, make 345,600 and pass the day limit
|
||||
of 300,000. Three busy at once make 21,600 an hour and pass the hour limit of
|
||||
20,000. Five busy at once make 600 a minute and pass the minute limit of 500.
|
||||
Their address is then banned, their running jobs fail, and each repeat within
|
||||
a day makes the ban longer. Newer runners ask for work less often. Put the
|
||||
runners' addresses in `SWWAF_RATE_LIMIT_EXEMPT_NETS`.
|
||||
|
||||
## What the Core Rule Set lets through
|
||||
|
||||
At the defaults, the Core Rule Set lets gitea's ordinary use through, apart from
|
||||
the refusals in the next section:
|
||||
|
||||
- browsing and views of files in a repository, with their history, blame and the
|
||||
file tree;
|
||||
- diffs, including their hidden lines and large files, and pull request review;
|
||||
- git's clone, fetch and push over HTTP;
|
||||
- signing in, including the return to the page a visitor came from, and sign-in
|
||||
with Git Credential Manager, git-credential-oauth or tea;
|
||||
- the API's calls for a file and its commits;
|
||||
- pushing and pulling container images and packages;
|
||||
- Actions runners, and artifacts uploaded with `actions/upload-artifact@v4`;
|
||||
- posting issues, pull requests, comments, wiki pages and files saved in the web
|
||||
editor, code included, since no body is read;
|
||||
- the page gitea shows after a change, with its message naming what was done.
|
||||
|
||||
## What the Core Rule Set still refuses
|
||||
|
||||
Each refusal answers only that request ("Bans" in `SPEC.md` says what it counts
|
||||
toward). The request log names the rule in `waf_rule_ids`, which
|
||||
`SWWAF_WAF_DISABLED_RULES` can switch off.
|
||||
|
||||
### A query string that reads as an attack
|
||||
|
||||
Most often this is a search. A search term is a query parameter, which the Core
|
||||
Rule Set checks against its lists of system files and commands (see the fourth
|
||||
change under "Attack detection" in `SPEC.md`). A release attachment uploaded
|
||||
through the API as `docker-compose.yml` is refused the same way. Examples:
|
||||
|
||||
- a name on those lists, such as `package.json`, `.gitignore` or
|
||||
`docker-compose.yml`;
|
||||
- text that starts with a command name, such as `python3` or `ssh key`, or has
|
||||
one right after a `/` in its first word, such as `feat/docker-support`;
|
||||
- a shell command with its options or a system path (`ls -la`, `sed -i`,
|
||||
`/bin/sh`), or a command in backticks;
|
||||
- script code (`fetch(`, `${VAR}`, `process.env`), HTML (`<img src=`) or an SQL
|
||||
statement (`SELECT * FROM users WHERE`);
|
||||
- a URL naming an IP address or `localhost`.
|
||||
|
||||
### A branch, tag or file name that starts with a command name
|
||||
|
||||
The Core Rule Set's list of commands (932260) holds names such as `docker-`,
|
||||
`python3`, `ansible`, `base64` and `whoami`. A query parameter holding a branch,
|
||||
tag or file name that starts with one is refused. So is one that has such a name
|
||||
right after a `/`, such as `feat/docker-support`, `fix/docker-build` or
|
||||
`renovate/docker-build-push-action-6.x`. gitea's own pages send the name in a
|
||||
query parameter that 930120, 932160 and 932260 still check:
|
||||
|
||||
- `refSubUrl`, the branch or tag of a directory listing. A listing sends it when
|
||||
it asks for its entries' last commits in a second request. It does so whenever
|
||||
gitea takes more than a second to work them out;
|
||||
- `name`, when a branch is deleted or restored on the branches page;
|
||||
- `tag`, when a release is started from a tag;
|
||||
- `template`, the file of an issue template, when an issue is opened from it;
|
||||
- `rule_name`, when a branch protection rule is opened for editing in the
|
||||
repository's settings or has just been saved.
|
||||
|
||||
What a visitor sees, for a branch named `docker-build`:
|
||||
|
||||
- A directory listing that makes that second request leaves those last commits
|
||||
out and shows an error.
|
||||
- The branches page shows an error instead of deleting or restoring the branch.
|
||||
- A release started from such a tag gets the `403` answer of `smallwebwaf` in
|
||||
place of its form. So does an issue opened from such a template, and such a
|
||||
rule opened for editing.
|
||||
- Saving such a rule stores it. gitea then returns to the branch settings page,
|
||||
which gets the `403` answer, so the save looks as if it failed.
|
||||
|
||||
### A file name in a query parameter that trips the code rules
|
||||
|
||||
The Core Rule Set's rules for Java, script, PowerShell and SQL find what they
|
||||
look for in some file names. This holds too in the ten query parameters that
|
||||
930120, 932160 and 932260 do not check (see the fourth change under "Attack
|
||||
detection" in `SPEC.md`). Examples:
|
||||
|
||||
- `runtime` in OpenJDK's `src/java.base/share/classes/java/lang/Runtime.java`
|
||||
(944110);
|
||||
- a Java class, as in `java.lang.String.md` (944130);
|
||||
- `document.write.js` (941180);
|
||||
- `process.env.d.ts` (934100);
|
||||
- `__proto__` (934130);
|
||||
- the PowerShell command `Install-Script`, as in `install-script.sh` (932120);
|
||||
- a database's name, as in PostgreSQL's `information_schema.sql` (942140).
|
||||
|
||||
For such a file, these requests are refused:
|
||||
|
||||
- the API's list of its commits (`path`);
|
||||
- the API's call for the contents of files that names it in its query (`body`);
|
||||
- the commit graph of that file alone (`file`);
|
||||
- the hidden lines of its diff (`path`), which stay hidden;
|
||||
- the sign-in page, for a visitor who opens a page of the file that needs
|
||||
signing in, such as its edit page (`redirect_to`). The visitor can still sign
|
||||
in by opening the sign-in page itself.
|
||||
|
||||
### A path
|
||||
|
||||
- A file name ending in `~`, or an `.xhtml` file whose path holds a space.
|
||||
- A path that holds what the Core Rule Set takes for a Java class (944130), as
|
||||
`java.lang.String.md` and
|
||||
`META-INF/services/org.apache.commons.logging.LogFactory` do.
|
||||
- A path that holds what it takes for Node.js code (934100), as
|
||||
`process.env.d.ts` does.
|
||||
- A path with a directory named `...` (930100, 930110), such as `.../notes.txt`.
|
||||
|
||||
For such a file, these are refused:
|
||||
|
||||
- the page of the file;
|
||||
- its raw file;
|
||||
- its history and its blame;
|
||||
- its edit page.
|
||||
|
||||
The directory that holds it lists it, and a clone carries it.
|
||||
|
||||
### A container image push that reuses a layer
|
||||
|
||||
A push can reuse a layer of an image docker has pushed to that gitea before.
|
||||
docker then asks gitea to take the layer from that image, naming it in the query
|
||||
parameter `from`, which 932260 checks. The request is refused when the image's
|
||||
owner starts with an entry on the list of commands. It is refused too when the
|
||||
image's name after the owner starts with one. Examples are `owner/python3-app`,
|
||||
`owner/docker-base`, and any image of an owner named `ansible` or `docker-team`.
|
||||
|
||||
What happens next depends on docker's image store. A docker that keeps its
|
||||
images in containerd's image store fails the push, and fails each new try the
|
||||
same way. A new install of Docker Engine 29 does this. A docker that keeps them
|
||||
in its own older image store uploads the layer instead. The push then succeeds
|
||||
after one refusal for each layer it reuses, and each of those refusals counts
|
||||
toward the error burst ("Attack detection" in `SPEC.md`).
|
||||
|
||||
### Artifact uploads from `actions/upload-artifact@v3`
|
||||
|
||||
This action sends the header `Content-Range`, which the Core Rule Set refuses
|
||||
(920450). The action sends two files at a time, does not retry a `403`, and
|
||||
sends no further file once one is refused. So an upload is refused once or
|
||||
twice, however many files it holds, and the step fails. One upload does not ban
|
||||
the runner. The error burst ("Attack detection" in `SPEC.md`) bans it after more
|
||||
than 30 such refusals from its address in a minute, as when many jobs upload at
|
||||
once. `actions/upload-artifact@v4` does not send the header.
|
||||
|
||||
## What body inspection refuses
|
||||
|
||||
With `SWWAF_WAF_BODY_LIMIT` set, the Core Rule Set reads request bodies too. In
|
||||
front of gitea it then refuses these, until their rule ids, named in the request
|
||||
log, are added to `SWWAF_WAF_DISABLED_RULES`:
|
||||
|
||||
- issue and comment text, wiki pages and files saved in the web editor that hold
|
||||
shell commands or code (932125, 932235, 932250 and others);
|
||||
- package descriptions that show code;
|
||||
- PyPI uploads (922130);
|
||||
- attachments named like `debug.log` or `config.yml` (932180).
|
||||
|
||||
## What `script/gitea-requests` sends
|
||||
|
||||
Of the requests above, the script sends:
|
||||
|
||||
- for each file named under "A file name in a query parameter that trips the
|
||||
code rules", the API's list of its commits;
|
||||
- for `Runtime.java` alone, the commit graph of that file, the hidden lines of
|
||||
its diff, and the API's call for the contents of files that names it;
|
||||
- the sign-in page for a visitor who opens the edit page of `document.write.js`;
|
||||
- the pages of `java.lang.String.md`,
|
||||
`META-INF/services/org.apache.commons.logging.LogFactory`, `process.env.d.ts`
|
||||
and `.../notes.txt`;
|
||||
- one container image push that reuses a layer, which it does not try again.
|
||||
|
||||
It does not send the raw file, history, blame or edit page of those four files,
|
||||
or the pages of the directories `META-INF/services` and `...`. It also sends a
|
||||
short sample of gitea's ordinary use:
|
||||
|
||||
- signing in;
|
||||
- pages of files and commits, and a diff;
|
||||
- the API;
|
||||
- a push, and clone and fetch over HTTP;
|
||||
- two container image pushes.
|
||||
@@ -1,37 +0,0 @@
|
||||
# What the image holds
|
||||
|
||||
The contents of the `smallwebwaf` image, and how each part is pinned.
|
||||
"Deployment" in [`SPEC.md`](../SPEC.md) says why each part is there and why it
|
||||
is pinned that way.
|
||||
|
||||
## Contents
|
||||
|
||||
- Ubuntu 26.04 LTS, pinned by digest.
|
||||
- `ca-certificates`, `nix-bin` and `runit` from Ubuntu, from the snapshot of its
|
||||
archive at `20261001T000000Z`.
|
||||
- Nix, with nixpkgs `nixos-26.05.11045.774debe7a0d1` set up for root as
|
||||
`nixpkgs`. No Nix daemon; `build-users-group =` in `/etc/nix/nix.conf`. Root's
|
||||
Nix profile, `/nix/var/nix/profiles/default/bin`, last on the `PATH`.
|
||||
Unpacked, nixpkgs takes about 500 MiB.
|
||||
- `runsvinit` as the entrypoint, built from `github.com/peterbourgon/runsvinit`
|
||||
at a fixed commit hash.
|
||||
- The `smallwebwaf` binary, and the user `smallwebwaf` (uid and gid 65532).
|
||||
- The service directory `/etc/service/smallwebwaf`, with the `run` script
|
||||
`share/smallwebwaf.run`.
|
||||
- `/var/lib/smallwebwaf` for the state files, and `/etc/smallwebwaf/rules.d`
|
||||
with the default rule file `share/rules.d/00-default.rules`.
|
||||
- `EXPOSE 8080`, and the `HEALTHCHECK` that runs `smallwebwaf healthcheck`.
|
||||
|
||||
## Pins
|
||||
|
||||
- Ubuntu's packages: the snapshot service, `--snapshot 20261001T000000Z` on both
|
||||
`apt-get update` and `apt-get install`. The SHA-256 hash of each of the
|
||||
snapshot's `InRelease` files is named in the `Dockerfile` and checked after
|
||||
`apt-get update`. The hashes are the amd64 archive's.
|
||||
- nixpkgs: the release file `nixexprs.tar.xz` of
|
||||
`nixos-26.05.11045.774debe7a0d1` on `releases.nixos.org`, named with its
|
||||
SHA-256 hash.
|
||||
- `runsvinit`: the archived repository, unchanged since 2015, at its last commit
|
||||
(`b4b2c78`, eight commits after `v2.0.0`), by commit hash, with a `go.mod` the
|
||||
build writes.
|
||||
- The Go image that builds `smallwebwaf`: by digest.
|
||||
@@ -1,92 +0,0 @@
|
||||
# The metrics
|
||||
|
||||
What `GET /_smallwebwaf/metrics` answers with. "Metrics endpoint" in
|
||||
[`SPEC.md`](../SPEC.md) describes the endpoint. Prometheus keeps the `instance`
|
||||
label as `exported_instance` unless the scrape sets `honor_labels: true`.
|
||||
|
||||
## Traffic
|
||||
|
||||
- `smallwebwaf_requests_total`, `smallwebwaf_request_bytes_total` and
|
||||
`smallwebwaf_response_bytes_total`, by `status_class` such as `2xx` and by
|
||||
`action`;
|
||||
- `smallwebwaf_request_duration_seconds` and
|
||||
`smallwebwaf_upstream_duration_seconds`, as histograms;
|
||||
- `smallwebwaf_requests_in_flight`.
|
||||
|
||||
## Limits and bans
|
||||
|
||||
- `smallwebwaf_rate_limit_hits_total`, by `window` and by `kind` (`requests`,
|
||||
`bytes` or `refusals`);
|
||||
- `smallwebwaf_size_and_time_limit_hits_total`, by `limit`, the setting passed;
|
||||
- `smallwebwaf_offences_total`, by `kind`;
|
||||
- `smallwebwaf_bans_made_total`, by `cause`;
|
||||
- `smallwebwaf_active_bans` and `smallwebwaf_permanent_bans`.
|
||||
|
||||
## Attack detection
|
||||
|
||||
- `smallwebwaf_rule_matches_total`, by `rule_id` and `action`, and
|
||||
`smallwebwaf_rules_loaded`;
|
||||
- `smallwebwaf_waf_matches_total`, by `mode` and `rule_id`, for the rules that
|
||||
fired.
|
||||
|
||||
## Countries and AS numbers
|
||||
|
||||
For the `SWWAF_METRICS_TOP_N` busiest of each, the rest as `other`:
|
||||
|
||||
- `smallwebwaf_country_requests_total`,
|
||||
`smallwebwaf_country_request_bytes_total`,
|
||||
`smallwebwaf_country_response_bytes_total` and
|
||||
`smallwebwaf_country_list_refusals_total`, by `country`;
|
||||
- `smallwebwaf_asn_requests_total`, `smallwebwaf_asn_request_bytes_total` and
|
||||
`smallwebwaf_asn_response_bytes_total`, by `asn`.
|
||||
|
||||
## Lookups
|
||||
|
||||
- `smallwebwaf_geojs_requests_total` and `smallwebwaf_geojs_failures_total`;
|
||||
- `smallwebwaf_geojs_unanswered_total`: the requests that needed an answer and
|
||||
went on without it;
|
||||
- with the file source,
|
||||
`smallwebwaf_lookup_database_last_read_timestamp_seconds` and
|
||||
`smallwebwaf_lookup_database_read_failures_total`.
|
||||
|
||||
## Reputation sources
|
||||
|
||||
By `source`: each list's URL, each zone, and `abuseipdb`:
|
||||
|
||||
- `smallwebwaf_reputation_hits_total`, `smallwebwaf_reputation_queries_total`
|
||||
and `smallwebwaf_reputation_failures_total`;
|
||||
- `smallwebwaf_reputation_last_fetch_timestamp_seconds`, for a list;
|
||||
- `smallwebwaf_reputation_daily_budget_remaining`, for AbuseIPDB.
|
||||
|
||||
## State files
|
||||
|
||||
By `file`:
|
||||
|
||||
- `smallwebwaf_state_file_writes_total`,
|
||||
`smallwebwaf_state_file_write_failures_total`,
|
||||
`smallwebwaf_state_file_last_write_timestamp_seconds` and
|
||||
`smallwebwaf_state_file_size_bytes`;
|
||||
- `smallwebwaf_state_file_edits_taken_in_total` and
|
||||
`smallwebwaf_state_file_edits_set_aside_total`;
|
||||
- `smallwebwaf_tracked_clients`: the clients in the table.
|
||||
|
||||
## Remote log, alerts and the process
|
||||
|
||||
- while a syslog server is set, `smallwebwaf_remote_log_lines_sent_total`,
|
||||
`smallwebwaf_remote_log_lines_dropped_total` and
|
||||
`smallwebwaf_remote_log_buffer_depth`;
|
||||
- by `destination`, `smallwebwaf_alerts_sent_total`,
|
||||
`smallwebwaf_alerts_failed_total`, `smallwebwaf_alerts_suppressed_total` and
|
||||
`smallwebwaf_alerts_dropped_total`;
|
||||
- Go's own `go_` metrics and the process's `process_` metrics.
|
||||
|
||||
## `servewaf`
|
||||
|
||||
`smallwebwaf servewaf` counts a denied request at its decision. It counts an
|
||||
allowed request when its report comes, with the status, bytes and times the
|
||||
report gives. These are not counted:
|
||||
|
||||
- a report itself;
|
||||
- an allowed request whose report never comes;
|
||||
- a question refused with `400`;
|
||||
- a question whose body stopped arriving.
|
||||
@@ -1,182 +0,0 @@
|
||||
# The request log's fields
|
||||
|
||||
The fields of the JSON object `smallwebwaf` writes for each request. "Request
|
||||
log" in [`SPEC.md`](../SPEC.md) says when a line is written and how the log is
|
||||
sent to a syslog server. A field that does not apply to a request is left out.
|
||||
|
||||
## Example
|
||||
|
||||
```
|
||||
{"type":"request","time":"2026-10-03T12:00:00.123Z","instance":"fsn1app1/gitea","client_ip":"203.0.113.9","method":"GET","scheme":"https","host":"app.example","path":"/","query":"","protocol":"HTTP/1.1","status":200,"request_bytes":0,"response_bytes":5120,"referer":"","user_agent":"curl/8.9.1","request_id":"7Q2NHZ4KJ3VXW5YB6R3MEFTD2A","peer_ip":"172.18.0.2","forwarded_for":"203.0.113.9","client_group":"203.0.113.9/32","asn":"AS64496","as_name":"Example Net","country":"DE","request_headers":{"accept":"*/*"},"response_content_type":"text/html; charset=utf-8","upstream_status":200,"action":"forward","counts":{"minute":1,"hour":12,"day":40,"minute_bytes":5120,"hour_bytes":61440,"day_bytes":204800},"duration_total":3.217,"duration_checks":0.041,"duration_upstream_connect":0.052,"duration_upstream_first_byte":2.874,"duration_upstream_total":3.104}
|
||||
```
|
||||
|
||||
## Every line
|
||||
|
||||
Every line has:
|
||||
|
||||
- `type`: `request` (the process's own messages have `process`);
|
||||
- the standard web log fields;
|
||||
- `request_id`, `peer_ip` and `client_group`;
|
||||
- `asn`, `as_name` and `country`;
|
||||
- `action` and `duration_total`.
|
||||
|
||||
## Standard web log fields
|
||||
|
||||
- `time`: when the request arrived, RFC 3339 with milliseconds, in UTC;
|
||||
- `instance`: `SWWAF_INSTANCE_NAME`;
|
||||
- `client_ip`, `method`, `scheme`, `host`, `path`, `query`, `protocol`;
|
||||
- `status`: what the client was sent, `0` if nothing;
|
||||
- `request_bytes` and `response_bytes`: body bytes each way;
|
||||
- `referer` and `user_agent`.
|
||||
|
||||
## Request detail
|
||||
|
||||
- `request_id`: the `X-Request-ID` a trusted proxy sent, or a new random one of
|
||||
26 letters and digits.
|
||||
- `peer_ip`: the TCP peer, normally traefik.
|
||||
- `forwarded_for`: the `X-Forwarded-For` header as received.
|
||||
- `client_group`: the client as rate limiting counts it, an IPv4 address as a
|
||||
/32 or an IPv6 group.
|
||||
- `asn`, `as_name` and `country`: the client's, as looked up. Each is empty
|
||||
when:
|
||||
- lookups are off;
|
||||
- the client is not looked up;
|
||||
- the source cannot place the client;
|
||||
- GeoJS had not answered by the time the request went on.
|
||||
- `content_type` and `content_length`: the request's own.
|
||||
- `request_headers`: the headers listed in `SWWAF_LOG_REQUEST_HEADERS`, by name
|
||||
in lower case.
|
||||
- `has_authorization` and `has_cookie`: true when the request had those headers.
|
||||
Their values are never logged.
|
||||
- `websocket`: true when the app switched the connection to another protocol.
|
||||
|
||||
## Response detail
|
||||
|
||||
- `upstream_status`: what the app answered. It differs from `status` when
|
||||
`smallwebwaf` answered itself.
|
||||
- `response_content_type`, `cache_control` and `location`: the answer's headers.
|
||||
- `aborted`: true when the client went away early.
|
||||
|
||||
## Decision
|
||||
|
||||
- `action`: what was done:
|
||||
- `forward`: passed to the app;
|
||||
- `denied`: refused for `SWWAF_DENY_NETS`, or for a reputation source with
|
||||
the action `deny`;
|
||||
- `banned`: refused under a ban, or for a `ban` rule, a trap path or a
|
||||
CrowdSec decision, each of which bans the client;
|
||||
- `country_denied`, `rate_limited`, `rule_blocked` and `waf_blocked`:
|
||||
refused by that check;
|
||||
- `too_large` and `timed_out`: a size or time limit was passed;
|
||||
- `upstream_error`: the app could not be reached, or its answer broke off;
|
||||
- `admin`: answered at one of `smallwebwaf`'s own endpoints.
|
||||
- `would_action`: in `observe` mode, what `enforce` mode would have done with a
|
||||
request it would have refused. `action` then names what was done, normally
|
||||
`forward`.
|
||||
- `limit_percent` and `limit_percent_setting`: the client's percentage of the
|
||||
rate limits and the setting that lowered it, when one did. `bytes_percent` and
|
||||
`bytes_percent_setting` do the same for the byte limits.
|
||||
- `counts`: the client's totals as rate limiting counts them, this request
|
||||
included. It is left out when rate limiting did not count the request. It
|
||||
holds:
|
||||
- `minute`, `hour` and `day`: requests;
|
||||
- `minute_bytes`, `hour_bytes` and `day_bytes`: bytes.
|
||||
|
||||
A count can have a fraction (see "Counting method" in
|
||||
[`SPEC.md`](../SPEC.md)).
|
||||
|
||||
- `limit_hit`: the limit broken:
|
||||
- `minute`, `hour` or `day` for a rate limit;
|
||||
- `minute_bytes`, `hour_bytes` or `day_bytes` for a byte limit;
|
||||
- `error_burst`.
|
||||
|
||||
A request whose bytes broke a limit was passed on whole, so its `action`
|
||||
stays `forward`.
|
||||
|
||||
- `rule_ids`: the rule file rules that matched, in order, up to the one that
|
||||
refused.
|
||||
- `waf_score` and `waf_rule_ids`: the Core Rule Set's score and the rules that
|
||||
matched, for a request it inspected.
|
||||
- `reputation`: the sources the client is on, in this order:
|
||||
- blocklist URLs;
|
||||
- the CrowdSec decision list's URL;
|
||||
- DNS blocklist zones;
|
||||
- `abuseipdb`.
|
||||
- `offence`: `limit`, for a request that broke a limit.
|
||||
- `ban_expires`: when the client's ban ends, for a request that made one or was
|
||||
refused under one: a time, or `permanent`.
|
||||
|
||||
## Timings
|
||||
|
||||
All in milliseconds:
|
||||
|
||||
- `duration_total`: from the request's headers being read to its log line;
|
||||
- `duration_checks`: to the end of the checks, before forwarding;
|
||||
- `duration_waf`: the part of that the Core Rule Set took;
|
||||
- `duration_upstream_connect`: from the hand-over to the app until there was a
|
||||
connection;
|
||||
- `duration_upstream_first_byte`: until the first byte of the answer;
|
||||
- `duration_upstream_total`: until the end of the answer.
|
||||
|
||||
## `servewaf`'s lines
|
||||
|
||||
`smallwebwaf servewaf` writes two lines for each question it answers, joined by
|
||||
`request_id`. The field `servewaf_line` says which of the two a line is:
|
||||
|
||||
- `decision`: the line it writes once it has answered the question;
|
||||
- `report`: the line it writes when `rproxy`'s report comes, with everything
|
||||
known. The repetition is intended.
|
||||
|
||||
A request to its own endpoints writes one line, without `servewaf_line`. A
|
||||
report writes no line but the `report` line of the request it names.
|
||||
|
||||
These questions write no line:
|
||||
|
||||
- a question refused with `400` (see "The question" under "`servewaf`" in
|
||||
[`SPEC.md`](../SPEC.md));
|
||||
- a question whose body stopped arriving, which gets no answer.
|
||||
|
||||
### The `decision` line
|
||||
|
||||
It has the fields above, with these meanings where they differ:
|
||||
|
||||
- `client_ip`: `X-Smallwebwaf-Client`.
|
||||
- `peer_ip`: `X-Smallwebwaf-Peer`.
|
||||
- `host`: the site the visitor asked for.
|
||||
- `protocol`: the question's own, `HTTP/1.1`.
|
||||
- `content_length`: `X-Smallwebwaf-Content-Length`.
|
||||
- `request_id`: the question's `X-Request-ID`.
|
||||
- `status`: what `servewaf` answered: `200` for `allow`, the status of `deny`,
|
||||
or `0` for `close`, as the visitor is then sent nothing.
|
||||
- `request_bytes`: how much of the question's body the Core Rule Set read.
|
||||
- `response_bytes`: the body of `servewaf`'s answer.
|
||||
- `action`: `forward` for `allow`, and `admin` for the `404` for a path under
|
||||
`/_smallwebwaf/`.
|
||||
- `aborted`: true when `rproxy` went away before the answer.
|
||||
- `duration_total`: from `servewaf` having read the question's body to the line.
|
||||
- `time`: when `servewaf` had read the question's body.
|
||||
|
||||
The line leaves out `upstream_status`, `websocket` and the
|
||||
`duration_upstream_...` fields, since `servewaf` passes nothing to the app.
|
||||
|
||||
The bytes of an allowed request are counted when its report comes. So `counts`
|
||||
holds the client's bytes without this request's.
|
||||
|
||||
### The `report` line
|
||||
|
||||
It has every field of the `decision` line, and `time` stays as the `decision`
|
||||
line gives it. These fields say what the report says:
|
||||
|
||||
- `status`: what the visitor got, `0` if nothing;
|
||||
- `request_bytes` and `response_bytes`: the body bytes each way, with what a
|
||||
WebSocket carried each way;
|
||||
- `duration_total`: from the request's headers being read by `rproxy` to the end
|
||||
of the answer;
|
||||
- `duration_upstream_total`: from the hand-over to the app to the end of the
|
||||
answer, for an allowed request.
|
||||
|
||||
For an allowed request, the report's counting also changes these fields:
|
||||
|
||||
- `counts`: the byte totals now include this request's;
|
||||
- `limit_hit`, `offence` and `ban_expires`: set when the report's bytes broke a
|
||||
byte limit.
|
||||
@@ -1,67 +0,0 @@
|
||||
# Rule files
|
||||
|
||||
The format of a rule file, and the default file the image ships. "Rule files" in
|
||||
[`SPEC.md`](../SPEC.md) says how the files are read, watched and mounted, and
|
||||
what each action leads to.
|
||||
|
||||
## The format
|
||||
|
||||
A rule is a line of four fields separated by spaces or tabs: an id, a target, an
|
||||
action and a regex. The regex runs to the end of the line. Blank lines and lines
|
||||
starting with `#` are ignored.
|
||||
|
||||
```
|
||||
# id target action regex
|
||||
env-file path ban (?i)^/\.env(\.[a-z]+)?$
|
||||
scanner-agent user_agent ban (?i)\b(sqlmap|nikto|nuclei|wpscan)\b
|
||||
```
|
||||
|
||||
- The id is letters, digits, `-` and `_`, unique across all files. The log line,
|
||||
the metrics, the alerts and a ban's notes name the rule by it.
|
||||
- The target is what the regex is matched against:
|
||||
- `path`: the path without the query;
|
||||
- `uri`: the path and the query together;
|
||||
- `query`: the query as sent;
|
||||
- `method`, `host`, `user_agent` and `referer`;
|
||||
- `header:<Name>`: any one request header but `Host` and
|
||||
`Transfer-Encoding`.
|
||||
- The action is `log`, `block` or `ban`.
|
||||
- The regex is in Go's syntax, RE2. It matches anywhere in the target unless
|
||||
anchored with `^` and `$`. `(?i)` at its front makes it ignore case.
|
||||
|
||||
## The default file
|
||||
|
||||
The image ships one file, `share/rules.d/00-default.rules`, installed as
|
||||
`/etc/smallwebwaf/rules.d/00-default.rules`:
|
||||
|
||||
```
|
||||
# 00-default.rules: probes no real visitor sends, anchored at the site root
|
||||
|
||||
# id target action regex
|
||||
env-file path ban (?i)^/\.env(\.[a-z]+)?$
|
||||
vcs-dir path ban (?i)^/\.(git|svn|hg|bzr)(/|$)
|
||||
secrets-dir path ban (?i)^/\.(aws|ssh|docker|kube)/
|
||||
secret-file path ban (?i)^/\.(htpasswd|htaccess|npmrc|netrc|pgpass|git-credentials|bash_history|DS_Store)$
|
||||
editor-dir path ban (?i)^/\.(vscode|idea)/
|
||||
backup-file path ban (?i)^/[^/]+\.(php(\.[a-z0-9]+|~)|sql(\.[a-z0-9]+)?)$
|
||||
log-file path ban (?i)^/(debug|error|access)\.log$
|
||||
compose-file path ban (?i)^/(docker-)?compose\.ya?ml$
|
||||
php-shell path ban (?i)^/(shell|c99|r57|wso|alfa)\.php$
|
||||
scanner-agent user_agent ban (?i)\b(sqlmap|nikto|nuclei|masscan|zgrab|wpscan)\b
|
||||
path-traversal uri block (\.\./){2,}
|
||||
empty-agent user_agent log ^$
|
||||
```
|
||||
|
||||
## A file of your own
|
||||
|
||||
An app's `Dockerfile` adds rules of its own in a file named to sort after the
|
||||
default, such as `50-gitea.rules`:
|
||||
|
||||
```
|
||||
# 50-gitea.rules: WordPress probes, which gitea never serves
|
||||
wp-probe path ban (?i)^/(wp-login\.php|xmlrpc\.php|wp-admin/)
|
||||
```
|
||||
|
||||
```dockerfile
|
||||
COPY 50-gitea.rules /etc/smallwebwaf/rules.d/50-gitea.rules
|
||||
```
|
||||
@@ -1,360 +0,0 @@
|
||||
# Settings
|
||||
|
||||
Every setting, its default and the values it takes, in the groups of "Settings"
|
||||
in [`SPEC.md`](../SPEC.md). That section gives the conventions, the settings
|
||||
that take `off`, the `_FILE` form, and what each group does and why.
|
||||
|
||||
## Forms of values
|
||||
|
||||
- A duration is Go's syntax, with `d` for days: `90s`, `15m`, `24h`, `7d`. Zero
|
||||
and negative durations are refused.
|
||||
- A size is a number of bytes with an optional `K`, `M` or `G`, which are powers
|
||||
of 1024, so `1K` is 1024 bytes. Examples: `512K`, `100M`, `5G`.
|
||||
- A count is a whole number of requests, such as `1000`.
|
||||
- A list is comma separated, with spaces around items ignored. An empty item is
|
||||
refused.
|
||||
- A netblock is an address and a prefix length, such as `10.0.0.0/8`. A bare
|
||||
address stands for itself alone.
|
||||
- A country is a two-letter code of ISO 3166-1, in either case: `de` and `DE`
|
||||
are the same. `xk` stands for Kosovo. A code that is not a country is refused,
|
||||
such as `nk` (North Korea is `kp`) or the withdrawn `su`.
|
||||
- An AS number is `AS` and the number, in either case, such as `AS64496`.
|
||||
- A percentage is a whole number from 0 to 100.
|
||||
- A path prefix starts with `/`, such as `/assets/`.
|
||||
|
||||
## Core
|
||||
|
||||
- `SWWAF_UPSTREAM_URL` (default `http://127.0.0.1:8081`): the app. An `http` or
|
||||
`https` URL with a host and an optional port, and nothing more.
|
||||
- `SWWAF_LISTEN_ADDR` (default `:8080`): where `smallwebwaf` listens: `:` and a
|
||||
port, such as `:9000`. Leave the host out (see "Ports" in
|
||||
[`SPEC.md`](../SPEC.md)).
|
||||
- `SWWAF_INSTANCE_NAME` (default: the container's host name): the `instance`
|
||||
name in every log line, alert and metric, such as `fsn1app1/gitea`. Valid
|
||||
UTF-8. No control character while `SWWAF_ALERT_NTFY_URL` is set. 1 to 48
|
||||
printable ASCII characters without a space while `SWWAF_LOG_REMOTE_URL` is set
|
||||
and `SWWAF_LOG_REMOTE_APP_NAME` is not.
|
||||
- `SWWAF_MODE` (default `enforce`): `enforce` or `observe`.
|
||||
- `SWWAF_TRUSTED_PROXIES` (default `10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`):
|
||||
the netblocks whose `X-Forwarded-For` is believed. Set but empty, it trusts
|
||||
nothing.
|
||||
- `SWWAF_IPV6_GROUP_PREFIX` (default `64`): the prefix length of an IPv6
|
||||
client's group, from 32 to 128.
|
||||
- `SWWAF_MAX_TRACKED_CLIENTS` (default `20000`): the most clients kept, a whole
|
||||
number above zero.
|
||||
- `SWWAF_MAX_BANS` (default `5000`): how many of its own bans `smallwebwaf`
|
||||
keeps, a whole number above zero.
|
||||
|
||||
## Tokens
|
||||
|
||||
- `SWWAF_ADMIN_TOKEN` (default unset): the token for the ban endpoints and
|
||||
`/_smallwebwaf/clients/<ip>`. At least 32 characters.
|
||||
- `SWWAF_METRICS_TOKEN` (default unset): the token for `/_smallwebwaf/metrics`.
|
||||
At least 32 characters.
|
||||
|
||||
## State files
|
||||
|
||||
- `SWWAF_STATE_DIR` (default `/var/lib/smallwebwaf`): the directory of the state
|
||||
files, an absolute path that exists and can be written.
|
||||
- `SWWAF_STATE_WRITE_DELAY` (default `10s`): how long `bans.json` waits to be
|
||||
written after a ban is made, lifted or made permanent. A duration.
|
||||
- `SWWAF_STATE_COUNTER_INTERVAL` (default `15m`): how often every state file is
|
||||
written. A duration.
|
||||
|
||||
## Logging
|
||||
|
||||
- `SWWAF_LOG_LEVEL` (default `info`): the least severe of `smallwebwaf`'s own
|
||||
messages that are written: `debug`, `info`, `warn` or `error`.
|
||||
- `SWWAF_LOG_REQUEST_HEADERS` (default
|
||||
`accept,accept-language,accept-encoding,content-type,origin,range`): the
|
||||
request headers recorded in the log line, in either case. `Host` and
|
||||
`Transfer-Encoding` are refused; the host is the field `host`.
|
||||
- `SWWAF_LOG_REMOTE_URL` (default unset): a syslog server that also gets every
|
||||
line written to stdout: `syslog+udp://`, `syslog+tcp://` or `syslog+tls://`,
|
||||
each with a host and a port and nothing more.
|
||||
- `SWWAF_LOG_REMOTE_TLS_CA_FILE` (default unset): a file of PEM certificates
|
||||
that a `syslog+tls` server's certificate must chain to, instead of the host's
|
||||
own. A file that cannot be read, or holds no certificate, is refused.
|
||||
- `SWWAF_LOG_REMOTE_BUFFER` (default `10000`): the most lines held while they
|
||||
wait to be sent, a whole number above zero.
|
||||
- `SWWAF_LOG_REMOTE_FACILITY` (default `local0`): the syslog facility: `kern`,
|
||||
`user`, `mail`, `daemon`, `auth`, `syslog`, `lpr`, `news`, `uucp`, `cron`,
|
||||
`authpriv`, `ftp`, or `local0` to `local7`.
|
||||
- `SWWAF_LOG_REMOTE_APP_NAME` (default: `SWWAF_INSTANCE_NAME`): the app name in
|
||||
each syslog record, 1 to 48 printable ASCII characters without a space.
|
||||
|
||||
## Metrics
|
||||
|
||||
- `SWWAF_METRICS_TOP_N` (default `50`): how many AS numbers and how many
|
||||
countries get series of their own. A whole number above zero.
|
||||
|
||||
## Static lists
|
||||
|
||||
- `SWWAF_ALLOW_NETS` (default empty): netblocks you trust, such as your
|
||||
monitoring (see step 3 in [`SPEC.md`](../SPEC.md)).
|
||||
- `SWWAF_RATE_LIMIT_EXEMPT_NETS` (default empty): netblocks whose clients are
|
||||
exempt from rate limiting and the byte limits.
|
||||
- `SWWAF_DENY_NETS` (default empty): netblocks whose clients are refused (see
|
||||
step 3 in [`SPEC.md`](../SPEC.md)).
|
||||
|
||||
## Request rate limits, per client
|
||||
|
||||
- `SWWAF_RATE_LIMIT_PER_MINUTE` (default `500`, about 8 a second): the most
|
||||
requests a client may make in a minute. A count or `off`. The default catches,
|
||||
within seconds, a client firing requests as fast as it can. It leaves room for
|
||||
a fleet of about 20 agents at its busiest, about 120 requests a minute. It
|
||||
also leaves room for a few quick first visits to gitea with an empty cache,
|
||||
about 40 files each.
|
||||
- `SWWAF_RATE_LIMIT_PER_HOUR` (default `20000`, about 5.5 a second): the most
|
||||
requests a client may make in an hour. A count or `off`. The default leaves
|
||||
room for an automation fleet at work. It catches a client that keeps up more
|
||||
than about 5.5 a second for an hour.
|
||||
- `SWWAF_RATE_LIMIT_PER_DAY` (default `300000`, about 3.5 a second): the most
|
||||
requests a client may make in a day. A count or `off`. The default catches a
|
||||
scraper that keeps up 4 a second around the clock.
|
||||
- Counting cannot catch a client that stays under all three limits (see "Request
|
||||
rate limits, per client" in [`SPEC.md`](../SPEC.md)).
|
||||
- `SWWAF_RATE_LIMIT_EXEMPT_PATHS` (default empty): path prefixes exempt from
|
||||
rate limiting and the byte limits, such as `/assets/`.
|
||||
|
||||
## Byte limits, per client
|
||||
|
||||
- `SWWAF_BYTES_LIMIT_PER_MINUTE` (default `4G`), `SWWAF_BYTES_LIMIT_PER_HOUR`
|
||||
(default `20G`) and `SWWAF_BYTES_LIMIT_PER_DAY` (default `200G`): the most
|
||||
bytes counted for a client in each window. A size or `off`.
|
||||
- `SWWAF_BYTES_COUNT` (default `both`): which body bytes the byte limits count:
|
||||
`response`, `request` or `both`.
|
||||
|
||||
## Size and time limits
|
||||
|
||||
- `SWWAF_CLIENT_REQUEST_TIMEOUT` (default `60s`): how long a client may take to
|
||||
send its request line and headers, and then, from the end of the headers, its
|
||||
body. A duration or `off`.
|
||||
- `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES` (default `32K`): the largest request
|
||||
line and headers. A size over `4K`; never `off`.
|
||||
- `SWWAF_CLIENT_IDLE_TIMEOUT` (default `120s`): how long a kept-open connection
|
||||
may wait for its next request before it is closed. A duration or `off`.
|
||||
- `SWWAF_CLIENT_RESPONSE_TIMEOUT` (default `30m`): how long the answer may take
|
||||
to reach the client, from the end of the request to the last byte. A duration
|
||||
or `off`.
|
||||
- `SWWAF_UPSTREAM_REQUEST_TIMEOUT` (default `60s`): how long connecting to the
|
||||
app and sending it the whole request may take. A duration or `off`.
|
||||
- `SWWAF_UPSTREAM_RESPONSE_TIMEOUT` (default `30m`): how long the app may take
|
||||
to send its whole answer. A duration or `off`.
|
||||
- `SWWAF_REQUEST_MAX_BYTES` (default `100M`): the largest request body. A size
|
||||
or `off`.
|
||||
- `SWWAF_RESPONSE_MAX_BYTES` (default `1G`): the largest response body. A size
|
||||
or `off`.
|
||||
|
||||
## Lookup of AS number and country
|
||||
|
||||
- `SWWAF_LOOKUP_SOURCE` (default `geojs`): `geojs`, `file` or `off`. `off` stops
|
||||
the start while a setting that needs a lookup is set:
|
||||
- the two country lists;
|
||||
- `SWWAF_ADD_LOOKUP_HEADERS`;
|
||||
- the four biased threshold lists and `SWWAF_ASN_LIMIT_PERCENT_URL`;
|
||||
- `SWWAF_UNKNOWN_LIMIT_PERCENT` below `100`;
|
||||
- an anomaly threshold per AS number.
|
||||
- `SWWAF_LOOKUP_DB_PATH` (default empty): the IPinfo Lite database file, in its
|
||||
`.mmdb` form, for `SWWAF_LOOKUP_SOURCE=file`.
|
||||
- `SWWAF_LOOKUP_TIMEOUT` (default `1s`): how long a request waits for its
|
||||
client's first answer from GeoJS, and how long a request to GeoJS may take. A
|
||||
duration; never `off`.
|
||||
- `SWWAF_ADD_LOOKUP_HEADERS` (default `false`): `true` passes the app the
|
||||
client's AS number in `X-Client-ASN` and its country in `X-Client-Country`.
|
||||
|
||||
## Country lists
|
||||
|
||||
- `SWWAF_DENIED_COUNTRIES` (default empty): countries whose clients are refused,
|
||||
such as `cn,ru,kp`.
|
||||
- `SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES` (default empty): when set, the only
|
||||
countries whose clients get through, such as `us,de`. A country on both lists
|
||||
stops the start.
|
||||
|
||||
## Biased thresholds
|
||||
|
||||
- `SWWAF_ASN_LIMIT_PERCENT` (default empty): a list of AS numbers, each with `:`
|
||||
and a percentage, such as `AS14061:50,AS16276:50,AS45102:25`. An AS number
|
||||
listed twice is refused.
|
||||
- `SWWAF_COUNTRY_LIMIT_PERCENT` (default empty): the same by country, such as
|
||||
`cn:25,ru:50`.
|
||||
- `SWWAF_ASN_BYTES_PERCENT` and `SWWAF_COUNTRY_BYTES_PERCENT` (default empty):
|
||||
the same, for the byte limits alone.
|
||||
- `SWWAF_UNKNOWN_LIMIT_PERCENT` (default `100`): the percentage for a client the
|
||||
lookup cannot place.
|
||||
- `SWWAF_ASN_LIMIT_PERCENT_URL` (default unset): the `http` or `https` URL of a
|
||||
text file with one AS number and percentage on each line, such as
|
||||
`AS64496:50`. A URL that is also in `SWWAF_BLOCKLIST_URLS` is refused.
|
||||
|
||||
## Attack detection
|
||||
|
||||
- `SWWAF_RULES_DIR` (default `/etc/smallwebwaf/rules.d`): the directory of the
|
||||
rule files.
|
||||
- `SWWAF_RULES_ENABLED` (default `true`): `false` reads no rule file.
|
||||
- `SWWAF_TRAP_PATHS` (default empty): paths only scanners ask for, such as
|
||||
`/wp-login.php,/xmlrpc.php`. Each starts with `/` and holds no `?`.
|
||||
- `SWWAF_WAF_MODE` (default `block`): `off`, `detect` or `block`.
|
||||
- `SWWAF_WAF_PARANOIA_LEVEL` (default `1`): the Core Rule Set's paranoia level,
|
||||
from 1 to 4.
|
||||
- `SWWAF_WAF_ANOMALY_THRESHOLD` (default `5`): the score at which a request is a
|
||||
match. A whole number above zero, or `off`.
|
||||
- `SWWAF_WAF_DISABLED_RULES` (default
|
||||
`920340,920420,920440,920640,930130,930140`): the ids of Core Rule Set rules
|
||||
to switch off. Each a whole number above zero. An id from 900000 to 900999 is
|
||||
refused: those are the rules that set the Core Rule Set up, and
|
||||
`smallwebwaf`'s own.
|
||||
- `SWWAF_WAF_EXEMPT_PATHS` (default empty): path prefixes the Core Rule Set does
|
||||
not inspect.
|
||||
- `SWWAF_WAF_BODY_LIMIT` (default `off`): `off`, or how much of a request body
|
||||
the Core Rule Set reads, a size of at most `1G`.
|
||||
- `SWWAF_ERROR_BURST_THRESHOLD` (default `30`): the most refusals of one client
|
||||
in a minute (see "Attack detection" in `SPEC.md`). A count or `off`.
|
||||
|
||||
## Ban lengths, response and scope
|
||||
|
||||
- `SWWAF_BAN_RESPONSE` (default `403`): `403`, `429` or `close`.
|
||||
- `SWWAF_LIMIT_BAN_DURATION` (default `1h`): the ban for a first broken limit. A
|
||||
duration.
|
||||
- `SWWAF_LIMIT_BAN_REPEAT_WINDOW` (default `24h`): the time after a ban ends
|
||||
within which breaking a limit again lengthens the next ban. A duration.
|
||||
- `SWWAF_MAX_BAN_DURATION` (default `7d`): the longest ban for a broken limit;
|
||||
one that would be longer is permanent. A duration.
|
||||
- `SWWAF_ATTACK_BAN_DURATION` (default `7d`): the ban for a first clear sign of
|
||||
attack. A duration.
|
||||
- `SWWAF_BAN_SCOPE_V4_PREFIX` (default `32`): how much a ban on an IPv4 client
|
||||
covers: the prefix length of its netblock, from 0 to 32.
|
||||
|
||||
## Reputation
|
||||
|
||||
- `SWWAF_BLOCKLIST_URLS` (default empty): blocklists to fetch, such as
|
||||
`https://www.spamhaus.org/drop/drop.txt`. Each an `http` or `https` URL
|
||||
without a user or a fragment. A URL listed twice is refused.
|
||||
- `SWWAF_BLOCKLIST_REFRESH` (default `24h`): how long a list waits after its
|
||||
last try before it is fetched again. A duration of `1h` or more.
|
||||
- `SWWAF_BLOCKLIST_ACTION` (default `deny`): `deny`, `limit:<percent>` or `log`.
|
||||
- `SWWAF_DNSBL_ZONES` (default empty): the DNS blocklists to ask, such as
|
||||
`dnsbl.dronebl.org`. Each a DNS name of at most 189 characters. A Spamhaus
|
||||
zone carries its key, as in `<key>.xbl.dq.spamhaus.net`. A zone listed twice
|
||||
is refused, whatever its case or key.
|
||||
- `SWWAF_DNSBL_RESOLVER` (default unset): the resolver to ask the zones through,
|
||||
such as `192.0.2.53` or `[2001:db8::53]:5353`. An IP address with an optional
|
||||
port, 53 when none is given. Unset, the host's resolver is used.
|
||||
- `SWWAF_ABUSEIPDB_KEY` (default unset): the key of your AbuseIPDB account. A
|
||||
control character in it is refused.
|
||||
- `SWWAF_ABUSEIPDB_MIN_SCORE` (default `75`): the least abuse confidence score,
|
||||
from 0 to 100, that is a hit.
|
||||
- `SWWAF_ABUSEIPDB_DAILY_BUDGET` (default `900`): the most checks made in a day,
|
||||
a whole number above zero. AbuseIPDB's free accounts may make 1,000.
|
||||
- `SWWAF_CROWDSEC_LAPI_URL` (default unset): the local API of a CrowdSec engine
|
||||
you run, such as `http://172.17.0.1:8080`. An `http` or `https` URL without a
|
||||
user or a fragment; its decision list is at `v1/decisions` under it. Set
|
||||
without the key, it is refused. So is a decision list URL that is also in
|
||||
`SWWAF_BLOCKLIST_URLS`, or that is `SWWAF_ASN_LIMIT_PERCENT_URL`.
|
||||
- `SWWAF_CROWDSEC_LAPI_KEY` (default unset): the key the engine gave for
|
||||
`smallwebwaf`, as `cscli bouncers add smallwebwaf` prints it. Set without the
|
||||
URL, it is refused.
|
||||
- `SWWAF_REPUTATION_ACTION` (default `limit:25`): `deny`, `limit:<percent>` or
|
||||
`log`, for DNS blocklist and AbuseIPDB hits.
|
||||
- `SWWAF_REPUTATION_CACHE_TTL` (default `24h`): how long a zone's verdict or
|
||||
AbuseIPDB's score is used. A duration.
|
||||
- `SWWAF_REPUTATION_TIMEOUT` (default `2s`): how long a query to a zone or a
|
||||
check with AbuseIPDB may take. A duration.
|
||||
|
||||
## Alerting
|
||||
|
||||
- `SWWAF_ALERT_WEBHOOK_URL` (default unset): the webhook that receives each
|
||||
alert as a JSON object. An `http` or `https` URL without a user or a fragment.
|
||||
- `SWWAF_ALERT_WEBHOOK_HEADERS` (default empty): headers sent with each post,
|
||||
such as `Authorization:Bearer 0123456789abcdef`. A list of name, `:` and
|
||||
value. A value cannot hold a comma.
|
||||
- `SWWAF_ALERT_SLACK_WEBHOOK_URL` (default unset): the Slack incoming webhook
|
||||
that receives each alert as a message, checked as the webhook URL is.
|
||||
- `SWWAF_ALERT_NTFY_URL` (default unset): the ntfy topic that receives each
|
||||
alert, as the topic's full URL, such as `https://ntfy.sh/my-alerts`, checked
|
||||
as the webhook URL is.
|
||||
- `SWWAF_ALERT_NTFY_TOKEN` (default unset): an ntfy access token. A control
|
||||
character in it is refused.
|
||||
- `SWWAF_ALERT_EVENTS` (default
|
||||
`ban,permanent_ban,waf_block,anomaly,reputation_hit,source_failure,file_error`):
|
||||
the event types alerts are sent for, from those in [`alerts.md`](alerts.md)
|
||||
but `summary`. Any other name is refused.
|
||||
- `SWWAF_ALERT_COOLDOWN` (default `15m`): how long after an alert is sent its
|
||||
repeats are held back. A duration or `off`.
|
||||
- `SWWAF_ALERT_MAX_PER_HOUR` (default `60`): the most alerts sent in an hour. A
|
||||
whole number above zero, or `off`.
|
||||
|
||||
## Anomaly thresholds
|
||||
|
||||
Each scope has four thresholds: the most requests and the most bytes counted in
|
||||
a minute and in an hour. Requests are counts, bytes are sizes. Each is `off` by
|
||||
default. The four thresholds per client are:
|
||||
|
||||
- `SWWAF_ANOMALY_CLIENT_REQUESTS_PER_MINUTE`;
|
||||
- `SWWAF_ANOMALY_CLIENT_REQUESTS_PER_HOUR`;
|
||||
- `SWWAF_ANOMALY_CLIENT_BYTES_PER_MINUTE`;
|
||||
- `SWWAF_ANOMALY_CLIENT_BYTES_PER_HOUR`.
|
||||
|
||||
The other scopes' names have another prefix before the same four endings, such
|
||||
as `REQUESTS_PER_MINUTE`:
|
||||
|
||||
- `SWWAF_ANOMALY_NET_...`: per netblock around a client. Two settings give the
|
||||
netblock's length:
|
||||
- `SWWAF_ANOMALY_NET_V4_PREFIX` (default `24`, from 0 to 32) for an IPv4
|
||||
client;
|
||||
- `SWWAF_ANOMALY_NET_V6_PREFIX` (default `48`, from 0 to 128) for an IPv6
|
||||
client.
|
||||
- `SWWAF_ANOMALY_ASN_...`: per AS number.
|
||||
- `SWWAF_ANOMALY_TOTAL_...`: the whole service.
|
||||
- `SWWAF_WATCH_...`: each netblock in `SWWAF_WATCH_NETS`, as a whole.
|
||||
`SWWAF_WATCH_NETS` (default empty) is a list of name, `=` and netblock, such
|
||||
as `office=203.0.113.0/24,scraper-x=198.51.100.0/22`. An item without `=` or a
|
||||
name, or with a bad netblock, is refused. So is a name used twice.
|
||||
|
||||
## Settings given as files
|
||||
|
||||
To give a token as a file, as "Tokens" under "Deployment" in
|
||||
[`SPEC.md`](../SPEC.md) describes, make the file on the host:
|
||||
|
||||
```sh
|
||||
mkdir -p /srv/app/tokens
|
||||
openssl rand -hex 32 > /srv/app/tokens/metrics
|
||||
chown 65532:65532 /srv/app/tokens/metrics
|
||||
chmod 0400 /srv/app/tokens/metrics
|
||||
```
|
||||
|
||||
Then mount its directory into the container read-only, and name the file:
|
||||
|
||||
```sh
|
||||
docker run \
|
||||
-v /srv/app/tokens:/etc/smallwebwaf/tokens:ro \
|
||||
-e SWWAF_METRICS_TOKEN_FILE=/etc/smallwebwaf/tokens/metrics \
|
||||
...
|
||||
```
|
||||
|
||||
## Fixed limits
|
||||
|
||||
A few limits are fixed rather than settings:
|
||||
|
||||
- at most 100,000 GeoJS answers are kept;
|
||||
- at most 100,000 DNS blocklist verdicts and 100,000 AbuseIPDB scores are kept;
|
||||
- at most 1,000 DNS blocklist queries are under way at once;
|
||||
- at most 20,000 anomaly counters are kept;
|
||||
- each alert destination's queue holds at most 1,000 alerts;
|
||||
- `servewaf` holds at most 10,000 answered requests awaiting their reports.
|
||||
|
||||
## `servewaf`
|
||||
|
||||
`smallwebwaf servewaf` uses every setting here but these, which it ignores:
|
||||
|
||||
- `SWWAF_UPSTREAM_URL` and `SWWAF_TRUSTED_PROXIES`;
|
||||
- the five timeouts under "Size and time limits";
|
||||
- `SWWAF_CLIENT_REQUEST_HEADER_MAX_BYTES`, `SWWAF_REQUEST_MAX_BYTES` and
|
||||
`SWWAF_RESPONSE_MAX_BYTES`.
|
||||
|
||||
An invalid value of one of these still stops the start, and the line of settings
|
||||
at start still shows them.
|
||||
|
||||
`SWWAF_ADD_LOOKUP_HEADERS` puts the headers on the answer to `rproxy`, not on a
|
||||
request to an app. `SWWAF_WAF_BODY_LIMIT` has an effect up to 64 KiB less one
|
||||
byte, whatever its value. A question carries no more of a body than 64 KiB (see
|
||||
"`servewaf`" in [`SPEC.md`](../SPEC.md)).
|
||||
@@ -1,104 +0,0 @@
|
||||
# State files
|
||||
|
||||
The five JSON files in `SWWAF_STATE_DIR`, and what each holds. "Persistent
|
||||
state" in [`SPEC.md`](../SPEC.md) says how they are written, read and edited.
|
||||
Each has a top-level `version` of 1.
|
||||
|
||||
## `bans.json`
|
||||
|
||||
Active and past bans: every ban an admin made, and up to `SWWAF_MAX_BANS` of
|
||||
those `smallwebwaf` made. Per entry:
|
||||
|
||||
- `netblock`, `start` and `expires` (`null` for a permanent ban);
|
||||
- `cause`: `attack`, `limit`, `admin` or `crowdsec`;
|
||||
- `reason`: a short reason;
|
||||
- `lifted`: when an admin lifted it, if one did;
|
||||
- `notes`: the notes below.
|
||||
|
||||
An entry an admin adds needs `netblock`, `start` and `expires`; `reason`,
|
||||
`notes` and `cause` may be left out, and the cause is then `admin`:
|
||||
|
||||
```json
|
||||
{
|
||||
"version": 1,
|
||||
"bans": [
|
||||
{
|
||||
"netblock": "203.0.113.0/24",
|
||||
"start": "2026-10-06T12:00:00Z",
|
||||
"expires": null,
|
||||
"reason": "probes for logins"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
"Bans" in [`SPEC.md`](../SPEC.md) says how an edit lifts, keeps or forgets a
|
||||
ban. The notes:
|
||||
|
||||
- `asn`, `as_name` and `country`: the client's, added when the lookup answers
|
||||
(empty when lookups are off);
|
||||
- for a broken limit, `kind` (`requests`, `bytes` or `refusals`), `limit`,
|
||||
`window` and `count`: what the limit was on, the limit, its window and the
|
||||
count reached;
|
||||
- `limit_percent` and `limit_percent_setting`: the client's percentage of that
|
||||
limit and the setting that gave it, when a setting lowered it;
|
||||
- for a clear sign of attack, `rule_id` and `target`, or `trap_path`;
|
||||
- `reputation`: the sources the client was on, each with its `source` and, for
|
||||
AbuseIPDB, its `score`;
|
||||
- `request`: the request that caused the ban, each text cut to 256 bytes:
|
||||
- `time`, `method` and `host`;
|
||||
- `path`, with its query string;
|
||||
- `status` and `user_agent`;
|
||||
- `requests` and `refused`: the netblock's requests since it was first seen, and
|
||||
how many this ban has refused so far, kept up to date while the ban lasts;
|
||||
- `earlier_bans`: how many earlier bans the netblock had, by cause.
|
||||
|
||||
## `clients.json`
|
||||
|
||||
Per client, one per line: its minute, hour and day counters, and its history
|
||||
since it was first seen:
|
||||
|
||||
- the first and last time seen;
|
||||
- the AS number, AS name and country last looked up, and when;
|
||||
- total requests, and how many were forwarded and how many refused;
|
||||
- bytes in each direction;
|
||||
- responses by status class;
|
||||
- offences by kind: `limit`, `attack`, `rule_blocked`, `waf_blocked` and
|
||||
`token_refused`.
|
||||
|
||||
## `lookups.json`
|
||||
|
||||
GeoJS answers, one per line:
|
||||
|
||||
- `client`;
|
||||
- `asn`, `as_name` and `country`;
|
||||
- `answered`: when GeoJS answered;
|
||||
- `used`: when the answer was last used.
|
||||
|
||||
## `reputation.json`
|
||||
|
||||
- the last good copy of each list fetched from a URL, with when it was last
|
||||
tried;
|
||||
- each DNS blocklist verdict still in use;
|
||||
- AbuseIPDB's scores, and the checks spent today.
|
||||
|
||||
## `alerts.json`
|
||||
|
||||
- the anomaly counters;
|
||||
- the alert cooldowns: for each event and netblock, source, file or anomaly
|
||||
scope, when it was last sent and the count of repeats held back since;
|
||||
- the alerts sent and held back in the current hour;
|
||||
- the alerts waiting for each destination.
|
||||
|
||||
## Sizes
|
||||
|
||||
- `clients.json`: about 1 KiB per client, about 20 MiB at the default
|
||||
`SWWAF_MAX_TRACKED_CLIENTS`. Written every 15 minutes, that is under 2 GiB of
|
||||
disk writes a day.
|
||||
- `bans.json`: about 1.2 KiB per ban, at most about 2.5 KiB, so about 6 MiB at
|
||||
the default `SWWAF_MAX_BANS`. Written every 15 minutes, a full file comes to
|
||||
about 600 MiB of writes a day, and a hundred new bans a day add about 600 MiB
|
||||
more.
|
||||
- `lookups.json`: about 150 bytes per answer, under 1.5 GiB of writes a day when
|
||||
full.
|
||||
- `reputation.json` and `alerts.json`: usually a few MiB or less.
|
||||
@@ -1,8 +1,8 @@
|
||||
// Package alerts sends alerts on bans, on traffic over an anomaly
|
||||
// threshold, on a source that fails and on a file with an error to each
|
||||
// destination set: to the webhook
|
||||
// SWWAF_ALERT_WEBHOOK_URL names, each as one JSON object, as "The JSON
|
||||
// object" in docs/alerts.md describes, to the Slack incoming
|
||||
// SWWAF_ALERT_WEBHOOK_URL names, each as one JSON object, as the "Alert
|
||||
// webhook schema" section of SPEC.md describes, to the Slack incoming
|
||||
// webhook SWWAF_ALERT_SLACK_WEBHOOK_URL names, as a message, and to the
|
||||
// ntfy topic SWWAF_ALERT_NTFY_URL names. A repeat within
|
||||
// SWWAF_ALERT_COOLDOWN is held back, and so is an alert past
|
||||
@@ -149,17 +149,15 @@ type Params struct {
|
||||
}
|
||||
|
||||
// Alert is one alert, as the webhook is sent it and alerts.json holds it,
|
||||
// with the fields of "The JSON object" in docs/alerts.md. ASN, ASName and
|
||||
// Country are, for a ban, the client's as the ban's notes give them. Host is
|
||||
// the site of the request that raised the alert, while one servewaf serves
|
||||
// many sites, and "" otherwise.
|
||||
// with the fields of the "Alert webhook schema" section of SPEC.md. ASN,
|
||||
// ASName and Country are, for a ban, the client's as the ban's notes give
|
||||
// them.
|
||||
//
|
||||
//nolint:tagliatelle // docs/alerts.md names the alert's fields in snake_case
|
||||
//nolint:tagliatelle // SPEC.md's alert webhook schema names its fields in snake_case
|
||||
type Alert struct {
|
||||
Instance string `json:"instance"`
|
||||
Time time.Time `json:"time"`
|
||||
Event string `json:"event"`
|
||||
Host string `json:"host,omitempty"`
|
||||
Client netip.Addr `json:"client"`
|
||||
Netblock netip.Prefix `json:"netblock"`
|
||||
ASN string `json:"asn"`
|
||||
@@ -902,16 +900,12 @@ func title(alert *Alert) string {
|
||||
}
|
||||
|
||||
// text returns the text of alert in Slack and ntfy: its reason, then a
|
||||
// line for each of its host, client, netblock and country, the file,
|
||||
// source, error and mode its detail gives, and its suppressed repeats, that
|
||||
// it has.
|
||||
// line for each of its client, netblock and country, the file, source,
|
||||
// error and mode its detail gives, and its suppressed repeats, that it
|
||||
// has.
|
||||
func text(alert *Alert) string {
|
||||
lines := []string{alert.Reason}
|
||||
|
||||
if alert.Host != "" {
|
||||
lines = append(lines, "host: "+alert.Host)
|
||||
}
|
||||
|
||||
if alert.Client.IsValid() {
|
||||
lines = append(lines, "client: "+alert.Client.String())
|
||||
}
|
||||
|
||||
@@ -1284,11 +1284,9 @@ func anAlertForEachEvent() []eventMessage {
|
||||
Event: alerts.EventWAFBlock, Client: netip.MustParseAddr("192.0.2.1"),
|
||||
Netblock: netip.MustParsePrefix("192.0.2.1/32"),
|
||||
Reason: "refused by the Core Rule Set",
|
||||
Host: "app.example",
|
||||
},
|
||||
"fsn1app1/gitea: waf_block", "default shield",
|
||||
"refused by the Core Rule Set\nhost: app.example\nclient: 192.0.2.1\n" +
|
||||
"netblock: 192.0.2.1/32",
|
||||
"refused by the Core Rule Set\nclient: 192.0.2.1\nnetblock: 192.0.2.1/32",
|
||||
},
|
||||
{
|
||||
alerts.Alert{
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
// Package anomaly counts requests and bytes over a minute and an hour, per
|
||||
// client, per surrounding netblock, per AS number, for the whole service
|
||||
// and per named netblock, and raises an anomaly alert for a count over its
|
||||
// threshold, as "Anomaly thresholds" under "Settings" in SPEC.md
|
||||
// describes. It refuses and bans nothing. At most 20,000 counters
|
||||
// threshold, as "Anomaly thresholds" under "Configuration surface" in
|
||||
// SPEC.md describes. It refuses and bans nothing. At most 20,000 counters
|
||||
// are kept, in memory, and written to alerts.json and read from it by the
|
||||
// state package.
|
||||
package anomaly
|
||||
@@ -103,8 +103,6 @@ type Counter struct {
|
||||
|
||||
// Request is a request that has ended, as the counters count it.
|
||||
type Request struct {
|
||||
// Host is the site the alert names, "" for none.
|
||||
Host string
|
||||
// Client is the client's address, and ClientGroup the client it is
|
||||
// counted as: its IPv4 address, or the IPv6 netblock of
|
||||
// SWWAF_IPV6_GROUP_PREFIX its address is in.
|
||||
@@ -398,7 +396,6 @@ func alertFor(r Request, k key, o overThreshold) alerts.Alert {
|
||||
|
||||
return alerts.Alert{
|
||||
Event: alerts.EventAnomaly,
|
||||
Host: r.Host,
|
||||
Client: r.Client,
|
||||
Netblock: k.netblock,
|
||||
ASN: r.ASN,
|
||||
|
||||
@@ -61,59 +61,6 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
|
||||
wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 1, second: 0, third: 1})
|
||||
}
|
||||
|
||||
func TestEarlierBansCountABanDroppedAfterAnAdminsBan(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
rules := defaultRules()
|
||||
rules.MaxBans = 2
|
||||
ledger := bans.New(rules)
|
||||
netblock := netip.MustParsePrefix("203.0.113.1/32")
|
||||
other := netip.MustParsePrefix("203.0.113.2/32")
|
||||
|
||||
// An admin bans the netblock for a minute, and it then breaks a limit
|
||||
// twice, each time after its last ban has ended.
|
||||
ledger.BanForAdmin(netblock, midnight(), midnight().Add(time.Minute),
|
||||
"probes for logins")
|
||||
first, _ := ledger.BanForLimit(netblock, midnight().Add(time.Minute), bans.Notes{})
|
||||
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
|
||||
|
||||
// A ban on another netblock fills the ledger, and the netblock's first
|
||||
// ban for a limit, its earliest that is not an admin's, goes.
|
||||
ledger.BanForLimit(other, second.Start, bans.Notes{})
|
||||
wantBans(t, ledger, map[netip.Prefix]int{netblock: 2, other: 1})
|
||||
|
||||
// The netblock's next ban still counts it.
|
||||
third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
|
||||
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 2, Admin: 1}) {
|
||||
t.Errorf("the third ban for a limit has earlier bans %+v, "+
|
||||
"want 2 for a limit and 1 an admin's", third.Notes.EarlierBans)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEarlierBansCountABanAnAdminKeepsOnce(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ledger := bans.New(defaultRules())
|
||||
netblock := netip.MustParsePrefix("203.0.113.1/32")
|
||||
|
||||
// The netblock breaks a limit twice, the second time after its first
|
||||
// ban has ended, and the second ban's notes count the first. An admin
|
||||
// then keeps the first ban.
|
||||
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
||||
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
|
||||
|
||||
kept := first
|
||||
kept.Cause = bans.CauseAdmin
|
||||
ledger.LoadEdit([]bans.Ban{kept, second})
|
||||
|
||||
// The netblock's next ban counts the kept ban once, as an admin's.
|
||||
third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
|
||||
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Admin: 1}) {
|
||||
t.Errorf("the third ban for a limit has earlier bans %+v, "+
|
||||
"want 1 for a limit and 1 an admin's", third.Notes.EarlierBans)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReasonOfTheBansSmallwebwafMakes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
+10
-35
@@ -687,51 +687,26 @@ func (l *Ledger) ban(
|
||||
}
|
||||
|
||||
// earlierBans returns how many bans a netblock with the bans held, oldest
|
||||
// first, has had, by cause. Each ban held gives a count: the bans its
|
||||
// notes count before it, and one for it and each ban held after it. A
|
||||
// ban's count misses a ban dropped after it to make room, and that of an
|
||||
// admin's ban added to bans.json without notes misses the bans before it,
|
||||
// so the count that names the most bans is taken, whole: a ban an admin
|
||||
// keeps by giving it CauseAdmin is still counted under its old cause in
|
||||
// later bans' notes, and taking each cause from another count would count
|
||||
// it twice.
|
||||
// first, has had, by cause: the first ban held counts the bans the
|
||||
// netblock had before that one, since dropped to make room, and each ban
|
||||
// held adds one.
|
||||
func earlierBans(held []Ban) EarlierBans {
|
||||
// fromHere counts ban and the bans held after it.
|
||||
var most, fromHere EarlierBans
|
||||
earlier := held[0].Notes.EarlierBans
|
||||
|
||||
for _, ban := range slices.Backward(held) {
|
||||
for _, ban := range held {
|
||||
switch ban.Cause {
|
||||
case CauseLimit:
|
||||
fromHere.Limit++
|
||||
earlier.Limit++
|
||||
case CauseAttack:
|
||||
fromHere.Attack++
|
||||
earlier.Attack++
|
||||
case CauseAdmin:
|
||||
fromHere.Admin++
|
||||
earlier.Admin++
|
||||
case CauseCrowdSec:
|
||||
fromHere.CrowdSec++
|
||||
}
|
||||
|
||||
notes := ban.Notes.EarlierBans
|
||||
count := EarlierBans{
|
||||
Limit: notes.Limit + fromHere.Limit,
|
||||
Attack: notes.Attack + fromHere.Attack,
|
||||
Admin: notes.Admin + fromHere.Admin,
|
||||
CrowdSec: notes.CrowdSec + fromHere.CrowdSec,
|
||||
}
|
||||
|
||||
// Of counts that name as many bans, the earlier ban's is taken: it
|
||||
// counts more of the bans held by the cause they have now.
|
||||
if count.total() >= most.total() {
|
||||
most = count
|
||||
earlier.CrowdSec++
|
||||
}
|
||||
}
|
||||
|
||||
return most
|
||||
}
|
||||
|
||||
// total returns how many bans e counts, of all causes.
|
||||
func (e EarlierBans) total() int {
|
||||
return e.Limit + e.Attack + e.Admin + e.CrowdSec
|
||||
return earlier
|
||||
}
|
||||
|
||||
// markChanged has Changed receive a value, unless one is waiting already.
|
||||
|
||||
@@ -502,17 +502,17 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
|
||||
UpstreamRequestTimeout: env.duration("SWWAF_UPSTREAM_REQUEST_TIMEOUT", "60s"),
|
||||
UpstreamResponseTimeout: env.duration("SWWAF_UPSTREAM_RESPONSE_TIMEOUT", "30m"),
|
||||
RequestMaxBytes: env.size("SWWAF_REQUEST_MAX_BYTES", "100M"),
|
||||
ResponseMaxBytes: env.size("SWWAF_RESPONSE_MAX_BYTES", "1G"),
|
||||
ResponseMaxBytes: env.size("SWWAF_RESPONSE_MAX_BYTES", "5G"),
|
||||
AllowNets: env.netblocks("SWWAF_ALLOW_NETS", ""),
|
||||
RateLimitExemptNets: env.netblocks("SWWAF_RATE_LIMIT_EXEMPT_NETS", ""),
|
||||
DenyNets: env.netblocks("SWWAF_DENY_NETS", ""),
|
||||
RateLimitPerMinute: env.count("SWWAF_RATE_LIMIT_PER_MINUTE", "500"),
|
||||
RateLimitPerHour: env.count("SWWAF_RATE_LIMIT_PER_HOUR", "20000"),
|
||||
RateLimitPerDay: env.count("SWWAF_RATE_LIMIT_PER_DAY", "300000"),
|
||||
RateLimitPerMinute: env.count("SWWAF_RATE_LIMIT_PER_MINUTE", "1000"),
|
||||
RateLimitPerHour: env.count("SWWAF_RATE_LIMIT_PER_HOUR", "10000"),
|
||||
RateLimitPerDay: env.count("SWWAF_RATE_LIMIT_PER_DAY", "50000"),
|
||||
RateLimitExemptPaths: env.pathPrefixes("SWWAF_RATE_LIMIT_EXEMPT_PATHS", ""),
|
||||
BytesLimitPerMinute: env.size("SWWAF_BYTES_LIMIT_PER_MINUTE", "4G"),
|
||||
BytesLimitPerMinute: env.size("SWWAF_BYTES_LIMIT_PER_MINUTE", "10G"),
|
||||
BytesLimitPerHour: env.size("SWWAF_BYTES_LIMIT_PER_HOUR", "20G"),
|
||||
BytesLimitPerDay: env.size("SWWAF_BYTES_LIMIT_PER_DAY", "200G"),
|
||||
BytesLimitPerDay: env.size("SWWAF_BYTES_LIMIT_PER_DAY", "50G"),
|
||||
BytesCount: env.bytesCount("SWWAF_BYTES_COUNT", "both"),
|
||||
LookupSource: env.lookupSource("SWWAF_LOOKUP_SOURCE", "geojs"),
|
||||
LookupDBPath: env.value("SWWAF_LOOKUP_DB_PATH", ""),
|
||||
|
||||
@@ -261,10 +261,10 @@ func TestDefaults(t *testing.T) {
|
||||
UpstreamRequestTimeout: time.Minute,
|
||||
UpstreamResponseTimeout: 30 * time.Minute,
|
||||
RequestMaxBytes: 100 << 20,
|
||||
ResponseMaxBytes: 1 << 30,
|
||||
RateLimitPerMinute: 500,
|
||||
RateLimitPerHour: 20000,
|
||||
RateLimitPerDay: 300000,
|
||||
ResponseMaxBytes: 5 << 30,
|
||||
RateLimitPerMinute: 1000,
|
||||
RateLimitPerHour: 10000,
|
||||
RateLimitPerDay: 50000,
|
||||
BanResponse: 403,
|
||||
LimitBanDuration: time.Hour,
|
||||
LimitBanRepeatWindow: 24 * time.Hour,
|
||||
@@ -284,8 +284,8 @@ func TestDefaults(t *testing.T) {
|
||||
LookupSource: defaultLookupSource, LookupTimeout: time.Second,
|
||||
})
|
||||
wantByteLimitSettings(t, cfg, config.Config{
|
||||
BytesLimitPerMinute: 4 << 30, BytesLimitPerHour: 20 << 30,
|
||||
BytesLimitPerDay: 200 << 30, BytesCount: "both",
|
||||
BytesLimitPerMinute: 10 << 30, BytesLimitPerHour: 20 << 30,
|
||||
BytesLimitPerDay: 50 << 30, BytesCount: "both",
|
||||
})
|
||||
|
||||
if cfg.UpstreamURL.String() != "http://127.0.0.1:8081" {
|
||||
@@ -2371,17 +2371,17 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
|
||||
upstreamRequestTimeout: "60s",
|
||||
upstreamResponseTimeout: "30m",
|
||||
requestMaxBytes: "100M",
|
||||
responseMaxBytes: "1G",
|
||||
responseMaxBytes: "5G",
|
||||
allowNets: "",
|
||||
rateLimitExemptNets: "",
|
||||
denyNets: "",
|
||||
rateLimitPerMinute: "500",
|
||||
rateLimitPerHour: "20000",
|
||||
rateLimitPerDay: "300000",
|
||||
rateLimitPerMinute: "1000",
|
||||
rateLimitPerHour: "10000",
|
||||
rateLimitPerDay: "50000",
|
||||
rateLimitExemptPaths: "",
|
||||
bytesLimitPerMinute: "4G",
|
||||
bytesLimitPerMinute: "10G",
|
||||
bytesLimitPerHour: "20G",
|
||||
bytesLimitPerDay: "200G",
|
||||
bytesLimitPerDay: "50G",
|
||||
bytesCount: "both",
|
||||
lookupSource: defaultLookupSource,
|
||||
lookupDBPath: "",
|
||||
|
||||
+23
-44
@@ -1,7 +1,6 @@
|
||||
// Package metrics keeps smallwebwaf's Prometheus metrics, as
|
||||
// docs/metrics.md lists them, and serves them in the Prometheus text
|
||||
// format, as the "Metrics endpoint" section of SPEC.md requires. No
|
||||
// metric carries a client's address.
|
||||
// Package metrics keeps smallwebwaf's Prometheus metrics, as the "Metrics
|
||||
// endpoint" section of SPEC.md lists them, and serves them in the
|
||||
// Prometheus text format. No metric carries a client's address.
|
||||
package metrics
|
||||
|
||||
import (
|
||||
@@ -403,21 +402,28 @@ func (m *Metrics) RequestEnded(
|
||||
line *requestlog.Line, limit string, duration, upstreamDuration time.Duration,
|
||||
) {
|
||||
m.inFlight.Dec()
|
||||
m.count(line, limit, duration, upstreamDuration)
|
||||
}
|
||||
|
||||
// RequestAllowed counts a question of rproxy's, answered allow, as no longer
|
||||
// under way. RequestReported counts the request it was about.
|
||||
func (m *Metrics) RequestAllowed() {
|
||||
m.inFlight.Dec()
|
||||
}
|
||||
class := statusClass(line.Status)
|
||||
m.requests.WithLabelValues(class, line.Action).Inc()
|
||||
m.requestBytes.WithLabelValues(class, line.Action).Add(float64(line.RequestBytes))
|
||||
m.responseBytes.WithLabelValues(class, line.Action).Add(float64(line.ResponseBytes))
|
||||
m.requestDuration.Observe(duration.Seconds())
|
||||
|
||||
// RequestReported counts a request rproxy passed to the app, as RequestEnded
|
||||
// does, from what its report says, which line holds.
|
||||
func (m *Metrics) RequestReported(
|
||||
line *requestlog.Line, duration, upstreamDuration time.Duration,
|
||||
) {
|
||||
m.count(line, "", duration, upstreamDuration)
|
||||
if upstreamDuration > 0 {
|
||||
m.upstreamDuration.Observe(upstreamDuration.Seconds())
|
||||
}
|
||||
|
||||
if limit != "" {
|
||||
m.sizeAndTimeLimitHits.WithLabelValues(limit).Inc()
|
||||
}
|
||||
|
||||
if line.Country != "" {
|
||||
m.countries.add(line.Country, line)
|
||||
}
|
||||
|
||||
if line.ASN != "" {
|
||||
m.asns.add(line.ASN, line)
|
||||
}
|
||||
}
|
||||
|
||||
// LimitHit counts a request that broke a rate limit, a byte limit or the
|
||||
@@ -486,33 +492,6 @@ func (m *Metrics) StateFileEditSetAside(name string) {
|
||||
m.stateFileEditsSetAside.WithLabelValues(name).Inc()
|
||||
}
|
||||
|
||||
// count counts a request that has ended, as RequestEnded describes.
|
||||
func (m *Metrics) count(
|
||||
line *requestlog.Line, limit string, duration, upstreamDuration time.Duration,
|
||||
) {
|
||||
class := statusClass(line.Status)
|
||||
m.requests.WithLabelValues(class, line.Action).Inc()
|
||||
m.requestBytes.WithLabelValues(class, line.Action).Add(float64(line.RequestBytes))
|
||||
m.responseBytes.WithLabelValues(class, line.Action).Add(float64(line.ResponseBytes))
|
||||
m.requestDuration.Observe(duration.Seconds())
|
||||
|
||||
if upstreamDuration > 0 {
|
||||
m.upstreamDuration.Observe(upstreamDuration.Seconds())
|
||||
}
|
||||
|
||||
if limit != "" {
|
||||
m.sizeAndTimeLimitHits.WithLabelValues(limit).Inc()
|
||||
}
|
||||
|
||||
if line.Country != "" {
|
||||
m.countries.add(line.Country, line)
|
||||
}
|
||||
|
||||
if line.ASN != "" {
|
||||
m.asns.add(line.ASN, line)
|
||||
}
|
||||
}
|
||||
|
||||
// addReputationQueries adds the counter of the queries to source, a DNSBL
|
||||
// zone, or of the checks of clients with AbuseIPDB, which count tells.
|
||||
func (m *Metrics) addReputationQueries(source prometheus.Labels, count func() int) {
|
||||
|
||||
@@ -41,15 +41,6 @@ var (
|
||||
errNotAddress = errors.New("is not an address, such as 203.0.113.9")
|
||||
)
|
||||
|
||||
// answerHealth answers a request for the health endpoint.
|
||||
func (rq *request) answerHealth() {
|
||||
rq.line.Action = requestlog.ActionAdmin
|
||||
// Set here rather than left to Go's server, which would set it only
|
||||
// after the log line has taken the response's headers.
|
||||
rq.out.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
_, _ = io.WriteString(rq.out, "ok\n")
|
||||
}
|
||||
|
||||
// answerAdmin answers a request for smallwebwaf itself, under
|
||||
// /_smallwebwaf/, once it has passed the checks. Each endpoint needs a
|
||||
// token, sent as Authorization: Bearer <token>: the metrics
|
||||
|
||||
@@ -15,11 +15,9 @@ func TestWithEveryAnomalyThresholdOffARequestIsNotCounted(t *testing.T) {
|
||||
// anomaly counters, nor a clock, and the request no response: reading
|
||||
// any of them to count the request panics.
|
||||
rq := &request{
|
||||
decision: decision{
|
||||
client: netip.MustParseAddr("203.0.113.9"),
|
||||
lookedUp: true,
|
||||
},
|
||||
h: &handler{config: &config.Config{LookupSource: "geojs"}},
|
||||
h: &handler{config: &config.Config{LookupSource: "geojs"}},
|
||||
client: netip.MustParseAddr("203.0.113.9"),
|
||||
lookedUp: true,
|
||||
}
|
||||
|
||||
defer func() {
|
||||
|
||||
@@ -323,7 +323,6 @@ func (rq *request) alertBan(ban bans.Ban) {
|
||||
|
||||
rq.h.alerts.Raise(alerts.Alert{
|
||||
Event: event,
|
||||
Host: rq.site,
|
||||
Client: rq.client,
|
||||
Netblock: ban.Netblock,
|
||||
ASN: ban.Notes.ASN,
|
||||
@@ -340,11 +339,11 @@ func (rq *request) alertBan(ban bans.Ban) {
|
||||
func (rq *request) noted(now time.Time, status int) bans.Request {
|
||||
return bans.Request{
|
||||
Time: now,
|
||||
Method: rq.line.Method,
|
||||
Host: rq.line.Host,
|
||||
Path: rq.requestURI,
|
||||
Method: rq.in.Method,
|
||||
Host: rq.in.Host,
|
||||
Path: rq.in.URL.RequestURI(),
|
||||
Status: status,
|
||||
UserAgent: rq.line.UserAgent,
|
||||
UserAgent: rq.in.UserAgent(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -25,8 +25,6 @@ const (
|
||||
userAgent = "ban-test/1.0"
|
||||
// permanent is the log line's ban_expires for a permanent ban.
|
||||
permanent = "permanent"
|
||||
// banResponseClose is the SWWAF_BAN_RESPONSE that closes the connection.
|
||||
banResponseClose = "close"
|
||||
)
|
||||
|
||||
func TestBrokenLimitBansTheClient(t *testing.T) {
|
||||
@@ -239,7 +237,7 @@ func TestBanResponseAnswersEveryRefusalButTheSizeLimits(t *testing.T) {
|
||||
{"", http.StatusForbidden},
|
||||
{"403", http.StatusForbidden},
|
||||
{"429", http.StatusTooManyRequests},
|
||||
{banResponseClose, 0},
|
||||
{"close", 0},
|
||||
} {
|
||||
t.Run(banResponse+"="+tc.setting, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
|
||||
@@ -213,20 +211,9 @@ func (w *responseWriter) FlushError() error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Hijack takes over the connection to the client, which ReverseProxy does
|
||||
// only to pass on the app's 101 Switching Protocols: that is then the
|
||||
// status sent.
|
||||
func (w *responseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
|
||||
conn, buffered, err := http.NewResponseController(w.ResponseWriter).Hijack()
|
||||
if err == nil {
|
||||
w.status = http.StatusSwitchingProtocols
|
||||
}
|
||||
|
||||
return conn, buffered, err
|
||||
}
|
||||
|
||||
// Unwrap lets http.ResponseController reach net/http's own
|
||||
// ResponseWriter for what responseWriter does not do itself.
|
||||
// ResponseWriter, which is how ReverseProxy takes over the connection of
|
||||
// an upgraded request.
|
||||
func (w *responseWriter) Unwrap() http.ResponseWriter {
|
||||
return w.ResponseWriter
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ import (
|
||||
// limit while the Core Rule Set reads it, which it notes nothing of.
|
||||
func (rq *request) checkCoreRuleSet() string {
|
||||
cfg := rq.h.config
|
||||
if cfg.WAFMode == config.WAFModeOff || pathExempt(rq.in, cfg.WAFExemptPaths) {
|
||||
if cfg.WAFMode == config.WAFModeOff || pathExempt(rq.in.URL, cfg.WAFExemptPaths) {
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -113,7 +113,6 @@ func (rq *request) alertWAFBlock(result waf.Result) {
|
||||
|
||||
rq.h.alerts.Raise(alerts.Alert{
|
||||
Event: alerts.EventWAFBlock,
|
||||
Host: rq.site,
|
||||
Client: rq.client,
|
||||
Netblock: rq.h.clientGroup(rq.client),
|
||||
ASN: rq.line.ASN,
|
||||
|
||||
@@ -86,75 +86,6 @@ func TestEveryClientIsLookedUpWithoutWaitingWhileNoSettingNeedsIt(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestThatWaitedForItsLookupIsCountedWhenItGoesOn(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// The country list has a new client's request wait for its answer,
|
||||
// which the stand-in for GeoJS gives only once released.
|
||||
geojsURL, asked, release := startHeldGeoJS(t)
|
||||
app := startApp(t, func(http.ResponseWriter, *http.Request) {})
|
||||
clk := &clock{now: time.Date(2026, 10, 6, 0, 0, 58, 0, time.UTC)}
|
||||
addr, out, server := startProxyWithClock(t, app.URL, geojsURL, clk.Now,
|
||||
map[string]string{
|
||||
trustedProxies: trustLocalhost,
|
||||
lookupTimeout: "1h",
|
||||
deniedCountries: "kp",
|
||||
})
|
||||
|
||||
requestFromDE := func() *http.Request {
|
||||
req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
|
||||
req.Header.Set(forwardedFor, fromDE)
|
||||
|
||||
return req
|
||||
}
|
||||
|
||||
// fromDE's first request arrives two seconds before a minute ends,
|
||||
// and waits for GeoJS.
|
||||
first, httpClient := requestFromDE(), newClient(t)
|
||||
firstEnded := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
res, err := httpClient.Do(first)
|
||||
if err == nil {
|
||||
err = readAnswer(res).err
|
||||
}
|
||||
|
||||
firstEnded <- err
|
||||
}()
|
||||
|
||||
waitUntil(func() bool { return len(asked()) == 1 })
|
||||
|
||||
// Half a second into the next minute, the answer is kept, as an edit
|
||||
// of lookups.json would keep it, while the first request still waits
|
||||
// for GeoJS. The second request goes on at once, and is counted.
|
||||
clk.advance(2500 * time.Millisecond)
|
||||
server.GeoJS.Load([]lookup.Answer{{
|
||||
Client: netip.MustParsePrefix(fromDE + "/32"), ASN: asnDE, ASName: asNameDE,
|
||||
Country: "DE", Answered: clk.Now(), Used: clk.Now(),
|
||||
}})
|
||||
wantStatus(t, do(t, requestFromDE()), http.StatusOK)
|
||||
|
||||
// GeoJS answers, and the first request goes on.
|
||||
release()
|
||||
|
||||
err := <-firstEnded
|
||||
if err != nil {
|
||||
t.Fatalf("first request: %v", err)
|
||||
}
|
||||
|
||||
wantLine(t, out.requestLines(t, 2)[1], http.StatusOK, requestlog.ActionForward)
|
||||
|
||||
// The third request's count holds all three: the first was counted
|
||||
// when it went on, in the minute under way, and the second's count
|
||||
// was kept.
|
||||
clk.advance(500 * time.Millisecond)
|
||||
wantStatus(t, do(t, requestFromDE()), http.StatusOK)
|
||||
|
||||
if got := out.requestLines(t, 3)[2].Counts.Minute; got != 3 {
|
||||
t.Errorf("the third request's count in the minute is %v, want 3", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASNumberAndNameInTheLogLineTheHistoryTheBanNotesAndTheAlert(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -207,7 +207,7 @@ func TestMetricsCountLimitsAndBans(t *testing.T) {
|
||||
rateLimitPerMinute: "1",
|
||||
rateLimitExemptNets: scraper,
|
||||
denyNets: denied,
|
||||
banResponse: banResponseClose,
|
||||
banResponse: "close",
|
||||
limitBanDuration: "1h",
|
||||
maxBanDuration: "2h",
|
||||
})
|
||||
|
||||
@@ -74,47 +74,6 @@ func TestPassesRequestAndAnswerUnchanged(t *testing.T) {
|
||||
wantRequestFields(t, line, addr, len(requestBody), len(answerBody))
|
||||
}
|
||||
|
||||
func TestPassesThePathAsSent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, path := range []string{
|
||||
// An encoded slash and a character Go escapes: |, ^, and é sent
|
||||
// as its UTF-8 bytes.
|
||||
"/a%2Fb|c", "/feat%2Fx/file^1", "/x%2Fy/café",
|
||||
// A path starting with //, which Go would send as a scheme and
|
||||
// host if it were put in URL.Opaque.
|
||||
"//a%2Fb",
|
||||
} {
|
||||
t.Run(path, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
saw := make(chan string, 1)
|
||||
app := startApp(t, func(_ http.ResponseWriter, r *http.Request) {
|
||||
saw <- r.RequestURI
|
||||
})
|
||||
addr, out := startProxy(t, app.URL, nil)
|
||||
|
||||
conn := dial(t, addr)
|
||||
send(t, conn, "GET "+path+"?q=1 HTTP/1.1\r\nHost: "+appHost+"\r\n\r\n")
|
||||
|
||||
got := readResponse(t, conn)
|
||||
if got.status != http.StatusOK {
|
||||
t.Fatalf("status %d, want %d", got.status, http.StatusOK)
|
||||
}
|
||||
|
||||
target := <-saw
|
||||
if target != path+"?q=1" {
|
||||
t.Errorf("app saw %s, want %s", target, path+"?q=1")
|
||||
}
|
||||
|
||||
line := out.requestLine(t)
|
||||
if line.Path != path {
|
||||
t.Errorf("log line has path %s, want %s", line.Path, path)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// wantAppSaw checks that the app received the test's request unchanged.
|
||||
func wantAppSaw(t *testing.T, saw appSaw, body []byte) {
|
||||
t.Helper()
|
||||
@@ -192,36 +151,6 @@ func wantRequestFields(t *testing.T, line logLine, host string, sent, received i
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddsNoContentTypeTheAppDidNotSend(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const typed = "text/plain"
|
||||
|
||||
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/typed" {
|
||||
w.Header().Set("Content-Type", typed)
|
||||
} else {
|
||||
// The key with no value keeps Go's server from adding one.
|
||||
w.Header()["Content-Type"] = nil
|
||||
}
|
||||
|
||||
_, _ = io.WriteString(w, "<!doctype html><title>page</title>")
|
||||
})
|
||||
addr, _ := startProxy(t, app.URL, nil)
|
||||
|
||||
got := get(t, addr, "/")
|
||||
if got.header.Values("Content-Type") != nil {
|
||||
t.Errorf("client got Content-Type %q, want none",
|
||||
got.header.Get("Content-Type"))
|
||||
}
|
||||
|
||||
got = get(t, addr, "/typed")
|
||||
if got.header.Get("Content-Type") != typed {
|
||||
t.Errorf("client got Content-Type %q, want %q",
|
||||
got.header.Get("Content-Type"), typed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamsTheRequestBody(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -380,23 +309,6 @@ func echoAfterUpgrade(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnswers502WhenTheAppSwitchesToAnotherProtocol(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// answerAfterUpgrade switches to websocket whatever the client asked
|
||||
// for, and ReverseProxy passes on a switch only to the protocol the
|
||||
// client asked for, here "websocket, foo".
|
||||
app := startApp(t, answerAfterUpgrade)
|
||||
addr, out := startProxy(t, app.URL, nil)
|
||||
|
||||
conn := dial(t, addr)
|
||||
send(t, conn, "GET /ws HTTP/1.1\r\nHost: app\r\n"+
|
||||
"Connection: Upgrade\r\nUpgrade: websocket, foo\r\n\r\n")
|
||||
|
||||
wantStatus(t, readResponse(t, conn), http.StatusBadGateway)
|
||||
wantLine(t, out.requestLine(t), http.StatusBadGateway, requestlog.ActionUpstreamError)
|
||||
}
|
||||
|
||||
func TestServerHasTheDefaultLimits(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
+33
-52
@@ -1,8 +1,6 @@
|
||||
// Package proxy runs the checks on each request and writes its request log
|
||||
// line. New passes each request it lets through to the app and the app's
|
||||
// answer back, unchanged, within the size and time limits. NewServeWAF instead
|
||||
// answers rproxy's question about each request, and takes rproxy's report once
|
||||
// the visitor's answer has ended, which writes the request a second log line.
|
||||
// Package proxy passes each request to the app and the app's answer back,
|
||||
// unchanged, within the size and time limits, and writes one request log
|
||||
// line for each request.
|
||||
package proxy
|
||||
|
||||
import (
|
||||
@@ -13,8 +11,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
lru "github.com/hashicorp/golang-lru/v2"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
@@ -23,6 +19,7 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
)
|
||||
@@ -56,8 +53,7 @@ const ClientsPath = "/_smallwebwaf/clients/"
|
||||
// Params are what New needs.
|
||||
type Params struct {
|
||||
Config *config.Config
|
||||
// RequestLog receives one JSON line per request, and from servewaf a second
|
||||
// when the request's report comes.
|
||||
// RequestLog receives one JSON line per request.
|
||||
RequestLog io.Writer
|
||||
// ProcessLog receives the process's own messages.
|
||||
ProcessLog *slog.Logger
|
||||
@@ -113,23 +109,6 @@ type Server struct {
|
||||
// SWWAF_CLIENT_REQUEST_TIMEOUT while the headers arrive; the proxy
|
||||
// applies the timeouts and size limits from then on.
|
||||
func New(params Params) *Server {
|
||||
h := newHandler(params)
|
||||
|
||||
return h.server(&http.Server{
|
||||
Handler: h,
|
||||
ReadHeaderTimeout: params.Config.ClientRequestTimeout,
|
||||
// Off is an IdleTimeout of 0, which Go's server replaces with
|
||||
// ReadTimeout: no limit, as long as ReadTimeout stays unset.
|
||||
IdleTimeout: params.Config.ClientIdleTimeout,
|
||||
// Go's server reads 4 KiB past MaxHeaderBytes before it
|
||||
// refuses, so the limit a client meets is the setting.
|
||||
MaxHeaderBytes: int(params.Config.ClientRequestHeaderMaxBytes - 4<<10),
|
||||
})
|
||||
}
|
||||
|
||||
// newHandler returns the handler with what every request shares, as the
|
||||
// settings in params make it.
|
||||
func newHandler(params Params) *handler {
|
||||
errorLog := slog.NewLogLogger(params.ProcessLog.Handler(), slog.LevelWarn)
|
||||
m := metrics.New(params.Config.MetricsTopN, params.Config.InstanceName)
|
||||
lists, dnsbl, abuseIPDB := newReputation(params, m)
|
||||
@@ -192,7 +171,29 @@ func newHandler(params Params) *handler {
|
||||
m.AddBansAndClients(h.ledger, h.limiter, params.Now)
|
||||
m.AddRules(params.Rules)
|
||||
|
||||
return h
|
||||
return &Server{
|
||||
Server: &http.Server{
|
||||
Addr: params.Config.ListenAddr,
|
||||
Handler: h,
|
||||
ReadHeaderTimeout: params.Config.ClientRequestTimeout,
|
||||
// Off is an IdleTimeout of 0, which Go's server replaces with
|
||||
// ReadTimeout: no limit, as long as ReadTimeout stays unset.
|
||||
IdleTimeout: params.Config.ClientIdleTimeout,
|
||||
// Go's server reads 4 KiB past MaxHeaderBytes before it
|
||||
// refuses, so the limit a client meets is the setting.
|
||||
MaxHeaderBytes: int(params.Config.ClientRequestHeaderMaxBytes - 4<<10),
|
||||
ErrorLog: errorLog,
|
||||
},
|
||||
Ledger: h.ledger,
|
||||
Limiter: h.limiter,
|
||||
GeoJS: h.geojs,
|
||||
Anomalies: h.anomalies,
|
||||
LookupFile: h.lookupFile,
|
||||
Lists: h.lists,
|
||||
DNSBL: h.dnsbl,
|
||||
AbuseIPDB: h.abuseIPDB,
|
||||
Metrics: m,
|
||||
}
|
||||
}
|
||||
|
||||
// newReputation returns the lists fetched from URLs, the DNSBL zones'
|
||||
@@ -273,10 +274,6 @@ type handler struct {
|
||||
rules *rules.Files
|
||||
coreRuleSet *waf.CoreRuleSet
|
||||
alerts *alerts.Queue
|
||||
// awaitingReport holds, in servewaf, the decision on each answered
|
||||
// question by its id until its report comes, and nil for a question still
|
||||
// being handled.
|
||||
awaitingReport *lru.Cache[string, *decision]
|
||||
}
|
||||
|
||||
// newTransport returns what carries requests to the app. It never goes
|
||||
@@ -302,7 +299,11 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// The health endpoint is answered at once, before any check, so that
|
||||
// a health checker is never refused. It does not ask the app.
|
||||
if r.Method == http.MethodGet && r.URL.Path == HealthPath {
|
||||
rq.answerHealth()
|
||||
rq.line.Action = requestlog.ActionAdmin
|
||||
// Set here rather than left to Go's server, which would set it only
|
||||
// after the log line has taken the response's headers.
|
||||
rq.out.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
_, _ = io.WriteString(rq.out, "ok\n")
|
||||
|
||||
return
|
||||
}
|
||||
@@ -338,23 +339,3 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
rq.forward(r.Context())
|
||||
}
|
||||
|
||||
// server returns the Server of h, which listens at SWWAF_LISTEN_ADDR and
|
||||
// logs through h, with what httpServer sets.
|
||||
func (h *handler) server(httpServer *http.Server) *Server {
|
||||
httpServer.Addr = h.config.ListenAddr
|
||||
httpServer.ErrorLog = h.errorLog
|
||||
|
||||
return &Server{
|
||||
Server: httpServer,
|
||||
Ledger: h.ledger,
|
||||
Limiter: h.limiter,
|
||||
GeoJS: h.geojs,
|
||||
Anomalies: h.anomalies,
|
||||
LookupFile: h.lookupFile,
|
||||
Lists: h.lists,
|
||||
DNSBL: h.dnsbl,
|
||||
AbuseIPDB: h.abuseIPDB,
|
||||
Metrics: h.metrics,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -284,16 +284,6 @@ func newProxy(
|
||||
) (*proxy.Server, *output, *alerts.Queue) {
|
||||
t.Helper()
|
||||
|
||||
return newServerWith(t, proxy.New, appURL, geojsURL, now, env)
|
||||
}
|
||||
|
||||
// newServerWith is newProxy for the server build makes.
|
||||
func newServerWith(
|
||||
t *testing.T, build func(proxy.Params) *proxy.Server,
|
||||
appURL, geojsURL string, now func() time.Time, env map[string]string,
|
||||
) (*proxy.Server, *output, *alerts.Queue) {
|
||||
t.Helper()
|
||||
|
||||
settings := map[string]string{
|
||||
"SWWAF_UPSTREAM_URL": appURL, rulesDir: t.TempDir(), instanceName: "app",
|
||||
wafMode: off,
|
||||
@@ -344,7 +334,7 @@ func newServerWith(
|
||||
}
|
||||
}
|
||||
|
||||
server := build(proxy.Params{
|
||||
server := proxy.New(proxy.Params{
|
||||
Config: cfg,
|
||||
RequestLog: out,
|
||||
ProcessLog: processLog,
|
||||
|
||||
@@ -185,9 +185,6 @@ func TestRateLimitCountsPathsThatAreNotExempt(t *testing.T) {
|
||||
"/assets/x%2Fy",
|
||||
"/assets/x%2fy",
|
||||
`/assets/x\y`,
|
||||
// The same with a character Go escapes, |, for which Go's
|
||||
// URL.EscapedPath gives /assets/x/y%7Cz, with no encoded slash.
|
||||
"/assets/x%2Fy|z",
|
||||
} {
|
||||
t.Run(sent, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -110,7 +110,6 @@ func (rq *request) noteHit(hit bans.ReputationHit, reason string) {
|
||||
rq.h.metrics.ReputationHit(hit.Source)
|
||||
rq.h.alerts.Raise(alerts.Alert{
|
||||
Event: alerts.EventReputationHit,
|
||||
Host: rq.site,
|
||||
Client: rq.client,
|
||||
Netblock: rq.h.clientGroup(rq.client),
|
||||
ASN: rq.line.ASN,
|
||||
|
||||
+63
-147
@@ -8,6 +8,7 @@ import (
|
||||
"net/http/httptrace"
|
||||
"net/http/httputil"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -21,7 +22,6 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||
)
|
||||
|
||||
// flushAfterEachWrite has ReverseProxy pass on each part of the app's
|
||||
@@ -39,20 +39,20 @@ type refusal struct {
|
||||
limit string
|
||||
}
|
||||
|
||||
// decision is what the checks note of a request: its log line, and what
|
||||
// counting the request needs once it has ended. servewaf keeps a copy of it
|
||||
// until the request's report comes. It holds no credentials, since the log line
|
||||
// never does, and neither the request's objects nor their bodies.
|
||||
type decision struct {
|
||||
line requestlog.Line
|
||||
client netip.Addr
|
||||
// requestURI is the path with the query, as a ban's notes keep it.
|
||||
requestURI string
|
||||
// site is, in servewaf, the host the visitor asked for, which the
|
||||
// alerts name, since one servewaf serves many sites. It is "" otherwise.
|
||||
site string
|
||||
// allowed is true for a question answered with DispositionAllow.
|
||||
allowed bool
|
||||
// request is one request on its way through smallwebwaf, from the moment
|
||||
// its headers have been read to its log line.
|
||||
type request struct {
|
||||
h *handler
|
||||
in *http.Request
|
||||
// rc sets the deadlines of the connection to the client.
|
||||
rc *http.ResponseController
|
||||
out *responseWriter
|
||||
body *requestBody // nil for a request without a body
|
||||
line requestlog.Line
|
||||
|
||||
client netip.Addr
|
||||
peer netip.Addr
|
||||
peerTrusted bool
|
||||
// lookedUp is true once the client's AS number and country have been
|
||||
// looked up, whether or not an answer was there, and lookupAnswer is
|
||||
// what the lookup gave then, the zero Answer while GeoJS had given none.
|
||||
@@ -65,37 +65,18 @@ type decision struct {
|
||||
counted bool
|
||||
limitPercent, bytesPercent percentage
|
||||
// attack is true for a request that matched a ban rule or asked for a
|
||||
// trap path, ruleBlocked for one a block rule refused, and wafBlocked for
|
||||
// one the Core Rule Set refused, each an offence its client's history
|
||||
// counts.
|
||||
attack, ruleBlocked, wafBlocked bool
|
||||
// reputation is the reputation sources that list the client, for the
|
||||
// notes of a ban the request makes.
|
||||
reputation []bans.ReputationHit
|
||||
}
|
||||
|
||||
// request is one request on its way through smallwebwaf, from the moment
|
||||
// its headers have been read to its log line.
|
||||
type request struct {
|
||||
decision
|
||||
|
||||
h *handler
|
||||
in *http.Request
|
||||
// rc sets the deadlines of the connection to the client.
|
||||
rc *http.ResponseController
|
||||
out *responseWriter
|
||||
body *requestBody // nil for a request without a body
|
||||
|
||||
peer netip.Addr
|
||||
peerTrusted bool
|
||||
// tokenRefused is true for a request refused for a missing or wrong
|
||||
// token, an offence its client's history counts.
|
||||
tokenRefused bool
|
||||
// trap path, ruleBlocked for one a block rule refused, wafBlocked for
|
||||
// one the Core Rule Set refused, and tokenRefused for one refused for a
|
||||
// missing or wrong token, each an offence its client's history counts.
|
||||
attack, ruleBlocked, wafBlocked, tokenRefused bool
|
||||
// blocklisted is true once a blocklist is found to list the client,
|
||||
// dnsblListed once a DNSBL zone's verdict is, and abuseIPDBHit once
|
||||
// AbuseIPDB's score of it is a hit.
|
||||
blocklisted, dnsblListed, abuseIPDBHit bool
|
||||
start time.Time
|
||||
// reputation is the reputation sources that list the client, for the
|
||||
// notes of a ban the request makes.
|
||||
reputation []bans.ReputationHit
|
||||
start time.Time
|
||||
// checked is when the checks were done, and upstreamStart when the
|
||||
// request was handed to the app.
|
||||
checked time.Time
|
||||
@@ -127,62 +108,50 @@ type request struct {
|
||||
answerStarted time.Time
|
||||
}
|
||||
|
||||
// newRequest starts handling r: it works out the client from the TCP peer
|
||||
// and X-Forwarded-For, and goes on as startRequest does.
|
||||
// newRequest starts handling r: it notes the time, counts the request as
|
||||
// under way, works out the client, and starts the log line with what is
|
||||
// known of the request.
|
||||
func (h *handler) newRequest(w http.ResponseWriter, r *http.Request) *request {
|
||||
peer := peerAddress(r)
|
||||
trusted := h.config.TrustedProxies
|
||||
client := clientAddress(peer, r.Header.Values("X-Forwarded-For"), trusted)
|
||||
|
||||
return h.startRequest(w, r, client, peer, isInside(peer, trusted))
|
||||
}
|
||||
|
||||
// startRequest starts handling r from client, which came through peer: it
|
||||
// notes the time, counts the request as under way, and starts the log line
|
||||
// with what is known of the request. peerTrusted says whether to believe
|
||||
// the request's id and scheme, which r's headers give.
|
||||
func (h *handler) startRequest(
|
||||
w http.ResponseWriter, r *http.Request, client, peer netip.Addr, peerTrusted bool,
|
||||
) *request {
|
||||
h.metrics.RequestStarted()
|
||||
|
||||
start := time.Now()
|
||||
peer := peerAddress(r)
|
||||
trusted := h.config.TrustedProxies
|
||||
peerTrusted := isInside(peer, trusted)
|
||||
forwardedFor := r.Header.Values("X-Forwarded-For")
|
||||
client := clientAddress(peer, forwardedFor, trusted)
|
||||
|
||||
rq := &request{
|
||||
decision: decision{
|
||||
client: client,
|
||||
requestURI: r.URL.RequestURI(),
|
||||
line: requestlog.Line{
|
||||
Time: requestlog.FormatTime(start),
|
||||
Instance: h.config.InstanceName,
|
||||
ClientIP: client.String(),
|
||||
Method: r.Method,
|
||||
Scheme: scheme(r, peerTrusted),
|
||||
Host: r.Host,
|
||||
Path: rules.Path(r),
|
||||
Query: r.URL.RawQuery,
|
||||
Protocol: r.Proto,
|
||||
Referer: r.Referer(),
|
||||
UserAgent: r.UserAgent(),
|
||||
RequestID: requestID(r, peerTrusted),
|
||||
PeerIP: peer.String(),
|
||||
ForwardedFor: strings.Join(forwardedFor, ", "),
|
||||
ClientGroup: h.clientGroup(client).String(),
|
||||
ContentType: r.Header.Get("Content-Type"),
|
||||
RequestHeaders: requestHeaders(r, h.config.LogRequestHeaders),
|
||||
HasAuthorization: len(r.Header.Values("Authorization")) > 0,
|
||||
HasCookie: len(r.Header.Values("Cookie")) > 0,
|
||||
Action: requestlog.ActionForward,
|
||||
},
|
||||
},
|
||||
h: h,
|
||||
in: r,
|
||||
rc: http.NewResponseController(w),
|
||||
out: &responseWriter{ResponseWriter: w},
|
||||
client: client,
|
||||
peer: peer,
|
||||
peerTrusted: peerTrusted,
|
||||
start: start,
|
||||
line: requestlog.Line{
|
||||
Time: requestlog.FormatTime(start),
|
||||
Instance: h.config.InstanceName,
|
||||
ClientIP: client.String(),
|
||||
Method: r.Method,
|
||||
Scheme: scheme(r, peerTrusted),
|
||||
Host: r.Host,
|
||||
Path: r.URL.EscapedPath(),
|
||||
Query: r.URL.RawQuery,
|
||||
Protocol: r.Proto,
|
||||
Referer: r.Referer(),
|
||||
UserAgent: r.UserAgent(),
|
||||
RequestID: requestID(r, peerTrusted),
|
||||
PeerIP: peer.String(),
|
||||
ForwardedFor: strings.Join(forwardedFor, ", "),
|
||||
ClientGroup: h.clientGroup(client).String(),
|
||||
ContentType: r.Header.Get("Content-Type"),
|
||||
RequestHeaders: requestHeaders(r, h.config.LogRequestHeaders),
|
||||
HasAuthorization: len(r.Header.Values("Authorization")) > 0,
|
||||
HasCookie: len(r.Header.Values("Cookie")) > 0,
|
||||
Action: requestlog.ActionForward,
|
||||
},
|
||||
}
|
||||
|
||||
// A length of -1 is a body whose length was not announced.
|
||||
@@ -292,12 +261,6 @@ func (rq *request) checkClient(ctx context.Context) string {
|
||||
|
||||
rq.lookUp(ctx)
|
||||
|
||||
// The lookup can wait for GeoJS, so the checks after it read the time
|
||||
// again: a request counted at the time it arrived could be counted
|
||||
// after a later one of its client's, which the rate limits would take
|
||||
// for the clock set back.
|
||||
now = rq.h.now()
|
||||
|
||||
if rq.countryDenied() {
|
||||
return requestlog.ActionCountryDenied
|
||||
}
|
||||
@@ -315,7 +278,7 @@ func (rq *request) checkClient(ctx context.Context) string {
|
||||
}
|
||||
|
||||
rq.counted = !isInside(rq.client, cfg.RateLimitExemptNets) &&
|
||||
!pathExempt(rq.in, cfg.RateLimitExemptPaths)
|
||||
!pathExempt(rq.in.URL, cfg.RateLimitExemptPaths)
|
||||
if rq.counted {
|
||||
rq.limitPercent, rq.bytesPercent = rq.limitPercentages()
|
||||
rq.line.LimitPercent, rq.line.LimitPercentSetting = rq.limitPercent.logged()
|
||||
@@ -338,7 +301,7 @@ func (rq *request) checkClient(ctx context.Context) string {
|
||||
return rq.checkCoreRuleSet()
|
||||
}
|
||||
|
||||
// pathExempt reports whether r is exempt under prefixes,
|
||||
// pathExempt reports whether a request for u is exempt under prefixes,
|
||||
// SWWAF_RATE_LIMIT_EXEMPT_PATHS or SWWAF_WAF_EXEMPT_PATHS: whether its
|
||||
// path as sent, the path the app receives, not percent-decoded, starts
|
||||
// with one of prefixes, so that /%61ssets/x is not under /assets/ for an
|
||||
@@ -347,9 +310,10 @@ func (rq *request) checkClient(ctx context.Context) string {
|
||||
// encoded slash (%2F or %2f), never is, since an app may act on it as a
|
||||
// path outside every prefix: /assets/..%2Flogin as /login, or /assets%2Fx
|
||||
// as one path segment, as Go's router does.
|
||||
func pathExempt(r *http.Request, prefixes []string) bool {
|
||||
decoded := r.URL.Path
|
||||
sent := rules.Path(r)
|
||||
func pathExempt(u *url.URL, prefixes []string) bool {
|
||||
decoded := u.Path
|
||||
// EscapedPath is the path as the app receives it, not decoded.
|
||||
sent := u.EscapedPath()
|
||||
|
||||
if strings.Contains(decoded, "..") || strings.Contains(decoded, `\`) ||
|
||||
strings.Contains(strings.ToLower(sent), "%2f") {
|
||||
@@ -405,17 +369,6 @@ func (rq *request) rewrite(pr *httputil.ProxyRequest) {
|
||||
// ReverseProxy drops query parameters it cannot parse; the app gets
|
||||
// the query as the client sent it.
|
||||
pr.Out.URL.RawQuery = pr.In.URL.RawQuery
|
||||
// ReverseProxy sends the path URL.EscapedPath gives, which, for a path
|
||||
// holding a character Go escapes, such as |, is the whole path decoded
|
||||
// and escaped again, %2F turned into /. Opaque is sent as it is, so the
|
||||
// app gets the path as the client sent it. Left out are a target that
|
||||
// is no path, such as *, and a path starting with //, which Go would
|
||||
// send in Opaque as a scheme and host.
|
||||
path := rules.Path(pr.In)
|
||||
if strings.HasPrefix(path, "/") && !strings.HasPrefix(path, "//") {
|
||||
pr.Out.URL.Opaque = path
|
||||
}
|
||||
|
||||
setForwardedHeaders(pr.In, pr.Out, rq.peer, rq.peerTrusted)
|
||||
pr.Out.Header.Set(requestIDHeader, rq.line.RequestID)
|
||||
pr.Out.Header.Del(asnHeader)
|
||||
@@ -436,10 +389,8 @@ func (rq *request) modifyResponse(res *http.Response) error {
|
||||
// timeouts. ReverseProxy writes this answer straight to the
|
||||
// connection it takes over, not through rq.out, and then copies
|
||||
// what passes each way through res.Body, the connection to the app.
|
||||
// It may still refuse the switch, as when the app's Upgrade is not
|
||||
// the client's, so the status is noted only once it has taken over
|
||||
// the connection, by rq.out's Hijack.
|
||||
rq.stopTimers()
|
||||
rq.out.status = res.StatusCode
|
||||
rq.line.Websocket = true
|
||||
|
||||
conn, ok := res.Body.(io.ReadWriteCloser)
|
||||
@@ -462,13 +413,6 @@ func (rq *request) modifyResponse(res *http.Response) error {
|
||||
return errResponseTooLarge
|
||||
}
|
||||
|
||||
// Go's server adds a Content-Type it guesses from the body to a
|
||||
// response whose headers have no Content-Type key. The key with no
|
||||
// value sends none, as the app sent none.
|
||||
if res.Header.Get("Content-Type") == "" {
|
||||
rq.out.Header()["Content-Type"] = nil
|
||||
}
|
||||
|
||||
res.Body = &responseBody{body: limitBody(res.Body, maxBytes), rq: rq}
|
||||
rq.startClientResponseTimeout()
|
||||
|
||||
@@ -478,13 +422,6 @@ func (rq *request) modifyResponse(res *http.Response) error {
|
||||
// answerError is ReverseProxy's ErrorHandler: the request could not be
|
||||
// passed to the app, or the app's answer cannot be passed on.
|
||||
func (rq *request) answerError(_ http.ResponseWriter, _ *http.Request, err error) {
|
||||
if rq.upgraded != nil {
|
||||
// The app switched protocols, but the switch was not passed on,
|
||||
// and ReverseProxy leaves the app's connection open when it
|
||||
// refuses it.
|
||||
_ = rq.upgraded.Close()
|
||||
}
|
||||
|
||||
refused := rq.refused.Load()
|
||||
if refused == nil {
|
||||
if rq.in.Context().Err() != nil {
|
||||
@@ -600,20 +537,10 @@ func (rq *request) finish() {
|
||||
}
|
||||
|
||||
// Counted before the log line is written, so that the metrics count
|
||||
// every request whose line is out. An allowed request is the exception:
|
||||
// its report gives the status, bytes and times the metrics count.
|
||||
if rq.allowed {
|
||||
rq.h.metrics.RequestAllowed()
|
||||
} else {
|
||||
rq.h.metrics.RequestEnded(line, limit, duration, upstreamDuration)
|
||||
}
|
||||
// every request whose line is out.
|
||||
rq.h.metrics.RequestEnded(line, limit, duration, upstreamDuration)
|
||||
|
||||
rq.writeLine()
|
||||
}
|
||||
|
||||
// writeLine writes the request's log line.
|
||||
func (rq *request) writeLine() {
|
||||
err := requestlog.Write(rq.h.requestLog, &rq.line)
|
||||
err := requestlog.Write(rq.h.requestLog, line)
|
||||
if err != nil {
|
||||
rq.h.processLog.Error("writing the request log failed", "error", err.Error())
|
||||
}
|
||||
@@ -675,7 +602,6 @@ func (rq *request) countAnomalies() {
|
||||
answer, _ := rq.answerAtTheEnd()
|
||||
|
||||
rq.h.anomalies.Count(rq.h.now(), anomaly.Request{
|
||||
Host: rq.site,
|
||||
Client: rq.client,
|
||||
ClientGroup: rq.h.clientGroup(rq.client),
|
||||
ASN: answer.ASN,
|
||||
@@ -778,16 +704,6 @@ func (rq *request) requestTimedOut(limit string) {
|
||||
}
|
||||
|
||||
if rq.body == nil || !rq.body.waiting.Load() {
|
||||
// Once the app has answered, the transport, ReverseProxy and Go's
|
||||
// server each close the client's body, and closing it reads what
|
||||
// is left of a body under 256 KiB without requestBody.Read, so
|
||||
// waiting is false while that read waits for the client. That
|
||||
// read ends now, before refuse: should smallwebwaf still answer
|
||||
// itself, answer then sets the deadline it gives its own answers.
|
||||
if !rq.answerStarted.IsZero() {
|
||||
rq.stopReadingBody(time.Now())
|
||||
}
|
||||
|
||||
rq.refuse(refusal{
|
||||
status: http.StatusGatewayTimeout,
|
||||
action: requestlog.ActionTimedOut,
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
)
|
||||
|
||||
func TestAnAnswerWithoutAContentTypeGetsNone(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, _ *http.Request) {
|
||||
rq := &request{
|
||||
h: &handler{config: &config.Config{}},
|
||||
out: &responseWriter{ResponseWriter: w},
|
||||
}
|
||||
app := &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Header: http.Header{},
|
||||
Body: http.NoBody,
|
||||
}
|
||||
|
||||
err := rq.modifyResponse(app)
|
||||
if err != nil {
|
||||
t.Errorf("modifyResponse: %v", err)
|
||||
}
|
||||
|
||||
// The headers go out with this, the first part of the body,
|
||||
// which Go's server would guess a Content-Type from.
|
||||
_, _ = io.WriteString(w, "<!doctype html><title>page</title>")
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, srv.URL,
|
||||
http.NoBody)
|
||||
if err != nil {
|
||||
t.Fatalf("new request: %v", err)
|
||||
}
|
||||
|
||||
res, err := srv.Client().Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
|
||||
_ = res.Body.Close()
|
||||
|
||||
if res.Header.Values("Content-Type") != nil {
|
||||
t.Errorf("client got Content-Type %q, want none",
|
||||
res.Header.Get("Content-Type"))
|
||||
}
|
||||
}
|
||||
@@ -1,464 +0,0 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
lru "github.com/hashicorp/golang-lru/v2"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// The headers of rproxy's question and of servewaf's answer, as
|
||||
// "servewaf" in SPEC.md gives them.
|
||||
const (
|
||||
// ClientHeader marks a request as rproxy's question, and gives the
|
||||
// visitor's address.
|
||||
ClientHeader = "X-Smallwebwaf-Client"
|
||||
// PeerHeader gives the address of rproxy's own TCP peer.
|
||||
PeerHeader = "X-Smallwebwaf-Peer"
|
||||
// ContentLengthHeader gives the body length the visitor announced.
|
||||
ContentLengthHeader = "X-Smallwebwaf-Content-Length"
|
||||
// DispositionHeader gives servewaf's answer.
|
||||
DispositionHeader = "X-Smallwebwaf-Disposition"
|
||||
)
|
||||
|
||||
// The values of DispositionHeader.
|
||||
const (
|
||||
// DispositionAllow tells rproxy to pass the request to the app.
|
||||
DispositionAllow = "allow"
|
||||
// DispositionDeny tells rproxy to give the visitor the answer's status,
|
||||
// headers and body.
|
||||
DispositionDeny = "deny"
|
||||
// DispositionClose tells rproxy to close the visitor's connection
|
||||
// without an answer, which SWWAF_BAN_RESPONSE asks for when it is close.
|
||||
// servewaf closing the question's connection instead would read as
|
||||
// servewaf being down, and rproxy would then let the request through.
|
||||
DispositionClose = "close"
|
||||
)
|
||||
|
||||
// ReportPath is where rproxy sends the report of each request servewaf
|
||||
// answered.
|
||||
const ReportPath = "/_smallwebwaf/report"
|
||||
|
||||
// QuestionBodyMaxBytes is the most of a visitor's body a question carries.
|
||||
const QuestionBodyMaxBytes = 64 << 10
|
||||
|
||||
const (
|
||||
// maxAwaitingReport is the most answered requests held until their
|
||||
// reports come. Past it, the request answered longest ago is dropped.
|
||||
maxAwaitingReport = 10000
|
||||
// maxRequestIDBytes is the longest id a question may give.
|
||||
maxRequestIDBytes = 128
|
||||
// reportBodyMaxBytes is the most of a report's body that is read. It is
|
||||
// far above any report's.
|
||||
reportBodyMaxBytes = 1 << 20
|
||||
// maxHTTPStatus is the highest status a report may give.
|
||||
maxHTTPStatus = 599
|
||||
// maxReportedBytes is the most bytes a report may give for each way, 1
|
||||
// PiB, so that adding them up cannot overflow.
|
||||
maxReportedBytes = 1 << 50
|
||||
// maxReportedMilliseconds is the longest duration a report may give,
|
||||
// about 31 years, so that it fits a time.Duration.
|
||||
maxReportedMilliseconds = 1e12
|
||||
// serveWAFTimeout bounds how long a request to servewaf takes to arrive,
|
||||
// and its answer to leave, and serveWAFIdleTimeout how long a kept-open
|
||||
// connection waits for its next request.
|
||||
serveWAFTimeout = 60 * time.Second
|
||||
serveWAFIdleTimeout = 120 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
errReportFieldMissing = errors.New("a field is missing")
|
||||
errReportValue = errors.New("a value is out of range")
|
||||
)
|
||||
|
||||
// Report is the JSON object rproxy posts to ReportPath once the visitor's
|
||||
// answer has ended, its fields named as the request log's of the same
|
||||
// meaning. RequestID is the question's id. Status is what the visitor got, 0
|
||||
// if nothing. The durations are in milliseconds.
|
||||
//
|
||||
//nolint:tagliatelle // docs/request-log.md names the fields in snake_case
|
||||
type Report struct {
|
||||
RequestID string `json:"request_id"`
|
||||
Status int `json:"status"`
|
||||
RequestBytes int64 `json:"request_bytes"`
|
||||
ResponseBytes int64 `json:"response_bytes"`
|
||||
DurationTotal float64 `json:"duration_total"`
|
||||
DurationUpstreamTotal float64 `json:"duration_upstream_total"`
|
||||
}
|
||||
|
||||
// NewServeWAF returns the server `smallwebwaf servewaf` runs. It answers
|
||||
// rproxy's questions with the checks New runs, takes rproxy's reports and
|
||||
// answers its own endpoints. It passes nothing to an app. rproxy applies
|
||||
// the size and time limits to the visitor, so their settings do not apply
|
||||
// here. Go's default limit on a request's line and headers, 1 MiB, stays:
|
||||
// it is far above rproxy's, and a question refused for its size would have
|
||||
// rproxy let the request through.
|
||||
func NewServeWAF(params Params) *Server {
|
||||
cfg := *params.Config
|
||||
cfg.RequestMaxBytes = 0
|
||||
cfg.ClientRequestTimeout = serveWAFTimeout
|
||||
// The server's WriteTimeout does this. It sets a new deadline for each
|
||||
// request, where a deadline a handler sets stays on a kept-open
|
||||
// connection, and would run out under a later request's answer.
|
||||
cfg.ClientResponseTimeout = 0
|
||||
// A question's body of QuestionBodyMaxBytes may be cut short, and the
|
||||
// Core Rule Set does not read a JSON or XML body over its limit.
|
||||
cfg.WAFBodyLimit = min(cfg.WAFBodyLimit, QuestionBodyMaxBytes-1)
|
||||
params.Config = &cfg
|
||||
|
||||
h := newHandler(params)
|
||||
|
||||
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
|
||||
if err != nil {
|
||||
panic(err) // New fails only for a size below one
|
||||
}
|
||||
|
||||
h.awaitingReport = awaitingReport
|
||||
|
||||
return h.server(&http.Server{
|
||||
Handler: http.HandlerFunc(h.serveWAF),
|
||||
ReadTimeout: serveWAFTimeout,
|
||||
WriteTimeout: serveWAFTimeout,
|
||||
IdleTimeout: serveWAFIdleTimeout,
|
||||
})
|
||||
}
|
||||
|
||||
// serveWAF handles one request to servewaf: a request carrying
|
||||
// ClientHeader is rproxy's question, and any other is for servewaf itself.
|
||||
func (h *handler) serveWAF(w http.ResponseWriter, r *http.Request) {
|
||||
if len(r.Header.Values(ClientHeader)) > 0 {
|
||||
h.answerQuestion(w, r)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
h.answerOwnEndpoint(w, r)
|
||||
}
|
||||
|
||||
// answerQuestion runs the checks on the visitor's request a question
|
||||
// carries, and answers rproxy what to do with it. A denied request's counting
|
||||
// is finished here, and an allowed one's by its report.
|
||||
func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
|
||||
client, body, ok := h.readQuestion(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
// One that cannot be read leaves the log line's peer_ip invalid.
|
||||
peer, _ := netip.ParseAddr(r.Header.Get(PeerHeader))
|
||||
|
||||
rq := h.startRequest(w, asTheVisitorSentIt(r, body), client.Unmap(), peer.Unmap(),
|
||||
true)
|
||||
// The last deferred runs first, so that what is held has the finished log
|
||||
// line. Go's server sends the answer only once this function has
|
||||
// returned, so the report cannot come before it is held.
|
||||
defer rq.holdForReport()
|
||||
defer rq.finish()
|
||||
|
||||
rq.site = r.Host
|
||||
rq.line.ServeWAFLine = requestlog.ServeWAFLineDecision
|
||||
refused := rq.check(r.Context())
|
||||
rq.checked = time.Now()
|
||||
underOwnPrefix := strings.HasPrefix(r.URL.Path, adminPrefix)
|
||||
|
||||
if refused == nil && !underOwnPrefix {
|
||||
// In observe mode, what enforce mode would have refused counts here.
|
||||
defer rq.countRefusal()
|
||||
|
||||
rq.allow()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// The last deferred runs first, as in ServeHTTP.
|
||||
defer rq.addToHistory()
|
||||
defer rq.countAnomalies()
|
||||
defer rq.countRefusal()
|
||||
|
||||
if refused != nil {
|
||||
rq.deny(*refused)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// The app never sees this prefix. servewaf's own endpoints are at its
|
||||
// own address, so this answers as a path with no endpoint does.
|
||||
rq.line.Action = requestlog.ActionAdmin
|
||||
rq.out.Header().Set(DispositionHeader, DispositionDeny)
|
||||
http.Error(rq.out, http.StatusText(http.StatusNotFound), http.StatusNotFound)
|
||||
}
|
||||
|
||||
// readQuestion returns the visitor's address and the body a question carries,
|
||||
// and whether servewaf takes the question. It answers a question it does not
|
||||
// take with 400 and no disposition, which rproxy treats as it treats no
|
||||
// answer. The id of a question it takes is held from here on, without a
|
||||
// decision yet, so that another question with that id is refused while this
|
||||
// one is being handled.
|
||||
//
|
||||
// The body is read before any check, since Go's server does not send an answer
|
||||
// until it has read the body. A question whose body stops arriving gets no
|
||||
// answer, and is not looked at.
|
||||
func (h *handler) readQuestion(
|
||||
w http.ResponseWriter, r *http.Request,
|
||||
) (netip.Addr, []byte, bool) {
|
||||
client, err := netip.ParseAddr(r.Header.Get(ClientHeader))
|
||||
if err != nil || client.Zone() != "" {
|
||||
h.refuseQuestion(w, ClientHeader+" is not an address without a zone")
|
||||
|
||||
return netip.Addr{}, nil, false
|
||||
}
|
||||
|
||||
id := r.Header.Get(requestIDHeader)
|
||||
if id == "" || len(id) > maxRequestIDBytes || !utf8.ValidString(id) {
|
||||
h.refuseQuestion(w, requestIDHeader+" is not a valid id")
|
||||
|
||||
return netip.Addr{}, nil, false
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, QuestionBodyMaxBytes))
|
||||
|
||||
if _, tooLarge := errors.AsType[*http.MaxBytesError](err); tooLarge {
|
||||
h.refuseQuestion(w, "the body is over 64 KiB")
|
||||
|
||||
return netip.Addr{}, nil, false
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
h.processLog.Warn("a question's body stopped arriving", "error", err.Error())
|
||||
// This panic has Go's server close the connection without an answer,
|
||||
// and log nothing.
|
||||
panic(http.ErrAbortHandler)
|
||||
}
|
||||
|
||||
held, _ := h.awaitingReport.ContainsOrAdd(id, nil)
|
||||
if held {
|
||||
h.refuseQuestion(w, requestIDHeader+" is the id of a request servewaf holds")
|
||||
|
||||
return netip.Addr{}, nil, false
|
||||
}
|
||||
|
||||
return client, body, true
|
||||
}
|
||||
|
||||
// refuseQuestion answers a question servewaf does not take, for reason.
|
||||
func (h *handler) refuseQuestion(w http.ResponseWriter, reason string) {
|
||||
h.processLog.Warn("a question is refused", "reason", reason)
|
||||
http.Error(w, reason, http.StatusBadRequest)
|
||||
}
|
||||
|
||||
// asTheVisitorSentIt returns a copy of r, a question, as the checks and the
|
||||
// log line are to see it: without the headers rproxy adds for servewaf, with
|
||||
// body, which has been read of r, and with the length the visitor announced,
|
||||
// as -1 when it announced none. The question's own length covers only the part
|
||||
// of the body it carries.
|
||||
func asTheVisitorSentIt(r *http.Request, body []byte) *http.Request {
|
||||
sent := r.Clone(r.Context())
|
||||
announced := sent.Header.Get(ContentLengthHeader)
|
||||
|
||||
sent.Header.Del(ClientHeader)
|
||||
sent.Header.Del(PeerHeader)
|
||||
sent.Header.Del(ContentLengthHeader)
|
||||
sent.Header.Del("Content-Length")
|
||||
|
||||
sent.ContentLength = -1
|
||||
|
||||
length, err := strconv.ParseInt(announced, 10, 64)
|
||||
if err == nil && length >= 0 {
|
||||
sent.ContentLength = length
|
||||
sent.Header.Set("Content-Length", announced)
|
||||
}
|
||||
|
||||
sent.Body = http.NoBody
|
||||
if len(body) > 0 {
|
||||
sent.Body = io.NopCloser(bytes.NewReader(body))
|
||||
}
|
||||
|
||||
return sent
|
||||
}
|
||||
|
||||
// allow answers a question with DispositionAllow and, while
|
||||
// SWWAF_ADD_LOOKUP_HEADERS is set, the client's AS number and country.
|
||||
func (rq *request) allow() {
|
||||
rq.allowed = true
|
||||
|
||||
header := rq.out.Header()
|
||||
header.Set(DispositionHeader, DispositionAllow)
|
||||
|
||||
if rq.h.config.AddLookupHeaders {
|
||||
setLookupHeaders(header, rq.line.ASN, rq.line.Country)
|
||||
}
|
||||
|
||||
rq.out.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
// holdForReport holds a copy of what the report's steps need, by the request's
|
||||
// id. A report only ever reaches that copy, never the request being handled.
|
||||
func (rq *request) holdForReport() {
|
||||
held := rq.decision
|
||||
rq.h.awaitingReport.Add(rq.line.RequestID, &held)
|
||||
}
|
||||
|
||||
// deny answers a question with what the visitor is to get for r.
|
||||
func (rq *request) deny(r refusal) {
|
||||
if r.status == 0 {
|
||||
// SWWAF_BAN_RESPONSE is close. Written past rq.out, since the log line's
|
||||
// status stays 0: the visitor is sent nothing.
|
||||
rq.refused.CompareAndSwap(nil, &r)
|
||||
rq.out.Header().Set(DispositionHeader, DispositionClose)
|
||||
rq.out.ResponseWriter.WriteHeader(http.StatusOK)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
rq.out.Header().Set(DispositionHeader, DispositionDeny)
|
||||
rq.answer(r)
|
||||
}
|
||||
|
||||
// answerOwnEndpoint answers a request to servewaf's own address. No check
|
||||
// runs on it: the address is on the private network, where the peers are
|
||||
// rproxy and the admin's tools, not visitors.
|
||||
func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodPost && r.URL.Path == ReportPath {
|
||||
h.takeReport(w, r)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
peer := peerAddress(r)
|
||||
|
||||
rq := h.startRequest(w, r, peer, peer, false)
|
||||
defer rq.finish()
|
||||
|
||||
if r.Method == http.MethodGet && r.URL.Path == HealthPath {
|
||||
rq.answerHealth()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
rq.answerAdmin()
|
||||
}
|
||||
|
||||
// takeReport answers POST ReportPath: it finishes the request the report
|
||||
// names, if it holds that request's decision, and ignores the report
|
||||
// otherwise.
|
||||
func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) {
|
||||
report, err := readReport(w, r)
|
||||
if err != nil {
|
||||
h.processLog.Warn("a report is refused", "error", err.Error())
|
||||
http.Error(w, "the body is not a report", http.StatusBadRequest)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
held, found := h.awaitingReport.Peek(report.RequestID)
|
||||
// Remove reports true to one report only, of several that name the id.
|
||||
if found && held != nil && h.awaitingReport.Remove(report.RequestID) {
|
||||
h.finishWith(held, report)
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// readReport reads a report from r's body, which must be one JSON object with
|
||||
// all of Report's fields, in the ranges SPEC.md gives.
|
||||
func readReport(w http.ResponseWriter, r *http.Request) (Report, error) {
|
||||
var report Report
|
||||
|
||||
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, reportBodyMaxBytes))
|
||||
if err != nil {
|
||||
return report, fmt.Errorf("read the body: %w", err)
|
||||
}
|
||||
|
||||
var fields map[string]json.RawMessage
|
||||
|
||||
err = json.Unmarshal(body, &fields)
|
||||
if err != nil {
|
||||
return report, fmt.Errorf("decode the body: %w", err)
|
||||
}
|
||||
|
||||
for _, name := range []string{
|
||||
"request_id", "status", "request_bytes", "response_bytes",
|
||||
"duration_total", "duration_upstream_total",
|
||||
} {
|
||||
value, there := fields[name]
|
||||
if !there || string(value) == "null" {
|
||||
return report, fmt.Errorf("%w: %s", errReportFieldMissing, name)
|
||||
}
|
||||
}
|
||||
|
||||
err = json.Unmarshal(body, &report)
|
||||
if err != nil {
|
||||
return report, fmt.Errorf("decode the body: %w", err)
|
||||
}
|
||||
|
||||
return report, report.checkRanges()
|
||||
}
|
||||
|
||||
// checkRanges returns an error for a value of the report that cannot be true.
|
||||
func (report *Report) checkRanges() error {
|
||||
switch {
|
||||
case report.Status != 0 &&
|
||||
(report.Status < http.StatusContinue || report.Status > maxHTTPStatus):
|
||||
return fmt.Errorf("%w: status", errReportValue)
|
||||
case report.RequestBytes < 0 || report.RequestBytes > maxReportedBytes:
|
||||
return fmt.Errorf("%w: request_bytes", errReportValue)
|
||||
case report.ResponseBytes < 0 || report.ResponseBytes > maxReportedBytes:
|
||||
return fmt.Errorf("%w: response_bytes", errReportValue)
|
||||
case report.DurationTotal < 0 || report.DurationTotal > maxReportedMilliseconds:
|
||||
return fmt.Errorf("%w: duration_total", errReportValue)
|
||||
case report.DurationUpstreamTotal < 0 ||
|
||||
report.DurationUpstreamTotal > maxReportedMilliseconds:
|
||||
return fmt.Errorf("%w: duration_upstream_total", errReportValue)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// finishWith finishes a request with what its report says, and writes the
|
||||
// request's last log line, which holds its decision's fields and the report's.
|
||||
// For an allowed request it first takes the steps ServeHTTP takes once a
|
||||
// request passed to the app has ended, with the status and bytes of the report
|
||||
// in place of those it would have seen. A denied request's counting was
|
||||
// finished at its decision.
|
||||
func (h *handler) finishWith(held *decision, report Report) {
|
||||
rq := &request{
|
||||
decision: *held,
|
||||
h: h,
|
||||
out: &responseWriter{status: report.Status, bytes: report.ResponseBytes},
|
||||
body: &requestBody{},
|
||||
}
|
||||
rq.body.bytes.Store(report.RequestBytes)
|
||||
|
||||
rq.line.ServeWAFLine = requestlog.ServeWAFLineReport
|
||||
rq.line.Status = report.Status
|
||||
rq.line.RequestBytes = report.RequestBytes
|
||||
rq.line.ResponseBytes = report.ResponseBytes
|
||||
rq.line.DurationTotal = report.DurationTotal
|
||||
|
||||
if rq.allowed {
|
||||
rq.line.DurationUpstreamTotal = &report.DurationUpstreamTotal
|
||||
// addToHistory goes by this for whether the request reached the app.
|
||||
rq.upstreamStart = time.Now()
|
||||
|
||||
rq.countBytes()
|
||||
rq.countAnomalies()
|
||||
rq.addToHistory()
|
||||
|
||||
rq.h.metrics.RequestReported(&rq.line,
|
||||
time.Duration(report.DurationTotal*float64(time.Millisecond)),
|
||||
time.Duration(report.DurationUpstreamTotal*float64(time.Millisecond)))
|
||||
}
|
||||
|
||||
rq.writeLine()
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -7,7 +7,6 @@ import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
@@ -198,25 +197,9 @@ func sendLargeBody(t *testing.T, addr string) net.Conn {
|
||||
func TestClientStopsSendingAfterTheAppHasAnswered(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// answeredAt is when the app answered, or nil if the bytes the client
|
||||
// sent never reached it.
|
||||
var answeredAt atomic.Pointer[time.Time]
|
||||
|
||||
// The app answers without reading the whole body, as an app does for
|
||||
// a path it does not serve, and only once smallwebwaf waits for the
|
||||
// rest of it: the app takes the bytes the client sent, and then waits
|
||||
// half the timeout, far longer than smallwebwaf takes to go on to
|
||||
// wait.
|
||||
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
||||
_, err := io.ReadFull(r.Body, make([]byte, len("the first bytes")))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
time.Sleep(shortTimeout / 2)
|
||||
answeredAt.Store(new(time.Now()))
|
||||
http.NotFound(w, r)
|
||||
})
|
||||
// The app answers at once without reading the body, as an app does
|
||||
// for a path it does not serve.
|
||||
app := startApp(t, http.NotFound)
|
||||
addr, out := startProxy(t, app.URL, map[string]string{
|
||||
clientRequestTimeout: shortTimeoutSetting,
|
||||
metricsToken: token,
|
||||
@@ -226,120 +209,13 @@ func TestClientStopsSendingAfterTheAppHasAnswered(t *testing.T) {
|
||||
got := readResponse(t, sendPartOfBody(t, addr))
|
||||
wantTimedOut(t, start)
|
||||
|
||||
// Close returns once the app has finished with the request.
|
||||
app.Close()
|
||||
|
||||
// The app's answer came before the time ran out, so the client is
|
||||
// sent that answer, not smallwebwaf's 408. Only a hold-up of the test
|
||||
// process can make it come later, and SPEC.md then asks for 408, as
|
||||
// smallwebwaf was waiting for the client, or for 504 if the bytes the
|
||||
// client sent had not reached the app.
|
||||
at := answeredAt.Load()
|
||||
want := http.StatusNotFound
|
||||
|
||||
switch {
|
||||
case at == nil:
|
||||
want = http.StatusGatewayTimeout
|
||||
case at.Sub(start) >= shortTimeout:
|
||||
want = http.StatusRequestTimeout
|
||||
}
|
||||
|
||||
wantStatus(t, got, want)
|
||||
wantLine(t, out.requestLine(t), want, requestlog.ActionTimedOut)
|
||||
// sent that answer, not smallwebwaf's 408.
|
||||
wantStatus(t, got, http.StatusNotFound)
|
||||
wantLine(t, out.requestLine(t), http.StatusNotFound, requestlog.ActionTimedOut)
|
||||
wantLimitHits(t, addr, clientRequestTimeout, 1)
|
||||
}
|
||||
|
||||
func TestClientStopsSendingWithLittleOfItsBodyLeft(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
// limit is the setting set to shortTimeout, which runs out, and
|
||||
// clientTimeout what SWWAF_CLIENT_REQUEST_TIMEOUT is set to when it
|
||||
// is not limit.
|
||||
limit, clientTimeout string
|
||||
}{
|
||||
{
|
||||
name: "client request timeout",
|
||||
limit: clientRequestTimeout,
|
||||
},
|
||||
{
|
||||
name: "upstream request timeout, client request timeout off",
|
||||
limit: upstreamRequestTimeout,
|
||||
clientTimeout: off,
|
||||
},
|
||||
{
|
||||
name: "upstream request timeout, client request timeout longer",
|
||||
limit: upstreamRequestTimeout,
|
||||
clientTimeout: longTimeoutSetting,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// appClosed is closed once smallwebwaf has closed its
|
||||
// connection to the app, which it does once it has the app's
|
||||
// answer.
|
||||
appClosed := make(chan struct{})
|
||||
|
||||
// The app answers without reading the rest of the body, and
|
||||
// says it closes the connection, only once smallwebwaf waits
|
||||
// for the client to send more: it takes the bytes the client
|
||||
// sent, and then waits half the timeout, far longer than
|
||||
// smallwebwaf takes to go on to wait.
|
||||
app := startApp(t, func(w http.ResponseWriter, _ *http.Request) {
|
||||
conn, buf, err := http.NewResponseController(w).Hijack()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
_, _ = io.ReadFull(buf.Reader, make([]byte, len("the first bytes")))
|
||||
|
||||
time.Sleep(shortTimeout / 2)
|
||||
|
||||
_, _ = buf.WriteString("HTTP/1.1 404 Not Found\r\n" +
|
||||
"Content-Length: 0\r\nConnection: close\r\n\r\n")
|
||||
_ = buf.Flush()
|
||||
_, _ = io.Copy(io.Discard, buf.Reader)
|
||||
_ = conn.Close()
|
||||
|
||||
close(appClosed)
|
||||
})
|
||||
|
||||
env := map[string]string{tc.limit: shortTimeoutSetting, metricsToken: token}
|
||||
if tc.clientTimeout != "" {
|
||||
env[clientRequestTimeout] = tc.clientTimeout
|
||||
}
|
||||
|
||||
addr, out := startProxy(t, app.URL, env)
|
||||
|
||||
// Less than 256 KiB of the body is left after the first bytes,
|
||||
// little enough for Go's server in smallwebwaf to read the rest
|
||||
// itself before it sends the app's answer.
|
||||
start := time.Now()
|
||||
conn := dial(t, addr)
|
||||
send(t, conn, "POST /upload HTTP/1.1\r\nHost: app\r\nContent-Length: 200000\r\n\r\n"+
|
||||
"the first bytes")
|
||||
|
||||
select {
|
||||
case <-appClosed:
|
||||
case <-time.After(waitLimit):
|
||||
t.Fatal("smallwebwaf did not close its connection to the app")
|
||||
}
|
||||
|
||||
// More of the body, which can no longer go to the app, and then
|
||||
// nothing.
|
||||
send(t, conn, strings.Repeat("x", 8<<10))
|
||||
|
||||
got := readResponse(t, conn)
|
||||
wantTimedOut(t, start)
|
||||
wantStatus(t, got, http.StatusNotFound)
|
||||
wantLine(t, out.requestLine(t), http.StatusNotFound, requestlog.ActionTimedOut)
|
||||
wantLimitHits(t, addr, tc.limit, 1)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppTooSlowToAnswer(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -206,7 +206,7 @@ type Hit struct {
|
||||
// Counts are a client's requests and bytes in the minute, the hour and
|
||||
// the day that end at a request, that request's included.
|
||||
//
|
||||
//nolint:tagliatelle // docs/request-log.md names the fields in snake_case
|
||||
//nolint:tagliatelle // SPEC.md's request log names its fields in snake_case
|
||||
type Counts struct {
|
||||
Minute float64 `json:"minute"`
|
||||
Hour float64 `json:"hour"`
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Package requestlog writes the lines smallwebwaf prints on stdout: one
|
||||
// JSON object per request, or two from servewaf, marked "type":"request", and
|
||||
// the process's own messages as JSON lines marked "type":"process".
|
||||
// JSON object per request, marked "type":"request", and the process's own
|
||||
// messages as JSON lines marked "type":"process".
|
||||
package requestlog
|
||||
|
||||
import (
|
||||
@@ -60,29 +60,17 @@ const OffenceLimit = "limit"
|
||||
// that broke the error burst.
|
||||
const LimitHitErrorBurst = "error_burst"
|
||||
|
||||
// The servewaf_line a request line names, which says which of servewaf's two
|
||||
// lines about a question it is.
|
||||
const (
|
||||
// ServeWAFLineDecision is servewaf's line at its decision.
|
||||
ServeWAFLineDecision = "decision"
|
||||
// ServeWAFLineReport is servewaf's line when rproxy's report comes, with
|
||||
// the decision's fields and the report's together.
|
||||
ServeWAFLineReport = "report"
|
||||
)
|
||||
|
||||
// timeLayout is RFC 3339 with milliseconds.
|
||||
const timeLayout = "2006-01-02T15:04:05.000Z07:00"
|
||||
|
||||
// Line is one request's line in the request log. The field names, and
|
||||
// their order, are those of docs/request-log.md. A field that may not
|
||||
// apply to a request is left out of its line when it does not.
|
||||
// their order, are those of the "Request log" section of SPEC.md. A field
|
||||
// that may not apply to a request is left out of its line when it does
|
||||
// not.
|
||||
//
|
||||
//nolint:tagliatelle // docs/request-log.md names the fields in snake_case
|
||||
//nolint:tagliatelle // SPEC.md's request log names its fields in snake_case
|
||||
type Line struct {
|
||||
Type string `json:"type"`
|
||||
// ServeWAFLine is ServeWAFLineDecision or ServeWAFLineReport on servewaf's
|
||||
// two lines about a question.
|
||||
ServeWAFLine string `json:"servewaf_line,omitempty"`
|
||||
|
||||
// The standard web log fields. Scheme is how the client reached
|
||||
// smallwebwaf, or the trusted proxy in front of it.
|
||||
|
||||
@@ -60,19 +60,8 @@ func TestHealthCheck(t *testing.T) {
|
||||
}, 0, "")
|
||||
}
|
||||
|
||||
// Another test can take a closed server's port at once, so the app's
|
||||
// address here is a multicast address, to which TCP cannot connect.
|
||||
wantHealthCheck(t, map[string]string{
|
||||
listenAddr: ":" + port,
|
||||
upstreamURL: "http://224.0.0.1",
|
||||
}, 1, "unhealthy: connect to the app: ")
|
||||
|
||||
// Listening on port 0 takes any free port, so nothing listens on port 0
|
||||
// itself.
|
||||
wantHealthCheck(t, map[string]string{listenAddr: ":0"}, 1,
|
||||
"unhealthy: ask smallwebwaf: ")
|
||||
wantHealthCheck(t, map[string]string{listenAddr: "8080"}, 1,
|
||||
"unhealthy: invalid setting: SWWAF_LISTEN_ADDR: ")
|
||||
app.Close()
|
||||
wantHealthCheck(t, env, 1, "unhealthy: connect to the app: ")
|
||||
|
||||
stop()
|
||||
|
||||
@@ -81,6 +70,10 @@ func TestHealthCheck(t *testing.T) {
|
||||
case <-time.After(waitLimit):
|
||||
t.Fatal("still running after being told to stop")
|
||||
}
|
||||
|
||||
wantHealthCheck(t, env, 1, "unhealthy: ask smallwebwaf: ")
|
||||
wantHealthCheck(t, map[string]string{listenAddr: "8080"}, 1,
|
||||
"unhealthy: invalid setting: SWWAF_LISTEN_ADDR: ")
|
||||
}
|
||||
|
||||
func TestHealthCheckRefusesAnArgument(t *testing.T) {
|
||||
|
||||
@@ -1,210 +0,0 @@
|
||||
package smallwebwaf_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/proxy"
|
||||
"sneak.berlin/go/smallwebwaf/internal/smallwebwaf"
|
||||
)
|
||||
|
||||
// commandForMain, while set in the environment, has this test binary run Main
|
||||
// with the command it gives, in place of the tests, so that a test can run
|
||||
// Main in a process of its own.
|
||||
const commandForMain = "SMALLWEBWAF_TEST_COMMAND"
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
command, set := os.LookupEnv(commandForMain)
|
||||
if !set {
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
os.Args = strings.Fields("smallwebwaf " + command)
|
||||
|
||||
os.Exit(smallwebwaf.Main(testVersion))
|
||||
}
|
||||
|
||||
func TestMainAnswersQuestionsOnlyAsServeWAF(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, tc := range []struct {
|
||||
command, disposition string
|
||||
}{
|
||||
{"servewaf", proxy.DispositionAllow},
|
||||
{"", ""},
|
||||
} {
|
||||
t.Run("command "+tc.command, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
binary, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatalf("find the test binary: %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 3*waitLimit)
|
||||
defer cancel()
|
||||
|
||||
out := &output{}
|
||||
cmd := exec.CommandContext(ctx, binary) //nolint:gosec // this test binary
|
||||
cmd.Stdout = out
|
||||
|
||||
cmd.Env = append(os.Environ(),
|
||||
commandForMain+"="+tc.command,
|
||||
listenAddr+"="+localhost+":0",
|
||||
upstreamURL+"="+startApp(t),
|
||||
stateDir+"="+t.TempDir(),
|
||||
rulesDir+"="+t.TempDir(),
|
||||
lookupSource+"=off")
|
||||
|
||||
err = cmd.Start()
|
||||
if err != nil {
|
||||
t.Fatalf("start: %v", err)
|
||||
}
|
||||
|
||||
t.Cleanup(func() {
|
||||
_ = cmd.Process.Kill()
|
||||
_ = cmd.Wait()
|
||||
})
|
||||
|
||||
addr, _ := out.line(t, "msg", "starting")["address"].(string)
|
||||
ask(t, addr, "/", http.StatusOK, tc.disposition)
|
||||
|
||||
err = cmd.Process.Signal(syscall.SIGTERM)
|
||||
if err != nil {
|
||||
t.Fatalf("signal: %v", err)
|
||||
}
|
||||
|
||||
err = cmd.Wait()
|
||||
if err != nil {
|
||||
t.Errorf("exit: %v; output:\n%s", err, out.text())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ask sends servewaf at addr rproxy's question about a request for path from
|
||||
// placed, checks the answer's status and disposition, and closes it.
|
||||
func ask(t *testing.T, addr, path string, status int, disposition string) {
|
||||
t.Helper()
|
||||
|
||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet,
|
||||
"http://"+addr+path, http.NoBody)
|
||||
if err != nil {
|
||||
t.Fatalf("new request: %v", err)
|
||||
}
|
||||
|
||||
req.Header.Set(proxy.ClientHeader, placed)
|
||||
req.Header.Set("X-Request-ID", rand.Text())
|
||||
|
||||
transport := &http.Transport{}
|
||||
defer transport.CloseIdleConnections()
|
||||
|
||||
res, err := (&http.Client{Transport: transport}).Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("question: %v", err)
|
||||
}
|
||||
|
||||
_, _ = io.Copy(io.Discard, res.Body)
|
||||
_ = res.Body.Close()
|
||||
|
||||
if res.StatusCode != status ||
|
||||
res.Header.Get(proxy.DispositionHeader) != disposition {
|
||||
t.Errorf("answered %d with disposition %q, want %d with %q", res.StatusCode,
|
||||
res.Header.Get(proxy.DispositionHeader), status, disposition)
|
||||
}
|
||||
}
|
||||
|
||||
// runServeWAF runs `smallwebwaf servewaf` with the settings in env, has use
|
||||
// send it requests at the address it listens on, stops it as SIGTERM does, and
|
||||
// checks that it stopped in order.
|
||||
func runServeWAF(t *testing.T, env map[string]string, use func(addr string)) {
|
||||
t.Helper()
|
||||
|
||||
ctx, stop := context.WithCancel(t.Context())
|
||||
out := &output{}
|
||||
exited := make(chan int, 1)
|
||||
|
||||
go func() {
|
||||
exited <- runAs(ctx, true, env, out)
|
||||
}()
|
||||
|
||||
addr, _ := out.line(t, "msg", "starting")["address"].(string)
|
||||
use(addr)
|
||||
stop()
|
||||
|
||||
select {
|
||||
case status := <-exited:
|
||||
if status != 0 {
|
||||
t.Fatalf("exit status %d, want 0; output:\n%s", status, out.text())
|
||||
}
|
||||
case <-time.After(waitLimit):
|
||||
t.Fatal("still running after being told to stop")
|
||||
}
|
||||
|
||||
out.line(t, "msg", "stopped")
|
||||
}
|
||||
|
||||
func TestServeWAFKeepsItsBansAcrossARestartAndNeedsNoApp(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := map[string]string{
|
||||
listenAddr: localhost + ":0",
|
||||
stateDir: t.TempDir(),
|
||||
rulesDir: filepath.Join("..", "..", "share", "rules.d"),
|
||||
}
|
||||
|
||||
runServeWAF(t, env, func(addr string) {
|
||||
ask(t, addr, "/", http.StatusOK, proxy.DispositionAllow)
|
||||
// The default rule file bans a probe for this.
|
||||
ask(t, addr, "/.env", http.StatusForbidden, proxy.DispositionDeny)
|
||||
})
|
||||
|
||||
runServeWAF(t, env, func(addr string) {
|
||||
ask(t, addr, "/", http.StatusForbidden, proxy.DispositionDeny)
|
||||
})
|
||||
}
|
||||
|
||||
func TestWithoutACommandTheClientHeaderIsNoQuestion(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
runUntilStopped(t, map[string]string{
|
||||
listenAddr: localhost + ":0",
|
||||
upstreamURL: startApp(t),
|
||||
stateDir: t.TempDir(),
|
||||
rulesDir: t.TempDir(),
|
||||
}, func(url string) {
|
||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, url,
|
||||
http.NoBody)
|
||||
if err != nil {
|
||||
t.Fatalf("new request: %v", err)
|
||||
}
|
||||
|
||||
req.Header.Set(proxy.ClientHeader, placed)
|
||||
|
||||
transport := &http.Transport{}
|
||||
defer transport.CloseIdleConnections()
|
||||
|
||||
res, err := (&http.Client{Transport: transport}).Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("request: %v", err)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(res.Body)
|
||||
_ = res.Body.Close()
|
||||
|
||||
if err != nil || string(body) != greeting ||
|
||||
res.Header.Get(proxy.DispositionHeader) != "" {
|
||||
t.Errorf("answered %q (%v) with disposition %q, want the app's greeting",
|
||||
body, err, res.Header.Get(proxy.DispositionHeader))
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,8 +1,7 @@
|
||||
// Package smallwebwaf runs the smallwebwaf process: it reads the settings,
|
||||
// the rule files, the lookup database and the state files, serves requests
|
||||
// until it is told to stop, and then stops in an orderly way, writing the
|
||||
// state files. It serves them in front of an app, or, as `smallwebwaf
|
||||
// servewaf`, by answering rproxy's questions about them.
|
||||
// state files.
|
||||
package smallwebwaf
|
||||
|
||||
import (
|
||||
@@ -46,15 +45,11 @@ type Params struct {
|
||||
LookupEnv func(string) (string, bool)
|
||||
// Stdout receives the request log and the process's own messages.
|
||||
Stdout io.Writer
|
||||
// ServeWAF makes it `smallwebwaf servewaf`, which answers rproxy's
|
||||
// questions, rather than the proxy in front of an app.
|
||||
ServeWAF bool
|
||||
}
|
||||
|
||||
// Main runs smallwebwaf until SIGTERM or SIGINT, and returns the
|
||||
// process's exit status. Run as `smallwebwaf servewaf`, it answers
|
||||
// rproxy's questions rather than pass requests to an app. Run as
|
||||
// `smallwebwaf healthcheck`, it is the container's health check instead.
|
||||
// process's exit status. Run as `smallwebwaf healthcheck`, it is the
|
||||
// container's health check instead.
|
||||
func Main(version string) int {
|
||||
if len(os.Args) > 1 && os.Args[1] == "healthcheck" {
|
||||
return HealthCheck(context.Background(), os.Args[2:], os.LookupEnv, os.Stderr)
|
||||
@@ -68,7 +63,6 @@ func Main(version string) int {
|
||||
Version: version,
|
||||
LookupEnv: os.LookupEnv,
|
||||
Stdout: os.Stdout,
|
||||
ServeWAF: len(os.Args) > 1 && os.Args[1] == "servewaf",
|
||||
})
|
||||
}
|
||||
|
||||
@@ -123,12 +117,7 @@ func Run(ctx context.Context, params Params) int {
|
||||
return 1
|
||||
}
|
||||
|
||||
newProxy := proxy.New
|
||||
if params.ServeWAF {
|
||||
newProxy = proxy.NewServeWAF
|
||||
}
|
||||
|
||||
server, err := newServer(newProxy, cfg, stdout, processLog, now, ruleFiles, alertQueue)
|
||||
server, err := newServer(cfg, stdout, processLog, now, ruleFiles, alertQueue)
|
||||
if err != nil {
|
||||
processLog.Error("cannot use the lookup database", "error", err.Error())
|
||||
|
||||
@@ -162,12 +151,10 @@ func Run(ctx context.Context, params Params) int {
|
||||
return serve(ctx, server, listener, files, ruleFiles, alertQueue, processLog)
|
||||
}
|
||||
|
||||
// newServer returns the server smallwebwaf runs, which newProxy makes, with
|
||||
// the metrics of the alerts, after reading the lookup database while
|
||||
// SWWAF_LOOKUP_SOURCE is file. A lookup database that cannot be read is an
|
||||
// error.
|
||||
// newServer returns the server smallwebwaf runs, with the metrics of the
|
||||
// alerts, after reading the lookup database while SWWAF_LOOKUP_SOURCE is
|
||||
// file. A lookup database that cannot be read is an error.
|
||||
func newServer(
|
||||
newProxy func(proxy.Params) *proxy.Server,
|
||||
cfg *config.Config, stdout io.Writer, processLog *slog.Logger,
|
||||
now func() time.Time, ruleFiles *rules.Files, alertQueue *alerts.Queue,
|
||||
) (*proxy.Server, error) {
|
||||
@@ -187,7 +174,7 @@ func newServer(
|
||||
}
|
||||
}
|
||||
|
||||
server := newProxy(proxy.Params{
|
||||
server := proxy.New(proxy.Params{
|
||||
Config: cfg,
|
||||
RequestLog: stdout,
|
||||
ProcessLog: processLog,
|
||||
|
||||
@@ -113,13 +113,6 @@ func (o *output) text() string {
|
||||
// returns its exit status. SWWAF_LOOKUP_SOURCE is off unless env sets it,
|
||||
// so that no test sends GeoJS its clients' addresses.
|
||||
func run(ctx context.Context, env map[string]string, out *output) int {
|
||||
return runAs(ctx, false, env, out)
|
||||
}
|
||||
|
||||
// runAs is run as `smallwebwaf servewaf` while serveWAF is true.
|
||||
func runAs(
|
||||
ctx context.Context, serveWAF bool, env map[string]string, out *output,
|
||||
) int {
|
||||
return smallwebwaf.Run(ctx, smallwebwaf.Params{
|
||||
Version: testVersion,
|
||||
LookupEnv: func(name string) (string, bool) {
|
||||
@@ -130,8 +123,7 @@ func runAs(
|
||||
|
||||
return value, ok
|
||||
},
|
||||
Stdout: out,
|
||||
ServeWAF: serveWAF,
|
||||
Stdout: out,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1248,13 +1240,13 @@ func wantStartingLine(t *testing.T, line map[string]any, appURL, dir string) {
|
||||
"SWWAF_UPSTREAM_REQUEST_TIMEOUT": "60s",
|
||||
"SWWAF_UPSTREAM_RESPONSE_TIMEOUT": "30m",
|
||||
"SWWAF_REQUEST_MAX_BYTES": "100M",
|
||||
"SWWAF_RESPONSE_MAX_BYTES": "1G",
|
||||
"SWWAF_RESPONSE_MAX_BYTES": "5G",
|
||||
"SWWAF_ALLOW_NETS": "",
|
||||
rateLimitExemptNets: "",
|
||||
"SWWAF_DENY_NETS": "",
|
||||
"SWWAF_RATE_LIMIT_PER_MINUTE": "500",
|
||||
"SWWAF_RATE_LIMIT_PER_HOUR": "20000",
|
||||
rateLimitPerDay: "300000",
|
||||
"SWWAF_RATE_LIMIT_PER_MINUTE": "1000",
|
||||
"SWWAF_RATE_LIMIT_PER_HOUR": "10000",
|
||||
rateLimitPerDay: "50000",
|
||||
"SWWAF_RATE_LIMIT_EXEMPT_PATHS": "",
|
||||
"SWWAF_DENIED_COUNTRIES": "",
|
||||
"SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES": "",
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
// Package state keeps smallwebwaf's state in JSON files in
|
||||
// SWWAF_STATE_DIR, as the "Persistent state" section of SPEC.md requires
|
||||
// and docs/state-files.md lists them:
|
||||
// SWWAF_STATE_DIR, as the "Persistent state" section of SPEC.md describes:
|
||||
// bans.json holds the bans, clients.json each client's counters and
|
||||
// history, lookups.json GeoJS's answers, reputation.json the last try and
|
||||
// last good copy of each list fetched from a URL, the DNSBL zones'
|
||||
@@ -62,7 +61,6 @@ var (
|
||||
// errMissing is for an entry without a field it needs.
|
||||
errMissing = errors.New("has no")
|
||||
errCause = errors.New("is not limit, attack, admin or crowdsec")
|
||||
errExpires = errors.New("is not after its start")
|
||||
errDestination = errors.New("is not webhook, slack or ntfy")
|
||||
errScope = errors.New("is not client, net, asn, total or watch")
|
||||
errWaitingList = errors.New(`waiting is a list, but now lists the alerts by ` +
|
||||
@@ -648,10 +646,8 @@ func (e BanEntry) ban() bans.Ban {
|
||||
// client, a start, from which the length of the netblock's next ban is
|
||||
// worked out, or an expires, which would make it permanent. A permanent
|
||||
// ban's expires is null, which Bans cannot tell from a missing one, so
|
||||
// each expires is read again as written. An expires not after the start
|
||||
// is refused too, since the netblock's next ban for a broken limit, three
|
||||
// times as long as this one, would then refuse nothing; and so is a cause
|
||||
// other than limit, attack, admin or crowdsec, most likely misspelt.
|
||||
// each expires is read again as written. A cause other than limit,
|
||||
// attack, admin or crowdsec, most likely misspelt, is refused too.
|
||||
func (f *bansFile) check(data []byte) error {
|
||||
var written struct {
|
||||
Bans []struct {
|
||||
@@ -672,8 +668,6 @@ func (f *bansFile) check(data []byte) error {
|
||||
return missing(i, "start")
|
||||
case written.Bans[i].Expires == nil:
|
||||
return missing(i, "expires")
|
||||
case entry.Expires != nil && !entry.Expires.After(entry.Start):
|
||||
return fmt.Errorf("entry %d's expires %w", i+1, errExpires)
|
||||
case entry.Cause != "" && entry.Cause != bans.CauseLimit &&
|
||||
entry.Cause != bans.CauseAttack && entry.Cause != bans.CauseAdmin &&
|
||||
entry.Cause != bans.CauseCrowdSec:
|
||||
|
||||
@@ -865,21 +865,6 @@ func TestBanWithAnotherCauseStopsTheStart(t *testing.T) {
|
||||
`: entry 4's cause "atack" is not limit, attack, admin or crowdsec`)
|
||||
}
|
||||
|
||||
func TestBanThatDoesNotEndAfterItsStartStopsTheStart(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// The first ban ends a second after its start; the second ends as it
|
||||
// starts, or an hour before.
|
||||
for _, expires := range []string{`"2026-10-06T00:00:00Z"`, `"2026-10-05T23:00:00Z"`} {
|
||||
wantRefused(t, bansJSON, `{"version": 1, "bans": [`+
|
||||
`{"netblock": "203.0.113.9/32", "start": "2026-10-06T00:00:00Z", `+
|
||||
`"expires": "2026-10-06T00:00:01Z"}, `+
|
||||
`{"netblock": "203.0.113.10/32", "start": "2026-10-06T00:00:00Z", `+
|
||||
`"expires": `+expires+`}]}`,
|
||||
`: entry 2's expires is not after its start`)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnknownVersionStopsTheStart(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -1019,8 +1004,8 @@ func TestEditJustBeforeAScheduledWriteSurvivesIt(t *testing.T) {
|
||||
edit(t, dir, bansJSON, permanentBansJSON)
|
||||
|
||||
// The write takes the edit in first, and writes it back. The second
|
||||
// ban, made after the admin opened the file, is lost, as the
|
||||
// "Persistent state" section of SPEC.md says.
|
||||
// ban, made after the admin opened the file, is lost, as "Edits
|
||||
// while running" in SPEC.md says.
|
||||
time.Sleep(time.Nanosecond)
|
||||
synctest.Wait()
|
||||
|
||||
|
||||
+2
-2
@@ -1,8 +1,8 @@
|
||||
// Package waf runs the OWASP Core Rule Set 4.25.0, through Coraza, on the
|
||||
// method, the URL with its query and the headers of a request, and on its
|
||||
// body while SWWAF_WAF_BODY_LIMIT is set, with the six changes smallwebwaf
|
||||
// makes to it, as "Attack detection" under "Settings" in SPEC.md describes
|
||||
// them. It reads no response.
|
||||
// makes to it, as "Attack detection" under "Configuration surface" in
|
||||
// SPEC.md describes them. It reads no response.
|
||||
//
|
||||
// smallwebwaf writes only to its state directory, so Coraza is built with
|
||||
// its no_fs_access tag, as the Dockerfile and script/build build it: of a
|
||||
|
||||
+6
-49
@@ -5,15 +5,12 @@
|
||||
# served through smallwebwaf, that a second one in a minute bans the
|
||||
# client, that a probe for /.env bans another client, which its next
|
||||
# request bans for good, that `sv stop` stops smallwebwaf in order, that
|
||||
# `docker stop` stops the container without having to kill it, that a
|
||||
# new container on the same volume still refuses the banned client, that
|
||||
# a container with its state files on a new volume mounted at the
|
||||
# directory SWWAF_STATE_DIR_FILE names becomes healthy, and that one whose
|
||||
# SWWAF_STATE_DIR does not exist logs smallwebwaf's message about it. The
|
||||
# `docker stop` stops the container without having to kill it, and that
|
||||
# a new container on the same volume still refuses the banned client. The
|
||||
# containers run with SWWAF_LOOKUP_SOURCE=off, so that no address is sent
|
||||
# to GeoJS. The containers, the volumes and both images are removed
|
||||
# however the script ends. Building the app needs network access, for
|
||||
# nixpkgs' binary cache. script/check does not run this.
|
||||
# to GeoJS. The containers, the volume and both images are removed however
|
||||
# the script ends. Building the app needs network access, for nixpkgs'
|
||||
# binary cache. script/check does not run this.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
@@ -26,15 +23,10 @@ IMAGE="$NAME-base"
|
||||
APP_IMAGE="$NAME-app"
|
||||
CONTAINER="$NAME"
|
||||
VOLUME="$NAME-state"
|
||||
# The volumes of the container whose state directory SWWAF_STATE_DIR_FILE
|
||||
# names: its state files, and the file that names their directory.
|
||||
MOVED_VOLUME="$NAME-moved-state"
|
||||
SETTINGS_VOLUME="$NAME-settings"
|
||||
|
||||
cleanup() {
|
||||
docker rm --force "$CONTAINER" >/dev/null 2>&1 || true
|
||||
docker volume rm --force "$VOLUME" "$MOVED_VOLUME" "$SETTINGS_VOLUME" \
|
||||
>/dev/null 2>&1 || true
|
||||
docker volume rm --force "$VOLUME" >/dev/null 2>&1 || true
|
||||
docker rmi --force "$APP_IMAGE" "$IMAGE" >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
@@ -84,27 +76,6 @@ start_container() {
|
||||
address="$(docker port "$CONTAINER" 8080/tcp)"
|
||||
}
|
||||
|
||||
# start_moved_container: run the app's container with the state files on
|
||||
# a new volume at /srv/state, which SWWAF_STATE_DIR_FILE names in a file
|
||||
# only the smallwebwaf user can read, as under "Settings given as files"
|
||||
# in docs/settings.md, and wait until it is healthy. smallwebwaf can write the
|
||||
# new volume only once the run script has given it to that user.
|
||||
start_moved_container() {
|
||||
docker volume create "$MOVED_VOLUME" >/dev/null
|
||||
docker volume create "$SETTINGS_VOLUME" >/dev/null
|
||||
docker run --rm --volume "$SETTINGS_VOLUME:/settings" \
|
||||
--entrypoint bash "$IMAGE" -c 'echo /srv/state > /settings/state-dir &&
|
||||
chown smallwebwaf:smallwebwaf /settings/state-dir &&
|
||||
chmod 0400 /settings/state-dir'
|
||||
docker run --detach --name "$CONTAINER" \
|
||||
--volume "$MOVED_VOLUME:/srv/state" \
|
||||
--volume "$SETTINGS_VOLUME:/etc/smallwebwaf/settings:ro" \
|
||||
--env SWWAF_STATE_DIR_FILE=/etc/smallwebwaf/settings/state-dir \
|
||||
--env SWWAF_LOOKUP_SOURCE=off \
|
||||
"$APP_IMAGE" >/dev/null
|
||||
wait_for "the health check did not pass" healthy
|
||||
}
|
||||
|
||||
# refused: a request to the container gets 403, SWWAF_BAN_RESPONSE's
|
||||
# default.
|
||||
refused() {
|
||||
@@ -169,20 +140,6 @@ main() {
|
||||
wait_for "smallwebwaf logged no request refused under the ban" \
|
||||
logged '"action":"banned"'
|
||||
echo "example-app: a new container on the same volume keeps the ban"
|
||||
|
||||
docker rm --force "$CONTAINER" >/dev/null
|
||||
start_moved_container
|
||||
echo "example-app: the run script gives the state files' directory named" \
|
||||
"by SWWAF_STATE_DIR_FILE to the smallwebwaf user"
|
||||
|
||||
docker rm --force "$CONTAINER" >/dev/null
|
||||
docker run --detach --name "$CONTAINER" \
|
||||
--env SWWAF_STATE_DIR=/srv/missing \
|
||||
--env SWWAF_LOOKUP_SOURCE=off \
|
||||
"$APP_IMAGE" >/dev/null
|
||||
wait_for "smallwebwaf did not report the missing state directory" \
|
||||
logged "SWWAF_STATE_DIR cannot be written"
|
||||
echo "example-app: smallwebwaf reports a state directory that does not exist"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
@@ -1,336 +0,0 @@
|
||||
#!/bin/sh
|
||||
# script/gitea-requests: build the image and run gitea 1.27.3 behind it,
|
||||
# with every setting of smallwebwaf at its default but SWWAF_UPSTREAM_URL,
|
||||
# which names gitea's container. Then send through it the gitea requests
|
||||
# that reading gitea's source found the Core Rule Set refuses, which
|
||||
# docs/gitea.md records, and a short sample of gitea's ordinary use:
|
||||
# signing in, pages of files and commits, a diff, the API, a push, clone
|
||||
# and fetch over HTTP, and two container image pushes. Last, print each
|
||||
# request smallwebwaf logged, with what it did with it and the ids of the
|
||||
# Core Rule Set's rules that matched it.
|
||||
# Every request comes from one client, docker's gateway, so more than
|
||||
# SWWAF_ERROR_BURST_THRESHOLD, 30, refusals in a minute would ban it, and
|
||||
# the requests after would show as banned. The containers, the network,
|
||||
# the images and the files the script makes are removed however it ends;
|
||||
# gitea's own image stays, as the base images of a build do. It needs
|
||||
# network access, and a docker daemon that pushes to a registry on
|
||||
# localhost over plain HTTP, as docker does by default. script/check does
|
||||
# not run this.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
|
||||
# gitea 1.27.3, 2026-08-29
|
||||
GITEA_IMAGE="gitea/gitea@sha256:87a67ee09d3ae0d1df5fda5dcda3e2a1f9236a45b0a59025d6e00e46adc43bef"
|
||||
|
||||
# Named after this run, so that runs in other clones on the same host
|
||||
# never touch each other's.
|
||||
NAME="$("$SCRIPT_DIR/projectname")-gitea-$$"
|
||||
IMAGE="$NAME-image"
|
||||
NETWORK="$NAME"
|
||||
SMALLWEBWAF="$NAME-smallwebwaf"
|
||||
GITEA="$NAME-gitea"
|
||||
|
||||
# gitea's one user, who owns the repository and the container images.
|
||||
PASSWORD="owner-password"
|
||||
|
||||
BROWSER="Mozilla/5.0 (X11; Linux x86_64; rv:131.0) Gecko/20100101 Firefox/131.0"
|
||||
|
||||
# The files the script makes, and smallwebwaf's address once it runs,
|
||||
# localhost and the port docker publishes it on: a host name, since the
|
||||
# Core Rule Set takes a Host that is an IP address for a warning (920350).
|
||||
WORK=""
|
||||
address=""
|
||||
|
||||
cleanup() {
|
||||
# So that a signal cannot stop it partway through.
|
||||
trap '' HUP INT TERM
|
||||
docker rm --force --volumes "$SMALLWEBWAF" "$GITEA" >/dev/null 2>&1 || true
|
||||
docker network rm "$NETWORK" >/dev/null 2>&1 || true
|
||||
docker rmi --force "$IMAGE" >/dev/null 2>&1 || true
|
||||
if [ -n "$address" ]; then
|
||||
docker rmi --force "$address/owner/python3-app:1" "$address/owner/app:1" \
|
||||
>/dev/null 2>&1 || true
|
||||
fi
|
||||
rm -rf "$WORK"
|
||||
}
|
||||
|
||||
fail() {
|
||||
echo "gitea-requests: $*" >&2
|
||||
report
|
||||
exit 1
|
||||
}
|
||||
|
||||
# wait_for <what fails> <command>...: run the command every second until
|
||||
# it succeeds, for at most a minute.
|
||||
wait_for() {
|
||||
failure="$1"
|
||||
shift
|
||||
tries=0
|
||||
until "$@"; do
|
||||
tries=$((tries + 1))
|
||||
[ "$tries" -lt 60 ] || fail "$failure"
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
healthy() {
|
||||
status="$(docker inspect --format '{{.State.Health.Status}}' "$SMALLWEBWAF")"
|
||||
[ "$status" = healthy ]
|
||||
}
|
||||
|
||||
# logged <text>...: a line of smallwebwaf's output holds every text, in
|
||||
# any order.
|
||||
logged() {
|
||||
lines="$(docker logs "$SMALLWEBWAF" 2>&1)"
|
||||
for text in "$@"; do
|
||||
lines="$(printf '%s\n' "$lines" | grep -F "$text")" || return 1
|
||||
done
|
||||
}
|
||||
|
||||
# send <label> <path> <curl option>...: send a request for path to
|
||||
# smallwebwaf, with label as its X-Request-ID, which the report gives,
|
||||
# and keep the body of the answer in $WORK/answer. The Core Rule Set
|
||||
# reads that header too, so a label names no file the cases name.
|
||||
send() {
|
||||
label="$1"
|
||||
path="$2"
|
||||
shift 2
|
||||
curl --silent --show-error --max-time 30 --output "$WORK/answer" \
|
||||
--header "X-Request-ID: $label" "$@" "http://$address$path"
|
||||
}
|
||||
|
||||
# page <label> <path> <curl option>...: send the request as a browser
|
||||
# does, with the cookies gitea has set. Its Referer is the repository's
|
||||
# page, which links to the others, and never a page the Core Rule Set
|
||||
# refused: its address would have the next request refused too.
|
||||
page() {
|
||||
label="$1"
|
||||
path="$2"
|
||||
shift 2
|
||||
send "$label" "$path" --compressed --user-agent "$BROWSER" \
|
||||
--header "Accept: text/html,application/xhtml+xml,*/*;q=0.8" \
|
||||
--header "Accept-Language: en-US,en;q=0.5" \
|
||||
--cookie "$WORK/cookies" --cookie-jar "$WORK/cookies" \
|
||||
--referer "http://$address/owner/repo" "$@"
|
||||
}
|
||||
|
||||
# api <label> <path> <curl option>...: send the request as a client of
|
||||
# gitea's API does, as the owner.
|
||||
api() {
|
||||
label="$1"
|
||||
path="$2"
|
||||
shift 2
|
||||
send "$label" "$path" --user "owner:$PASSWORD" \
|
||||
--header "Accept: application/json" "$@"
|
||||
}
|
||||
|
||||
# repo_git <git argument>...: run git as the owner, without the host's
|
||||
# own git settings, so that nothing it does is kept outside $WORK.
|
||||
repo_git() {
|
||||
GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 git \
|
||||
-c user.name=owner -c user.email=owner@example.invalid "$@"
|
||||
}
|
||||
|
||||
# lines: forty lines of text, enough for a diff to hide some of them.
|
||||
lines() {
|
||||
i=1
|
||||
while [ "$i" -le 40 ]; do
|
||||
echo "line $i"
|
||||
i=$((i + 1))
|
||||
done
|
||||
}
|
||||
|
||||
# make_repository: create the repository owner/repo, holding a file of
|
||||
# each name the cases name, and push it, which is the push over HTTP of
|
||||
# the sample. The second commit adds a line at the end of Runtime.java,
|
||||
# so that its diff hides the lines above.
|
||||
make_repository() {
|
||||
status="$(api create-repository /api/v1/user/repos \
|
||||
--header "Content-Type: application/json" --data '{"name":"repo"}' \
|
||||
--write-out '%{http_code}')"
|
||||
[ "$status" = 201 ] || fail "creating the repository was answered $status"
|
||||
|
||||
dir="$WORK/repo"
|
||||
repo_git init --quiet --initial-branch=main "$dir"
|
||||
for file in \
|
||||
src/java.base/share/classes/java/lang/Runtime.java \
|
||||
java.lang.String.md \
|
||||
META-INF/services/org.apache.commons.logging.LogFactory \
|
||||
document.write.js \
|
||||
process.env.d.ts \
|
||||
test/fixtures/__proto__.json \
|
||||
install-script.sh \
|
||||
src/backend/catalog/information_schema.sql \
|
||||
.../notes.txt \
|
||||
README.md; do
|
||||
mkdir -p "$dir/$(dirname "$file")"
|
||||
lines >"$dir/$file"
|
||||
repo_git -C "$dir" add "$file"
|
||||
done
|
||||
repo_git -C "$dir" commit --quiet --message "Add the files"
|
||||
echo "line 41" >>"$dir/src/java.base/share/classes/java/lang/Runtime.java"
|
||||
repo_git -C "$dir" commit --quiet --all --message "Add a line"
|
||||
|
||||
repo_git -C "$dir" -c "http.extraHeader=X-Request-ID: git-push" push --quiet \
|
||||
"http://owner:$PASSWORD@$address/owner/repo.git" main ||
|
||||
fail "the push of the repository failed"
|
||||
}
|
||||
|
||||
# browse: sign in, then open pages as a person does. The cases are those
|
||||
# of https://git.eeqj.de/sneak/smallwebwaf/issues/30.
|
||||
browse() {
|
||||
# Case: gitea sends a visitor who is not signed in to the sign-in
|
||||
# page, with the page asked for in redirect_to, when that page needs
|
||||
# signing in, as a file's edit page does.
|
||||
page issue-30-case /owner/repo/_edit/main/document.write.js --location
|
||||
|
||||
page sign-in /user/login
|
||||
status="$(page sign-in /user/login --header "Origin: http://$address" \
|
||||
--data-urlencode user_name=owner --data-urlencode "password=$PASSWORD" \
|
||||
--write-out '%{http_code}')"
|
||||
[ "$status" = 303 ] || fail "signing in was answered $status"
|
||||
|
||||
page browse /
|
||||
page browse /owner/repo
|
||||
page browse /owner/repo/src/branch/main/README.md
|
||||
page browse /owner/repo/commits/branch/main
|
||||
page browse "/owner/repo/commit/$(repo_git -C "$WORK/repo" rev-parse HEAD)"
|
||||
|
||||
# Case: expanding the lines the diff hides, by the link the diff's
|
||||
# page gives for it.
|
||||
excerpt="$(sed -n 's/.*data-fetch-url="\([^"]*blob_excerpt[^"]*\)".*/\1/p' \
|
||||
"$WORK/answer" | head -n 1 | sed 's/&/\&/g')"
|
||||
[ -n "$excerpt" ] || fail "the diff's page has no link to its hidden lines"
|
||||
page issue-30-case "$excerpt"
|
||||
|
||||
# Cases: the commit graph of one file, and the pages of files whose
|
||||
# path the Core Rule Set takes for an attack.
|
||||
page issue-30-case "/owner/repo/graph?file=src/java.base/share/classes/java/lang/Runtime.java"
|
||||
page issue-30-case /owner/repo/src/branch/main/META-INF/services/org.apache.commons.logging.LogFactory
|
||||
page issue-30-case /owner/repo/src/branch/main/java.lang.String.md
|
||||
page issue-30-case /owner/repo/src/branch/main/process.env.d.ts
|
||||
page issue-30-case /owner/repo/src/branch/main/.../notes.txt
|
||||
}
|
||||
|
||||
# call_api: the API's calls the cases name: the commits of a file, for a
|
||||
# file of each name the Core Rule Set takes for an attack in the query
|
||||
# parameter path, and the file-contents call, which takes its list of
|
||||
# files as JSON in the query parameter body.
|
||||
call_api() {
|
||||
for file in \
|
||||
src/java.base/share/classes/java/lang/Runtime.java \
|
||||
java.lang.String.md \
|
||||
document.write.js \
|
||||
process.env.d.ts \
|
||||
test/fixtures/__proto__.json \
|
||||
install-script.sh \
|
||||
src/backend/catalog/information_schema.sql; do
|
||||
api issue-30-case /api/v1/repos/owner/repo/commits --get --data-urlencode "path=$file"
|
||||
done
|
||||
api issue-30-case /api/v1/repos/owner/repo/file-contents --get \
|
||||
--data-urlencode 'body={"files":["src/java.base/share/classes/java/lang/Runtime.java"]}'
|
||||
}
|
||||
|
||||
# push_images: push a container image to owner/python3-app, then the same
|
||||
# image to owner/app. For the second, the case, docker asks gitea to take
|
||||
# its layer from owner/python3-app, naming it in the query parameter
|
||||
# from. docker sends no X-Request-ID, so its requests have the ids
|
||||
# smallwebwaf gives them. The image is one small file, imported rather
|
||||
# than built.
|
||||
push_images() {
|
||||
echo "$NAME" >"$WORK/layer.txt"
|
||||
tar -cf - -C "$WORK" layer.txt |
|
||||
docker import - "$address/owner/python3-app:1" >/dev/null
|
||||
docker tag "$address/owner/python3-app:1" "$address/owner/app:1"
|
||||
|
||||
# The login is written here rather than made with docker login, which
|
||||
# would keep it in the host's credential helper when it has one.
|
||||
mkdir "$WORK/docker"
|
||||
printf '{"auths":{"%s":{"auth":"%s"}}}\n' "$address" \
|
||||
"$(printf 'owner:%s' "$PASSWORD" | base64)" >"$WORK/docker/config.json"
|
||||
for image in "$address/owner/python3-app:1" "$address/owner/app:1"; do
|
||||
DOCKER_CONFIG="$WORK/docker" docker push --quiet "$image" >/dev/null ||
|
||||
echo "gitea-requests: the push of $image failed" >&2
|
||||
done
|
||||
}
|
||||
|
||||
# clone_and_fetch: the clone and fetch over HTTP of the sample.
|
||||
clone_and_fetch() {
|
||||
repo_git -c "http.extraHeader=X-Request-ID: git-clone" clone --quiet \
|
||||
"http://owner:$PASSWORD@$address/owner/repo.git" "$WORK/clone" ||
|
||||
fail "the clone failed"
|
||||
repo_git -C "$WORK/clone" -c "http.extraHeader=X-Request-ID: git-fetch" fetch --quiet ||
|
||||
fail "the fetch failed"
|
||||
}
|
||||
|
||||
# field <name> <line>: the text of the field name in line, a line of the
|
||||
# request log, or nothing when it has none.
|
||||
field() {
|
||||
printf '%s\n' "$2" | sed -n "s/.*\"$1\":\"\([^\"]*\)\".*/\1/p"
|
||||
}
|
||||
|
||||
# report: print a line for each request smallwebwaf logged, but its
|
||||
# health checks: what it did with the request, the ids of the Core Rule
|
||||
# Set's rules that matched it, its X-Request-ID, and the request.
|
||||
# waf_blocked is a refusal by the Core Rule Set, and forward a request
|
||||
# passed on to gitea, whatever gitea answered.
|
||||
report() {
|
||||
docker logs "$SMALLWEBWAF" 2>/dev/null | grep '"type":"request"' |
|
||||
grep -v '"action":"admin"' | sed 's/\\u0026/\&/g' >"$WORK/requests"
|
||||
while IFS= read -r line; do
|
||||
ids="$(printf '%s\n' "$line" | sed -n 's/.*"waf_rule_ids":\[\([0-9,]*\)\].*/\1/p')"
|
||||
query="$(field query "$line")"
|
||||
printf '%-11s %-13s %-17s %s %s%s\n' "$(field action "$line")" "${ids:--}" \
|
||||
"$(field request_id "$line")" "$(field method "$line")" \
|
||||
"$(field path "$line")" "${query:+?$query}"
|
||||
done <"$WORK/requests"
|
||||
echo "gitea-requests: $(grep -c . "$WORK/requests") requests," \
|
||||
"$(grep -c '"action":"waf_blocked"' "$WORK/requests") refused by the Core Rule Set"
|
||||
}
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
trap cleanup EXIT
|
||||
trap 'exit 1' HUP INT TERM
|
||||
WORK="$(mktemp -d)"
|
||||
|
||||
docker build --no-cache -t "$IMAGE" .
|
||||
|
||||
# smallwebwaf first, so that gitea can be given the address it is
|
||||
# reached at, and gitea's image pulled before, so that gitea starts
|
||||
# within the minute smallwebwaf's health check waits for it.
|
||||
docker pull --quiet "$GITEA_IMAGE" >/dev/null
|
||||
docker network create "$NETWORK" >/dev/null
|
||||
docker run --detach --name "$SMALLWEBWAF" --network "$NETWORK" \
|
||||
--publish 127.0.0.1::8080 \
|
||||
--env SWWAF_UPSTREAM_URL="http://$GITEA:3000" \
|
||||
"$IMAGE" >/dev/null
|
||||
published="$(docker port "$SMALLWEBWAF" 8080/tcp)"
|
||||
address="localhost:${published##*:}"
|
||||
|
||||
docker run --detach --name "$GITEA" --network "$NETWORK" \
|
||||
--env GITEA__security__INSTALL_LOCK=true \
|
||||
--env GITEA__server__ROOT_URL="http://$address/" \
|
||||
"$GITEA_IMAGE" >/dev/null
|
||||
# smallwebwaf's health check passes once gitea takes connections.
|
||||
wait_for "the health check did not pass" healthy
|
||||
docker exec --user git "$GITEA" gitea admin user create --admin \
|
||||
--username owner --password "$PASSWORD" --email owner@example.invalid \
|
||||
--must-change-password=false >/dev/null
|
||||
|
||||
make_repository
|
||||
browse
|
||||
call_api
|
||||
push_images
|
||||
clone_and_fetch
|
||||
|
||||
# The fetch's last request is the last of all; once it is logged, so
|
||||
# is every other.
|
||||
wait_for "smallwebwaf logged no line for the fetch" \
|
||||
logged '"request_id":"git-fetch"' '"method":"POST"'
|
||||
report
|
||||
}
|
||||
|
||||
main "$@"
|
||||
+3
-23
@@ -5,32 +5,12 @@ set -euo pipefail
|
||||
# exits; the wait spaces out the restarts. The state directory and every
|
||||
# file in it are given to the smallwebwaf user, so that a volume mounted
|
||||
# there needs no change of owner; chown -R changes a symbolic link itself,
|
||||
# never what it points to. When SWWAF_STATE_DIR, in either form, cannot be
|
||||
# read or is not the absolute path of a directory, no owner is changed,
|
||||
# and smallwebwaf starts and stops with a message naming the setting. A
|
||||
# change of owner that fails, as on a volume mounted read-only, does not
|
||||
# end the script either: smallwebwaf reports a directory it cannot write.
|
||||
# exec, so that the signal `sv stop` sends reaches smallwebwaf itself.
|
||||
# never what it points to. exec, so that the signal `sv stop` sends
|
||||
# reaches smallwebwaf itself.
|
||||
main() {
|
||||
sleep 1
|
||||
local dir
|
||||
if dir="$(state_dir)" && [[ "$dir" == /* && -d "$dir" ]]; then
|
||||
chown -R smallwebwaf:smallwebwaf "$dir" || true
|
||||
fi
|
||||
chown -R smallwebwaf:smallwebwaf "${SWWAF_STATE_DIR:-/var/lib/smallwebwaf}"
|
||||
exec chpst -u smallwebwaf:smallwebwaf /usr/local/bin/smallwebwaf
|
||||
}
|
||||
|
||||
# state_dir prints SWWAF_STATE_DIR, /var/lib/smallwebwaf when it is unset,
|
||||
# or what is in the file SWWAF_STATE_DIR_FILE names, less every newline at
|
||||
# its end. It fails when both are set or the file cannot be read.
|
||||
state_dir() {
|
||||
if [[ ! -v SWWAF_STATE_DIR_FILE ]]; then
|
||||
printf '%s\n' "${SWWAF_STATE_DIR-/var/lib/smallwebwaf}"
|
||||
elif [[ ! -v SWWAF_STATE_DIR ]]; then
|
||||
cat -- "$SWWAF_STATE_DIR_FILE" 2>/dev/null
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
Reference in New Issue
Block a user