Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 sneak
|
1b949fddc6 |
No files matched your search
+25
-4
@@ -133,11 +133,32 @@ RUN mkdir /etc/sysusers.d \
|
||||
&& echo 'u _runit-log - "runit svlogd user" /nonexistent /sbin/nologin' \
|
||||
> /etc/sysusers.d/runit.conf
|
||||
|
||||
# ca-certificates, nix-bin and runit, from Ubuntu's archive. apt fetches
|
||||
# them over plain HTTP and checks each one against the archive's signed
|
||||
# package lists, so no CA certificates are needed.
|
||||
RUN apt-get update \
|
||||
# ca-certificates, nix-bin and runit, from Ubuntu's archive as it was at
|
||||
# the snapshot moment, which is never earlier than the Ubuntu image above.
|
||||
# apt checks every package against the snapshot's InRelease files, and
|
||||
# this step checks those against the hashes named here, which are those
|
||||
# of the amd64 archive: other architectures use Ubuntu's ports archive.
|
||||
# apt also fetches the live archive's InRelease files, which change daily
|
||||
# and which the install does not use. The snapshot service is HTTPS only
|
||||
# and this image has no CA certificates yet, so this step uses the Go
|
||||
# image's. The snapshot service answers 503 at times; apt retries a
|
||||
# failed download, waiting twice as long before each retry, and this
|
||||
# step gives it five retries, about half a minute in all, where its
|
||||
# default of three gives up after seven seconds.
|
||||
RUN --mount=type=bind,from=builder,source=/etc/ssl/certs/ca-certificates.crt,target=/tmp/go-image-ca.crt \
|
||||
apt-get update --snapshot 20261001T000000Z \
|
||||
-o Acquire::https::CaInfo=/tmp/go-image-ca.crt \
|
||||
-o Acquire::Retries=5 \
|
||||
&& printf '%s\n' \
|
||||
'45f95ce276cdba3e41870516a130e03c58b8b7a79e9546b0efe9e526d255740c snapshot.ubuntu.com_ubuntu_20261001T000000Z_dists_resolute_InRelease' \
|
||||
'802e675dd9de4c7f3916434a95e7c1d8eec0e82886622d7805ab19a2c6fe0365 snapshot.ubuntu.com_ubuntu_20261001T000000Z_dists_resolute-updates_InRelease' \
|
||||
'64b3353f0bd4970b4f7271962245bcea9ff24d4cc7bea16b433f8a60e42ca3dd snapshot.ubuntu.com_ubuntu_20261001T000000Z_dists_resolute-backports_InRelease' \
|
||||
'1d5041572116a8b23aabf79ac7439ad8af83d57ad3fb0f9aa0d4523ec10c5908 snapshot.ubuntu.com_ubuntu_20261001T000000Z_dists_resolute-security_InRelease' \
|
||||
| (cd /var/lib/apt/lists && sha256sum --check --strict) \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends \
|
||||
--snapshot 20261001T000000Z \
|
||||
-o Acquire::https::CaInfo=/tmp/go-image-ca.crt \
|
||||
-o Acquire::Retries=5 \
|
||||
ca-certificates nix-bin runit \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
.PHONY: bootstrap setup test lint fmt fmt-check tidy check docker hooks build \
|
||||
run example-app gitea-requests gitea-setup
|
||||
run example-app gitea-requests
|
||||
|
||||
# Makefile targets are thin shims; the implementations live in script/
|
||||
# per the scripts-to-rule-them-all pattern (see the Entrypoints section
|
||||
# of README.md). tidy writes go.mod and go.sum as `go mod tidy` does,
|
||||
# which test checks. build and run are for working on the code by hand;
|
||||
# example-app checks the image with an app built on it; gitea-requests
|
||||
# reports the gitea requests the Core Rule Set refuses at the defaults;
|
||||
# gitea-setup checks the setup in deploy/gitea.
|
||||
# reports the gitea requests the Core Rule Set refuses at the defaults.
|
||||
|
||||
bootstrap:
|
||||
@script/bootstrap
|
||||
@@ -50,6 +49,3 @@ example-app:
|
||||
|
||||
gitea-requests:
|
||||
@script/gitea-requests
|
||||
|
||||
gitea-setup:
|
||||
@script/gitea-setup
|
||||
@@ -26,10 +26,10 @@ It protects against:
|
||||
|
||||
- Scrapers that fetch every page of a site, all day. Each client gets a limit on
|
||||
requests and on bytes per minute, per hour and per day. The defaults leave
|
||||
room for a person reading a site for hours. They catch a scraper that keeps up
|
||||
one request every 2 seconds. A client over a limit is refused and banned: for
|
||||
a while at first, for three times as long at each repeat, and permanently when
|
||||
it keeps at it.
|
||||
room for programmatic use of an API, such as an automation fleet, and catch a
|
||||
client firing requests as fast as it can within seconds. A client over a limit
|
||||
is refused and banned: for a while at first, for three times as long at each
|
||||
repeat, and permanently when it keeps at it.
|
||||
- Scanners that probe for secrets, such as `/.env` or `/.git/config`. The
|
||||
default rule file bans such a client at its first probe, and its next request
|
||||
during the ban makes the ban permanent. Your own rules, or a list of trap
|
||||
@@ -69,7 +69,8 @@ Not built, and not planned:
|
||||
### The image
|
||||
|
||||
No image is published. Build it from a clone. `make docker` runs the tests and
|
||||
the linter, then builds the image, tagged `smallwebwaf`.
|
||||
the linter, then builds the image, tagged `smallwebwaf`. It builds for amd64, on
|
||||
an amd64 host only.
|
||||
|
||||
```sh
|
||||
git clone https://git.eeqj.de/sneak/smallwebwaf.git
|
||||
@@ -236,13 +237,13 @@ Every setting the code reads, with its default:
|
||||
| `SWWAF_ALLOW_NETS` | empty | Netblocks whose clients skip every check, such as your monitoring. |
|
||||
| `SWWAF_RATE_LIMIT_EXEMPT_NETS` | empty | Netblocks whose clients the request and byte limits neither count nor refuse. |
|
||||
| `SWWAF_DENY_NETS` | empty | Netblocks whose clients are refused. |
|
||||
| `SWWAF_RATE_LIMIT_PER_MINUTE` | `120` | The most requests a client may make in a minute, a count or `off`; one more bans it. |
|
||||
| `SWWAF_RATE_LIMIT_PER_HOUR` | `1500` | The most requests a client may make in an hour, a count or `off`; one more bans it. |
|
||||
| `SWWAF_RATE_LIMIT_PER_DAY` | `6000` | The most requests a client may make in a day, a count or `off`; one more bans it. |
|
||||
| `SWWAF_RATE_LIMIT_PER_MINUTE` | `500` | The most requests a client may make in a minute, a count or `off`; one more bans it. |
|
||||
| `SWWAF_RATE_LIMIT_PER_HOUR` | `20000` | The most requests a client may make in an hour, a count or `off`; one more bans it. |
|
||||
| `SWWAF_RATE_LIMIT_PER_DAY` | `300000` | The most requests a client may make in a day, a count or `off`; one more bans it. |
|
||||
| `SWWAF_RATE_LIMIT_EXEMPT_PATHS` | empty | Path prefixes whose requests the request and byte limits neither count nor refuse, such as `/assets/`. |
|
||||
| `SWWAF_BYTES_LIMIT_PER_MINUTE` | `off` | The most bytes a client's requests may carry in a minute, a size or `off`; the request that takes a client past any byte limit bans it. |
|
||||
| `SWWAF_BYTES_LIMIT_PER_HOUR` | `50G` | The most bytes a client's requests may carry in an hour, a size or `off`. |
|
||||
| `SWWAF_BYTES_LIMIT_PER_DAY` | `100G` | The most bytes a client's requests may carry in a day, a size or `off`. |
|
||||
| `SWWAF_BYTES_LIMIT_PER_MINUTE` | `4G` | The most bytes a client's requests may carry in a minute, a size or `off`; the request that takes a client past any byte limit bans it. |
|
||||
| `SWWAF_BYTES_LIMIT_PER_HOUR` | `20G` | The most bytes a client's requests may carry in an hour, a size or `off`. |
|
||||
| `SWWAF_BYTES_LIMIT_PER_DAY` | `200G` | The most bytes a client's requests may carry in a day, a size or `off`. |
|
||||
| `SWWAF_BYTES_COUNT` | `both` | Which body bytes the byte limits count: `response`, `request` or `both`. |
|
||||
| `SWWAF_ERROR_BURST_THRESHOLD` | `30` | The most refusals by a rule file, a trap path or the Core Rule Set a client may get in a minute, a count or `off`; one more bans it. |
|
||||
| `SWWAF_BAN_RESPONSE` | `403` | How a refused client is answered: `403`, `429` or `close`, which drops the connection. |
|
||||
@@ -407,10 +408,10 @@ services:
|
||||
SWWAF_RATE_LIMIT_EXEMPT_NETS: 203.0.113.0/24,198.51.100.7
|
||||
# A netblock that sends nothing wanted.
|
||||
SWWAF_DENY_NETS: 198.51.100.128/25
|
||||
# Tighter limits for everyone else: a small static site.
|
||||
SWWAF_RATE_LIMIT_PER_MINUTE: "60"
|
||||
SWWAF_RATE_LIMIT_PER_HOUR: "600"
|
||||
SWWAF_RATE_LIMIT_PER_DAY: "3000"
|
||||
# Tighter limits for everyone else: a site with no API.
|
||||
SWWAF_RATE_LIMIT_PER_MINUTE: "120"
|
||||
SWWAF_RATE_LIMIT_PER_HOUR: "3000"
|
||||
SWWAF_RATE_LIMIT_PER_DAY: "30000"
|
||||
# Hosting networks and one country get a share of those limits.
|
||||
SWWAF_ASN_LIMIT_PERCENT: AS14061:50,AS16276:25
|
||||
SWWAF_COUNTRY_LIMIT_PERCENT: cn:25
|
||||
@@ -431,9 +432,10 @@ volumes:
|
||||
gitea behind traefik, set up for a code forge:
|
||||
|
||||
- the upload limits raised, in size and in time;
|
||||
- the response size limit raised, for clones over 1 GiB;
|
||||
- the response size limit and the byte limits raised, for clones over 1 GiB;
|
||||
- the Actions runners exempt from counting;
|
||||
- the default request and byte limits kept.
|
||||
- the default request limits kept, since they already leave room for API
|
||||
clients.
|
||||
|
||||
`traefik` is the network traefik is on, made outside this file.
|
||||
|
||||
@@ -487,6 +489,11 @@ services:
|
||||
SWWAF_INSTANCE_NAME: fsn1/servewaf
|
||||
# The Actions runners ask for work all day; never count them.
|
||||
SWWAF_RATE_LIMIT_EXEMPT_NETS: 10.0.4.0/24
|
||||
# Raised with rproxy's response limit, to four times the default
|
||||
# each, so that one clone of that size breaks no byte limit.
|
||||
SWWAF_BYTES_LIMIT_PER_MINUTE: 16G
|
||||
SWWAF_BYTES_LIMIT_PER_HOUR: 80G
|
||||
SWWAF_BYTES_LIMIT_PER_DAY: 800G
|
||||
# gitea never serves these; a request for one bans the client.
|
||||
SWWAF_TRAP_PATHS: /wp-login.php,/xmlrpc.php,/wp-admin/
|
||||
volumes:
|
||||
@@ -504,9 +511,8 @@ volumes:
|
||||
servewaf-state:
|
||||
```
|
||||
|
||||
`deploy/gitea` holds a complete setup of gitea, traefik included, as a Docker
|
||||
Compose file. [`docs/gitea.md`](docs/gitea.md) explains it, and records what the
|
||||
defaults do in front of gitea, request by request.
|
||||
[`docs/gitea.md`](docs/gitea.md) records what the defaults do in front of gitea,
|
||||
request by request.
|
||||
|
||||
### `rproxy` beside the app
|
||||
|
||||
@@ -656,8 +662,8 @@ aside as `<name>.bad`, and never stops the running process (see
|
||||
|
||||
### The image and its processes
|
||||
|
||||
- The image is built on Ubuntu 26.04 LTS, pinned by digest. Its Ubuntu packages
|
||||
come from Ubuntu's archive, and nixpkgs is fixed at one release. An app's
|
||||
- The image is built on Ubuntu 26.04 LTS, pinned by digest, with its packages
|
||||
from a snapshot of Ubuntu's archive and nixpkgs fixed at one release. An app's
|
||||
`Dockerfile` installs a package with `nix-env -iA nixpkgs.<name>` and gets the
|
||||
same package each time it is built (see [`docs/image.md`](docs/image.md)).
|
||||
- runit, a service supervisor, runs `smallwebwaf` as the `smallwebwaf` user (uid
|
||||
@@ -779,10 +785,6 @@ on the host.
|
||||
Core Rule Set is known to refuse, and prints what `servewaf` did with each.
|
||||
Needs network access, and a docker daemon that pushes to a registry on
|
||||
`localhost` over plain HTTP. `make gitea-requests`.
|
||||
- `script/gitea-setup`: builds the image, brings up the setup in `deploy/gitea`,
|
||||
and checks it as [`docs/gitea.md`](docs/gitea.md) describes. Then it removes
|
||||
the containers, the network, the volumes and the image it built. Needs network
|
||||
access. `make gitea-setup`.
|
||||
|
||||
## TODO
|
||||
|
||||
@@ -806,8 +808,7 @@ on the host.
|
||||
- [`docs/state-files.md`](docs/state-files.md): the state files and what each
|
||||
holds.
|
||||
- [`docs/image.md`](docs/image.md): what the image holds and how it is pinned.
|
||||
- [`docs/gitea.md`](docs/gitea.md): `smallwebwaf` in front of gitea: the setup
|
||||
in `deploy/gitea`, and what the defaults do there.
|
||||
- [`docs/gitea.md`](docs/gitea.md): `smallwebwaf` in front of gitea.
|
||||
|
||||
## License
|
||||
|
||||
|
||||
@@ -123,9 +123,7 @@ runs steps 3 to 13, and does the counting of step 15. The first step that
|
||||
produces a final answer ends the checks. Step 15 runs for each request
|
||||
`servewaf` answers with `allow`, `deny` or `close`. `rproxy`'s size and time
|
||||
limits apply to the whole exchange. Each step names the section that holds its
|
||||
rules. Each check is a heuristic. Its name, in parentheses below, is the value
|
||||
of the request log's `heuristic` field. `refusal_limit` and `byte_limits` give
|
||||
theirs in `after_answer_heuristic` instead.
|
||||
rules.
|
||||
|
||||
1. `GET /_smallwebwaf/healthz` is answered at once by `rproxy`, before any
|
||||
check, so a health checker never needs an exemption and is never refused.
|
||||
@@ -143,9 +141,9 @@ theirs in `after_answer_heuristic` instead.
|
||||
around its address, a /64 by default, since one abuser usually controls a
|
||||
whole /64. A client is therefore one IPv4 address or one IPv6 group.
|
||||
3. The static lists. A client in `SWWAF_ALLOW_NETS` skips steps 4 to 12, the
|
||||
byte limits and the error burst. A client in `SWWAF_DENY_NETS` is refused
|
||||
(`deny_nets`); a client in both lists is let through.
|
||||
4. Bans. A client under a ban is refused (`active_ban`; see "Bans").
|
||||
byte limits and the error burst. A client in `SWWAF_DENY_NETS` is refused; a
|
||||
client in both lists is let through.
|
||||
4. Bans. A client under a ban is refused (see "Bans").
|
||||
5. The lookup, unless `SWWAF_LOOKUP_SOURCE` is `off` (see "Lookup of AS number
|
||||
and country"). The request goes on at once. It waits for the first GeoJS
|
||||
answer, up to `SWWAF_LOOKUP_TIMEOUT`, only when one of these needs it:
|
||||
@@ -155,26 +153,22 @@ theirs in `after_answer_heuristic` instead.
|
||||
|
||||
A request that ends before the answer comes is logged without it.
|
||||
|
||||
6. The country lists (`countries`; see "Country lists").
|
||||
7. The reputation sources (see "Reputation"), in this order:
|
||||
- A client on a fetched blocklist gets `SWWAF_BLOCKLIST_ACTION`
|
||||
(`blocklists`).
|
||||
- A client on the CrowdSec decision list is banned (`crowdsec`).
|
||||
- A client listed by a DNS blocklist gets `SWWAF_REPUTATION_ACTION`
|
||||
(`dnsbl`).
|
||||
- A client scored at least `SWWAF_ABUSEIPDB_MIN_SCORE` by AbuseIPDB gets
|
||||
`SWWAF_REPUTATION_ACTION` (`abuseipdb`).
|
||||
6. The country lists (see "Country lists").
|
||||
7. The reputation sources (see "Reputation"). A client on a fetched blocklist
|
||||
gets `SWWAF_BLOCKLIST_ACTION`. A client on the CrowdSec decision list is
|
||||
banned. A client listed by a DNS blocklist, or scored at least
|
||||
`SWWAF_ABUSEIPDB_MIN_SCORE` by AbuseIPDB, gets `SWWAF_REPUTATION_ACTION`.
|
||||
8. The client's limit percentage (see "Biased thresholds").
|
||||
9. Rate limiting (`rate_limits`; see "Request rate limits, per client"). A
|
||||
request over a limit times the percentage is refused and bans the client.
|
||||
10. The trap paths (`trap_paths`; see "Attack detection"). A request for one is
|
||||
refused and bans the client.
|
||||
11. The rule files (`rule_files`; see "Rule files"). Matching stops at the first
|
||||
rule that refuses.
|
||||
12. The header limits (`header_limits`; see "Answering in time"), and then the
|
||||
Core Rule Set (`core_rule_set`; see "Attack detection"). A Core Rule Set
|
||||
match is refused with `403` while `SWWAF_WAF_MODE` is `block`, and only
|
||||
logged and alerted while it is `detect`.
|
||||
9. Rate limiting (see "Request rate limits, per client"). A request over a limit
|
||||
times the percentage is refused and bans the client.
|
||||
10. The trap paths (see "Attack detection"). A request for one is refused and
|
||||
bans the client.
|
||||
11. The rule files (see "Rule files"). Matching stops at the first rule that
|
||||
refuses.
|
||||
12. The header limits (see "Answering in time"), and then the Core Rule Set (see
|
||||
"Attack detection"). A Core Rule Set match is refused with `403` while
|
||||
`SWWAF_WAF_MODE` is `block`, and only logged and alerted while it is
|
||||
`detect`.
|
||||
13. A path under `/_smallwebwaf/` is answered `404`, so that the app never sees
|
||||
that prefix.
|
||||
14. `rproxy` passes the request to `SWRP_UPSTREAM_URL`, streaming (see "Passing
|
||||
@@ -188,14 +182,14 @@ theirs in `after_answer_heuristic` instead.
|
||||
are added to the byte counters, as `SWWAF_BYTES_COUNT` says. For a
|
||||
WebSocket, what it carried each way counts once it closes. A client whose
|
||||
total passes a byte limit times the percentage has broken that limit and
|
||||
is banned (`byte_limits`). The response in progress is not cut off.
|
||||
is banned. The response in progress is not cut off.
|
||||
- For a request it allowed, the client's history and the anomaly counters
|
||||
are updated, and the alert thresholds are evaluated.
|
||||
|
||||
For a request it refused, `servewaf` adds nothing to the byte limits. It
|
||||
updates the history and the anomaly counters at its decision. It counts the
|
||||
refusals the error burst counts at the decision too (`refusal_limit`; see
|
||||
"Attack detection").
|
||||
refusals the error burst counts at the decision too (see "Attack
|
||||
detection").
|
||||
|
||||
In `SWWAF_MODE=observe`, every refusal above is logged as what would have
|
||||
happened, nothing is refused, and `servewaf` makes no ban of its own. A path
|
||||
@@ -403,31 +397,32 @@ Conventions:
|
||||
|
||||
### Request rate limits, per client
|
||||
|
||||
Breaking one bans the client. The defaults let people read a site and stop
|
||||
scrapers that send many requests slowly over hours or days. The hour limit is
|
||||
the main one.
|
||||
Breaking one bans the client. The defaults suit programmatic use of an API, such
|
||||
as an automation fleet on another host, and still stop abusive bots. A fleet of
|
||||
20 agents, each making a request every 10 seconds at its busiest, makes 120
|
||||
requests a minute and 7,200 an hour. Working around the clock, it makes 172,800
|
||||
a day. It stays under all three limits.
|
||||
|
||||
- Per minute: 120, 2 requests a second. It leaves room for a person opening two
|
||||
pages at once with an empty cache. A gitea page opened with an empty cache
|
||||
loads 20 to 60 files, and a handful once cached.
|
||||
- Per hour: 1,500. A person reading code in gitea for a whole hour, a page every
|
||||
20 seconds, makes about 900 requests. A scraper making one request every 2
|
||||
seconds makes 1,800 an hour, more than the limit.
|
||||
- Per day: 6,000, four hours at the hour limit. A scraper making one request
|
||||
every 10 seconds makes 8,640 a day, more than the limit.
|
||||
- Per minute: 500, about 8 requests a second. It leaves room for the fleet's 120
|
||||
a minute and a few quick first visits with an empty cache. A gitea page opened
|
||||
with an empty cache loads about 40 files. A client firing requests as fast as
|
||||
it can passes the limit within seconds.
|
||||
- Per hour: 20,000, about 5.5 requests a second for a whole hour. A client that
|
||||
keeps up 6 a second makes 21,600 in an hour, and passes it, at 360 a minute,
|
||||
under the minute limit. The fleet's 7,200 an hour is about a third of it.
|
||||
- Per day: 300,000, about 3.5 requests a second around the clock. A scraper that
|
||||
keeps up 4 a second makes 345,600 a day, and passes it. At 240 a minute and
|
||||
14,400 an hour, it stays under the other two limits.
|
||||
|
||||
Counting cannot catch a client that stays under all three limits, about 4
|
||||
requests a minute around the clock. Only the other checks can catch an abuser
|
||||
Counting cannot catch a client that stays under all three limits, such as one
|
||||
that keeps up 3 requests a second. Only the other checks can catch an abuser
|
||||
that slow. They include the trap paths, the rule files, the Core Rule Set and
|
||||
the blocklists. An abuser that trips none of them is not caught.
|
||||
|
||||
Some legitimate clients pass the limits:
|
||||
|
||||
- An Actions runner asks gitea for work all day and can pass the limits.
|
||||
- Everyone behind one shared address, such as an office, shares one client's
|
||||
limits.
|
||||
|
||||
Their addresses belong in `SWWAF_RATE_LIMIT_EXEMPT_NETS`.
|
||||
the blocklists. An abuser that trips none of them is not caught. A machine that
|
||||
keeps up more than 500 requests a minute passes the minute limit. One that keeps
|
||||
up more than about 5.5 a second for an hour passes the hour limit. One that
|
||||
keeps up more than about 3.5 a second all day passes the day limit, as two busy
|
||||
Actions runners on one address do. Such a machine's address belongs in
|
||||
`SWWAF_RATE_LIMIT_EXEMPT_NETS`.
|
||||
|
||||
`SWWAF_RATE_LIMIT_EXEMPT_PATHS` takes path prefixes out of the count, such as
|
||||
static assets or health checks; their bytes are not counted either. A prefix is
|
||||
@@ -440,10 +435,12 @@ matched the same way.
|
||||
|
||||
### Byte limits, per client
|
||||
|
||||
By default there is no minute limit, so a client can download in a burst at full
|
||||
speed. The hour limit defaults to 50 GiB and the day limit to 100 GiB. They
|
||||
allow several downloads of a few GiB each. A response's bytes are counted when
|
||||
it ends (see step 15).
|
||||
The hour and day limits sit about twice above what 40 people busy with heavy web
|
||||
apps move together on one address. Each moves about 256 MiB an hour, for ten
|
||||
hours a day. A response's bytes are counted when it ends (see step 15). So the
|
||||
minute limit holds three of the largest responses, with their requests, ending
|
||||
in the same minute. No single download then breaks a limit. An operator who
|
||||
raises `SWRP_MAX_RESPONSE_SIZE` must raise the byte limits too.
|
||||
|
||||
### Lookup of AS number and country
|
||||
|
||||
@@ -1547,11 +1544,20 @@ It sets no `ENTRYPOINT` or `USER` of its own. The container must start
|
||||
|
||||
- The image is Ubuntu's newest long-term support release, pinned by digest, and
|
||||
moves to the next LTS release when that ships. Its packages come from Ubuntu's
|
||||
archive, whose packages change with every update.
|
||||
snapshot service, the archive as it was at the moment the Dockerfile names,
|
||||
rather than the live archive, whose packages change with every update. The
|
||||
snapshot is never older than the pinned Ubuntu image, since older packages can
|
||||
need older versions of packages the image holds, and it moves forward with the
|
||||
image's digest. The Dockerfile names the hash of each of the snapshot's
|
||||
`InRelease` files, which cover every package list and so every package, and
|
||||
the build checks them before installing. The live archive's `InRelease` files,
|
||||
which `apt-get update` also fetches and the install does not use, are left out
|
||||
of the check. The hashes are the amd64 archive's, so the image is built for
|
||||
amd64 only.
|
||||
- The Ubuntu image has no CA certificates. Without them Nix cannot download
|
||||
packages and `servewaf` cannot reach GeoJS, so `ca-certificates` is installed
|
||||
by name. apt fetches the packages over plain HTTP and checks each one against
|
||||
the archive's signed package lists, so the install needs no CA certificates.
|
||||
by name. The snapshot itself is fetched over HTTPS with the CA certificates of
|
||||
the Go image that builds `smallwebwaf`, itself pinned by digest.
|
||||
- nixpkgs comes from a release file on `releases.nixos.org`, named with its
|
||||
hash, not from a GitHub archive, whose bytes can change. Root uses Nix
|
||||
directly; no Nix daemon runs. `nix-bin` creates no build users, so
|
||||
|
||||
@@ -1,135 +0,0 @@
|
||||
# Gitea behind traefik, with rproxy and servewaf in front of it.
|
||||
# docs/gitea.md says what each setting here does and why. Before starting it:
|
||||
# - set SMALLWEBWAF_IMAGE to the smallwebwaf image, named by digest;
|
||||
# - replace git.example.org with the site's host name;
|
||||
# - replace each 10.99.0 with the first three numbers of a /24 subnet that
|
||||
# no other network on the host uses. The /24 must lie inside 10.0.0.0/8,
|
||||
# 172.16.0.0/12 or 192.168.0.0/16, because rproxy believes
|
||||
# X-Forwarded-For only from those by default. Replace each fd10:99 with
|
||||
# the first two hextets of an unused /64 subnet whose third and fourth
|
||||
# hextets are 0, as in fd10:99::/64;
|
||||
# - set GITEA_SECRET_KEY to the output of
|
||||
# `docker run --rm <gitea's image below> gitea generate secret SECRET_KEY`,
|
||||
# once per install, and keep it.
|
||||
# Then run `docker compose up --detach` in this directory.
|
||||
# script/gitea-setup checks this setup.
|
||||
|
||||
name: gitea
|
||||
|
||||
x-logging: &logging
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: 10m
|
||||
max-file: "5"
|
||||
|
||||
services:
|
||||
traefik:
|
||||
# traefik v3.7.17, 2026-11-03
|
||||
image: traefik@sha256:68ae5fe5b64ca166704b746a9e8afcd385e36b3f569a5691401bf652d57b3748
|
||||
# traefik replaces the X-Forwarded-For header a visitor sends. Keep it
|
||||
# so: otherwise a visitor could name an address on the shared network,
|
||||
# and skip servewaf's checks.
|
||||
command:
|
||||
- --providers.docker.exposedbydefault=false
|
||||
- --entrypoints.web.address=:80
|
||||
- --entrypoints.web.http.redirections.entrypoint.to=websecure
|
||||
- --entrypoints.websecure.address=:443
|
||||
# Until a certificate resolver is added here, as for your other
|
||||
# sites, traefik answers with a certificate it made itself.
|
||||
- --entrypoints.websecure.http.tls=true
|
||||
# traefik's own limit of 60 seconds for a whole request would cut
|
||||
# off an upload long before rproxy's limit of 30 minutes.
|
||||
- --entrypoints.websecure.transport.respondingTimeouts.readTimeout=30m
|
||||
ports:
|
||||
- 80:80
|
||||
- 443:443
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
networks:
|
||||
shared:
|
||||
# Containers on the shared network find traefik under the
|
||||
# site's name, and so reach it from their own address.
|
||||
aliases:
|
||||
- git.example.org
|
||||
restart: unless-stopped
|
||||
|
||||
gitea:
|
||||
# gitea 1.27.3, 2026-08-29
|
||||
image: gitea/gitea@sha256:87a67ee09d3ae0d1df5fda5dcda3e2a1f9236a45b0a59025d6e00e46adc43bef
|
||||
environment:
|
||||
GITEA__server__ROOT_URL: https://git.example.org/
|
||||
# Gitea starts at its defaults, with SQLite, instead of showing its
|
||||
# install page to whoever opens it first.
|
||||
GITEA__security__INSTALL_LOCK: "true"
|
||||
GITEA__security__SECRET_KEY: ${GITEA_SECRET_KEY:?set GITEA_SECRET_KEY, see docs/gitea.md}
|
||||
# Accounts are made with `gitea admin user create`.
|
||||
GITEA__service__DISABLE_REGISTRATION: "true"
|
||||
# rproxy's addresses. Gitea then takes each visitor's address from
|
||||
# the X-Real-Ip header traefik sets and rproxy passes on.
|
||||
GITEA__security__REVERSE_PROXY_TRUSTED_PROXIES: 10.99.0.2,fd10:99::2
|
||||
volumes:
|
||||
- gitea-data:/data
|
||||
networks:
|
||||
- shared
|
||||
restart: unless-stopped
|
||||
|
||||
rproxy:
|
||||
image: ${SMALLWEBWAF_IMAGE:?set SMALLWEBWAF_IMAGE to the smallwebwaf image}
|
||||
environment:
|
||||
SWRP_WAF_BACKEND: http://servewaf:8080
|
||||
SWRP_UPSTREAM_URL: http://gitea:3000
|
||||
# Pushes, LFS objects, packages and release attachments of up to
|
||||
# 2 GiB, sent in up to 30 minutes.
|
||||
SWRP_MAX_REQUEST_SIZE: 2G
|
||||
SWRP_MAX_REQUEST_TIME_MS: "1800000"
|
||||
# 15 minutes for a clone or a download, half the default.
|
||||
SWRP_MAX_RESPONSE_TIME_MS: "900000"
|
||||
labels:
|
||||
traefik.enable: "true"
|
||||
traefik.http.routers.gitea.rule: Host(`git.example.org`)
|
||||
traefik.http.routers.gitea.entrypoints: websecure
|
||||
traefik.http.services.gitea.loadbalancer.server.port: "8080"
|
||||
networks:
|
||||
shared:
|
||||
ipv4_address: 10.99.0.2
|
||||
ipv6_address: fd10:99::2
|
||||
logging: *logging
|
||||
restart: unless-stopped
|
||||
|
||||
servewaf:
|
||||
image: ${SMALLWEBWAF_IMAGE:?set SMALLWEBWAF_IMAGE to the smallwebwaf image}
|
||||
environment:
|
||||
SMALLWEBWAF_COMMAND: servewaf
|
||||
# The containers on the shared network skip the checks. Add the
|
||||
# address of an automation fleet on another host here too; without
|
||||
# that, it is counted like any client.
|
||||
SWWAF_ALLOW_NETS: 10.99.0.128/25,fd10:99:0:0:8000::/65
|
||||
# In block mode the Core Rule Set refuses everyday Gitea searches
|
||||
# and file names, and 31 refusals in a minute ban the address.
|
||||
SWWAF_WAF_MODE: detect
|
||||
volumes:
|
||||
- servewaf-state:/var/lib/smallwebwaf
|
||||
networks:
|
||||
- shared
|
||||
logging: *logging
|
||||
restart: unless-stopped
|
||||
|
||||
networks:
|
||||
# The network traefik and Gitea share. Its containers get addresses in the
|
||||
# upper half of each subnet, which servewaf lets skip the checks. Each
|
||||
# subnet's gateway, the host's own address on it, stays out of that half:
|
||||
# requests docker passes on from a published port can come from it.
|
||||
shared:
|
||||
enable_ipv6: true
|
||||
ipam:
|
||||
config:
|
||||
- subnet: 10.99.0.0/24
|
||||
ip_range: 10.99.0.128/25
|
||||
gateway: 10.99.0.1
|
||||
- subnet: fd10:99::/64
|
||||
ip_range: fd10:99:0:0:8000::/65
|
||||
gateway: fd10:99::1
|
||||
|
||||
volumes:
|
||||
gitea-data:
|
||||
servewaf-state:
|
||||
+49
-169
@@ -1,177 +1,51 @@
|
||||
# smallwebwaf in front of gitea
|
||||
|
||||
`deploy/gitea/compose.yaml` runs gitea behind traefik, with `rproxy` and
|
||||
`servewaf` in front of it. This document says what that setup sets and why, and
|
||||
what the defaults would do without it. [`SPEC.md`](../SPEC.md) says how
|
||||
`smallwebwaf` behaves.
|
||||
|
||||
## The setup
|
||||
|
||||
Four containers share one Docker network, the shared network:
|
||||
|
||||
- traefik, which routes the site's HTTPS requests to `rproxy`, and redirects
|
||||
plain HTTP to HTTPS;
|
||||
- `rproxy`, in front of gitea;
|
||||
- gitea 1.27.3, as its image comes;
|
||||
- `servewaf`, with a volume for its state files.
|
||||
|
||||
To start it:
|
||||
|
||||
1. In `deploy/gitea/compose.yaml`, replace `git.example.org` with the site's
|
||||
host name.
|
||||
2. Replace each `10.99.0` with the first three numbers of a `/24` subnet that no
|
||||
other network on the host uses. The `/24` must lie inside `10.0.0.0/8`,
|
||||
`172.16.0.0/12` or `192.168.0.0/16`, because `rproxy` believes
|
||||
`X-Forwarded-For` only from those by default. Replace each `fd10:99` with the
|
||||
first two hextets of an unused `/64` subnet whose third and fourth hextets
|
||||
are 0, as in `fd10:99::/64`.
|
||||
3. Set `SMALLWEBWAF_IMAGE` to the `smallwebwaf` image, named by digest.
|
||||
4. Set `GITEA_SECRET_KEY` to the output of
|
||||
`docker run --rm <gitea's image> gitea generate secret SECRET_KEY`, with
|
||||
gitea's image as `deploy/gitea/compose.yaml` names it. Do this once per
|
||||
install, and keep the key.
|
||||
5. Run `docker compose up --detach` in `deploy/gitea`.
|
||||
6. Make gitea's first user with `gitea admin user create`, in gitea's container,
|
||||
as the user `git`.
|
||||
|
||||
Sign-up is off: accounts are made with `gitea admin user create`. gitea starts
|
||||
at its defaults, with SQLite. traefik answers with a certificate it made itself
|
||||
until you add a certificate resolver, as for your other sites. SSH to gitea does
|
||||
not pass through `smallwebwaf`; publish gitea's SSH port yourself if you use it.
|
||||
|
||||
`make gitea-setup` checks the setup. It builds the image, brings the setup up
|
||||
under a name of its own, and checks:
|
||||
|
||||
- that a page load, a push and a clone sent through traefik pass through
|
||||
`rproxy`, and that `servewaf` checks them;
|
||||
- that gitea logs the visitor's address, not `rproxy`'s;
|
||||
- that a request from a container on the shared network skips the checks;
|
||||
- that with `servewaf` stopped, gitea still answers through `rproxy`.
|
||||
|
||||
## Uploads and downloads
|
||||
|
||||
An upload is a request:
|
||||
|
||||
- a git push;
|
||||
- an LFS object;
|
||||
- a container image layer;
|
||||
- a package file;
|
||||
- a release attachment.
|
||||
|
||||
The setup lets an upload of up to 2 GiB, sent in up to 30 minutes, through:
|
||||
|
||||
- `SWRP_MAX_REQUEST_SIZE` is `2G`. At the default, an upload over 100 MiB is cut
|
||||
off.
|
||||
- `SWRP_MAX_REQUEST_TIME_MS` is `1800000`, 30 minutes. At the default, an upload
|
||||
that takes over 60 seconds is cut off.
|
||||
- traefik's `readTimeout` is 30 minutes. traefik's own default of 60 seconds for
|
||||
a whole request would cut off an upload first.
|
||||
|
||||
A clone or a download is a response:
|
||||
|
||||
- `SWRP_MAX_RESPONSE_TIME_MS` is `900000`, 15 minutes, half the default. A clone
|
||||
or download that takes longer is cut off.
|
||||
- `SWRP_MAX_RESPONSE_SIZE` stays at its default, 1 GiB. A repository whose clone
|
||||
is larger needs it raised.
|
||||
|
||||
The byte limits stay at their defaults: no minute limit, 50 GiB an hour and 100
|
||||
GiB a day.
|
||||
|
||||
## The checks
|
||||
|
||||
`SWWAF_WAF_MODE` is `detect`. The Core Rule Set then logs a match, with its rule
|
||||
ids, and refuses nothing. In `block` mode, the default, it refuses everyday
|
||||
gitea searches and file names. The section "What the Core Rule Set refuses in
|
||||
`block` mode" below lists them. Each such refusal counts toward the error burst,
|
||||
so 31 in one minute ban the address.
|
||||
|
||||
Every other check stays at its default. A client may make 120 requests a minute,
|
||||
1,500 an hour and 6,000 a day. Archive downloads and blame and history pages are
|
||||
what scrapers hammer, and these limits do most of the work there.
|
||||
|
||||
## The shared network
|
||||
|
||||
Containers on the shared network skip `servewaf`'s checks:
|
||||
|
||||
- The shared network's subnets are `10.99.0.0/24` and `fd10:99::/64`. Docker
|
||||
gives its containers addresses in the upper half of each, `10.99.0.128/25` and
|
||||
`fd10:99:0:0:8000::/65`, which `SWWAF_ALLOW_NETS` names.
|
||||
- traefik is known on the shared network by the site's host name. So a container
|
||||
there that asks for the site reaches traefik from its own address. An
|
||||
automation fleet in such a container skips the checks.
|
||||
- Each subnet's gateway, `10.99.0.1` and `fd10:99::1`, is the host's own address
|
||||
on it. Requests docker passes on from a published port can come from the
|
||||
gateway, so the gateways are left out of `SWWAF_ALLOW_NETS`. The network has
|
||||
IPv6, so a visitor over IPv6 reaches traefik from its own address, and
|
||||
`servewaf` checks it as itself.
|
||||
- A container on the shared network that asks for the site at the host's public
|
||||
address arrives from the gateway. It is checked like any visitor.
|
||||
- A fleet on another host can have its address added to `SWWAF_ALLOW_NETS`.
|
||||
Without that, it is counted like any client.
|
||||
- traefik replaces the `X-Forwarded-For` a visitor sends, by default. Keep it
|
||||
so: otherwise a visitor could name an address on the shared network, and skip
|
||||
the checks.
|
||||
- Anything on the shared network can reach gitea and `servewaf` around `rproxy`.
|
||||
Put only containers you trust on it.
|
||||
|
||||
## gitea's settings
|
||||
|
||||
- `REVERSE_PROXY_TRUSTED_PROXIES` names `rproxy`'s fixed addresses, `10.99.0.2`
|
||||
and `fd10:99::2`. gitea then takes each visitor's address from the `X-Real-Ip`
|
||||
header, which traefik sets and `rproxy` passes on. Without it, gitea logs
|
||||
every visitor as `rproxy`.
|
||||
- `INSTALL_LOCK` is `true`, as "The setup" above says.
|
||||
|
||||
## Methods
|
||||
|
||||
`rproxy` passes on only these methods: `GET`, `HEAD`, `POST`, `OPTIONS`, `PUT`,
|
||||
`DELETE`, `PATCH` and `QUERY`. gitea 1.27.3 needs no other. Its routes use:
|
||||
|
||||
- pages and the API: `GET`, `HEAD`, `POST`, `PUT`, `PATCH` and `DELETE`;
|
||||
- git's push, clone and fetch over HTTP: `GET`, `POST` and `OPTIONS`;
|
||||
- LFS: `GET`, `POST` and `PUT`;
|
||||
- the container registry and the other package registries: `GET`, `HEAD`,
|
||||
`POST`, `PUT`, `PATCH` and `DELETE`;
|
||||
- Actions runners: `POST`.
|
||||
|
||||
## The log
|
||||
|
||||
`rproxy` and `servewaf` write each query string to the log whole. A token sent
|
||||
in the query string, as `?token=`, is then in the log. Send it in the
|
||||
`Authorization` header instead.
|
||||
|
||||
## Actions runners
|
||||
|
||||
A runner on the shared network skips the checks. One on another host is counted
|
||||
like any client. A runner older than version 0.4 (April 2026) asks for work
|
||||
every 2 seconds, 1,800 requests an hour. While it runs a job, it reports the
|
||||
job's log and state every second instead, two requests a second.
|
||||
|
||||
So one idle runner makes 1,800 requests an hour, more than the hour limit of
|
||||
1,500. A busy one, at two requests a second, passes the minute limit of 120
|
||||
within two minutes. Its address is then banned, its running jobs fail, and each
|
||||
repeat within a day makes the ban longer. Newer runners ask for work less often.
|
||||
Put the runners' addresses in `SWWAF_RATE_LIMIT_EXEMPT_NETS`.
|
||||
|
||||
## What the Core Rule Set refuses in `block` mode
|
||||
|
||||
These notes record what the Core Rule Set does in front of gitea in `block`
|
||||
mode, at the other defaults. They are observations, not requirements.
|
||||
`script/gitea-requests` runs gitea behind `rproxy` and `servewaf` at the
|
||||
defaults. It sends through them a sample of gitea's ordinary use and the
|
||||
These notes record what `smallwebwaf` does in front of gitea at the default
|
||||
settings. They are observations, not requirements; the requirements are in
|
||||
[`SPEC.md`](../SPEC.md). `script/gitea-requests` runs gitea behind `rproxy` and
|
||||
`servewaf`, and sends through them a sample of gitea's ordinary use and the
|
||||
requests named below. It reports each request the Core Rule Set refuses, so that
|
||||
these notes can be checked again when gitea or the Core Rule Set changes
|
||||
version. In `detect` mode, as the setup runs it, the same requests are logged
|
||||
with their rule ids, and pass.
|
||||
version.
|
||||
|
||||
Each refusal answers only that request ("Bans" in `SPEC.md` says what it counts
|
||||
toward). The request log names the rule in `waf_rule_ids`, which
|
||||
`SWWAF_WAF_DISABLED_RULES` can switch off.
|
||||
## What the defaults cover
|
||||
|
||||
### What it lets through
|
||||
- SSH access to gitea does not pass through `smallwebwaf` and is not protected
|
||||
by it.
|
||||
- A clone is a response. It fits the defaults of 30 minutes and 1 GiB unless the
|
||||
repository is large or the link slow. A repository whose clone is larger than
|
||||
1 GiB needs `SWRP_MAX_RESPONSE_SIZE` raised, and the byte limits with it
|
||||
("Byte limits, per client" in `SPEC.md`).
|
||||
- A push is a request, and so is every other upload:
|
||||
- a git push;
|
||||
- an LFS object;
|
||||
- a container image layer;
|
||||
- a package file;
|
||||
- a release attachment.
|
||||
|
||||
The Core Rule Set lets gitea's ordinary use through, apart from the refusals in
|
||||
the sections after this one:
|
||||
At the defaults, an upload larger than 100 MiB or taking more than 60
|
||||
seconds is cut off ("`rproxy`'s limits" in `SPEC.md`). A gitea that takes
|
||||
large uploads needs `SWRP_MAX_REQUEST_SIZE` and `SWRP_MAX_REQUEST_TIME_MS`
|
||||
raised to fit.
|
||||
|
||||
- Archive download and blame or history pages are what scrapers hammer. The
|
||||
request limits do most of the work there.
|
||||
- An Actions runner that reaches gitea through `smallwebwaf` sends requests all
|
||||
day. One older than version 0.4 (April 2026) asks for work every 2 seconds,
|
||||
43,200 requests a day. While it runs a job, it reports the job's log and state
|
||||
every second instead, two requests a second. So one runner busy all day sends
|
||||
120 requests a minute and 172,800 a day, under the request limits. Two such
|
||||
runners on one address, each busy all day, make 345,600 and pass the day limit
|
||||
of 300,000. Three busy at once make 21,600 an hour and pass the hour limit of
|
||||
20,000. Five busy at once make 600 a minute and pass the minute limit of 500.
|
||||
Their address is then banned, their running jobs fail, and each repeat within
|
||||
a day makes the ban longer. Newer runners ask for work less often. Put the
|
||||
runners' addresses in `SWWAF_RATE_LIMIT_EXEMPT_NETS`.
|
||||
|
||||
## What the Core Rule Set lets through
|
||||
|
||||
At the defaults, the Core Rule Set lets gitea's ordinary use through, apart from
|
||||
the refusals in the next section:
|
||||
|
||||
- browsing and views of files in a repository, with their history, blame and the
|
||||
file tree;
|
||||
@@ -186,6 +60,12 @@ the sections after this one:
|
||||
editor, code included, since no body is read;
|
||||
- the page gitea shows after a change, with its message naming what was done.
|
||||
|
||||
## What the Core Rule Set still refuses
|
||||
|
||||
Each refusal answers only that request ("Bans" in `SPEC.md` says what it counts
|
||||
toward). The request log names the rule in `waf_rule_ids`, which
|
||||
`SWWAF_WAF_DISABLED_RULES` can switch off.
|
||||
|
||||
### A query string that reads as an attack
|
||||
|
||||
Most often this is a search. A search term is a query parameter, which the Core
|
||||
@@ -303,7 +183,7 @@ the runner. The error burst ("Attack detection" in `SPEC.md`) bans it after more
|
||||
than 30 such refusals from its address in a minute, as when many jobs upload at
|
||||
once. `actions/upload-artifact@v4` does not send the header.
|
||||
|
||||
### What body inspection refuses
|
||||
## What body inspection refuses
|
||||
|
||||
With `SWWAF_WAF_BODY_LIMIT` set, the Core Rule Set reads request bodies too. In
|
||||
front of gitea it then refuses these, until their rule ids, named in the request
|
||||
@@ -315,7 +195,7 @@ log, are added to `SWWAF_WAF_DISABLED_RULES`:
|
||||
- PyPI uploads (922130);
|
||||
- attachments named like `debug.log` or `config.yml` (932180).
|
||||
|
||||
### What `script/gitea-requests` sends
|
||||
## What `script/gitea-requests` sends
|
||||
|
||||
Of the requests above, the script sends:
|
||||
|
||||
|
||||
+6
-3
@@ -8,7 +8,8 @@ is pinned that way.
|
||||
## Contents
|
||||
|
||||
- Ubuntu 26.04 LTS, pinned by digest.
|
||||
- `ca-certificates`, `nix-bin` and `runit` from Ubuntu's archive.
|
||||
- `ca-certificates`, `nix-bin` and `runit` from Ubuntu, from the snapshot of its
|
||||
archive at `20261001T000000Z`.
|
||||
- Nix, with nixpkgs `nixos-26.05.11045.774debe7a0d1` set up for root as
|
||||
`nixpkgs`. No Nix daemon; `build-users-group =` in `/etc/nix/nix.conf`. Root's
|
||||
Nix profile, `/nix/var/nix/profiles/default/bin`, last on the `PATH`.
|
||||
@@ -25,8 +26,10 @@ is pinned that way.
|
||||
|
||||
## Pins
|
||||
|
||||
- Ubuntu's packages are not pinned: they come from the live archive, whose
|
||||
packages change with every update.
|
||||
- Ubuntu's packages: the snapshot service, `--snapshot 20261001T000000Z` on both
|
||||
`apt-get update` and `apt-get install`. The SHA-256 hash of each of the
|
||||
snapshot's `InRelease` files is named in the `Dockerfile` and checked after
|
||||
`apt-get update`. The hashes are the amd64 archive's.
|
||||
- nixpkgs: the release file `nixexprs.tar.xz` of
|
||||
`nixos-26.05.11045.774debe7a0d1` on `releases.nixos.org`, named with its
|
||||
SHA-256 hash.
|
||||
|
||||
@@ -107,35 +107,6 @@ Every one of these lines has:
|
||||
- `would_action`: in `observe` mode, what `enforce` mode would have done with a
|
||||
request it would have refused. `action` then names what was done, normally
|
||||
`forward`.
|
||||
- `heuristic`: the heuristic that refused the request, in `observe` mode too. It
|
||||
is left out when none did.
|
||||
- `heuristic_figures`: an object of the figures that heuristic acted on. Each
|
||||
heuristic and its figures:
|
||||
- `deny_nets`: `netblock`, the entry of `SWWAF_DENY_NETS` the client is in;
|
||||
- `active_ban`: `netblock`, the ban's netblock, and `ban_expires`;
|
||||
- `countries`: `country`, and `setting`, the country list that refused it;
|
||||
- `blocklists`: `lists`, the blocklist URLs that list the client;
|
||||
- `crowdsec`: `scenario`, the decision's scenario, and `expires`, when the
|
||||
decision ends;
|
||||
- `dnsbl`: `zones`, the DNS blocklist zones that list the client, their keys
|
||||
masked;
|
||||
- `abuseipdb`: `score`, and `min_score`, `SWWAF_ABUSEIPDB_MIN_SCORE`;
|
||||
- `rate_limits`: `window`, `count` and `limit`;
|
||||
- `trap_paths`: `path`, the trap path asked for;
|
||||
- `rule_files`: `rule_id` and `rule_action`, of the rule that refused it;
|
||||
- `header_limits`: `fields`, `fields_limit`, `cookies` and `cookies_limit`;
|
||||
- `core_rule_set`: `score`, and `threshold`, `SWWAF_WAF_ANOMALY_THRESHOLD`;
|
||||
- `refusal_limit`: `window`, `count` and `limit`, of the error burst;
|
||||
- `byte_limits`: `window`, `count` and `limit`, the count in bytes.
|
||||
- `after_answer_heuristic`: the heuristic that banned the client after
|
||||
`servewaf` answered the request, in `observe` mode too: `refusal_limit` or
|
||||
`byte_limits`.
|
||||
- `after_answer_heuristic_figures`: an object of the figures that heuristic
|
||||
acted on, as listed above.
|
||||
|
||||
`byte_limits` bans when the report comes, so only the `report` line names
|
||||
it.
|
||||
|
||||
- `limit_percent` and `limit_percent_setting`: the client's percentage of the
|
||||
rate limits and the setting that lowered it, when one did. `bytes_percent` and
|
||||
`bytes_percent_setting` do the same for the byte limits.
|
||||
|
||||
+15
-18
@@ -99,22 +99,19 @@ chooses between the two (see "The image's command" below).
|
||||
|
||||
## Request rate limits, per client
|
||||
|
||||
- `SWWAF_RATE_LIMIT_PER_MINUTE` (default `120`, 2 a second): the most requests a
|
||||
client may make in a minute. A count or `off`. The default leaves room for a
|
||||
person opening two gitea pages at once with an empty cache, up to 60 files
|
||||
each.
|
||||
- `SWWAF_RATE_LIMIT_PER_HOUR` (default `1500`): the most requests a client may
|
||||
make in an hour. A count or `off`. It is the main limit against scrapers. A
|
||||
person reading code in gitea for an hour, a page every 20 seconds, makes
|
||||
about 900. A scraper making one request every 2 seconds makes 1,800 an hour,
|
||||
more than the limit.
|
||||
- `SWWAF_RATE_LIMIT_PER_DAY` (default `6000`): the most requests a client may
|
||||
make in a day. A count or `off`. The default is four hours at the hour limit.
|
||||
A scraper making one request every 10 seconds makes 8,640 a day, more than the
|
||||
limit.
|
||||
- An Actions runner asks gitea for work all day and can pass the limits. Put the
|
||||
runners' addresses in `SWWAF_RATE_LIMIT_EXEMPT_NETS`. Do the same for a
|
||||
network where many people share one address, such as an office.
|
||||
- `SWWAF_RATE_LIMIT_PER_MINUTE` (default `500`, about 8 a second): the most
|
||||
requests a client may make in a minute. A count or `off`. The default catches,
|
||||
within seconds, a client firing requests as fast as it can. It leaves room for
|
||||
a fleet of about 20 agents at its busiest, about 120 requests a minute. It
|
||||
also leaves room for a few quick first visits to gitea with an empty cache,
|
||||
about 40 files each.
|
||||
- `SWWAF_RATE_LIMIT_PER_HOUR` (default `20000`, about 5.5 a second): the most
|
||||
requests a client may make in an hour. A count or `off`. The default leaves
|
||||
room for an automation fleet at work. It catches a client that keeps up more
|
||||
than about 5.5 a second for an hour.
|
||||
- `SWWAF_RATE_LIMIT_PER_DAY` (default `300000`, about 3.5 a second): the most
|
||||
requests a client may make in a day. A count or `off`. The default catches a
|
||||
scraper that keeps up 4 a second around the clock.
|
||||
- Counting cannot catch a client that stays under all three limits (see "Request
|
||||
rate limits, per client" in [`SPEC.md`](../SPEC.md)).
|
||||
- `SWWAF_RATE_LIMIT_EXEMPT_PATHS` (default empty): path prefixes exempt from
|
||||
@@ -122,8 +119,8 @@ chooses between the two (see "The image's command" below).
|
||||
|
||||
## Byte limits, per client
|
||||
|
||||
- `SWWAF_BYTES_LIMIT_PER_MINUTE` (default `off`), `SWWAF_BYTES_LIMIT_PER_HOUR`
|
||||
(default `50G`) and `SWWAF_BYTES_LIMIT_PER_DAY` (default `100G`): the most
|
||||
- `SWWAF_BYTES_LIMIT_PER_MINUTE` (default `4G`), `SWWAF_BYTES_LIMIT_PER_HOUR`
|
||||
(default `20G`) and `SWWAF_BYTES_LIMIT_PER_DAY` (default `200G`): the most
|
||||
bytes counted for a client in each window. A size or `off`.
|
||||
- `SWWAF_BYTES_COUNT` (default `both`): which body bytes the byte limits count:
|
||||
`response`, `request` or `both`.
|
||||
|
||||
@@ -468,13 +468,13 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
|
||||
AllowNets: env.netblocks("SWWAF_ALLOW_NETS", ""),
|
||||
RateLimitExemptNets: env.netblocks("SWWAF_RATE_LIMIT_EXEMPT_NETS", ""),
|
||||
DenyNets: env.netblocks("SWWAF_DENY_NETS", ""),
|
||||
RateLimitPerMinute: env.count("SWWAF_RATE_LIMIT_PER_MINUTE", "120"),
|
||||
RateLimitPerHour: env.count("SWWAF_RATE_LIMIT_PER_HOUR", "1500"),
|
||||
RateLimitPerDay: env.count("SWWAF_RATE_LIMIT_PER_DAY", "6000"),
|
||||
RateLimitPerMinute: env.count("SWWAF_RATE_LIMIT_PER_MINUTE", "500"),
|
||||
RateLimitPerHour: env.count("SWWAF_RATE_LIMIT_PER_HOUR", "20000"),
|
||||
RateLimitPerDay: env.count("SWWAF_RATE_LIMIT_PER_DAY", "300000"),
|
||||
RateLimitExemptPaths: env.pathPrefixes("SWWAF_RATE_LIMIT_EXEMPT_PATHS", ""),
|
||||
BytesLimitPerMinute: env.size("SWWAF_BYTES_LIMIT_PER_MINUTE", "off"),
|
||||
BytesLimitPerHour: env.size("SWWAF_BYTES_LIMIT_PER_HOUR", "50G"),
|
||||
BytesLimitPerDay: env.size("SWWAF_BYTES_LIMIT_PER_DAY", "100G"),
|
||||
BytesLimitPerMinute: env.size("SWWAF_BYTES_LIMIT_PER_MINUTE", "4G"),
|
||||
BytesLimitPerHour: env.size("SWWAF_BYTES_LIMIT_PER_HOUR", "20G"),
|
||||
BytesLimitPerDay: env.size("SWWAF_BYTES_LIMIT_PER_DAY", "200G"),
|
||||
BytesCount: env.bytesCount("SWWAF_BYTES_COUNT", "both"),
|
||||
LookupSource: env.lookupSource("SWWAF_LOOKUP_SOURCE", "geojs"),
|
||||
LookupDBPath: env.value("SWWAF_LOOKUP_DB_PATH", ""),
|
||||
|
||||
@@ -244,9 +244,9 @@ func TestDefaults(t *testing.T) {
|
||||
wantSettings(t, cfg, config.Config{
|
||||
ListenAddr: ":8080",
|
||||
Observe: false,
|
||||
RateLimitPerMinute: 120,
|
||||
RateLimitPerHour: 1500,
|
||||
RateLimitPerDay: 6000,
|
||||
RateLimitPerMinute: 500,
|
||||
RateLimitPerHour: 20000,
|
||||
RateLimitPerDay: 300000,
|
||||
BanResponse: 403,
|
||||
LimitBanDuration: time.Hour,
|
||||
LimitBanRepeatWindow: 24 * time.Hour,
|
||||
@@ -266,8 +266,8 @@ func TestDefaults(t *testing.T) {
|
||||
LookupSource: defaultLookupSource, LookupTimeout: 800 * time.Millisecond,
|
||||
})
|
||||
wantByteLimitSettings(t, cfg, config.Config{
|
||||
BytesLimitPerMinute: 0, BytesLimitPerHour: 50 << 30,
|
||||
BytesLimitPerDay: 100 << 30, BytesCount: "both",
|
||||
BytesLimitPerMinute: 4 << 30, BytesLimitPerHour: 20 << 30,
|
||||
BytesLimitPerDay: 200 << 30, BytesCount: "both",
|
||||
})
|
||||
|
||||
wantNetblocks(t, cfg.AllowNets)
|
||||
@@ -308,7 +308,7 @@ func TestNoSingleRequestBreaksAByteLimitAtTheDefaults(t *testing.T) {
|
||||
for _, limit := range []int64{
|
||||
cfg.BytesLimitPerMinute, cfg.BytesLimitPerHour, cfg.BytesLimitPerDay,
|
||||
} {
|
||||
if limit != 0 && largest > limit { // 0 is off
|
||||
if largest > limit {
|
||||
t.Errorf("a request of %d bytes breaks the byte limit of %d", largest, limit)
|
||||
}
|
||||
}
|
||||
@@ -2271,12 +2271,12 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
|
||||
rateLimitExemptNets: "",
|
||||
denyNets: "",
|
||||
rateLimitPerMinute: "45",
|
||||
rateLimitPerHour: "1500",
|
||||
rateLimitPerDay: "6000",
|
||||
rateLimitPerHour: "20000",
|
||||
rateLimitPerDay: "300000",
|
||||
rateLimitExemptPaths: "",
|
||||
bytesLimitPerMinute: "off",
|
||||
bytesLimitPerHour: "50G",
|
||||
bytesLimitPerDay: "100G",
|
||||
bytesLimitPerMinute: "4G",
|
||||
bytesLimitPerHour: "20G",
|
||||
bytesLimitPerDay: "200G",
|
||||
bytesCount: "both",
|
||||
lookupSource: defaultLookupSource,
|
||||
lookupDBPath: "",
|
||||
|
||||
@@ -131,16 +131,6 @@ type Line struct {
|
||||
// ActionRuleBlocked, ActionWAFBlocked, or, in servewaf, ActionTooLarge
|
||||
// for the header limits.
|
||||
WouldAction string `json:"would_action,omitempty"`
|
||||
// Heuristic is the name of the heuristic that refused the request, in
|
||||
// observe mode too, and HeuristicFigures are the figures it acted on, by
|
||||
// name.
|
||||
Heuristic string `json:"heuristic,omitempty"`
|
||||
HeuristicFigures map[string]any `json:"heuristic_figures,omitempty"`
|
||||
// AfterAnswerHeuristic and AfterAnswerHeuristicFigures are the same for
|
||||
// the heuristic that banned the client after servewaf answered the
|
||||
// request.
|
||||
AfterAnswerHeuristic string `json:"after_answer_heuristic,omitempty"`
|
||||
AfterAnswerHeuristicFigures map[string]any `json:"after_answer_heuristic_figures,omitempty"` //nolint:lll // a struct tag cannot be split
|
||||
// LimitPercent and LimitPercentSetting are, for a request the rate
|
||||
// limits counted whose client a biased threshold gives a percentage of
|
||||
// the rate limits below 100, that percentage and the setting that gave
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
package servewaf
|
||||
|
||||
// The heuristic abuseipdb. It reads SWWAF_ABUSEIPDB_KEY,
|
||||
// SWWAF_ABUSEIPDB_MIN_SCORE and SWWAF_REPUTATION_ACTION.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// abuseIPDB notes AbuseIPDB, as noteHit does, with the score, when its score
|
||||
// of the client is a hit, and refuses the request when SWWAF_REPUTATION_ACTION
|
||||
// is deny, as dnsbl does for a zone. While SWWAF_ABUSEIPDB_KEY is unset it does
|
||||
// nothing. A client without a score is checked in the background, by the
|
||||
// request's address, if its history counts an offence, and the request does
|
||||
// not wait for the answer. The score is then used for each address of the
|
||||
// client. Its figures are the score and SWWAF_ABUSEIPDB_MIN_SCORE.
|
||||
type abuseIPDB struct{}
|
||||
|
||||
func (abuseIPDB) name() string { return "abuseipdb" }
|
||||
|
||||
func (abuseIPDB) check(ctx context.Context, rq *request, _ time.Time) *verdict {
|
||||
if rq.h.config.AbuseIPDBKey == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
client := rq.h.clientGroup(rq.client)
|
||||
held, _ := rq.h.limiter.Client(client)
|
||||
offender := held.History.Offences != ratelimit.Offences{}
|
||||
|
||||
score, hit := rq.h.abuseIPDB.Hit(ctx, client, rq.client, offender)
|
||||
if !hit {
|
||||
return nil
|
||||
}
|
||||
|
||||
rq.abuseIPDBHit = true
|
||||
rq.noteHit(bans.ReputationHit{Source: reputation.AbuseIPDBSource, Score: &score},
|
||||
"scored by AbuseIPDB at or over SWWAF_ABUSEIPDB_MIN_SCORE")
|
||||
|
||||
if rq.h.config.ReputationAction != deny {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &verdict{
|
||||
action: requestlog.ActionDenied,
|
||||
figures: map[string]any{
|
||||
figureScore: score, "min_score": rq.h.config.AbuseIPDBMinScore,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -91,18 +91,6 @@ func TestSixteenThousandCookiesAreRefusedWellWithinTheTimeLimit(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeaderLimitRefusalNamesItsHeuristicCountsAndLimits(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s := startServeWAF(t, "", nil)
|
||||
req := s.question(siteA, client, "/")
|
||||
fillHeaders(req, headerFieldLimit+1)
|
||||
do(t, req)
|
||||
|
||||
wantHeuristic(t, s.out.requestLine(t), "header_limits",
|
||||
`{"fields": 101, "fields_limit": 100, "cookies": 0, "cookies_limit": 50}`)
|
||||
}
|
||||
|
||||
func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
+79
-25
@@ -1,10 +1,7 @@
|
||||
package servewaf
|
||||
|
||||
// The heuristic active_ban, and the code that makes bans, which several
|
||||
// heuristics share. active_ban reads SWWAF_MODE; the bans are the ban ledger's.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"time"
|
||||
|
||||
@@ -21,15 +18,11 @@ func (rq *request) banResponse(action string) *refusal {
|
||||
return &refusal{status: rq.h.config.BanResponse, action: action}
|
||||
}
|
||||
|
||||
// activeBan refuses a client a ban covers: a ban on a netblock the client is
|
||||
// in, in force at now. It notes for the log line when that ban ends. A request
|
||||
// that makes the ban permanent, or in observe mode would have, raises the alert
|
||||
// for it. Its figures are the ban's netblock and when it ends.
|
||||
type activeBan struct{}
|
||||
|
||||
func (activeBan) name() string { return "active_ban" }
|
||||
|
||||
func (activeBan) check(_ context.Context, rq *request, now time.Time) *verdict {
|
||||
// banned reports whether a ban on a netblock the client is in covers the
|
||||
// request at now, and notes for the log line when that ban ends. A
|
||||
// request that makes the ban permanent, or in observe mode would have,
|
||||
// raises the alert for it.
|
||||
func (rq *request) banned(now time.Time) bool {
|
||||
check := rq.h.ledger.Check
|
||||
if rq.h.config.Observe {
|
||||
check = rq.h.ledger.Find // the ban refuses nothing, and stays as it is
|
||||
@@ -45,24 +38,85 @@ func (activeBan) check(_ context.Context, rq *request, now time.Time) *verdict {
|
||||
rq.alertBan(ban)
|
||||
}
|
||||
|
||||
if !banned {
|
||||
return nil
|
||||
return banned
|
||||
}
|
||||
|
||||
// limitBroken counts the request for the rate limits at now, notes the
|
||||
// client's counts for the log line, and reports whether the request takes
|
||||
// the client over a rate limit, as its limit percentage lowers it, which
|
||||
// breaks it.
|
||||
func (rq *request) limitBroken(now time.Time) bool {
|
||||
counts, hit, over := rq.h.limiter.Count(rq.h.clientGroup(rq.client), now,
|
||||
rq.limitPercent.percent)
|
||||
rq.line.Counts = counts
|
||||
|
||||
if over {
|
||||
rq.banForLimit(now, hit, rq.h.config.BanResponse)
|
||||
}
|
||||
|
||||
return &verdict{
|
||||
action: requestlog.ActionBanned,
|
||||
figures: map[string]any{
|
||||
figureNetblock: ban.Netblock.String(), "ban_expires": rq.line.BanExpires,
|
||||
},
|
||||
return over
|
||||
}
|
||||
|
||||
// countBytes counts the request's bytes, as countedBytes gives them, for
|
||||
// the byte limits, once its report has come, and notes the client's byte totals
|
||||
// for the log line; its requests stay there as the rate limits counted them.
|
||||
// Only an allowed request has them counted, and only one the rate limits
|
||||
// counted; in observe mode, not one that enforce mode would have refused. Bytes
|
||||
// that take the client over a byte limit, as its limit percentage for the byte
|
||||
// limits lowers it, break it; the response was passed on whole.
|
||||
func (rq *request) countBytes() {
|
||||
if !rq.counted || rq.line.WouldAction != "" {
|
||||
return
|
||||
}
|
||||
|
||||
now := rq.h.now()
|
||||
|
||||
counts, hit, over := rq.h.limiter.CountBytes(rq.h.clientGroup(rq.client), now,
|
||||
rq.countedBytes(), rq.bytesPercent.percent)
|
||||
rq.line.Counts.MinuteBytes = counts.MinuteBytes
|
||||
rq.line.Counts.HourBytes = counts.HourBytes
|
||||
rq.line.Counts.DayBytes = counts.DayBytes
|
||||
|
||||
if over {
|
||||
rq.banForLimit(now, hit, rq.out.status)
|
||||
}
|
||||
}
|
||||
|
||||
// limitFigures are the figures of a heuristic that broke hit, a limit: its
|
||||
// window, the count and the limit.
|
||||
func limitFigures(hit ratelimit.Hit) map[string]any {
|
||||
return map[string]any{
|
||||
figureWindow: hit.Window, figureCount: hit.Count, figureLimit: hit.Limit,
|
||||
// countRefusal counts the request for the error burst once it has been
|
||||
// answered, if servewaf refused it after a rule file match, a trap path or a
|
||||
// Core Rule Set match, and in observe mode if enforce mode would have: more than
|
||||
// SWWAF_ERROR_BURST_THRESHOLD such refusals of the client within a minute
|
||||
// break a limit. A client in SWWAF_ALLOW_NETS, which the checks skip, is
|
||||
// not counted, and nothing is while the threshold is off.
|
||||
func (rq *request) countRefusal() {
|
||||
cfg := rq.h.config
|
||||
if cfg.ErrorBurstThreshold == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
if !rq.attack && !rq.ruleBlocked && !rq.wafBlocked {
|
||||
return
|
||||
}
|
||||
|
||||
now := rq.h.now()
|
||||
|
||||
hit, over := rq.h.limiter.CountRefusal(rq.h.clientGroup(rq.client), now,
|
||||
cfg.ErrorBurstThreshold)
|
||||
if !over {
|
||||
return
|
||||
}
|
||||
|
||||
// What the client was sent, or in observe mode would have been.
|
||||
status := rq.out.status
|
||||
|
||||
switch rq.line.WouldAction {
|
||||
case requestlog.ActionRuleBlocked, requestlog.ActionWAFBlocked:
|
||||
status = http.StatusForbidden
|
||||
case requestlog.ActionBanned:
|
||||
status = cfg.BanResponse
|
||||
}
|
||||
|
||||
rq.banForLimit(now, hit, status)
|
||||
}
|
||||
|
||||
// countedBytes returns the request's bytes, once it has ended, as the
|
||||
|
||||
@@ -228,22 +228,6 @@ func TestBannedClientIsRefusedBeforeItsCountryIsLookedUp(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefusalUnderABanNamesItsHeuristicNetblockAndEnd(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, clk, _ := startWithClock(t, "", map[string]string{
|
||||
rateLimitPerMinute: "1", banScopeV4Prefix: "24",
|
||||
})
|
||||
expires := requestlog.FormatTime(clk.Now().Add(time.Hour))
|
||||
|
||||
s.get(client, http.StatusOK, requestlog.ActionForward)
|
||||
s.get(client, http.StatusForbidden, requestlog.ActionRateLimited)
|
||||
|
||||
line := s.get(otherClient, http.StatusForbidden, requestlog.ActionBanned)
|
||||
wantHeuristic(t, line, "active_ban",
|
||||
`{"netblock": "`+clientsNet+`", "ban_expires": "`+expires+`"}`)
|
||||
}
|
||||
|
||||
func TestBanResponseAnswersEveryRefusalButTheSizeLimits(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
package servewaf
|
||||
|
||||
// The heuristic blocklists. It reads SWWAF_BLOCKLIST_ACTION, and the blocklists
|
||||
// through the reputation lists.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// blocklists notes the blocklists that list the client, as noteListed does,
|
||||
// and refuses the request when SWWAF_BLOCKLIST_ACTION is deny. Being limit, it
|
||||
// lowers the client's limits instead (see limitPercentages), and being log, it
|
||||
// does nothing more. Its figure is the blocklists that list the client.
|
||||
type blocklists struct{}
|
||||
|
||||
func (blocklists) name() string { return "blocklists" }
|
||||
|
||||
func (blocklists) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
||||
listedBy := rq.h.lists.ListedBy(rq.client)
|
||||
rq.blocklisted = len(listedBy) > 0
|
||||
rq.noteListed(listedBy, "listed by a blocklist")
|
||||
|
||||
if !rq.refusedByBlocklists(listedBy) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &verdict{
|
||||
action: requestlog.ActionDenied,
|
||||
figures: map[string]any{"lists": listedBy},
|
||||
}
|
||||
}
|
||||
|
||||
// refusedByBlocklists reports whether the heuristic blocklists refuses a
|
||||
// client, given listedBy, the blocklists that list it: it does when one lists
|
||||
// it and SWWAF_BLOCKLIST_ACTION is deny.
|
||||
func (rq *request) refusedByBlocklists(listedBy []string) bool {
|
||||
return len(listedBy) > 0 && rq.h.config.BlocklistAction == deny
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
package servewaf
|
||||
|
||||
// The heuristic byte_limits. It reads SWWAF_BYTES_COUNT, and the byte limits
|
||||
// and biased thresholds through the limiter and limitPercentages.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// byteLimits counts the request's bytes, as countedBytes gives them, for the
|
||||
// byte limits, once its report has come, and notes the client's byte totals for
|
||||
// the log line; its requests stay there as the rate limits counted them. Only
|
||||
// an allowed request has them counted, and only one the rate limits counted; in
|
||||
// observe mode, not one that enforce mode would have refused. Bytes that take
|
||||
// the client over a byte limit, as its limit percentage for the byte limits
|
||||
// lowers it, break it, which bans the client; the response was passed on whole.
|
||||
// Its figures are the window, the count and the limit.
|
||||
type byteLimits struct{}
|
||||
|
||||
func (byteLimits) name() string { return "byte_limits" }
|
||||
|
||||
func (byteLimits) check(_ context.Context, rq *request, now time.Time) *verdict {
|
||||
if !rq.counted || rq.line.WouldAction != "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
counts, hit, over := rq.h.limiter.CountBytes(rq.h.clientGroup(rq.client), now,
|
||||
rq.countedBytes(), rq.bytesPercent.percent)
|
||||
rq.line.Counts.MinuteBytes = counts.MinuteBytes
|
||||
rq.line.Counts.HourBytes = counts.HourBytes
|
||||
rq.line.Counts.DayBytes = counts.DayBytes
|
||||
|
||||
if !over {
|
||||
return nil
|
||||
}
|
||||
|
||||
rq.banForLimit(now, hit, rq.out.status)
|
||||
|
||||
return &verdict{action: requestlog.ActionBanned, figures: limitFigures(hit)}
|
||||
}
|
||||
@@ -98,21 +98,6 @@ func TestEachByteLimitBansOnceTheResponseHasEnded(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestByteLimitBanNamesItsHeuristicWindowCountAndLimit(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, _ := startWithAnswers(t, map[string]string{bytesLimitPerMinute: byteLimit})
|
||||
|
||||
line, _ := s.download()
|
||||
wantAfterAnswerHeuristic(t, line, "", "null")
|
||||
|
||||
// Two answers of 70 bytes are over the limit of 99.
|
||||
line, _ = s.download()
|
||||
wantHeuristic(t, line, "", "null")
|
||||
wantAfterAnswerHeuristic(t, line, "byte_limits",
|
||||
`{"window": "minute", "count": 140, "limit": 99}`)
|
||||
}
|
||||
|
||||
func TestResponseOverAByteLimitByItselfIsPassedOnWhole(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -1,10 +1,6 @@
|
||||
package servewaf
|
||||
|
||||
// The heuristic core_rule_set. It reads SWWAF_WAF_MODE, SWWAF_WAF_EXEMPT_PATHS
|
||||
// and SWWAF_WAF_ANOMALY_THRESHOLD.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
@@ -13,21 +9,18 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
)
|
||||
|
||||
// coreRuleSet inspects the request with the Core Rule Set, unless
|
||||
// checkCoreRuleSet inspects the request with the Core Rule Set, unless
|
||||
// SWWAF_WAF_MODE is off or SWWAF_WAF_EXEMPT_PATHS exempts its path, as
|
||||
// pathExempt decides, and notes the rules it matched and its score in the
|
||||
// log line, and the rules in the metrics. A score at or over
|
||||
// SWWAF_WAF_ANOMALY_THRESHOLD is a match: it raises the waf_block alert,
|
||||
// and in block mode refuses the request with 403, which is an offence its
|
||||
// client's history counts. Its figures are the score and the threshold.
|
||||
type coreRuleSet struct{}
|
||||
|
||||
func (coreRuleSet) name() string { return "core_rule_set" }
|
||||
|
||||
func (coreRuleSet) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
||||
// and in block mode refuses the request, which is an offence its client's
|
||||
// history counts, and so returns ActionWAFBlocked. It returns "" for a
|
||||
// request it does not refuse.
|
||||
func (rq *request) checkCoreRuleSet() string {
|
||||
cfg := rq.h.config
|
||||
if cfg.WAFMode == config.WAFModeOff || pathExempt(rq.in, cfg.WAFExemptPaths) {
|
||||
return nil
|
||||
return ""
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
@@ -47,21 +40,18 @@ func (coreRuleSet) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
||||
|
||||
threshold := cfg.WAFAnomalyThreshold
|
||||
if threshold == 0 || result.Score < threshold {
|
||||
return nil
|
||||
return ""
|
||||
}
|
||||
|
||||
rq.alertWAFBlock(result)
|
||||
|
||||
if cfg.WAFMode == config.WAFModeDetect {
|
||||
return nil
|
||||
return ""
|
||||
}
|
||||
|
||||
rq.wafBlocked = true
|
||||
|
||||
return &verdict{
|
||||
action: requestlog.ActionWAFBlocked,
|
||||
figures: map[string]any{figureScore: result.Score, "threshold": threshold},
|
||||
}
|
||||
return requestlog.ActionWAFBlocked
|
||||
}
|
||||
|
||||
// alertWAFBlock raises the waf_block alert for the request, which the Core
|
||||
|
||||
@@ -110,16 +110,6 @@ func TestCoreRuleSetRefusesAttacksInBlockModeAndOnlyLogsThemInDetectMode(t *test
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoreRuleSetRefusalNamesItsHeuristicScoreAndThreshold(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, _, _ := startWithClock(t, "", map[string]string{wafMode: block})
|
||||
|
||||
line := s.request(client, sqlInjection, http.StatusForbidden,
|
||||
requestlog.ActionWAFBlocked)
|
||||
wantHeuristic(t, line, "core_rule_set", `{"score": 5, "threshold": 5}`)
|
||||
}
|
||||
|
||||
func TestOrdinaryRequestIsInspectedAndPassed(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -1,42 +1,21 @@
|
||||
package servewaf
|
||||
|
||||
// The heuristic countries. It reads SWWAF_DENIED_COUNTRIES and
|
||||
// SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// countries refuses a request by the client's country as it was looked up. A
|
||||
// client without a country, or whose country cannot be found, is refused only
|
||||
// by SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES. Its figures are the country and the
|
||||
// setting that refused it.
|
||||
type countries struct{}
|
||||
|
||||
func (countries) name() string { return "countries" }
|
||||
|
||||
func (countries) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
||||
// countryDenied reports whether the country lists refuse the request, by
|
||||
// the client's country as it was looked up. A client without a country,
|
||||
// or whose country cannot be found, is refused only by
|
||||
// SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES.
|
||||
func (rq *request) countryDenied() bool {
|
||||
denied := rq.h.config.DeniedCountries
|
||||
allowed := rq.h.config.ExclusivelyAllowedCountries
|
||||
country := rq.line.Country
|
||||
|
||||
var setting string
|
||||
|
||||
switch {
|
||||
case slices.Contains(denied, country):
|
||||
setting = "SWWAF_DENIED_COUNTRIES"
|
||||
case len(allowed) > 0 && !slices.Contains(allowed, country):
|
||||
setting = "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES"
|
||||
default:
|
||||
return nil
|
||||
if slices.Contains(denied, country) {
|
||||
return true
|
||||
}
|
||||
|
||||
return &verdict{
|
||||
action: requestlog.ActionCountryDenied,
|
||||
figures: map[string]any{"country": country, "setting": setting},
|
||||
}
|
||||
return len(allowed) > 0 && !slices.Contains(allowed, country)
|
||||
}
|
||||
@@ -90,23 +90,6 @@ func TestCountryLists(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountryRefusalNamesItsHeuristicCountryAndSetting(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
geojsURL, _ := startGeoJS(t)
|
||||
s, _, _ := startWithClock(t, geojsURL, map[string]string{
|
||||
lookupTimeout: "1h", deniedCountries: "kp", allowedCountries: "de",
|
||||
})
|
||||
|
||||
line := s.get(fromKP, http.StatusForbidden, requestlog.ActionCountryDenied)
|
||||
wantHeuristic(t, line, "countries",
|
||||
`{"country": "KP", "setting": "SWWAF_DENIED_COUNTRIES"}`)
|
||||
|
||||
line = s.get(unplaced, http.StatusForbidden, requestlog.ActionCountryDenied)
|
||||
wantHeuristic(t, line, "countries",
|
||||
`{"country": "", "setting": "SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES"}`)
|
||||
}
|
||||
|
||||
func TestCountryRefusalKeepsARequestFromTheApp(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
package servewaf
|
||||
|
||||
// The heuristic crowdsec. It reads SWWAF_CROWDSEC_LAPI_URL, and the decisions
|
||||
// through the reputation lists.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// crowdSec refuses a request when a decision of the CrowdSec decision list on
|
||||
// the client is in force at now. It then notes the list, as noteHit does, and
|
||||
// bans the client until that decision ends. Its figures are the decision's
|
||||
// scenario and when it ends.
|
||||
type crowdSec struct{}
|
||||
|
||||
func (crowdSec) name() string { return "crowdsec" }
|
||||
|
||||
func (crowdSec) check(_ context.Context, rq *request, now time.Time) *verdict {
|
||||
decision, listed := rq.crowdSecDecisionInForce(now)
|
||||
if !listed {
|
||||
return nil
|
||||
}
|
||||
|
||||
rq.noteHit(bans.ReputationHit{Source: rq.h.config.CrowdSecDecisionsURL},
|
||||
"listed by the CrowdSec decision list")
|
||||
rq.banForCrowdSec(now, decision)
|
||||
|
||||
return &verdict{
|
||||
action: requestlog.ActionBanned,
|
||||
figures: map[string]any{
|
||||
"scenario": decision.Scenario,
|
||||
"expires": requestlog.FormatTime(decision.Expires),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// crowdSecDecisionInForce returns the decision of the CrowdSec decision list on
|
||||
// the client that is in force at now, and reports whether there is one, which
|
||||
// the heuristic crowdsec refuses the request for.
|
||||
func (rq *request) crowdSecDecisionInForce(
|
||||
now time.Time,
|
||||
) (reputation.Decision, bool) {
|
||||
return rq.h.lists.CrowdSecDecision(rq.client, now)
|
||||
}
|
||||
@@ -95,21 +95,6 @@ func TestClientTheCrowdSecDecisionListListsIsBannedUntilTheDecisionEnds(t *testi
|
||||
float64(fetched.Unix()))
|
||||
}
|
||||
|
||||
func TestCrowdSecBanNamesItsHeuristicScenarioAndEnd(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, clk, server, _ := startWithAlerts(t, map[string]string{
|
||||
crowdSecURL: lapi, crowdSecKey: bouncerKey,
|
||||
})
|
||||
loadDecisions(t, server, clk.Now(), `[{"duration": "4h0m0s", `+
|
||||
`"scenario": "crowdsecurity/ssh-bf", "scope": "Ip", "type": "ban", `+
|
||||
`"value": "`+client+`"}]`)
|
||||
|
||||
line := s.get(client, http.StatusForbidden, requestlog.ActionBanned)
|
||||
wantHeuristic(t, line, "crowdsec", `{"scenario": "crowdsecurity/ssh-bf", `+
|
||||
`"expires": "`+requestlog.FormatTime(clk.Now().Add(4*time.Hour))+`"}`)
|
||||
}
|
||||
|
||||
func TestEndedCrowdSecDecisionNoLongerBansThoughTheCopyStillHoldsIt(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
package servewaf
|
||||
|
||||
// The heuristic deny_nets. It reads SWWAF_DENY_NETS.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// denyNets refuses a client in SWWAF_DENY_NETS. Its figure is the netblock of
|
||||
// the setting the client is in.
|
||||
type denyNets struct{}
|
||||
|
||||
func (denyNets) name() string { return "deny_nets" }
|
||||
|
||||
func (denyNets) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
||||
for _, netblock := range rq.h.config.DenyNets {
|
||||
if netblock.Contains(rq.client) {
|
||||
return &verdict{
|
||||
action: requestlog.ActionDenied,
|
||||
figures: map[string]any{figureNetblock: netblock.String()},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
package servewaf
|
||||
|
||||
// The heuristic dnsbl. It reads SWWAF_REPUTATION_ACTION, and the DNSBL zones'
|
||||
// verdicts through the DNSBL checker.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// dnsbl notes the DNSBL zones whose verdict lists the client, as noteListed
|
||||
// does, and refuses the request when SWWAF_REPUTATION_ACTION is deny. Being
|
||||
// limit, it lowers the client's limits instead (see limitPercentages), and
|
||||
// being log, it does nothing more. A zone without a verdict on the client is
|
||||
// asked about it in the background, and the request does not wait for the
|
||||
// answer. Its figure is the zones that list the client, their keys masked.
|
||||
type dnsbl struct{}
|
||||
|
||||
func (dnsbl) name() string { return "dnsbl" }
|
||||
|
||||
func (dnsbl) check(ctx context.Context, rq *request, _ time.Time) *verdict {
|
||||
listedBy := rq.h.dnsbl.ListedBy(ctx, rq.client)
|
||||
rq.dnsblListed = len(listedBy) > 0
|
||||
rq.noteListed(listedBy, "listed by a DNSBL zone")
|
||||
|
||||
if !rq.dnsblListed || rq.h.config.ReputationAction != deny {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &verdict{
|
||||
action: requestlog.ActionDenied,
|
||||
figures: map[string]any{"zones": listedBy},
|
||||
}
|
||||
}
|
||||
@@ -106,25 +106,6 @@ func TestErrorBurstBreaksAtOneOverTheThreshold(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefusalLimitBanNamesItsHeuristicWindowCountAndLimit(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, _, _ := startForErrorBurst(t, "1", nil)
|
||||
|
||||
const ruleFigures = `{"rule_id": "blocked", "rule_action": "block"}`
|
||||
|
||||
line := blockRule.send(s, client)
|
||||
wantHeuristic(t, line, "rule_files", ruleFigures)
|
||||
wantAfterAnswerHeuristic(t, line, "", "null")
|
||||
|
||||
// The second refusal is the request refusal_limit bans for, and the line
|
||||
// names both.
|
||||
line = blockRule.send(s, client)
|
||||
wantHeuristic(t, line, "rule_files", ruleFigures)
|
||||
wantAfterAnswerHeuristic(t, line, "refusal_limit",
|
||||
`{"window": "minute", "count": 2, "limit": 1}`)
|
||||
}
|
||||
|
||||
func TestErrorBurstBanNotesHistoryAndMetrics(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -1,71 +0,0 @@
|
||||
package servewaf
|
||||
|
||||
// The heuristic header_limits. It reads no setting: its limits are fixed.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// The header limits: the most cookies and header fields a request may have
|
||||
// when the checks reach the Core Rule Set. Real clients never reach them.
|
||||
const (
|
||||
maxCookies = 50
|
||||
maxHeaderFields = 100
|
||||
)
|
||||
|
||||
// headerLimits refuses a request over the header limits, with 431. Its figures
|
||||
// are the request's header fields and cookies, and the limit of each.
|
||||
type headerLimits struct{}
|
||||
|
||||
func (headerLimits) name() string { return "header_limits" }
|
||||
|
||||
func (headerLimits) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
||||
fields, cookies := rq.headerCounts()
|
||||
if !overHeaderLimits(fields, cookies) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &verdict{
|
||||
action: requestlog.ActionTooLarge,
|
||||
figures: map[string]any{
|
||||
"fields": fields, "fields_limit": maxHeaderFields,
|
||||
"cookies": cookies, "cookies_limit": maxCookies,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// overHeaderLimits reports whether a request with these counts of header fields
|
||||
// and cookies, as headerCounts counts them, is over the header limits: it has
|
||||
// more than maxHeaderFields fields or more than maxCookies cookies.
|
||||
func overHeaderLimits(fields, cookies int) bool {
|
||||
return fields > maxHeaderFields || cookies > maxCookies
|
||||
}
|
||||
|
||||
// headerCounts returns the request's header fields, each value of a header sent
|
||||
// twice being one, and its cookies, each part of a Cookie header between
|
||||
// semicolons being one unless it is empty or only spaces and tabs. The Core
|
||||
// Rule Set runs each of its rules on each cookie, so that thousands of small
|
||||
// cookies keep it busy for over a second. It trims only spaces and control
|
||||
// characters, of which a header value can hold only tabs, so a part of Unicode
|
||||
// spaces is a cookie to it.
|
||||
func (rq *request) headerCounts() (int, int) {
|
||||
fields, cookies := 0, 0
|
||||
|
||||
for _, values := range rq.in.Header {
|
||||
fields += len(values)
|
||||
}
|
||||
|
||||
for _, value := range rq.in.Header.Values("Cookie") {
|
||||
for part := range strings.SplitSeq(value, ";") {
|
||||
if strings.Trim(part, " \t") != "" {
|
||||
cookies++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return fields, cookies
|
||||
}
|
||||
@@ -1,133 +0,0 @@
|
||||
package servewaf
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// To add a heuristic, write one file with a type that implements heuristic,
|
||||
// and add one line for it to one of the lists below. Its settings go in
|
||||
// config.Config, and the comment at the top of its file names them.
|
||||
|
||||
// heuristic is one of the checks that decide what servewaf does with a request.
|
||||
type heuristic interface {
|
||||
// name is the heuristic's name, which the log line gives.
|
||||
name() string
|
||||
// check looks at the request at now, and returns nil to let it through.
|
||||
// ctx is the request's own context.
|
||||
check(ctx context.Context, rq *request, now time.Time) *verdict
|
||||
}
|
||||
|
||||
// verdict is a heuristic refusing a request, or banning its client: the action
|
||||
// the log line names, one of requestlog's Action constants, and the figures the
|
||||
// heuristic acted on, by name, such as the count and the limit.
|
||||
type verdict struct {
|
||||
action string
|
||||
figures map[string]any
|
||||
}
|
||||
|
||||
// The names of the figures that more than one heuristic gives.
|
||||
const (
|
||||
figureWindow = "window"
|
||||
figureCount = "count"
|
||||
figureLimit = "limit"
|
||||
figureScore = "score"
|
||||
figureNetblock = "netblock"
|
||||
)
|
||||
|
||||
// The lists of heuristics, each in the order its heuristics run. A client in
|
||||
// SWWAF_ALLOW_NETS skips them all.
|
||||
|
||||
// beforeLookup are the heuristics that run before the client's AS number and
|
||||
// country are looked up, so that a client they refuse is not looked up.
|
||||
func beforeLookup() []heuristic {
|
||||
return []heuristic{
|
||||
denyNets{},
|
||||
activeBan{},
|
||||
}
|
||||
}
|
||||
|
||||
// afterLookup are the heuristics that run once the lookup has been made.
|
||||
func afterLookup() []heuristic {
|
||||
return []heuristic{
|
||||
countries{},
|
||||
blocklists{},
|
||||
crowdSec{},
|
||||
dnsbl{},
|
||||
abuseIPDB{},
|
||||
rateLimits{},
|
||||
trapPaths{},
|
||||
ruleFiles{},
|
||||
headerLimits{},
|
||||
coreRuleSet{},
|
||||
}
|
||||
}
|
||||
|
||||
// afterAnswer are the heuristics that run on a request once servewaf has
|
||||
// answered it, before its decision line is written. They ban the client, and do
|
||||
// not change what the request got: their verdict's action is not used.
|
||||
func afterAnswer() []heuristic {
|
||||
return []heuristic{
|
||||
refusalLimit{},
|
||||
}
|
||||
}
|
||||
|
||||
// onReport are the heuristics that run on an allowed request when rproxy's
|
||||
// report of it comes, before its report line is written. Like afterAnswer, they
|
||||
// only ban.
|
||||
func onReport() []heuristic {
|
||||
return []heuristic{
|
||||
byteLimits{},
|
||||
}
|
||||
}
|
||||
|
||||
// runAfterAnswer runs the heuristics of afterAnswer on the request, at the time
|
||||
// it is called.
|
||||
func (rq *request) runAfterAnswer(ctx context.Context) {
|
||||
rq.runBanningHeuristics(ctx, afterAnswer())
|
||||
}
|
||||
|
||||
// runHeuristics runs list, beforeLookup or afterLookup, on the request at now,
|
||||
// and returns the first verdict, or nil when none gives one. The log line's
|
||||
// heuristic and heuristic_figures name the heuristic that gave it, and give its
|
||||
// figures. ctx is the request's own context.
|
||||
func (rq *request) runHeuristics(
|
||||
ctx context.Context, list []heuristic, now time.Time,
|
||||
) *verdict {
|
||||
name, v := rq.firstVerdict(ctx, list, now)
|
||||
if v != nil {
|
||||
rq.line.Heuristic = name
|
||||
rq.line.HeuristicFigures = v.figures
|
||||
}
|
||||
|
||||
return v
|
||||
}
|
||||
|
||||
// runBanningHeuristics runs list, afterAnswer or onReport, on the request at
|
||||
// the time it is called. The log line's after_answer_heuristic and
|
||||
// after_answer_heuristic_figures name the heuristic that gave a verdict, and
|
||||
// give its figures, so that heuristic and heuristic_figures still name the one
|
||||
// that refused the request. ctx is the request's own context.
|
||||
func (rq *request) runBanningHeuristics(ctx context.Context, list []heuristic) {
|
||||
name, v := rq.firstVerdict(ctx, list, rq.h.now())
|
||||
if v != nil {
|
||||
rq.line.AfterAnswerHeuristic = name
|
||||
rq.line.AfterAnswerHeuristicFigures = v.figures
|
||||
}
|
||||
}
|
||||
|
||||
// firstVerdict runs list on the request at now, in order, until a heuristic
|
||||
// gives a verdict, and returns that heuristic's name and its verdict, or "" and
|
||||
// nil when none gives one. ctx is the request's own context.
|
||||
func (rq *request) firstVerdict(
|
||||
ctx context.Context, list []heuristic, now time.Time,
|
||||
) (string, *verdict) {
|
||||
for _, each := range list {
|
||||
v := each.check(ctx, rq, now)
|
||||
if v != nil {
|
||||
return each.name(), v
|
||||
}
|
||||
}
|
||||
|
||||
return "", nil
|
||||
}
|
||||
@@ -14,18 +14,6 @@ import (
|
||||
// observe is the value of SWWAF_MODE for observe mode.
|
||||
const observe = "observe"
|
||||
|
||||
func TestObserveModeNamesTheHeuristicThatWouldHaveRefused(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, _, _ := startWithClock(t, "", map[string]string{
|
||||
mode: observe, denyNets: client,
|
||||
})
|
||||
|
||||
line := s.get(client, http.StatusOK, requestlog.ActionForward)
|
||||
wantWouldAction(t, line, requestlog.ActionDenied)
|
||||
wantHeuristic(t, line, "deny_nets", `{"netblock": "`+client+`/32"}`)
|
||||
}
|
||||
|
||||
func TestObserveModeForwardsWhatEnforceModeRefuses(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -1,65 +0,0 @@
|
||||
package servewaf
|
||||
|
||||
// The heuristic rate_limits. It reads SWWAF_RATE_LIMIT_EXEMPT_NETS,
|
||||
// SWWAF_RATE_LIMIT_EXEMPT_PATHS and SWWAF_BAN_RESPONSE, and the rate limits and
|
||||
// biased thresholds through the limiter and limitPercentages.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// rateLimits counts the request for the rate limits, unless
|
||||
// countedByRateLimits exempts it, by the client's limit percentages, which it
|
||||
// notes for the log line with the client's counts. A request that takes the
|
||||
// client over a rate limit, as its limit percentage lowers it, breaks it: it is
|
||||
// refused, and bans the client. A request exempt from the rate limits is exempt
|
||||
// from the byte limits too. Its figures are the window, the count and the
|
||||
// limit.
|
||||
type rateLimits struct{}
|
||||
|
||||
func (rateLimits) name() string { return "rate_limits" }
|
||||
|
||||
func (rateLimits) check(_ context.Context, rq *request, now time.Time) *verdict {
|
||||
rq.counted = rq.countedByRateLimits()
|
||||
if !rq.counted {
|
||||
return nil
|
||||
}
|
||||
|
||||
rq.limitPercent, rq.bytesPercent = rq.limitPercentages()
|
||||
rq.line.LimitPercent, rq.line.LimitPercentSetting = rq.limitPercent.logged()
|
||||
rq.line.BytesPercent, rq.line.BytesPercentSetting = rq.bytesPercent.logged()
|
||||
|
||||
counts, hit, over := rq.h.limiter.Count(rq.h.clientGroup(rq.client), now,
|
||||
rq.limitPercent.percent)
|
||||
rq.line.Counts = counts
|
||||
|
||||
if !over {
|
||||
return nil
|
||||
}
|
||||
|
||||
rq.banForLimit(now, hit, rq.h.config.BanResponse)
|
||||
|
||||
return &verdict{action: requestlog.ActionRateLimited, figures: limitFigures(hit)}
|
||||
}
|
||||
|
||||
// overAWholeRateLimit reports whether the heuristic rate_limits will refuse the
|
||||
// request at now, whatever its client's limit percentages: the rate limits
|
||||
// count it, and one more request from its client takes it over a whole rate
|
||||
// limit. It counts nothing.
|
||||
func (rq *request) overAWholeRateLimit(now time.Time) bool {
|
||||
return rq.countedByRateLimits() &&
|
||||
rq.h.limiter.OverARateLimit(rq.h.clientGroup(rq.client), now)
|
||||
}
|
||||
|
||||
// countedByRateLimits reports whether the rate limits count the request: its
|
||||
// client is not in SWWAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt
|
||||
// under SWWAF_RATE_LIMIT_EXEMPT_PATHS.
|
||||
func (rq *request) countedByRateLimits() bool {
|
||||
cfg := rq.h.config
|
||||
|
||||
return !isInside(rq.client, cfg.RateLimitExemptNets) &&
|
||||
!pathExempt(rq.in, cfg.RateLimitExemptPaths)
|
||||
}
|
||||
@@ -72,17 +72,6 @@ func TestRateLimitRefusesBeforeTheApp(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateLimitRefusalNamesItsHeuristicWindowCountAndLimit(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, _, _ := startWithClock(t, "", map[string]string{rateLimitPerMinute: "1"})
|
||||
|
||||
s.get(client, http.StatusOK, requestlog.ActionForward)
|
||||
|
||||
line := s.get(client, http.StatusForbidden, requestlog.ActionRateLimited)
|
||||
wantHeuristic(t, line, "rate_limits", `{"window": "minute", "count": 2, "limit": 1}`)
|
||||
}
|
||||
|
||||
func TestIPv6GroupPrefixSetsTheClientTheLimitsCount(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
package servewaf
|
||||
|
||||
// The heuristic refusal_limit. It reads SWWAF_ERROR_BURST_THRESHOLD and
|
||||
// SWWAF_BAN_RESPONSE.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// refusalLimit counts the request once it has been answered, if servewaf
|
||||
// refused it after a rule file match, a trap path or a Core Rule Set match, and
|
||||
// in observe mode if enforce mode would have: more than
|
||||
// SWWAF_ERROR_BURST_THRESHOLD such refusals of the client within a minute break
|
||||
// a limit, which bans the client. A client in SWWAF_ALLOW_NETS, which the
|
||||
// heuristics skip, is not counted, and nothing is while the threshold is off.
|
||||
// Its figures are the window, the count and the limit.
|
||||
type refusalLimit struct{}
|
||||
|
||||
func (refusalLimit) name() string { return "refusal_limit" }
|
||||
|
||||
func (refusalLimit) check(_ context.Context, rq *request, now time.Time) *verdict {
|
||||
cfg := rq.h.config
|
||||
if cfg.ErrorBurstThreshold == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if !rq.attack && !rq.ruleBlocked && !rq.wafBlocked {
|
||||
return nil
|
||||
}
|
||||
|
||||
hit, over := rq.h.limiter.CountRefusal(rq.h.clientGroup(rq.client), now,
|
||||
cfg.ErrorBurstThreshold)
|
||||
if !over {
|
||||
return nil
|
||||
}
|
||||
|
||||
// What the client was sent, or in observe mode would have been.
|
||||
status := rq.out.status
|
||||
|
||||
switch rq.line.WouldAction {
|
||||
case requestlog.ActionRuleBlocked, requestlog.ActionWAFBlocked:
|
||||
status = http.StatusForbidden
|
||||
case requestlog.ActionBanned:
|
||||
status = cfg.BanResponse
|
||||
}
|
||||
|
||||
rq.banForLimit(now, hit, status)
|
||||
|
||||
return &verdict{action: requestlog.ActionBanned, figures: limitFigures(hit)}
|
||||
}
|
||||
@@ -1,17 +1,91 @@
|
||||
package servewaf
|
||||
|
||||
// What the reputation heuristics share: blocklists, crowdsec, dnsbl and
|
||||
// abuseipdb.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
)
|
||||
|
||||
// deny is the SWWAF_BLOCKLIST_ACTION and the SWWAF_REPUTATION_ACTION that
|
||||
// refuses the requests of a client a source lists.
|
||||
const deny = "deny"
|
||||
|
||||
// blocklistDenied notes the blocklists that list the client, as
|
||||
// noteListed does, and reports whether SWWAF_BLOCKLIST_ACTION, being deny,
|
||||
// refuses the request. Being limit, it lowers the client's limits instead
|
||||
// (see limitPercentages), and being log, it does nothing more.
|
||||
func (rq *request) blocklistDenied() bool {
|
||||
listedBy := rq.h.lists.ListedBy(rq.client)
|
||||
rq.blocklisted = len(listedBy) > 0
|
||||
rq.noteListed(listedBy, "listed by a blocklist")
|
||||
|
||||
return rq.blocklisted && rq.h.config.BlocklistAction == deny
|
||||
}
|
||||
|
||||
// crowdSecBanned reports whether a decision of the CrowdSec decision list
|
||||
// on the client is in force at now. If one is, it notes the list, as
|
||||
// noteHit does, and bans the client until that decision ends.
|
||||
func (rq *request) crowdSecBanned(now time.Time) bool {
|
||||
decision, listed := rq.h.lists.CrowdSecDecision(rq.client, now)
|
||||
if !listed {
|
||||
return false
|
||||
}
|
||||
|
||||
rq.noteHit(bans.ReputationHit{Source: rq.h.config.CrowdSecDecisionsURL},
|
||||
"listed by the CrowdSec decision list")
|
||||
rq.banForCrowdSec(now, decision)
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// dnsblDenied notes the DNSBL zones whose verdict lists the client, as
|
||||
// noteListed does, and reports whether SWWAF_REPUTATION_ACTION, being
|
||||
// deny, refuses the request. Being limit, it lowers the client's limits
|
||||
// instead (see limitPercentages), and being log, it does nothing more. A
|
||||
// zone without a verdict on the client is asked about it in the
|
||||
// background, and the request does not wait for the answer. ctx is the
|
||||
// request's own context.
|
||||
func (rq *request) dnsblDenied(ctx context.Context) bool {
|
||||
listedBy := rq.h.dnsbl.ListedBy(ctx, rq.client)
|
||||
rq.dnsblListed = len(listedBy) > 0
|
||||
rq.noteListed(listedBy, "listed by a DNSBL zone")
|
||||
|
||||
return rq.dnsblListed && rq.h.config.ReputationAction == deny
|
||||
}
|
||||
|
||||
// abuseIPDBDenied notes AbuseIPDB, as noteHit does, with the score, when
|
||||
// its score of the client is a hit, and reports whether
|
||||
// SWWAF_REPUTATION_ACTION, being deny, refuses the request, as dnsblDenied
|
||||
// does for a zone. While SWWAF_ABUSEIPDB_KEY is unset it does nothing. A
|
||||
// client without a score is checked in the background, by the request's
|
||||
// address, if its history counts an offence, and the request does not
|
||||
// wait for the answer. The score is then used for each address of the
|
||||
// client. ctx is the request's own context.
|
||||
func (rq *request) abuseIPDBDenied(ctx context.Context) bool {
|
||||
if rq.h.config.AbuseIPDBKey == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
client := rq.h.clientGroup(rq.client)
|
||||
held, _ := rq.h.limiter.Client(client)
|
||||
offender := held.History.Offences != ratelimit.Offences{}
|
||||
|
||||
score, hit := rq.h.abuseIPDB.Hit(ctx, client, rq.client, offender)
|
||||
if !hit {
|
||||
return false
|
||||
}
|
||||
|
||||
rq.abuseIPDBHit = true
|
||||
rq.noteHit(bans.ReputationHit{Source: reputation.AbuseIPDBSource, Score: &score},
|
||||
"scored by AbuseIPDB at or over SWWAF_ABUSEIPDB_MIN_SCORE")
|
||||
|
||||
return rq.h.config.ReputationAction == deny
|
||||
}
|
||||
|
||||
// noteListed notes each of sources, the URLs of the blocklists or the
|
||||
// DNSBL zones, their keys masked, that list the client, as noteHit does,
|
||||
// with reason.
|
||||
|
||||
@@ -118,56 +118,6 @@ func TestEachBlocklistActionForAListedAddressAndAListedNetblock(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReputationRefusalNamesItsHeuristicAndWhatListedTheClient(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, tc := range []struct {
|
||||
heuristic string
|
||||
env map[string]string
|
||||
load func(*testing.T, *servewaf.Server)
|
||||
figures string
|
||||
}{
|
||||
{
|
||||
"blocklists",
|
||||
map[string]string{blocklistURLs: dropURL, blocklistAction: actionDeny},
|
||||
func(t *testing.T, server *servewaf.Server) {
|
||||
t.Helper()
|
||||
loadLists(t, server, map[string][]string{dropURL: {client}})
|
||||
},
|
||||
`{"lists": ["` + dropURL + `"]}`,
|
||||
},
|
||||
{
|
||||
"dnsbl",
|
||||
map[string]string{
|
||||
dnsblZones: dnsblZone, dnsblResolver: noResolver,
|
||||
reputationAction: actionDeny,
|
||||
},
|
||||
func(_ *testing.T, server *servewaf.Server) {
|
||||
loadVerdicts(server, map[string][]string{client: {dnsblZone}})
|
||||
},
|
||||
`{"zones": ["` + dnsblZone + `"]}`,
|
||||
},
|
||||
{
|
||||
"abuseipdb",
|
||||
map[string]string{abuseIPDBKey: accountKey, reputationAction: actionDeny},
|
||||
func(_ *testing.T, server *servewaf.Server) {
|
||||
loadScores(server, map[string]int64{client: 80})
|
||||
},
|
||||
`{"score": 80, "min_score": 75}`,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.heuristic, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, server, _ := startWithLookups(t, tc.env)
|
||||
tc.load(t, server)
|
||||
|
||||
line := s.get(client, http.StatusForbidden, requestlog.ActionDenied)
|
||||
wantHeuristic(t, line, tc.heuristic, tc.figures)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlocklistsComeAfterTheCountryListsAndSkipAllowNets(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -160,20 +160,16 @@ func requestHeaders(r *http.Request, names []string) map[string]string {
|
||||
}
|
||||
|
||||
// check is the one place where a request can be refused. It returns nil to let
|
||||
// the request through. The heuristics of checkClient decide: a request they
|
||||
// refuse is answered with SWWAF_BAN_RESPONSE, 403 for a block rule or the Core
|
||||
// Rule Set, or 431 for the header limits. In observe mode nothing is refused,
|
||||
// and the log line names what enforce mode would have done. ctx is the
|
||||
// request's own context.
|
||||
// the request through. The checks of checkClient decide: a request they refuse
|
||||
// is answered with SWWAF_BAN_RESPONSE, 403 for a block rule or the Core Rule Set,
|
||||
// or 431 for the header limits. In observe mode nothing is refused, and the log
|
||||
// line names what enforce mode would have done. ctx is the request's own context.
|
||||
func (rq *request) check(ctx context.Context) *refusal {
|
||||
v := rq.checkClient(ctx)
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
action := v.action
|
||||
action := rq.checkClient(ctx)
|
||||
|
||||
switch {
|
||||
case action == "":
|
||||
return nil
|
||||
case rq.h.config.Observe:
|
||||
rq.line.WouldAction = action
|
||||
|
||||
@@ -187,21 +183,35 @@ func (rq *request) check(ctx context.Context) *refusal {
|
||||
}
|
||||
}
|
||||
|
||||
// checkClient runs the heuristics of beforeLookup on the request, then looks
|
||||
// up the client's AS number and country, then runs those of afterLookup. It
|
||||
// returns the first verdict, or nil when none gives one. A client in
|
||||
// SWWAF_ALLOW_NETS skips them all, and is not looked up. ctx is the request's
|
||||
// own context.
|
||||
func (rq *request) checkClient(ctx context.Context) *verdict {
|
||||
if isInside(rq.client, rq.h.config.AllowNets) {
|
||||
return nil
|
||||
// checkClient runs the checks on the request's client, and returns the
|
||||
// action of the first that refuses the request, or "" when none does. A
|
||||
// client in SWWAF_ALLOW_NETS skips them, and is not looked up. For any
|
||||
// other client, SWWAF_DENY_NETS comes first, then a ban on its netblock,
|
||||
// so that a client either refuses is not looked up, then the lookup of
|
||||
// its AS number and country, then the country lists, then the blocklists,
|
||||
// then the CrowdSec decision list, which bans the client it lists, then
|
||||
// the DNSBL zones' verdicts, and then AbuseIPDB's score; a request any of
|
||||
// them refuses is not counted for the rate limits. Then come the
|
||||
// rate limits, unless the client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the
|
||||
// request's path is exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that
|
||||
// every other request is counted, each of them by the client's limit
|
||||
// percentages, then SWWAF_TRAP_PATHS, then the rule files, then the header
|
||||
// limits, and last the Core Rule Set. A request exempt from the rate limits
|
||||
// is exempt from the byte limits too. ctx is the request's own context.
|
||||
func (rq *request) checkClient(ctx context.Context) string {
|
||||
cfg := rq.h.config
|
||||
if isInside(rq.client, cfg.AllowNets) {
|
||||
return ""
|
||||
}
|
||||
|
||||
now := rq.h.now()
|
||||
|
||||
v := rq.runHeuristics(ctx, beforeLookup(), now)
|
||||
if v != nil {
|
||||
return v
|
||||
if isInside(rq.client, cfg.DenyNets) {
|
||||
return requestlog.ActionDenied
|
||||
}
|
||||
|
||||
if rq.banned(now) {
|
||||
return requestlog.ActionBanned
|
||||
}
|
||||
|
||||
rq.lookUp(ctx, now)
|
||||
@@ -210,7 +220,59 @@ func (rq *request) checkClient(ctx context.Context) *verdict {
|
||||
// again: a request counted at the time it arrived could be counted
|
||||
// after a later one of its client's, which the rate limits would take
|
||||
// for the clock set back.
|
||||
return rq.runHeuristics(ctx, afterLookup(), rq.h.now())
|
||||
now = rq.h.now()
|
||||
|
||||
if rq.countryDenied() {
|
||||
return requestlog.ActionCountryDenied
|
||||
}
|
||||
|
||||
if rq.blocklistDenied() {
|
||||
return requestlog.ActionDenied
|
||||
}
|
||||
|
||||
if rq.crowdSecBanned(now) {
|
||||
return requestlog.ActionBanned
|
||||
}
|
||||
|
||||
if rq.dnsblDenied(ctx) || rq.abuseIPDBDenied(ctx) {
|
||||
return requestlog.ActionDenied
|
||||
}
|
||||
|
||||
rq.counted = rq.countedByRateLimits()
|
||||
if rq.counted {
|
||||
rq.limitPercent, rq.bytesPercent = rq.limitPercentages()
|
||||
rq.line.LimitPercent, rq.line.LimitPercentSetting = rq.limitPercent.logged()
|
||||
rq.line.BytesPercent, rq.line.BytesPercentSetting = rq.bytesPercent.logged()
|
||||
}
|
||||
|
||||
if rq.counted && rq.limitBroken(now) {
|
||||
return requestlog.ActionRateLimited
|
||||
}
|
||||
|
||||
if rq.trapPath(now) {
|
||||
return requestlog.ActionBanned
|
||||
}
|
||||
|
||||
action := rq.checkRules(now)
|
||||
if action != "" {
|
||||
return action
|
||||
}
|
||||
|
||||
if rq.overHeaderLimits() {
|
||||
return requestlog.ActionTooLarge
|
||||
}
|
||||
|
||||
return rq.checkCoreRuleSet()
|
||||
}
|
||||
|
||||
// countedByRateLimits reports whether the rate limits count the request: its
|
||||
// client is not in SWWAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt
|
||||
// under SWWAF_RATE_LIMIT_EXEMPT_PATHS.
|
||||
func (rq *request) countedByRateLimits() bool {
|
||||
cfg := rq.h.config
|
||||
|
||||
return !isInside(rq.client, cfg.RateLimitExemptNets) &&
|
||||
!pathExempt(rq.in, cfg.RateLimitExemptPaths)
|
||||
}
|
||||
|
||||
// pathExempt reports whether r is exempt under prefixes,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package servewaf_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
@@ -273,48 +272,6 @@ func wantTimings(t *testing.T, line logLine, want ...string) {
|
||||
}
|
||||
}
|
||||
|
||||
// wantHeuristic checks that line names heuristic as the one that refused the
|
||||
// request, with figures, a JSON object. A heuristic of "" with figures "null" is
|
||||
// a line that names none.
|
||||
func wantHeuristic(t *testing.T, line logLine, heuristic, figures string) {
|
||||
t.Helper()
|
||||
|
||||
if line.Heuristic != heuristic ||
|
||||
!reflect.DeepEqual(line.HeuristicFigures, decodeFigures(t, figures)) {
|
||||
t.Errorf("log line has heuristic %q with heuristic_figures %v, want %q with %s",
|
||||
line.Heuristic, line.HeuristicFigures, heuristic, figures)
|
||||
}
|
||||
}
|
||||
|
||||
// wantAfterAnswerHeuristic checks after_answer_heuristic and
|
||||
// after_answer_heuristic_figures as wantHeuristic checks heuristic and
|
||||
// heuristic_figures.
|
||||
func wantAfterAnswerHeuristic(t *testing.T, line logLine, heuristic, figures string) {
|
||||
t.Helper()
|
||||
|
||||
if line.AfterAnswerHeuristic != heuristic ||
|
||||
!reflect.DeepEqual(line.AfterAnswerHeuristicFigures, decodeFigures(t, figures)) {
|
||||
t.Errorf("log line has after_answer_heuristic %q with "+
|
||||
"after_answer_heuristic_figures %v, want %q with %s",
|
||||
line.AfterAnswerHeuristic, line.AfterAnswerHeuristicFigures, heuristic, figures)
|
||||
}
|
||||
}
|
||||
|
||||
// decodeFigures decodes figures, a JSON object of a heuristic's figures, as
|
||||
// the log line's figures are decoded.
|
||||
func decodeFigures(t *testing.T, figures string) map[string]any {
|
||||
t.Helper()
|
||||
|
||||
var decoded map[string]any
|
||||
|
||||
err := json.Unmarshal([]byte(figures), &decoded)
|
||||
if err != nil {
|
||||
t.Fatalf("decode %s: %v", figures, err)
|
||||
}
|
||||
|
||||
return decoded
|
||||
}
|
||||
|
||||
// microseconds is a timing in whole microseconds.
|
||||
func microseconds(milliseconds float64) int64 {
|
||||
return int64(math.Round(milliseconds * 1000))
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
package servewaf
|
||||
|
||||
// The heuristic rule_files. It reads the rules of the rule files, which
|
||||
// SWWAF_RULES_DIR and SWWAF_RULES_ENABLED name.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||
)
|
||||
|
||||
// ruleFiles checks the request against the rules of the rule files at now,
|
||||
// notes the ids of those it matches in the log line, and refuses the request
|
||||
// when the last rule it matches refuses it: with ActionRuleBlocked for a block
|
||||
// rule, and ActionBanned for a ban rule. A ban rule bans the client's netblock
|
||||
// for a clear sign of attack, or in observe mode raises the alert for the ban
|
||||
// it would have made. Either rule's match is noted as an offence, for the
|
||||
// client's history. Its figures are the rule's id and action.
|
||||
type ruleFiles struct{}
|
||||
|
||||
func (ruleFiles) name() string { return "rule_files" }
|
||||
|
||||
func (ruleFiles) check(_ context.Context, rq *request, now time.Time) *verdict {
|
||||
matched := rq.h.rules.Match(rq.in)
|
||||
|
||||
for _, rule := range matched {
|
||||
rq.line.RuleIDs = append(rq.line.RuleIDs, rule.ID)
|
||||
rq.h.metrics.RuleMatched(rule.ID, rule.Action)
|
||||
}
|
||||
|
||||
if len(matched) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Only the last rule matched can refuse the request.
|
||||
last := matched[len(matched)-1]
|
||||
figures := map[string]any{"rule_id": last.ID, "rule_action": last.Action}
|
||||
|
||||
switch last.Action {
|
||||
case rules.ActionBlock:
|
||||
rq.ruleBlocked = true
|
||||
|
||||
return &verdict{action: requestlog.ActionRuleBlocked, figures: figures}
|
||||
case rules.ActionBan:
|
||||
rq.attack = true
|
||||
rq.banForAttack(now, bans.Notes{RuleID: last.ID, Target: last.Target})
|
||||
|
||||
return &verdict{action: requestlog.ActionBanned, figures: figures}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -93,21 +93,6 @@ func TestEachRuleAction(t *testing.T) {
|
||||
s.get(client, http.StatusTooManyRequests, requestlog.ActionBanned)
|
||||
}
|
||||
|
||||
func TestRuleRefusalNamesItsHeuristicAndTheRule(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, _, _ := startWithClock(t, "", map[string]string{
|
||||
rulesDir: writeRules(t, "noted path log ^/\n"+testRules),
|
||||
})
|
||||
|
||||
line := s.request(client, "/blocked", http.StatusForbidden,
|
||||
requestlog.ActionRuleBlocked)
|
||||
wantHeuristic(t, line, "rule_files", `{"rule_id": "blocked", "rule_action": "block"}`)
|
||||
|
||||
line = s.request(client, "/.env", http.StatusForbidden, requestlog.ActionBanned)
|
||||
wantHeuristic(t, line, "rule_files", `{"rule_id": "probe", "rule_action": "ban"}`)
|
||||
}
|
||||
|
||||
func TestNextClearSignOfAttackAfterABanBansPermanently(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
package servewaf
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||
)
|
||||
|
||||
// trapPath reports whether the request asks for a path in
|
||||
// SWWAF_TRAP_PATHS: its path as a path rule sees it, without the query,
|
||||
// as sent or once percent-decoded, is one of them. Such a request is a
|
||||
// clear sign of attack, as a ban rule's match is: it bans the client's
|
||||
// netblock, or in observe mode raises the alert for the ban it would have
|
||||
// made. The ban's notes give the path as sent.
|
||||
func (rq *request) trapPath(now time.Time) bool {
|
||||
path := rules.Path(rq.in)
|
||||
if !slices.Contains(rq.h.config.TrapPaths, path) &&
|
||||
!slices.Contains(rq.h.config.TrapPaths, rules.DecodeOnce(path)) {
|
||||
return false
|
||||
}
|
||||
|
||||
rq.attack = true
|
||||
rq.banForAttack(now, bans.Notes{TrapPath: path})
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// checkRules checks the request against the rules of the rule files at
|
||||
// now, notes the ids of those it matches in the log line, and returns the
|
||||
// action of the rule that refuses it, ActionRuleBlocked for a block rule
|
||||
// and ActionBanned for a ban rule, or "" when none does. A ban rule bans
|
||||
// the client's netblock for a clear sign of attack, or in observe mode
|
||||
// raises the alert for the ban it would have made. Either rule's match
|
||||
// is noted as an offence, for the client's history.
|
||||
func (rq *request) checkRules(now time.Time) string {
|
||||
matched := rq.h.rules.Match(rq.in)
|
||||
|
||||
for _, rule := range matched {
|
||||
rq.line.RuleIDs = append(rq.line.RuleIDs, rule.ID)
|
||||
rq.h.metrics.RuleMatched(rule.ID, rule.Action)
|
||||
}
|
||||
|
||||
if len(matched) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Only the last rule matched can refuse the request.
|
||||
switch last := matched[len(matched)-1]; last.Action {
|
||||
case rules.ActionBlock:
|
||||
rq.ruleBlocked = true
|
||||
|
||||
return requestlog.ActionRuleBlocked
|
||||
case rules.ActionBan:
|
||||
rq.attack = true
|
||||
rq.banForAttack(now, bans.Notes{RuleID: last.ID, Target: last.Target})
|
||||
|
||||
return requestlog.ActionBanned
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,6 @@ package servewaf
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -53,6 +52,13 @@ const ReportPath = "/_smallwebwaf/report"
|
||||
// QuestionBodyMaxBytes is the most of a visitor's body a question carries.
|
||||
const QuestionBodyMaxBytes = 64 << 10
|
||||
|
||||
// The header limits: the most cookies and header fields a request may have
|
||||
// when the checks reach the Core Rule Set. Real clients never reach them.
|
||||
const (
|
||||
maxCookies = 50
|
||||
maxHeaderFields = 100
|
||||
)
|
||||
|
||||
const (
|
||||
// maxAwaitingReport is the most answered requests held until their
|
||||
// reports come. Past it, the request answered longest ago is dropped.
|
||||
@@ -141,20 +147,48 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// refusedWhateverTheAnswer reports whether the request is one that does not wait
|
||||
// for GeoJS, since the checks will refuse it at now whatever GeoJS answers about
|
||||
// its client: the heuristic crowdsec, header_limits, blocklists or rate_limits
|
||||
// will refuse it, as a function in that heuristic's file says. In observe mode,
|
||||
// which refuses nothing, none is. rproxy lets a request through when servewaf
|
||||
// answers too late.
|
||||
// its client: it is over the header limits, a blocklist lists its client while
|
||||
// SWWAF_BLOCKLIST_ACTION is deny, the CrowdSec decision list lists it, or it is
|
||||
// over a whole rate limit. In observe mode, which refuses nothing, none is.
|
||||
// rproxy lets a request through when servewaf answers too late.
|
||||
func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
|
||||
if rq.h.config.Observe {
|
||||
return false
|
||||
}
|
||||
|
||||
_, decided := rq.crowdSecDecisionInForce(now)
|
||||
lists := rq.h.lists
|
||||
_, decided := lists.CrowdSecDecision(rq.client, now)
|
||||
|
||||
return decided || overHeaderLimits(rq.headerCounts()) ||
|
||||
rq.refusedByBlocklists(rq.h.lists.ListedBy(rq.client)) ||
|
||||
rq.overAWholeRateLimit(now)
|
||||
return decided || rq.overHeaderLimits() ||
|
||||
(rq.h.config.BlocklistAction == deny && len(lists.ListedBy(rq.client)) > 0) ||
|
||||
(rq.countedByRateLimits() &&
|
||||
rq.h.limiter.OverARateLimit(rq.h.clientGroup(rq.client), now))
|
||||
}
|
||||
|
||||
// overHeaderLimits reports whether servewaf refuses the request for its
|
||||
// headers: it has more than maxHeaderFields fields, each value of a header sent
|
||||
// twice being one, or more than maxCookies cookies, each part of a Cookie header
|
||||
// between semicolons being one unless it is empty or only spaces and tabs. The
|
||||
// Core Rule Set runs each of its rules on each cookie, so that thousands of small
|
||||
// cookies keep it busy for over a second. It trims only spaces and control
|
||||
// characters, of which a header value can hold only tabs, so a part of Unicode
|
||||
// spaces is a cookie to it.
|
||||
func (rq *request) overHeaderLimits() bool {
|
||||
fields, cookies := 0, 0
|
||||
|
||||
for _, values := range rq.in.Header {
|
||||
fields += len(values)
|
||||
}
|
||||
|
||||
for _, value := range rq.in.Header.Values("Cookie") {
|
||||
for part := range strings.SplitSeq(value, ";") {
|
||||
if strings.Trim(part, " \t") != "" {
|
||||
cookies++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return fields > maxHeaderFields || cookies > maxCookies
|
||||
}
|
||||
|
||||
// answerQuestion runs the checks on the visitor's request a question
|
||||
@@ -185,18 +219,18 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
if refused == nil && !underOwnPrefix {
|
||||
// In observe mode, what enforce mode would have refused counts here.
|
||||
defer rq.runAfterAnswer(r.Context())
|
||||
defer rq.countRefusal()
|
||||
|
||||
rq.allow()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// The last deferred runs first: runAfterAnswer before addToHistory, so that
|
||||
// a broken refusal limit is in the client's history.
|
||||
// The last deferred runs first: countRefusal before addToHistory, so that a
|
||||
// broken error burst is in the client's history.
|
||||
defer rq.addToHistory()
|
||||
defer rq.countAnomalies()
|
||||
defer rq.runAfterAnswer(r.Context())
|
||||
defer rq.countRefusal()
|
||||
|
||||
if refused != nil {
|
||||
rq.deny(*refused)
|
||||
@@ -378,7 +412,7 @@ func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) {
|
||||
// Remove reports true to one report only, of several that name the id. A
|
||||
// question still being handled is held as nil.
|
||||
if found && held != nil && h.awaitingReport.Remove(report.RequestID) {
|
||||
h.finishWith(r.Context(), held, report)
|
||||
h.finishWith(held, report)
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
@@ -441,11 +475,11 @@ func (report *Report) checkRanges() error {
|
||||
|
||||
// finishWith finishes a request with what its report says, and writes the
|
||||
// request's last log line, which holds its decision's fields and the report's.
|
||||
// For an allowed request it first runs the heuristics of onReport, and adds the
|
||||
// request to its client's history, the anomaly counters and the metrics, with
|
||||
// the status and bytes the report gives. A denied request's counting was
|
||||
// finished at its decision. ctx is the report's own context.
|
||||
func (h *handler) finishWith(ctx context.Context, held *decision, report Report) {
|
||||
// For an allowed request it first counts the request's bytes, and adds it to its
|
||||
// client's history, the anomaly counters and the metrics, with the status and
|
||||
// bytes the report gives. A denied request's counting was finished at its
|
||||
// decision.
|
||||
func (h *handler) finishWith(held *decision, report Report) {
|
||||
rq := &request{
|
||||
decision: *held,
|
||||
h: h,
|
||||
@@ -462,7 +496,7 @@ func (h *handler) finishWith(ctx context.Context, held *decision, report Report)
|
||||
if rq.allowed {
|
||||
rq.line.DurationUpstreamTotal = &report.DurationUpstreamTotal
|
||||
|
||||
rq.runBanningHeuristics(ctx, onReport())
|
||||
rq.countBytes()
|
||||
rq.countAnomalies()
|
||||
rq.addToHistory()
|
||||
|
||||
|
||||
@@ -104,15 +104,6 @@ func TestDenyNetsRefuseBeforeTheLookup(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDenyNetsRefusalNamesItsHeuristicAndNetblock(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, _, _ := startWithClock(t, "", map[string]string{denyNets: clientsNet})
|
||||
|
||||
line := s.get(client, http.StatusForbidden, requestlog.ActionDenied)
|
||||
wantHeuristic(t, line, "deny_nets", `{"netblock": "`+clientsNet+`"}`)
|
||||
}
|
||||
|
||||
func TestRequestRefusedByDenyNetsIsNotCounted(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
package servewaf
|
||||
|
||||
// The heuristic trap_paths. It reads SWWAF_TRAP_PATHS.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||
)
|
||||
|
||||
// trapPaths refuses a request for a path in SWWAF_TRAP_PATHS: its path as a
|
||||
// path rule sees it, without the query, as sent or once percent-decoded, is one
|
||||
// of them. Such a request is a clear sign of attack, as a ban rule's match is:
|
||||
// it bans the client's netblock, or in observe mode raises the alert for the
|
||||
// ban it would have made. The ban's notes give the path as sent, and so does
|
||||
// its figure.
|
||||
type trapPaths struct{}
|
||||
|
||||
func (trapPaths) name() string { return "trap_paths" }
|
||||
|
||||
func (trapPaths) check(_ context.Context, rq *request, now time.Time) *verdict {
|
||||
path := rules.Path(rq.in)
|
||||
if !slices.Contains(rq.h.config.TrapPaths, path) &&
|
||||
!slices.Contains(rq.h.config.TrapPaths, rules.DecodeOnce(path)) {
|
||||
return nil
|
||||
}
|
||||
|
||||
rq.attack = true
|
||||
rq.banForAttack(now, bans.Notes{TrapPath: path})
|
||||
|
||||
return &verdict{
|
||||
action: requestlog.ActionBanned,
|
||||
figures: map[string]any{"path": path},
|
||||
}
|
||||
}
|
||||
@@ -83,16 +83,6 @@ func TestTrapPathBansAsABanRuleDoes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrapPathRefusalNamesItsHeuristicAndPath(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, _, _ := startWithClock(t, "", map[string]string{trapPaths: trapPathList})
|
||||
|
||||
line := s.request(client, "/wp-login.php?redirect_to=x", http.StatusForbidden,
|
||||
requestlog.ActionBanned)
|
||||
wantHeuristic(t, line, "trap_paths", `{"path": "/wp-login.php"}`)
|
||||
}
|
||||
|
||||
func TestTrapPathMatchesThePathDecodedOnce(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -1225,9 +1225,9 @@ func wantStartingLine(t *testing.T, line map[string]any, dir string) {
|
||||
"SWWAF_ALLOW_NETS": "",
|
||||
rateLimitExemptNets: "",
|
||||
"SWWAF_DENY_NETS": "",
|
||||
"SWWAF_RATE_LIMIT_PER_MINUTE": "120",
|
||||
"SWWAF_RATE_LIMIT_PER_HOUR": "1500",
|
||||
rateLimitPerDay: "6000",
|
||||
"SWWAF_RATE_LIMIT_PER_MINUTE": "500",
|
||||
"SWWAF_RATE_LIMIT_PER_HOUR": "20000",
|
||||
rateLimitPerDay: "300000",
|
||||
"SWWAF_RATE_LIMIT_EXEMPT_PATHS": "",
|
||||
"SWWAF_DENIED_COUNTRIES": "",
|
||||
"SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES": "",
|
||||
|
||||
@@ -1,216 +0,0 @@
|
||||
#!/bin/sh
|
||||
# script/gitea-setup: build the image, bring up deploy/gitea under a project
|
||||
# name of its own, and check it:
|
||||
# - that a page load, a push and a clone sent through traefik pass through
|
||||
# rproxy, and that servewaf checks them;
|
||||
# - that gitea logs the visitor's address, not rproxy's;
|
||||
# - that a request from a container on the shared network skips the checks;
|
||||
# - that with servewaf stopped, gitea still answers through rproxy.
|
||||
# It publishes no port of traefik's, since this host's ports 80 and 443 may be
|
||||
# taken. It reaches traefik at its address on the shared network instead, so
|
||||
# its own requests come from the network's gateway, which servewaf checks like
|
||||
# any visitor. Two runs at once on one docker host cannot both start, since
|
||||
# both would use the subnet compose.yaml names. The containers, the network,
|
||||
# the volumes, the image and the files the script makes are removed however it
|
||||
# ends; the gitea and traefik images stay, as the base images of a build do.
|
||||
# It needs network access. script/check does not run this.
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
|
||||
|
||||
# Named after this run, so that runs in other clones on the same host
|
||||
# never touch each other's.
|
||||
NAME="$("$SCRIPT_DIR/projectname")-gitea-setup-$$"
|
||||
IMAGE="$NAME-image"
|
||||
NETWORK="${NAME}_shared"
|
||||
VISITOR="$NAME-visitor"
|
||||
|
||||
# The site's host name and gitea's image, as deploy/gitea/compose.yaml names
|
||||
# them. gitea's image also runs the container on the shared network.
|
||||
SITE="git.example.org"
|
||||
# gitea 1.27.3, 2026-08-29
|
||||
GITEA_IMAGE="gitea/gitea@sha256:87a67ee09d3ae0d1df5fda5dcda3e2a1f9236a45b0a59025d6e00e46adc43bef"
|
||||
|
||||
# gitea's one user, who owns the repository.
|
||||
PASSWORD="owner-password"
|
||||
|
||||
# The files the script makes, gitea's secret key for this run, and traefik's
|
||||
# and the gateway's addresses on the shared network once the setup runs.
|
||||
WORK=""
|
||||
secret_key=""
|
||||
traefik=""
|
||||
gateway=""
|
||||
|
||||
cleanup() {
|
||||
# So that a signal cannot stop it partway through.
|
||||
trap '' HUP INT TERM
|
||||
docker rm --force "$VISITOR" >/dev/null 2>&1 || true
|
||||
if [ -n "$WORK" ]; then
|
||||
compose down --volumes --remove-orphans --timeout 5 >/dev/null 2>&1 || true
|
||||
rm -rf "$WORK"
|
||||
fi
|
||||
docker rmi --force "$IMAGE" >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
# compose <argument>...: docker compose on the setup, under this run's project
|
||||
# name, with the image built here, this run's secret key and no port of
|
||||
# traefik's published.
|
||||
compose() {
|
||||
SMALLWEBWAF_IMAGE="$IMAGE" GITEA_SECRET_KEY="$secret_key" \
|
||||
docker compose --project-name "$NAME" \
|
||||
--file "$ROOT/deploy/gitea/compose.yaml" --file "$WORK/no-ports.yaml" "$@"
|
||||
}
|
||||
|
||||
fail() {
|
||||
echo "gitea-setup: $*; the last lines of each container's output:" >&2
|
||||
compose logs --no-color --tail 30 >&2 || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
# wait_for <what fails> <command>...: run the command every second until
|
||||
# it succeeds, for at most a minute.
|
||||
wait_for() {
|
||||
failure="$1"
|
||||
shift
|
||||
tries=0
|
||||
until "$@"; do
|
||||
tries=$((tries + 1))
|
||||
[ "$tries" -lt 60 ] || fail "$failure"
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
# logged <service> <text>...: a line of the service's output holds every text,
|
||||
# in any order.
|
||||
logged() {
|
||||
lines="$(compose logs --no-color --no-log-prefix "$1" 2>&1)"
|
||||
shift
|
||||
for text in "$@"; do
|
||||
lines="$(printf '%s\n' "$lines" | grep -F -e "$text")" || return 1
|
||||
done
|
||||
}
|
||||
|
||||
# send <path> <curl option>...: send a request for path to the site, at
|
||||
# traefik's address, and print the status of the answer, 000 for none. traefik
|
||||
# answers with a certificate it made itself, since the setup names no
|
||||
# certificate resolver.
|
||||
send() {
|
||||
path="$1"
|
||||
shift
|
||||
curl --silent --show-error --insecure --max-time 30 --output /dev/null \
|
||||
--write-out '%{http_code}' --resolve "$SITE:443:$traefik" "$@" \
|
||||
"https://$SITE$path" || true
|
||||
}
|
||||
|
||||
# routed: traefik passes a request for the site's front page on to gitea.
|
||||
routed() {
|
||||
[ "$(send / 2>/dev/null)" = 200 ]
|
||||
}
|
||||
|
||||
# site_git <git argument>...: run git as the owner, without the host's own git
|
||||
# settings, reaching the site at traefik's address. git gives up on a transfer
|
||||
# slower than 1000 bytes a second for 30 seconds.
|
||||
site_git() {
|
||||
GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1 git \
|
||||
-c user.name=owner -c user.email=owner@example.invalid \
|
||||
-c http.sslVerify=false -c "http.curloptResolve=$SITE:443:$traefik" \
|
||||
-c http.lowSpeedLimit=1000 -c http.lowSpeedTime=30 "$@"
|
||||
}
|
||||
|
||||
# address_of <service>: print the service's address on the shared network.
|
||||
address_of() {
|
||||
docker inspect --format "{{(index .NetworkSettings.Networks \"$NETWORK\").IPAddress}}" \
|
||||
"$(compose ps --quiet "$1")"
|
||||
}
|
||||
|
||||
# passed_through <method> <path>: rproxy passed a request from the gateway for
|
||||
# path to gitea on servewaf's allow, and servewaf checked it: its report line
|
||||
# gives the client's counts and the Core Rule Set's score.
|
||||
passed_through() {
|
||||
wait_for "rproxy logged no $1 $2 that servewaf allowed" logged rproxy \
|
||||
"\"method\":\"$1\"" "\"path\":\"$2\"" "\"client_ip\":\"$gateway\"" \
|
||||
'"disposition":"allow"'
|
||||
wait_for "servewaf logged no check of $1 $2" logged servewaf \
|
||||
'"servewaf_line":"report"' "\"method\":\"$1\"" "\"path\":\"$2\"" \
|
||||
"\"client_ip\":\"$gateway\"" '"action":"forward"' '"counts":' '"waf_score":'
|
||||
}
|
||||
|
||||
main() {
|
||||
cd "$ROOT"
|
||||
trap cleanup EXIT
|
||||
trap 'exit 1' HUP INT TERM
|
||||
WORK="$(mktemp -d)"
|
||||
printf 'services:\n traefik:\n ports: !reset []\n' >"$WORK/no-ports.yaml"
|
||||
|
||||
docker build --no-cache -t "$IMAGE" .
|
||||
secret_key="$(docker run --rm "$GITEA_IMAGE" gitea generate secret SECRET_KEY)"
|
||||
|
||||
# rproxy is healthy once gitea takes connections.
|
||||
compose up --detach --quiet-pull --wait --wait-timeout 120 ||
|
||||
fail "the setup did not start"
|
||||
traefik="$(address_of traefik)"
|
||||
gateway="$(docker network inspect --format '{{(index .IPAM.Config 0).Gateway}}' "$NETWORK")"
|
||||
wait_for "traefik did not route the site to rproxy" routed
|
||||
|
||||
compose exec --user git gitea gitea admin user create --admin \
|
||||
--username owner --password "$PASSWORD" --email owner@example.invalid \
|
||||
--must-change-password=false >/dev/null || fail "creating gitea's user failed"
|
||||
status="$(send /api/v1/user/repos --user "owner:$PASSWORD" \
|
||||
--header "Content-Type: application/json" --data '{"name":"repo"}')"
|
||||
[ "$status" = 201 ] || fail "creating the repository was answered $status"
|
||||
|
||||
status="$(send /owner/repo)"
|
||||
[ "$status" = 200 ] || fail "the repository's page was answered $status"
|
||||
passed_through GET /owner/repo
|
||||
echo "gitea-setup: a page load passes through rproxy, and servewaf checks it"
|
||||
|
||||
wait_for "gitea did not log the visitor's address" logged gitea \
|
||||
"GET /owner/repo for $gateway:"
|
||||
echo "gitea-setup: gitea logs the visitor's address, not rproxy's"
|
||||
|
||||
site_git init --quiet --initial-branch=main "$WORK/repo"
|
||||
echo "hello" >"$WORK/repo/README.md"
|
||||
site_git -C "$WORK/repo" add README.md
|
||||
site_git -C "$WORK/repo" commit --quiet --message "Add a README"
|
||||
site_git -C "$WORK/repo" push --quiet \
|
||||
"https://owner:$PASSWORD@$SITE/owner/repo.git" main ||
|
||||
fail "the push failed"
|
||||
passed_through POST /owner/repo.git/git-receive-pack
|
||||
echo "gitea-setup: a push passes through rproxy, and servewaf checks it"
|
||||
|
||||
site_git clone --quiet "https://owner:$PASSWORD@$SITE/owner/repo.git" \
|
||||
"$WORK/clone" || fail "the clone failed"
|
||||
[ "$(cat "$WORK/clone/README.md")" = hello ] || fail "the clone lacks the README"
|
||||
passed_through POST /owner/repo.git/git-upload-pack
|
||||
echo "gitea-setup: a clone passes through rproxy, and servewaf checks it"
|
||||
|
||||
# A container on the shared network, as sandcastle is, asks for the site
|
||||
# by its name, and so reaches traefik from its own address.
|
||||
status="$(docker run --rm --name "$VISITOR" --network "$NETWORK" \
|
||||
--entrypoint curl "$GITEA_IMAGE" --silent --insecure --max-time 30 \
|
||||
--output /dev/null --write-out '%{http_code}' "https://$SITE/api/v1/version" ||
|
||||
true)"
|
||||
[ "$status" = 200 ] || fail "the request from the shared network was answered $status"
|
||||
wait_for "servewaf logged no request from the shared network" logged servewaf \
|
||||
'"servewaf_line":"report"' '"path":"/api/v1/version"' '"action":"forward"'
|
||||
line="$(compose logs --no-color --no-log-prefix servewaf |
|
||||
grep -F '"servewaf_line":"report"' | grep -F '"path":"/api/v1/version"')"
|
||||
case "$line" in
|
||||
*"\"client_ip\":\"$gateway\""*)
|
||||
fail "the request from the shared network came from the gateway" ;;
|
||||
*'"counts":'* | *'"waf_score":'*)
|
||||
fail "servewaf checked the request from the shared network" ;;
|
||||
esac
|
||||
echo "gitea-setup: a request from the shared network skips the checks"
|
||||
|
||||
compose stop servewaf >/dev/null 2>&1 || fail "servewaf did not stop"
|
||||
status="$(send /owner/repo)"
|
||||
[ "$status" = 200 ] ||
|
||||
fail "with servewaf stopped, the repository's page was answered $status"
|
||||
wait_for "rproxy logged no request SWRP_DEFAULT_DISPOSITION decided" \
|
||||
logged rproxy '"path":"/owner/repo"' '"default_disposition":true'
|
||||
echo "gitea-setup: with servewaf stopped, gitea answers through rproxy"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in new issue
Block a user