Compare commits

..
Author SHA1 Message Date
clawbot 334a082a42 sww: lift waf's check time limit and rproxy's wait in tests (closes #418)
check / check (push) Successful in 5m33s
Under heavy host load the sww tests got 503 where they expected 200 or
403: waf refused requests whose trap paths, rule files and Core Rule Set
took longer than 500 ms, and rproxy gave up on waf after 2 s.

waf.Params and sww.Params gain BesideLookupTimeLimit; zero keeps the
500 ms limit, so the binary is unchanged. The sww tests' run gives waf
an hour, and startRProxy sets SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS to
60000, as the rproxy package's tests do. waf's own tests set the limit
through the new field instead of SetBesideLookupTimeLimit, which is gone.

Model: opus-5-5
2026-11-19 08:48:15 +00:00
64 changed files with 1123 additions and 3276 deletions

No files matched your search

+34 -184
View File
@@ -75,8 +75,7 @@ Together, `rproxy` and `waf` protect against:
- give the clients of chosen networks and countries a fraction of every - give the clients of chosen networks and countries a fraction of every
limit, so that they are banned sooner; limit, so that they are banned sooner;
- refuse or limit clients on a blocklist, a DNS blocklist, or with a high - refuse or limit clients on a blocklist, a DNS blocklist, or with a high
AbuseIPDB score; AbuseIPDB score.
- ban the clients on AbuseIPDB's blacklist before they send a request.
For the operator, it: For the operator, it:
@@ -285,10 +284,6 @@ account, a key, a token or an alert destination stays off until you give one.
`SWW_WAF_BYTES_COUNTED` says. Headers are never counted. What the visitor sent `SWW_WAF_BYTES_COUNTED` says. Headers are never counted. What the visitor sent
over a WebSocket counts as request bytes, and what the app sent as response over a WebSocket counts as request bytes, and what the app sent as response
bytes. bytes.
- An anomaly is more requests or bytes than an `SWW_WAF_ANOMALY_` threshold
allows. Crossing a threshold sends one `anomaly` alert to the alert
destinations, and does nothing else: no client is banned, rate limited or
blocked.
- An invalid setting stops the process at start with one log line naming the - An invalid setting stops the process at start with one log line naming the
variable and what is wrong with the value. The settings in effect are logged variable and what is wrong with the value. The settings in effect are logged
on one line at start, with tokens, keys and webhook paths shown as `********`. on one line at start, with tokens, keys and webhook paths shown as `********`.
@@ -296,7 +291,7 @@ account, a key, a token or an alert destination stays off until you give one.
Every setting the code reads, with its default: Every setting the code reads, with its default:
| Variable | Default | What it does | | Variable | Default | What it does |
| --------------------------------------------------- | ---------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | ------------------------------------------------- | ---------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `SWW_COMMAND` | `rproxy` | Which command the image runs, and its health check checks: `rproxy` or `waf`. | | `SWW_COMMAND` | `rproxy` | Which command the image runs, and its health check checks: `rproxy` or `waf`. |
| `SWW_RPROXY_WAF_BACKEND` | required | `waf`'s URL, such as `http://waf:8080`. | | `SWW_RPROXY_WAF_BACKEND` | required | `waf`'s URL, such as `http://waf:8080`. |
| `SWW_RPROXY_UPSTREAM_URL` | `http://127.0.0.1:8081` | The app's URL: a scheme, a host and an optional port, nothing more. | | `SWW_RPROXY_UPSTREAM_URL` | `http://127.0.0.1:8081` | The app's URL: a scheme, a host and an optional port, nothing more. |
@@ -313,7 +308,7 @@ Every setting the code reads, with its default:
| `SWW_WAF_MODE` | `enforce` | `observe` refuses nothing and makes no ban; each log line says what `enforce` would have done. | | `SWW_WAF_MODE` | `enforce` | `observe` refuses nothing and makes no ban; each log line says what `enforce` would have done. |
| `SWW_WAF_IPV6_GROUP_PREFIX` | `64` | The prefix length of the IPv6 netblock that counts as one client, from 32 to 128. | | `SWW_WAF_IPV6_GROUP_PREFIX` | `64` | The prefix length of the IPv6 netblock that counts as one client, from 32 to 128. |
| `SWW_WAF_MAX_TRACKED_CLIENTS` | `20000` | The most clients kept, in memory and in `clients.json`; the least recently seen is dropped first. | | `SWW_WAF_MAX_TRACKED_CLIENTS` | `20000` | The most clients kept, in memory and in `clients.json`; the least recently seen is dropped first. |
| `SWW_WAF_MAX_BANS` | `5000` | The most bans `waf` keeps for broken limits and attacks; bans an admin made and bans for AbuseIPDB's blacklist do not count. | | `SWW_WAF_MAX_BANS` | `5000` | The most bans `waf` keeps of its own; bans an admin made do not count. |
| `SWW_WAF_ALLOW_NETS` | empty | Netblocks whose clients skip every check, such as your monitoring. | | `SWW_WAF_ALLOW_NETS` | empty | Netblocks whose clients skip every check, such as your monitoring. |
| `SWW_WAF_RATE_LIMIT_EXEMPT_NETS` | empty | Netblocks whose clients are exempt from rate limiting and the byte limits. | | `SWW_WAF_RATE_LIMIT_EXEMPT_NETS` | empty | Netblocks whose clients are exempt from rate limiting and the byte limits. |
| `SWW_WAF_DENY_NETS` | empty | Netblocks whose clients are refused. | | `SWW_WAF_DENY_NETS` | empty | Netblocks whose clients are refused. |
@@ -361,15 +356,12 @@ Every setting the code reads, with its default:
| `SWW_WAF_BLOCKLIST_ACTION` | `deny` | What a client on a blocklist gets: `deny`, `limit:<percent>` of every limit, or `log`. | | `SWW_WAF_BLOCKLIST_ACTION` | `deny` | What a client on a blocklist gets: `deny`, `limit:<percent>` of every limit, or `log`. |
| `SWW_WAF_DNSBL_ZONES` | empty | DNS blocklists asked about each new client in the background, such as `dnsbl.dronebl.org`. | | `SWW_WAF_DNSBL_ZONES` | empty | DNS blocklists asked about each new client in the background, such as `dnsbl.dronebl.org`. |
| `SWW_WAF_DNSBL_RESOLVER` | unset | The resolver the zones are asked through, an IP address with an optional port; unset, the host's resolver is used. | | `SWW_WAF_DNSBL_RESOLVER` | unset | The resolver the zones are asked through, an IP address with an optional port; unset, the host's resolver is used. |
| `SWW_WAF_ABUSEIPDB_KEY` | unset | The key of your AbuseIPDB account. Set, a client that has committed an offence is checked in the background, and the clients on AbuseIPDB's blacklist are banned. | | `SWW_WAF_ABUSEIPDB_KEY` | unset | The key of your AbuseIPDB account; set, a client that has committed an offence is checked in the background. |
| `SWW_WAF_ABUSEIPDB_MIN_SCORE` | `75` | The least AbuseIPDB abuse confidence score, from 0 to 100, that is a hit. | | `SWW_WAF_ABUSEIPDB_MIN_SCORE` | `75` | The least AbuseIPDB abuse confidence score, from 0 to 100, that is a hit. |
| `SWW_WAF_ABUSEIPDB_DAILY_BUDGET` | `900` | The most AbuseIPDB checks made in a day. | | `SWW_WAF_ABUSEIPDB_DAILY_BUDGET` | `900` | The most AbuseIPDB checks made in a day. |
| `SWW_WAF_DNSBL_ABUSEIPDB_ACTION` | `limit:25` | What a client listed by a DNS blocklist, or scored a hit by AbuseIPDB, gets: `deny`, `limit:<percent>` or `log`. | | `SWW_WAF_DNSBL_ABUSEIPDB_ACTION` | `limit:25` | What a client listed by a DNS blocklist, or scored a hit by AbuseIPDB, gets: `deny`, `limit:<percent>` or `log`. |
| `SWW_WAF_IP_REPUTATION_CACHE_TTL` | `24h` | How long a DNS blocklist verdict or an AbuseIPDB score is used. | | `SWW_WAF_IP_REPUTATION_CACHE_TTL` | `24h` | How long a DNS blocklist verdict or an AbuseIPDB score is used. |
| `SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS` | `2000` | How long a DNS blocklist query or an AbuseIPDB check may take, in milliseconds. | | `SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS` | `2000` | How long a DNS blocklist query or an AbuseIPDB check may take, in milliseconds. |
| `SWW_WAF_ABUSEIPDB_THRESHOLD` | `90` | The least AbuseIPDB score, from 25 to 100, at which a client on its blacklist is banned. Do not set it below 50 to 75: lower scores list ordinary visitors. |
| `SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS` | `60` | How often AbuseIPDB's blacklist is fetched again, in minutes. |
| `SWW_WAF_ABUSEIPDB_BAN_DAYS` | `30` | How long a client on AbuseIPDB's blacklist is banned, in days. |
| `SWW_WAF_STATE_DIR` | `/var/lib/sww` | The directory of the state files, an absolute path. | | `SWW_WAF_STATE_DIR` | `/var/lib/sww` | The directory of the state files, an absolute path. |
| `SWW_WAF_BANS_WRITE_DELAY` | `10s` | How long `bans.json` waits to be written after a ban is made, lifted or made permanent. | | `SWW_WAF_BANS_WRITE_DELAY` | `10s` | How long `bans.json` waits to be written after a ban is made, lifted or made permanent. |
| `SWW_WAF_STATE_WRITE_INTERVAL` | `15m` | How often every state and cache file is written. | | `SWW_WAF_STATE_WRITE_INTERVAL` | `15m` | How often every state and cache file is written. |
@@ -392,29 +384,29 @@ Every setting the code reads, with its default:
| `SWW_WAF_ALERT_EVENTS` | `ban,permanent_ban,core_rule_set_match,anomaly,reputation_hit,source_failure,file_error` | The events alerts are sent for; the hourly `summary` is sent whatever it says. | | `SWW_WAF_ALERT_EVENTS` | `ban,permanent_ban,core_rule_set_match,anomaly,reputation_hit,source_failure,file_error` | The events alerts are sent for; the hourly `summary` is sent whatever it says. |
| `SWW_WAF_ALERT_COOLDOWN` | `15m` | How long after an alert its repeats are held back and counted, a duration or `off`. | | `SWW_WAF_ALERT_COOLDOWN` | `15m` | How long after an alert its repeats are held back and counted, a duration or `off`. |
| `SWW_WAF_ALERT_MAX_PER_HOUR` | `60` | The most alerts sent in an hour, a number or `off`; the rest are counted in the hour's `summary`. | | `SWW_WAF_ALERT_MAX_PER_HOUR` | `60` | The most alerts sent in an hour, a number or `off`; the rest are counted in the hour's `summary`. |
| `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when one client makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. | | `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_MINUTE` | `off` | Alert when one client makes more requests than this in a minute, a count. |
| `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when one client makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. | | `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_HOUR` | `off` | Alert when one client makes more requests than this in an hour, a count. |
| `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one client in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. | | `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for one client in a minute, a size. |
| `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one client in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. | | `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for one client in an hour, a size. |
| `SWW_WAF_ANOMALY_NET_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when the netblock around a client makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. | | `SWW_WAF_ANOMALY_NET_REQUESTS_PER_MINUTE` | `off` | Alert when the netblock around a client makes more requests than this in a minute, a count. |
| `SWW_WAF_ANOMALY_NET_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when the netblock around a client makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. | | `SWW_WAF_ANOMALY_NET_REQUESTS_PER_HOUR` | `off` | Alert when the netblock around a client makes more requests than this in an hour, a count. |
| `SWW_WAF_ANOMALY_NET_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the netblock around a client in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. | | `SWW_WAF_ANOMALY_NET_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for the netblock around a client in a minute, a size. |
| `SWW_WAF_ANOMALY_NET_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the netblock around a client in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. | | `SWW_WAF_ANOMALY_NET_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for the netblock around a client in an hour, a size. |
| `SWW_WAF_ANOMALY_NET_V4_PREFIX` | `24` | The prefix length of the netblock around an IPv4 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 32. Those thresholds only send alerts. | | `SWW_WAF_ANOMALY_NET_V4_PREFIX` | `24` | The prefix length of the netblock around an IPv4 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 32. |
| `SWW_WAF_ANOMALY_NET_V6_PREFIX` | `48` | The prefix length of the netblock around an IPv6 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 128. Those thresholds only send alerts. | | `SWW_WAF_ANOMALY_NET_V6_PREFIX` | `48` | The prefix length of the netblock around an IPv6 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 128. |
| `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when one AS number makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. | | `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE` | `off` | Alert when one AS number makes more requests than this in a minute, a count. |
| `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when one AS number makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. | | `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR` | `off` | Alert when one AS number makes more requests than this in an hour, a count. |
| `SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one AS number in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. | | `SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for one AS number in a minute, a size. |
| `SWW_WAF_ANOMALY_ASN_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one AS number in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. | | `SWW_WAF_ANOMALY_ASN_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for one AS number in an hour, a size. |
| `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when the whole service gets more requests than this in a minute, a count. No client is banned, rate limited or blocked. | | `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_MINUTE` | `off` | Alert when the whole service gets more requests than this in a minute, a count. |
| `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when the whole service gets more requests than this in an hour, a count. No client is banned, rate limited or blocked. | | `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_HOUR` | `off` | Alert when the whole service gets more requests than this in an hour, a count. |
| `SWW_WAF_ANOMALY_TOTAL_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the whole service in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. | | `SWW_WAF_ANOMALY_TOTAL_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for the whole service in a minute, a size. |
| `SWW_WAF_ANOMALY_TOTAL_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the whole service in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. | | `SWW_WAF_ANOMALY_TOTAL_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for the whole service in an hour, a size. |
| `SWW_WAF_ANOMALY_WATCHED_NETS` | empty | Named netblocks the `SWW_WAF_ANOMALY_WATCHED_NET_` thresholds count each as a whole, such as `office=203.0.113.0/24,scraper-x=198.51.100.0/22`. Those thresholds only send alerts. | | `SWW_WAF_ANOMALY_WATCHED_NETS` | empty | Named netblocks the `SWW_WAF_ANOMALY_WATCHED_NET_` thresholds count each as a whole, such as `office=203.0.113.0/24,scraper-x=198.51.100.0/22`. |
| `SWW_WAF_ANOMALY_WATCHED_NET_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when a watched netblock makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. | | `SWW_WAF_ANOMALY_WATCHED_NET_REQUESTS_PER_MINUTE` | `off` | Alert when a watched netblock makes more requests than this in a minute, a count. |
| `SWW_WAF_ANOMALY_WATCHED_NET_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when a watched netblock makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. | | `SWW_WAF_ANOMALY_WATCHED_NET_REQUESTS_PER_HOUR` | `off` | Alert when a watched netblock makes more requests than this in an hour, a count. |
| `SWW_WAF_ANOMALY_WATCHED_NET_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for a watched netblock in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. | | `SWW_WAF_ANOMALY_WATCHED_NET_BYTES_PER_MINUTE` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for a watched netblock in a minute, a size. |
| `SWW_WAF_ANOMALY_WATCHED_NET_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for a watched netblock in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. | | `SWW_WAF_ANOMALY_WATCHED_NET_BYTES_PER_HOUR` | `off` | Alert when more than this many bytes, as `SWW_WAF_BYTES_COUNTED` counts them, are counted for a watched netblock in an hour, a size. |
[`docs/settings.md`](docs/settings.md) gives the values each setting takes in [`docs/settings.md`](docs/settings.md) gives the values each setting takes in
full, and the limits that are fixed rather than settings. full, and the limits that are fixed rather than settings.
@@ -424,8 +416,8 @@ full, and the limits that are fixed rather than settings.
Each example is a complete `compose.yaml`. `smallwebwaf` is the tag Each example is a complete `compose.yaml`. `smallwebwaf` is the tag
`make docker` gives the image; on another host, use the name you pushed it `make docker` gives the image; on another host, use the name you pushed it
under, by digest. Docker Compose puts the services on a private network of its under, by digest. Docker Compose puts the services on a private network of its
own, where each reaches the others by service name. Each example but "On the own, where each reaches the others by service name. Each example publishes
open internet" publishes `rproxy`'s port on the host, for trying it out. `rproxy`'s port on the host, for trying it out.
### Smallest setup ### Smallest setup
@@ -457,142 +449,6 @@ volumes:
waf-state: waf-state:
``` ```
### On the open internet
One gitea at `git.example.com`, with traefik in front of `rproxy`. traefik gets
the certificate from Let's Encrypt and is the only service with published ports.
`waf` sets what most sites set. More `waf` settings follow, commented out, each
with what it does and when to set it.
```yaml
services:
traefik:
# traefik v3.7.19, 2026-11-13
image: traefik@sha256:a6c0194dda77bcb804824580f36fa7edc56e315b1acfe933366c8b52844a78a4
restart: unless-stopped
command:
- --providers.docker=true
- --providers.docker.exposedByDefault=false
- --entryPoints.web.address=:80
- --entryPoints.websecure.address=:443
# Redirect plain HTTP to HTTPS.
- --entryPoints.web.http.redirections.entryPoint.to=websecure
- --entryPoints.web.http.redirections.entryPoint.scheme=https
# A Let's Encrypt certificate for each site, by the TLS-ALPN
# challenge on port 443.
- --entryPoints.websecure.http.tls.certResolver=le
- --certificatesResolvers.le.acme.email=hostmaster@example.com
- --certificatesResolvers.le.acme.storage=/letsencrypt/acme.json
- --certificatesResolvers.le.acme.tlsChallenge=true
# Give a request 10 minutes to arrive, rather than 1, for the
# large git pushes the rproxy lines below accept.
# - --entryPoints.websecure.transport.respondingTimeouts.readTimeout=600s
ports:
- "80:80"
- "443:443"
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
- acme:/letsencrypt
app:
# Your app: here a gitea, which listens on port 3000.
image: registry.example/app@sha256:0000000000000000000000000000000000000000000000000000000000000000
restart: unless-stopped
rproxy:
image: smallwebwaf
restart: unless-stopped
environment:
SWW_RPROXY_WAF_BACKEND: http://waf:8080
SWW_RPROXY_UPSTREAM_URL: http://app:3000
# Accept git pushes up to 1 GiB, rather than 100 MiB, sent in
# up to 10 minutes, rather than 1, with traefik's readTimeout
# line above.
# SWW_RPROXY_MAX_REQUEST_SIZE: 1G
# SWW_RPROXY_MAX_REQUEST_TIME_MS: "600000"
labels:
- traefik.enable=true
- traefik.http.routers.git.rule=Host(`git.example.com`)
- traefik.http.routers.git.entryPoints=websecure
- traefik.http.services.git.loadBalancer.server.port=8080
waf:
image: smallwebwaf
restart: unless-stopped
environment:
SWW_COMMAND: waf
# The name in every log line, alert and metric. The default,
# the container's host name, is a random id under Compose.
SWW_WAF_INSTANCE_NAME: host1/waf
# The app each hostname serves. Only that app's fixes to the
# Core Rule Set apply there.
SWW_WAF_APPS: git.example.com:gitea
# Monitoring skips every check.
SWW_WAF_ALLOW_NETS: 192.0.2.10
# The office and the CI runner, which would exceed the rate
# limits, are never rate limited.
SWW_WAF_RATE_LIMIT_EXEMPT_NETS: 203.0.113.0/24,198.51.100.7
# WordPress paths, which only scanners ask a gitea for. One
# request for one bans the client.
SWW_WAF_TRAP_PATHS: /wp-login.php,/xmlrpc.php
# Spamhaus's list of netblocks hijacked or leased by
# criminals. Their clients are refused.
SWW_WAF_BLOCKLIST_URLS: https://www.spamhaus.org/drop/drop.txt
# The token for the ban endpoints, to lift a ban without
# editing bans.json.
SWW_WAF_ADMIN_TOKEN_FILE: /etc/sww/tokens/admin
# Alerts, such as one for each ban, to an ntfy topic.
SWW_WAF_ALERT_NTFY_URL: https://ntfy.example.com/smallwebwaf
# Refuse nothing and ban no one, for the first days. Each log
# line says what enforce mode would have done.
# SWW_WAF_MODE: observe
# The access token for an ntfy topic that is not public.
# SWW_WAF_ALERT_NTFY_TOKEN_FILE: /etc/sww/tokens/ntfy
# The token for /_sww/metrics, for Prometheus. Unset, the
# metrics are off.
# SWW_WAF_METRICS_TOKEN_FILE: /etc/sww/tokens/metrics
# 600 requests an hour rather than 1500, to ban slower
# scrapers. A fast human reader can reach it too.
# SWW_WAF_RATE_LIMIT_PER_HOUR: "600"
# A quarter of every limit for clients in two hosting
# networks, where scrapers run rather than people.
# SWW_WAF_TAG_LIMIT_PERCENT: asn=14061:25,asn=16276:25
# For a site only for people in a few countries. Clients
# elsewhere are refused, and so are any the lookup cannot place.
# SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES: de,at,ch
# Close a refused request's connection rather than answer
# 403. traefik then answers 502, so a scraper cannot tell a
# ban from a broken site.
# SWW_WAF_BAN_RESPONSE: close
# A copy of every log line to a syslog server.
# SWW_WAF_LOG_REMOTE_URL: syslog+tls://logs.example.com:6514
volumes:
- waf-state:/var/lib/sww
- /srv/sww/tokens:/etc/sww/tokens:ro
volumes:
acme:
waf-state:
```
Change these before the first start:
- `git.example.com`, in the router rule and in `SWW_WAF_APPS`;
- the email address for Let's Encrypt;
- the ntfy topic's URL;
- the app's image and port;
- the netblocks, which are documentation addresses.
Make the admin token file first, as for any token file:
```sh
mkdir -p /srv/sww/tokens
openssl rand -hex 32 > /srv/sww/tokens/admin
chown 65532:65532 /srv/sww/tokens/admin
chmod 0400 /srv/sww/tokens/admin
```
### Custom per-address limits ### Custom per-address limits
For a site with no API: For a site with no API:
@@ -770,10 +626,8 @@ status for a size or time limit, such as `413` for a body too large.
- A client is one IPv4 address or one IPv6 netblock of - A client is one IPv4 address or one IPv6 netblock of
`SWW_WAF_IPV6_GROUP_PREFIX`, since one abuser usually controls a whole /64. `SWW_WAF_IPV6_GROUP_PREFIX`, since one abuser usually controls a whole /64.
- Each limit is counted over a sliding window with a few integers per client. - Each limit is counted over a sliding window with a few integers per client.
The table of clients and the bans for limits and attacks each have a bound The table of clients and the table of bans each have a bound
(`SWW_WAF_MAX_TRACKED_CLIENTS`, `SWW_WAF_MAX_BANS`). The bans for AbuseIPDB's (`SWW_WAF_MAX_TRACKED_CLIENTS`, `SWW_WAF_MAX_BANS`), so memory is bounded too.
blacklist are bounded by the clients it listed over the last
`SWW_WAF_ABUSEIPDB_BAN_DAYS` days. So memory is bounded too.
- A ban refuses every request from its netblock until it ends. Requests refused - A ban refuses every request from its netblock until it ends. Requests refused
under a ban count toward no limit. A permanent ban ends only when an admin under a ban count toward no limit. A permanent ban ends only when an admin
lifts it. lifts it.
@@ -896,8 +750,6 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
- `cmd/sww`: the binary, which only calls `internal/sww`. - `cmd/sww`: the binary, which only calls `internal/sww`.
- `internal/sww`: the process, as the command it is given, and the health check. - `internal/sww`: the process, as the command it is given, and the health check.
It is the only package that imports `go.uber.org/fx`, which builds the parts
of `waf` and `rproxy`.
- `internal/config`: reads the settings, the one place they are read. - `internal/config`: reads the settings, the one place they are read.
- `internal/rproxy`: the reverse proxy in front of one app. - `internal/rproxy`: the reverse proxy in front of one app.
- `internal/waf`: `waf`'s server, which runs the checks and answers its own - `internal/waf`: `waf`'s server, which runs the checks and answers its own
@@ -906,11 +758,9 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
`internal/coreruleset`, `internal/geojs`, `internal/lookup`, `internal/coreruleset`, `internal/geojs`, `internal/lookup`,
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and `internal/reputation`, `internal/routewatch`, `internal/anomaly` and
`internal/alerts`: one check or source each. `internal/alerts`: one check or source each.
- `internal/state`: the state and cache files. - `internal/state`, `internal/requestlog`, `internal/remotelog` and
- `internal/requestlog`: the request log. `internal/metrics`: the state and cache files, the log lines, the syslog
- `internal/processlog`: the process's own messages. sender and the metrics.
- `internal/remotelog`: the syslog sender.
- `internal/metrics`: the metrics.
## Rationale ## Rationale
+20 -52
View File
@@ -69,9 +69,7 @@ never as a time.
that command's settings, and runs `internal/rproxy` until `SIGTERM` or that command's settings, and runs `internal/rproxy` until `SIGTERM` or
`SIGINT` (see "`rproxy`"). `sww healthcheck` is the image's health check. `SIGINT` (see "`rproxy`"). `sww healthcheck` is the image's health check.
With no command, or one it does not know, the binary prints the commands With no command, or one it does not know, the binary prints the commands
and exits with status 2. `internal/sww` is the only package that imports and exits with status 2.
`go.uber.org/fx`. fx builds the parts of `waf` and `rproxy`, and starts
and stops their background work.
- `internal/config`: reads the settings, the one place they are read. - `internal/config`: reads the settings, the one place they are read.
- `internal/waf`: `waf`'s server. It runs the checks in `check`, and holds - `internal/waf`: `waf`'s server. It runs the checks in `check`, and holds
`waf`'s question, answer and report. It answers `waf`'s own endpoints in `waf`'s question, answer and report. It answers `waf`'s own endpoints in
@@ -94,8 +92,7 @@ never as a time.
to count and report failures. to count and report failures.
- `internal/lookup`: looks clients up in the lookup database. - `internal/lookup`: looks clients up in the lookup database.
- `internal/reputation`: fetches the lists, asks the DNS blocklists and - `internal/reputation`: fetches the lists, asks the DNS blocklists and
AbuseIPDB in the background, and keeps their answers. It also fetches AbuseIPDB in the background, and keeps their answers.
AbuseIPDB's blacklist, which `internal/waf` bans.
- `internal/routewatch`: asks a routewatch instance about clients in the - `internal/routewatch`: asks a routewatch instance about clients in the
background, and keeps its answers. background, and keeps its answers.
- `internal/anomaly`: the anomaly counters and their alerts. - `internal/anomaly`: the anomaly counters and their alerts.
@@ -104,10 +101,8 @@ never as a time.
its own. its own.
- `internal/state`: reads the state and cache files at start, takes in edits - `internal/state`: reads the state and cache files at start, takes in edits
while running, and writes the files when due and at the stop. while running, and writes the files when due and at the stop.
- `internal/requestlog`: the request log, as JSON lines on stdout. - `internal/requestlog` and `internal/remotelog`: the lines on stdout, and
- `internal/processlog`: the process lines on stdout, which hold the sending them to the syslog server.
process's own messages.
- `internal/remotelog`: sends the lines on stdout to the syslog server.
- `internal/metrics`: the metrics, in the Prometheus text format. - `internal/metrics`: the metrics, in the Prometheus text format.
- `Dockerfile`: the lint and test phases, then the image. - `Dockerfile`: the lint and test phases, then the image.
`deploy/example-app` is an app built on the image, with `rproxy` in its `deploy/example-app` is an app built on the image, with `rproxy` in its
@@ -243,15 +238,13 @@ apply.
load. load.
- Memory is bounded. When the table of clients holds - Memory is bounded. When the table of clients holds
`SWW_WAF_MAX_TRACKED_CLIENTS`, the least recently seen client is dropped `SWW_WAF_MAX_TRACKED_CLIENTS`, the least recently seen client is dropped
first, with its history. The bans for broken limits and clear signs of attack first, with its history. The bans are bounded by `SWW_WAF_MAX_BANS` (see
are bounded by `SWW_WAF_MAX_BANS` (see "Bans"). Those for AbuseIPDB's "Bans"). The GeoJS answers, the reputation verdicts and scores, the DNS
blacklist are bounded by the clients it listed over the last queries under way and the anomaly counters have fixed bounds, listed under
`SWW_WAF_ABUSEIPDB_BAN_DAYS` days (see "Reputation"). The GeoJS answers, the "Fixed limits" in `docs/settings.md`. Past its bound, the GeoJS answer or
reputation verdicts and scores, the DNS queries under way and the anomaly anomaly counter used longest ago goes first, and the verdict or score fetched
counters have fixed bounds, listed under "Fixed limits" in `docs/settings.md`. longest ago. Past the bound on DNS queries under way, a zone is asked at the
Past its bound, the GeoJS answer or anomaly counter used longest ago goes client's next request.
first, and the verdict or score fetched longest ago. Past the bound on DNS
queries under way, a zone is asked at the client's next request.
## Bans ## Bans
@@ -306,9 +299,6 @@ to a ban, each by its own rule:
Once it ends, only new requests can break a limit again. The client's Once it ends, only new requests can break a limit again. The client's
history keeps its totals. history keeps its totals.
A client on AbuseIPDB's blacklist is banned before it sends a request, with the
cause `abuseipdb` (see "Reputation").
A Core Rule Set match refuses only the request it matched, with `403` and A Core Rule Set match refuses only the request it matched, with `403` and
without a ban, because the Core Rule Set has false positives. A `block` rule without a ban, because the Core Rule Set has false positives. A `block` rule
does the same. Those refusals count toward the refusal limit, so a client that does the same. Those refusals count toward the refusal limit, so a client that
@@ -335,12 +325,11 @@ does not count toward a longer ban; it refuses nothing. Deleting its entry from
A permanent ban does not run out. It ends when an admin lifts it, or when `waf` A permanent ban does not run out. It ends when an admin lifts it, or when `waf`
drops it from a full table, as below. When `waf` holds `SWW_WAF_MAX_BANS` bans drops it from a full table, as below. When `waf` holds `SWW_WAF_MAX_BANS` bans
for broken limits and clear signs of attack, it drops the earliest such ban of of its own, it drops the earliest ban of one netblock: the netblock that has
one netblock: the netblock that has gone longest without a request. A past, gone longest without a request. A past, active or permanent ban is dropped
active or permanent ban is dropped alike. A scanner whose ban was dropped that alike. A scanner whose ban was dropped that way is refused and banned again by
way is refused and banned again by its next probe. Bans whose cause is `admin` its next probe. A ban whose cause is `admin` is never dropped and does not count
or `abuseipdb` do not count toward `SWW_WAF_MAX_BANS` and are never dropped to toward `SWW_WAF_MAX_BANS`, so such bans can never fill the table.
make room, so they can never push out the others.
A request refused under a ban that runs out is logged with the action A request refused under a ban that runs out is logged with the action
`temporarily_banned`. Under a permanent ban the action is `permanently_banned`. `temporarily_banned`. Under a permanent ban the action is `permanently_banned`.
@@ -2047,26 +2036,6 @@ host, sharing one address, would often fetch too close together.
- AbuseIPDB answers other than `200`; - AbuseIPDB answers other than `200`;
- the answer gives no score; - the answer gives no score;
- no answer comes within `SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS`. - no answer comes within `SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS`.
- While `SWW_WAF_ABUSEIPDB_KEY` is set, AbuseIPDB's blacklist is fetched at the
start and every `SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS` minutes
(default `60`). It lists the clients AbuseIPDB scores at
`SWW_WAF_ABUSEIPDB_THRESHOLD` (default `90`) or more. The blacklist itself is
not kept.
- Each client on it is banned for `SWW_WAF_ABUSEIPDB_BAN_DAYS` days (default
`30`), with the cause `abuseipdb`. The ban covers the client's netblock,
as any ban does.
- A netblock that already has an active ban is left as it is. So is one
whose `abuseipdb` ban an admin lifted, until that ban would have ended.
- An `abuseipdb` ban that has ended is forgotten at the next of these:
- a fetch of the blacklist in `enforce` mode;
- a read of `bans.json`, at the start or after an admin's edit.
- In `observe` mode the blacklist makes no ban.
- The threshold takes 25 to 100. It should not be set below 50 to 75, since
lower scores list ordinary visitors. AbuseIPDB serves free accounts the
blacklist at the threshold 100, whatever is asked for.
- AbuseIPDB limits the fetches of the blacklist a day: 5 for a free account.
A fetch over the limit fails, and is logged and alerted. The default
refresh fetches it 24 times a day.
- `SWW_WAF_DNSBL_ABUSEIPDB_ACTION`, for DNS blocklist and AbuseIPDB hits, is - `SWW_WAF_DNSBL_ABUSEIPDB_ACTION`, for DNS blocklist and AbuseIPDB hits, is
`limit:25` by default. Such verdicts are less certain than a blocklist, so a `limit:25` by default. Such verdicts are less certain than a blocklist, so a
listed client gets a quarter of every request and byte limit rather than a listed client gets a quarter of every request and byte limit rather than a
@@ -2081,10 +2050,9 @@ host, sharing one address, would often fetch too close together.
- Alerts go to a webhook as JSON, and to Slack and ntfy as formatted messages. - Alerts go to a webhook as JSON, and to Slack and ntfy as formatted messages.
`docs/alerts.md` lists the events, the JSON object and the messages. `docs/alerts.md` lists the events, the JSON object and the messages.
`SWW_WAF_ALERT_EVENTS` chooses which of the seven event types are sent; the `SWW_WAF_ALERT_EVENTS` chooses which of the seven event types are sent; the
hourly `summary` is sent whatever it says. The bans an admin makes, and the hourly `summary` is sent whatever it says. The bans an admin makes raise no
bans for AbuseIPDB's blacklist, raise no alert. In `observe` mode, a request alert. In `observe` mode, a request that would have made a ban still raises
that would have made a ban still raises the ban's alert, with `mode` set to the ban's alert, with `mode` set to `observe` in its `detail`.
`observe` in its `detail`.
- A repeat is an alert with the same event, and the same netblock, source, file - A repeat is an alert with the same event, and the same netblock, source, file
or anomaly scope, as one sent less than `SWW_WAF_ALERT_COOLDOWN` before. An or anomaly scope, as one sent less than `SWW_WAF_ALERT_COOLDOWN` before. An
anomaly scope is one of the things counted under "Anomaly thresholds". A anomaly scope is one of the things counted under "Anomaly thresholds". A
@@ -2232,7 +2200,7 @@ lists the files, what each holds, and their sizes.
these stop the process with a message naming the file and position: these stop the process with a message naming the file and position:
- a file that does not parse, or has an unknown `version`; - a file that does not parse, or has an unknown `version`;
- an entry without a field it needs; - an entry without a field it needs;
- a ban whose cause is not `limit`, `attack`, `admin` or `abuseipdb`; - a ban whose cause is not one of the four;
- a ban whose `expires` is not after its `start`. - a ban whose `expires` is not after its `start`.
Starting with empty state would silently forgive every repeat offender. Starting with empty state would silently forgive every repeat offender.
+2 -3
View File
@@ -6,7 +6,7 @@ back or dropped.
## Events ## Events
- `ban`: a ban `waf` made for a broken limit or a clear sign of attack. - `ban`: a ban `waf` made.
- `permanent_ban`: a permanent ban it made, or an existing ban made permanent by - `permanent_ban`: a permanent ban it made, or an existing ban made permanent by
a request. a request.
- `core_rule_set_match`: the Core Rule Set scored a request at or over the - `core_rule_set_match`: the Core Rule Set scored a request at or over the
@@ -68,8 +68,7 @@ on one line:
"earlier_bans": { "earlier_bans": {
"limit": 0, "limit": 0,
"attack": 0, "attack": 0,
"admin": 0, "admin": 0
"abuseipdb": 0
} }
} }
}, },
+2 -14
View File
@@ -42,8 +42,8 @@ chooses between the two (see "The image's command" below).
client's group, from 32 to 128. client's group, from 32 to 128.
- `SWW_WAF_MAX_TRACKED_CLIENTS` (default `20000`): the most clients kept, a - `SWW_WAF_MAX_TRACKED_CLIENTS` (default `20000`): the most clients kept, a
whole number above zero. whole number above zero.
- `SWW_WAF_MAX_BANS` (default `5000`): the most bans `waf` keeps for broken - `SWW_WAF_MAX_BANS` (default `5000`): how many of its own bans `waf` keeps, a
limits and attacks, a whole number above zero. whole number above zero.
## Tokens ## Tokens
@@ -338,13 +338,6 @@ tag that no request can get stops the start. A request can get only these:
or AbuseIPDB's score is used. A duration. or AbuseIPDB's score is used. A duration.
- `SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS` (default `2000`): how long a query to a - `SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS` (default `2000`): how long a query to a
zone or a check with AbuseIPDB may take. A time in milliseconds. zone or a check with AbuseIPDB may take. A time in milliseconds.
- `SWW_WAF_ABUSEIPDB_THRESHOLD` (default `90`): the least abuse confidence score
of the clients on AbuseIPDB's blacklist, a whole number from 25 to 100. It
should not be set below 50 to 75, since lower scores list ordinary visitors.
- `SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS` (default `60`): how often
the blacklist is fetched again, in minutes. A whole number above zero.
- `SWW_WAF_ABUSEIPDB_BAN_DAYS` (default `30`): how long the ban of a client on
the blacklist lasts, in days. A whole number above zero.
## Alerting ## Alerting
@@ -371,11 +364,6 @@ tag that no request can get stops the start. A request can get only these:
## Anomaly thresholds ## Anomaly thresholds
An anomaly is more requests or bytes than one of these thresholds allows.
Crossing a threshold sends one `anomaly` alert to the destinations under
"Alerting", and does nothing else: no client is banned, rate limited or blocked.
`SWW_WAF_ALERT_COOLDOWN` holds back repeats of that alert.
Each scope has four thresholds: the most requests and the most body bytes Each scope has four thresholds: the most requests and the most body bytes
counted in a minute and in an hour. The bytes are those `SWW_WAF_BYTES_COUNTED` counted in a minute and in an hour. The bytes are those `SWW_WAF_BYTES_COUNTED`
names. Requests are counts, bytes are sizes. Each is `off` by default. The four names. Requests are counts, bytes are sizes. Each is `off` by default. The four
+5 -19
View File
@@ -21,22 +21,11 @@ top-level `version` of 1.
## `bans.json` ## `bans.json`
Active and past bans: Active and past bans: every ban an admin made, and up to `SWW_WAF_MAX_BANS` of
those `waf` made. Per entry:
- every ban an admin made;
- up to `SWW_WAF_MAX_BANS` of those `waf` made for a broken limit or an attack;
- the bans for AbuseIPDB's blacklist.
A ban for AbuseIPDB's blacklist that has ended is forgotten at the next of
these:
- a fetch of the blacklist in `enforce` mode;
- a read of `bans.json`, at the start or after an admin's edit.
Per entry:
- `netblock`, `start` and `expires` (`null` for a permanent ban); - `netblock`, `start` and `expires` (`null` for a permanent ban);
- `cause`: `attack`, `limit`, `admin` or `abuseipdb`; - `cause`: `attack`, `limit` or `admin`;
- `reason`: a short reason; - `reason`: a short reason;
- `lifted`: when an admin lifted it, if one did; - `lifted`: when an admin lifted it, if one did;
- `notes`: the notes below. - `notes`: the notes below.
@@ -71,8 +60,7 @@ ban. The notes:
- for a clear sign of attack, `rule_id` and `target`, or `trap_path`; - for a clear sign of attack, `rule_id` and `target`, or `trap_path`;
- `reputation`: the sources the client was on, each with its `source` and, for - `reputation`: the sources the client was on, each with its `source` and, for
AbuseIPDB, its `score`; AbuseIPDB, its `score`;
- `request`: the request that caused the ban, each text cut to 256 bytes, and - `request`: the request that caused the ban, each text cut to 256 bytes:
empty for a ban for AbuseIPDB's blacklist:
- `time`, `method` and `host`; - `time`, `method` and `host`;
- `path`, with its query string; - `path`, with its query string;
- `status` and `user_agent`; - `status` and `user_agent`;
@@ -163,9 +151,7 @@ A cache file. routewatch's answers from the last hour, one per line:
- `bans.json`: about 1.2 KiB per ban, at most about 2.5 KiB, so about 6 MiB at - `bans.json`: about 1.2 KiB per ban, at most about 2.5 KiB, so about 6 MiB at
the default `SWW_WAF_MAX_BANS`. Written every 15 minutes, a full file comes to the default `SWW_WAF_MAX_BANS`. Written every 15 minutes, a full file comes to
about 600 MiB of writes a day, and a hundred new bans a day add about 600 MiB about 600 MiB of writes a day, and a hundred new bans a day add about 600 MiB
more. A ban for AbuseIPDB's blacklist takes about 1 KiB. A free account's more.
blacklist lists 10,000 clients, so it adds at least 10 MiB, and 1 GiB of
writes a day.
- `lookups.json`: about 150 bytes per answer, under 1.5 GiB of writes a day when - `lookups.json`: about 150 bytes per answer, under 1.5 GiB of writes a day when
full. full.
- `routewatch.json`: about 120 bytes per answer, at most about 12 MiB. - `routewatch.json`: about 120 bytes per answer, at most about 12 MiB.
-4
View File
@@ -12,7 +12,6 @@ require (
github.com/oschwald/maxminddb-golang/v2 v2.7.0 github.com/oschwald/maxminddb-golang/v2 v2.7.0
github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_golang v1.24.1
github.com/spf13/viper v1.21.0 github.com/spf13/viper v1.21.0
go.uber.org/fx v1.24.0
golang.org/x/sys v0.48.0 golang.org/x/sys v0.48.0
) )
@@ -45,9 +44,6 @@ require (
github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect
github.com/valllabh/ocsf-schema-golang v1.0.3 // indirect github.com/valllabh/ocsf-schema-golang v1.0.3 // indirect
go.uber.org/dig v1.19.0 // indirect
go.uber.org/multierr v1.10.0 // indirect
go.uber.org/zap v1.26.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
golang.org/x/net v0.60.0 // indirect golang.org/x/net v0.60.0 // indirect
-8
View File
@@ -97,16 +97,8 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/valllabh/ocsf-schema-golang v1.0.3 h1:eR8k/3jP/OOqB8LRCtdJ4U+vlgd/gk5y3KMXoodrsrw= github.com/valllabh/ocsf-schema-golang v1.0.3 h1:eR8k/3jP/OOqB8LRCtdJ4U+vlgd/gk5y3KMXoodrsrw=
github.com/valllabh/ocsf-schema-golang v1.0.3/go.mod h1:sZ3as9xqm1SSK5feFWIR2CuGeGRhsM7TR1MbpBctzPk= github.com/valllabh/ocsf-schema-golang v1.0.3/go.mod h1:sZ3as9xqm1SSK5feFWIR2CuGeGRhsM7TR1MbpBctzPk=
go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4=
go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE=
go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg=
go.uber.org/fx v1.24.0/go.mod h1:AmDeGyS+ZARGKM4tlH4FY2Jr63VjbEDJHtqXTGP5hbo=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
-165
View File
@@ -1,165 +0,0 @@
package bans_test
import (
"net/netip"
"testing"
"time"
"sneak.berlin/go/smallwebwaf/internal/bans"
)
func TestBlacklistedNetblockIsBannedUntilTheGivenEnd(t *testing.T) {
t.Parallel()
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
now := midnight()
expires := now.Add(30 * day)
made := ledger.BanForAbuseIPDB(
now, expires, map[netip.Prefix]int64{netblock: 95},
)
if made != 1 || ledger.Made(bans.CauseAbuseIPDB) != 1 {
t.Fatalf("made %d bans, %d counted, want 1 and 1",
made, ledger.Made(bans.CauseAbuseIPDB))
}
ban, banned, madePermanent := ledger.Check(netblock.Addr(), now)
if !banned || madePermanent || !ban.Expires.Equal(expires) ||
ban.Cause != bans.CauseAbuseIPDB ||
ban.Reason != "on AbuseIPDB's blacklist with a score of 95" ||
len(ban.Notes.Reputation) != 1 ||
ban.Notes.Reputation[0].Source != "abuseipdb" ||
*ban.Notes.Reputation[0].Score != 95 {
t.Errorf(
"ban %+v, banned %t, made permanent %t, want the blacklist's "+
"ban until %s",
ban,
banned,
madePermanent,
expires,
)
}
if _, banned, _ := ledger.Check(netblock.Addr(), expires); banned {
t.Error("the ban refuses a request at its end")
}
}
func TestBlacklistLeavesANetblockWithAnActiveBanAsItIs(t *testing.T) {
t.Parallel()
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
now := midnight()
scores := map[netip.Prefix]int64{netblock: 100}
limitBan, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 0 {
t.Errorf("made %d bans over a ban for a limit, want none", made)
}
// Once that ban has ended the netblock is banned, and the blacklist
// fetched again makes no second ban while the first lasts.
later := limitBan.Expires
if made := ledger.BanForAbuseIPDB(later, later.Add(30*day), scores); made != 1 {
t.Errorf("made %d bans after the ban for a limit, want 1", made)
}
later = later.Add(time.Hour)
if made := ledger.BanForAbuseIPDB(later, later.Add(30*day), scores); made != 0 {
t.Errorf("made %d bans while the blacklist's lasts, want none", made)
}
wantBans(t, ledger, map[netip.Prefix]int{netblock: 2})
if got := ledger.Bans(netblock)[1].Notes.EarlierBans; got != (bans.EarlierBans{
Limit: 1,
}) {
t.Errorf("earlier bans %+v, want 1 for a limit", got)
}
}
func TestLiftedBlacklistBanHoldsUntilItsEnd(t *testing.T) {
t.Parallel()
ledger := bans.New(defaultParams())
netblock := netip.MustParsePrefix("203.0.113.9/32")
now := midnight()
expires := now.Add(30 * day)
scores := map[netip.Prefix]int64{netblock: 100}
ledger.BanForAbuseIPDB(now, expires, scores)
ledger.Lift(netblock.Addr(), now)
// The admin lifted it: the blacklist fetched again until its end bans
// nothing.
later := expires.Add(-time.Hour)
if made := ledger.BanForAbuseIPDB(later, later.Add(30*day), scores); made != 0 {
t.Errorf("made %d bans before the lifted ban's end, want none", made)
}
// From its end on, the blacklist bans the netblock again, and the
// lifted ban, ended, is forgotten.
if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), scores); made != 1 {
t.Errorf("made %d bans at the lifted ban's end, want 1", made)
}
wantBans(t, ledger, map[netip.Prefix]int{netblock: 1})
}
func TestEndedBlacklistBansAreForgottenAtTheNextFetch(t *testing.T) {
t.Parallel()
ledger := bans.New(defaultParams())
listed := netip.MustParsePrefix("203.0.113.9/32")
limited := netip.MustParsePrefix("203.0.113.10/32")
now := midnight()
expires := now.Add(30 * day)
ledger.BanForAbuseIPDB(now, expires, map[netip.Prefix]int64{listed: 100})
ledger.BanForLimit(limited, now, bans.Notes{})
// listed is off the blacklist when it is fetched after the ban's end.
if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), nil); made != 0 {
t.Errorf("made %d bans for an empty blacklist, want none", made)
}
wantBans(t, ledger, map[netip.Prefix]int{listed: 0, limited: 1})
}
func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) {
t.Parallel()
params := defaultParams()
params.MaxBans = 1
ledger := bans.New(params)
limited := netip.MustParsePrefix("203.0.113.1/32")
attacker := netip.MustParsePrefix("203.0.113.2/32")
now := midnight()
scores := map[netip.Prefix]int64{}
for i := range 3 {
scores[netip.PrefixFrom(netip.AddrFrom4([4]byte{198, 51, 100, byte(i)}), 32)] = 100
}
ledger.BanForLimit(limited, now, bans.Notes{})
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 3 {
t.Errorf("made %d bans for 3 netblocks, want 3", made)
}
wantBans(t, ledger, map[netip.Prefix]int{limited: 1})
// The ban for an attack takes the place of the one for a limit, and no
// blacklist ban goes.
ledger.BanForAttack(attacker, now, bans.Notes{})
want := map[netip.Prefix]int{limited: 0, attacker: 1}
for netblock := range scores {
want[netblock] = 1
}
wantBans(t, ledger, want)
}
+28 -155
View File
@@ -1,9 +1,8 @@
// Package bans is the ban ledger: the bans sww makes on the // Package bans is the ban ledger: the bans sww makes on the
// netblocks of clients that break a rate limit, a byte limit or the refusal // netblocks of clients that break a rate limit, a byte limit or the refusal
// limit or show a clear sign of attack, or are on AbuseIPDB's blacklist, and // limit or show a clear sign of attack, and those an admin makes, with their
// those an admin makes, with their notes, as the "Bans" section of SPEC.md // notes, as the "Bans" section of SPEC.md describes. The bans are kept in
// describes. The bans are kept in memory, and written to bans.json and read // memory, and written to bans.json and read from it by the state package.
// from it by the state package.
package bans package bans
import ( import (
@@ -27,9 +26,6 @@ const (
// CauseAdmin is a ban an admin made, or one sww made that an // CauseAdmin is a ban an admin made, or one sww made that an
// admin keeps. It is never dropped. // admin keeps. It is never dropped.
CauseAdmin = "admin" CauseAdmin = "admin"
// CauseAbuseIPDB is a ban sww made for a client on AbuseIPDB's
// blacklist. It is forgotten once it has ended.
CauseAbuseIPDB = "abuseipdb"
) )
// repeatFactor is how many times as long as the netblock's last ban a ban // repeatFactor is how many times as long as the netblock's last ban a ban
@@ -54,11 +50,10 @@ type Params struct {
// AttackBanDuration is how long a first ban for a clear sign of attack // AttackBanDuration is how long a first ban for a clear sign of attack
// lasts. // lasts.
AttackBanDuration time.Duration AttackBanDuration time.Duration
// MaxBans is the most bans held whose cause is CauseLimit or // MaxBans is the most bans held whose cause is not CauseAdmin, at
// CauseAttack, at least one. Past it, the earliest such ban of the // least one. Past it, the earliest such ban of the netblock that has
// netblock that has gone longest without a request is dropped. Bans // gone longest without a request is dropped. Bans whose cause is
// whose cause is CauseAdmin or CauseAbuseIPDB are held besides, and // CauseAdmin are held besides, and never dropped.
// never dropped this way.
MaxBans int MaxBans int
} }
@@ -68,11 +63,10 @@ type Ban struct {
Start time.Time Start time.Time
// Expires is when the ban ends, zero for a permanent ban. // Expires is when the ban ends, zero for a permanent ban.
Expires time.Time Expires time.Time
// Cause is CauseLimit, CauseAttack, CauseAdmin or CauseAbuseIPDB. // Cause is CauseLimit, CauseAttack or CauseAdmin.
Cause string Cause string
// Reason is a short text: for a ban sww made, the limit broken, the // Reason is a short text: for a ban sww made, the limit broken or
// rule that matched or AbuseIPDB's score; for an admin's, what the // the rule that matched; for an admin's, what the admin wrote.
// admin wrote.
Reason string Reason string
// Lifted is when an admin lifted the ban, zero while no admin has. A // Lifted is when an admin lifted the ban, zero while no admin has. A
// lifted ban refuses nothing, and does not make the netblock's next // lifted ban refuses nothing, and does not make the netblock's next
@@ -133,8 +127,7 @@ type Notes struct {
// log's reputation names them. It is left out when none did. // log's reputation names them. It is left out when none did.
Reputation []ReputationHit `json:"reputation,omitempty"` Reputation []ReputationHit `json:"reputation,omitempty"`
// Request is the request that broke the limit, or whose bytes broke // Request is the request that broke the limit, or whose bytes broke
// it, or that was the clear sign of attack. A ban for AbuseIPDB's // it, or that was the clear sign of attack.
// blacklist has none, and leaves it zero.
Request Request `json:"request"` Request Request `json:"request"`
// Requests is how many requests the netblock has sent since it was // Requests is how many requests the netblock has sent since it was
// first seen, and Refused how many of them the ban has refused so // first seen, and Refused how many of them the ban has refused so
@@ -160,7 +153,6 @@ type EarlierBans struct {
Limit int `json:"limit"` Limit int `json:"limit"`
Attack int `json:"attack"` Attack int `json:"attack"`
Admin int `json:"admin"` Admin int `json:"admin"`
AbuseIPDB int `json:"abuseipdb"`
} }
// Request is a request in a ban's notes. Each text is cut to 256 bytes. // Request is a request in a ban's notes. Each text is cut to 256 bytes.
@@ -188,8 +180,8 @@ type Ledger struct {
// netblocks holds each banned netblock's bans, oldest first. Check and // netblocks holds each banned netblock's bans, oldest first. Check and
// Find make each netblock they find the most recently seen. // Find make each netblock they find the most recently seen.
netblocks *simplelru.LRU[netip.Prefix, *[]Ban] netblocks *simplelru.LRU[netip.Prefix, *[]Ban]
// held is how many bans netblocks holds whose cause is CauseLimit or // held is how many bans netblocks holds whose cause is not CauseAdmin,
// CauseAttack, at most params.MaxBans. // at most params.MaxBans.
held int held int
// made is how many bans have been made since the start, by cause: by // made is how many bans have been made since the start, by cause: by
// the ledger, and by an admin, through BanForAdmin or in an edit of // the ledger, and by an admin, through BanForAdmin or in an edit of
@@ -206,9 +198,8 @@ type Ledger struct {
// New returns a Ledger with no ban yet. // New returns a Ledger with no ban yet.
func New(params Params) *Ledger { func New(params Params) *Ledger {
// The ledger drops bans itself, and never those whose cause is // The ledger drops bans itself, and never those whose cause is
// CauseAdmin or CauseAbuseIPDB to make room, however many there are, so // CauseAdmin, however many there are, so the LRU has no limit of its
// the LRU has no limit of its own: it keeps the netblocks in the order // own: it keeps the netblocks in the order they were last seen.
// they were last seen.
netblocks, err := simplelru.NewLRU[netip.Prefix, *[]Ban]( netblocks, err := simplelru.NewLRU[netip.Prefix, *[]Ban](
math.MaxInt, math.MaxInt,
nil, nil,
@@ -438,88 +429,6 @@ func (l *Ledger) BanForAdmin(
return ban return ban
} }
// BanForAbuseIPDB bans each netblock in scores at now until expires, and
// returns how many bans it made. scores are the netblocks of the clients
// on AbuseIPDB's blacklist, each with the highest score of its clients.
// A netblock is left as it is while a ban on it is active, or while a ban
// for the blacklist that an admin lifted has not reached its end: the
// admin's lift holds that long. Each ban whose cause is CauseAbuseIPDB
// and that has ended is forgotten first, so that the bans held for the
// blacklist are those of the clients listed within one ban's length.
func (l *Ledger) BanForAbuseIPDB(
now, expires time.Time, scores map[netip.Prefix]int64,
) int {
l.mu.Lock()
defer l.mu.Unlock()
changed := l.forgetEndedAbuseIPDBBans(now)
made := 0
for netblock, score := range scores {
netblock = netblock.Masked()
var notes Notes
if held, found := l.netblocks.Peek(netblock); found {
if activeBan(*held, now) != nil ||
liftedAbuseIPDBBan(*held, now) {
continue
}
notes.EarlierBans = earlierBans(*held)
}
// The source is named as the request log names AbuseIPDB.
notes.Reputation = []ReputationHit{
{Source: CauseAbuseIPDB, Score: &score},
}
l.add(Ban{
Netblock: netblock,
Start: now,
Expires: expires,
Cause: CauseAbuseIPDB,
Reason: fmt.Sprintf(
"on AbuseIPDB's blacklist with a score of %d", score,
),
Notes: notes,
})
made++
}
l.made[CauseAbuseIPDB] += made
if changed || made > 0 {
l.markChanged()
}
return made
}
// ForgetEndedAbuseIPDBBans forgets the bans whose cause is CauseAbuseIPDB
// and that have ended at now, as BanForAbuseIPDB does first. The state
// package calls it each time it reads bans.json, so that these bans are
// also forgotten while no fetch of the blacklist reaches the ledger.
func (l *Ledger) ForgetEndedAbuseIPDBBans(now time.Time) {
l.mu.Lock()
defer l.mu.Unlock()
if l.forgetEndedAbuseIPDBBans(now) {
l.markChanged()
}
}
// liftedAbuseIPDBBan reports whether bans, a netblock's, hold a ban whose
// cause is CauseAbuseIPDB that an admin lifted, and whose end is not
// reached at now.
func liftedAbuseIPDBBan(bans []Ban, now time.Time) bool {
return slices.ContainsFunc(bans, func(ban Ban) bool {
return ban.Cause == CauseAbuseIPDB && !ban.Lifted.IsZero() &&
(ban.Permanent() || now.Before(ban.Expires))
})
}
// Lift lifts, at now, every ban active then on a netblock client is in, // Lift lifts, at now, every ban active then on a netblock client is in,
// as an admin does, and returns those bans. A lifted ban is kept, refuses // as an admin does, and returns those bans. A lifted ban is kept, refuses
// nothing, and does not make the netblock's next ban longer. // nothing, and does not make the netblock's next ban longer.
@@ -606,9 +515,9 @@ func (l *Ledger) AddLookup(
} }
// Made returns how many bans for cause have been made since the start: // Made returns how many bans for cause have been made since the start:
// for CauseLimit, CauseAttack and CauseAbuseIPDB, by the ledger; for // for CauseLimit and CauseAttack, by the ledger; for CauseAdmin, by an
// CauseAdmin, by an admin, with BanForAdmin or in an edit of bans.json, as // admin, with BanForAdmin or in an edit of bans.json, as LoadEdit counts
// LoadEdit counts them. The bans read from bans.json at the start are not among them. // them. The bans read from bans.json at the start are not among them.
func (l *Ledger) Made(cause string) int { func (l *Ledger) Made(cause string) int {
l.mu.Lock() l.mu.Lock()
defer l.mu.Unlock() defer l.mu.Unlock()
@@ -665,8 +574,8 @@ func (l *Ledger) Snapshot() []Ban {
// seen. A ban without a cause is an admin's, and gets CauseAdmin. Each // seen. A ban without a cause is an admin's, and gets CauseAdmin. Each
// netblock is masked to its length, so that 203.0.113.9/24 is // netblock is masked to its length, so that 203.0.113.9/24 is
// 203.0.113.0/24, and each text in the notes is cut to 256 bytes. Past // 203.0.113.0/24, and each text in the notes is cut to 256 bytes. Past
// MaxBans the earliest bans that count toward it are dropped, as when they // MaxBans the earliest bans whose cause is not CauseAdmin are dropped, as
// are made. // when they are made.
func (l *Ledger) Load(bans []Ban) { func (l *Ledger) Load(bans []Ban) {
l.mu.Lock() l.mu.Lock()
defer l.mu.Unlock() defer l.mu.Unlock()
@@ -730,35 +639,6 @@ func (l *Ledger) holds(netblock netip.Prefix, start time.Time) bool {
}) })
} }
// forgetEndedAbuseIPDBBans forgets each ban whose cause is CauseAbuseIPDB
// and whose end is reached at now, lifted or not, and each netblock left
// without a ban. It reports whether it forgot one.
func (l *Ledger) forgetEndedAbuseIPDBBans(now time.Time) bool {
forgot := false
for _, netblock := range l.netblocks.Keys() {
bans, _ := l.netblocks.Peek(netblock)
kept := slices.DeleteFunc(*bans, func(ban Ban) bool {
return ban.Cause == CauseAbuseIPDB && !ban.Permanent() &&
!now.Before(ban.Expires)
})
if len(kept) == len(*bans) {
continue
}
forgot = true
if len(kept) == 0 {
l.netblocks.Remove(netblock)
} else {
*bans = kept
}
}
return forgot
}
// ban bans netblock at now for cause, CauseLimit or CauseAttack, with // ban bans netblock at now for cause, CauseLimit or CauseAttack, with
// reason and notes, as BanForLimit and BanForAttack describe, and returns // reason and notes, as BanForLimit and BanForAttack describe, and returns
// the ban, and whether it made it. Unless keep is true, the ban is not // the ban, and whether it made it. Unless keep is true, the ban is not
@@ -834,8 +714,6 @@ func earlierBans(held []Ban) EarlierBans {
fromHere.Attack++ fromHere.Attack++
case CauseAdmin: case CauseAdmin:
fromHere.Admin++ fromHere.Admin++
case CauseAbuseIPDB:
fromHere.AbuseIPDB++
} }
notes := ban.Notes.EarlierBans notes := ban.Notes.EarlierBans
@@ -843,7 +721,6 @@ func earlierBans(held []Ban) EarlierBans {
Limit: notes.Limit + fromHere.Limit, Limit: notes.Limit + fromHere.Limit,
Attack: notes.Attack + fromHere.Attack, Attack: notes.Attack + fromHere.Attack,
Admin: notes.Admin + fromHere.Admin, Admin: notes.Admin + fromHere.Admin,
AbuseIPDB: notes.AbuseIPDB + fromHere.AbuseIPDB,
} }
// Of counts that name as many bans, the earlier ban's is taken: it // Of counts that name as many bans, the earlier ban's is taken: it
@@ -858,7 +735,7 @@ func earlierBans(held []Ban) EarlierBans {
// total returns how many bans e counts, of all causes. // total returns how many bans e counts, of all causes.
func (e EarlierBans) total() int { func (e EarlierBans) total() int {
return e.Limit + e.Attack + e.Admin + e.AbuseIPDB return e.Limit + e.Attack + e.Admin
} }
// markChanged has Changed receive a value, unless one is waiting already. // markChanged has Changed receive a value, unless one is waiting already.
@@ -929,9 +806,9 @@ func (l *Ledger) covering(client netip.Addr) []*[]Ban {
// add adds ban to its netblock's bans, after the last, and makes its // add adds ban to its netblock's bans, after the last, and makes its
// netblock the most recently seen. With MaxBans held, it drops one first, // netblock the most recently seen. With MaxBans held, it drops one first,
// unless ban does not count toward MaxBans. // unless ban's cause is CauseAdmin, which does not count toward MaxBans.
func (l *Ledger) add(ban Ban) { func (l *Ledger) add(ban Ban) {
counted := countsTowardMaxBans(ban) counted := ban.Cause != CauseAdmin
if counted && l.held == l.params.MaxBans { if counted && l.held == l.params.MaxBans {
l.dropOne() l.dropOne()
} }
@@ -1012,13 +889,7 @@ func (l *Ledger) attackExpiry(held []Ban, now time.Time) time.Time {
return now.Add(l.params.AttackBanDuration) return now.Add(l.params.AttackBanDuration)
} }
// countsTowardMaxBans reports whether ban counts toward MaxBans: its cause // dropOne drops the earliest ban whose cause is not CauseAdmin of the
// is CauseLimit or CauseAttack.
func countsTowardMaxBans(ban Ban) bool {
return ban.Cause == CauseLimit || ban.Cause == CauseAttack
}
// dropOne drops the earliest ban that counts toward MaxBans of the
// netblock that has gone longest without a request, of those that hold // netblock that has gone longest without a request, of those that hold
// such a ban, and the netblock with it if that was its only ban. It is // such a ban, and the netblock with it if that was its only ban. It is
// called with at least one such ban held. It looks at each netblock once // called with at least one such ban held. It looks at each netblock once
@@ -1027,9 +898,11 @@ func (l *Ledger) dropOne() {
for range l.netblocks.Len() { for range l.netblocks.Len() {
netblock, bans, _ := l.netblocks.GetOldest() netblock, bans, _ := l.netblocks.GetOldest()
i := slices.IndexFunc(*bans, countsTowardMaxBans) i := slices.IndexFunc(*bans, func(ban Ban) bool {
return ban.Cause != CauseAdmin
})
if i < 0 { if i < 0 {
// None of its bans is dropped to make room. Get makes // Its bans are all an admin's, and never dropped. Get makes
// it the most recently seen, so that the next netblock is // it the most recently seen, so that the next netblock is
// looked at; when it was seen matters only for dropping a // looked at; when it was seen matters only for dropping a
// ban, and a ban added to it makes it the most recently seen // ban, and a ban added to it makes it the most recently seen
+32 -154
View File
@@ -75,7 +75,7 @@ type WAF struct {
BytesLimitPerMinute int64 BytesLimitPerMinute int64
BytesLimitPerHour int64 BytesLimitPerHour int64
BytesLimitPerDay int64 BytesLimitPerDay int64
BytesCounted BytesCounted BytesCounted string
// LookupSource is where each client's AS number and country are looked // LookupSource is where each client's AS number and country are looked
// up (SWW_WAF_LOOKUP_SOURCE): geojs, file, or off for nowhere. // up (SWW_WAF_LOOKUP_SOURCE): geojs, file, or off for nowhere.
// LookupDBPath is the lookup database, the IPinfo Lite file looked up in // LookupDBPath is the lookup database, the IPinfo Lite file looked up in
@@ -86,7 +86,7 @@ type WAF struct {
// AddLookupHeaders is true when the app is passed the client's AS number // AddLookupHeaders is true when the app is passed the client's AS number
// and country in headers (SWW_WAF_ADD_LOOKUP_HEADERS), by default unless // and country in headers (SWW_WAF_ADD_LOOKUP_HEADERS), by default unless
// LookupSource is off. // LookupSource is off.
LookupSource LookupSource LookupSource string
LookupDBPath string LookupDBPath string
ASNAndCountryLookupTimeout time.Duration ASNAndCountryLookupTimeout time.Duration
AddLookupHeaders bool AddLookupHeaders bool
@@ -131,7 +131,7 @@ type WAF struct {
// BlocklistLimitPercent is the percentage of every limit it gets. // BlocklistLimitPercent is the percentage of every limit it gets.
BlocklistURLs []string BlocklistURLs []string
BlocklistRefresh time.Duration BlocklistRefresh time.Duration
BlocklistAction ReputationAction BlocklistAction string
BlocklistLimitPercent int64 BlocklistLimitPercent int64
// DNSBLZones are the DNSBL zones clients are asked about // DNSBLZones are the DNSBL zones clients are asked about
// (SWW_WAF_DNSBL_ZONES), through DNSBLResolver (SWW_WAF_DNSBL_RESOLVER), or // (SWW_WAF_DNSBL_ZONES), through DNSBLResolver (SWW_WAF_DNSBL_RESOLVER), or
@@ -154,18 +154,10 @@ type WAF struct {
AbuseIPDBKey string AbuseIPDBKey string
AbuseIPDBMinScore int64 AbuseIPDBMinScore int64
AbuseIPDBDailyBudget int AbuseIPDBDailyBudget int
DNSBLAbuseIPDBAction ReputationAction DNSBLAbuseIPDBAction string
DNSBLAbuseIPDBLimitPercent int64 DNSBLAbuseIPDBLimitPercent int64
IPReputationCacheTTL time.Duration IPReputationCacheTTL time.Duration
DNSBLAbuseIPDBTimeout time.Duration DNSBLAbuseIPDBTimeout time.Duration
// While AbuseIPDBKey is set, AbuseIPDB's blacklist of the clients it
// scores AbuseIPDBThreshold or more (SWW_WAF_ABUSEIPDB_THRESHOLD), from
// 25 to 100, is fetched at the start and every AbuseIPDBBlacklistRefresh
// (SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS). Each client on it
// is banned for AbuseIPDBBanDuration (SWW_WAF_ABUSEIPDB_BAN_DAYS).
AbuseIPDBThreshold int64
AbuseIPDBBlacklistRefresh time.Duration
AbuseIPDBBanDuration time.Duration
// BanResponse is the status a refused client is answered with, 403 // BanResponse is the status a refused client is answered with, 403
// or 429, or 0 to close the connection without an answer // or 429, or 0 to close the connection without an answer
// (SWW_WAF_BAN_RESPONSE). It answers a banned client, a request that // (SWW_WAF_BAN_RESPONSE). It answers a banned client, a request that
@@ -237,7 +229,7 @@ type WAF struct {
// reads none. // reads none.
// CoreRuleSetExclusionsFile is the path of the file of its exclusions // CoreRuleSetExclusionsFile is the path of the file of its exclusions
// (SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE), "" while there is none. // (SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE), "" while there is none.
CoreRuleSetMode CoreRuleSetMode CoreRuleSetMode string
CoreRuleSetParanoiaLevel int CoreRuleSetParanoiaLevel int
CoreRuleSetScoreThreshold int CoreRuleSetScoreThreshold int
CoreRuleSetDisabledRules []int CoreRuleSetDisabledRules []int
@@ -316,66 +308,20 @@ type WAF struct {
settings []slog.Attr settings []slog.Attr
} }
// BytesCounted is which body bytes count toward the byte limits and the
// byte anomaly thresholds, a value of SWW_WAF_BYTES_COUNTED.
type BytesCounted string
// The values of SWW_WAF_BYTES_COUNTED.
const (
// BytesCountedResponse counts the response's body bytes.
BytesCountedResponse BytesCounted = "response"
// BytesCountedRequest counts the request's body bytes.
BytesCountedRequest BytesCounted = "request"
// BytesCountedBoth counts the body bytes of both.
BytesCountedBoth BytesCounted = "both"
)
// LookupSource is where each client's AS number and country are looked up,
// a value of SWW_WAF_LOOKUP_SOURCE.
type LookupSource string
// The values of SWW_WAF_LOOKUP_SOURCE.
const (
// LookupSourceGeoJS asks GeoJS.
LookupSourceGeoJS LookupSource = "geojs"
// LookupSourceFile looks clients up in the lookup database, the file
// SWW_WAF_LOOKUP_DB_PATH names.
LookupSourceFile LookupSource = "file"
// LookupSourceOff looks no client up.
LookupSourceOff LookupSource = "off"
)
// ReputationAction is what is done with a client a reputation source
// lists, a value of SWW_WAF_BLOCKLIST_ACTION and of
// SWW_WAF_DNSBL_ABUSEIPDB_ACTION.
type ReputationAction string
// The values of SWW_WAF_BLOCKLIST_ACTION and SWW_WAF_DNSBL_ABUSEIPDB_ACTION.
const (
// ReputationActionDeny refuses the client's requests.
ReputationActionDeny ReputationAction = "deny"
// ReputationActionLimit gives the client a percentage of every limit,
// written limit:<percent> in the setting.
ReputationActionLimit ReputationAction = "limit"
// ReputationActionLog only notes the hit in the log line and raises an
// alert.
ReputationActionLog ReputationAction = "log"
)
// CoreRuleSetMode is what the Core Rule Set does, a value of
// SWW_WAF_CORE_RULE_SET_MODE.
type CoreRuleSetMode string
// The values of SWW_WAF_CORE_RULE_SET_MODE. // The values of SWW_WAF_CORE_RULE_SET_MODE.
const ( const (
// CoreRuleSetModeOff runs no request through the Core Rule Set. // CoreRuleSetModeOff runs no request through the Core Rule Set.
CoreRuleSetModeOff CoreRuleSetMode = "off" CoreRuleSetModeOff = off
// CoreRuleSetModeDetect logs and alerts a match, and refuses nothing. // CoreRuleSetModeDetect logs and alerts a match, and refuses nothing.
CoreRuleSetModeDetect CoreRuleSetMode = "detect" CoreRuleSetModeDetect = "detect"
// CoreRuleSetModeBlock refuses a match with 403. // CoreRuleSetModeBlock refuses a match with 403.
CoreRuleSetModeBlock CoreRuleSetMode = "block" CoreRuleSetModeBlock = "block"
) )
// fileSource is the SWW_WAF_LOOKUP_SOURCE that looks clients up in the
// lookup database, the file SWW_WAF_LOOKUP_DB_PATH names.
const fileSource = "file"
const ( const (
day = 24 * time.Hour day = 24 * time.Hour
ipv4Bits = 32 ipv4Bits = 32
@@ -384,9 +330,6 @@ const (
// netblock a provider is usually given: a shorter one would make one // netblock a provider is usually given: a shorter one would make one
// client of the customers of several providers. // client of the customers of several providers.
minIPv6GroupPrefix = 32 minIPv6GroupPrefix = 32
// minAbuseIPDBThreshold is the lowest score AbuseIPDB's blacklist can
// be cut at.
minAbuseIPDBThreshold = 25
// minTokenLength is the fewest characters a token may have. // minTokenLength is the fewest characters a token may have.
minTokenLength = 32 minTokenLength = 32
// maxParanoiaLevel is the Core Rule Set's highest paranoia level. // maxParanoiaLevel is the Core Rule Set's highest paranoia level.
@@ -503,13 +446,6 @@ var (
errNotPercent = errors.New( errNotPercent = errors.New(
"is not a percentage, a whole number from 0 to 100", "is not a percentage, a whole number from 0 to 100",
) )
errNotAbuseIPDBThreshold = errors.New(
"is not a whole number from 25 to 100, the scores AbuseIPDB's " +
"blacklist can be cut at",
)
errNotWholeUnits = errors.New(
"is not a whole number above zero, such as 30, that fits a duration",
)
errListedTwice = errors.New("is listed twice") errListedTwice = errors.New("is listed twice")
errNotListURL = errors.New( errNotListURL = errors.New(
"is not an http or https URL without a user or a fragment, " + "is not an http or https URL without a user or a fragment, " +
@@ -604,13 +540,10 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
), ),
BytesLimitPerHour: env.size("SWW_WAF_BYTES_LIMIT_PER_HOUR", "50G"), BytesLimitPerHour: env.size("SWW_WAF_BYTES_LIMIT_PER_HOUR", "50G"),
BytesLimitPerDay: env.size("SWW_WAF_BYTES_LIMIT_PER_DAY", "100G"), BytesLimitPerDay: env.size("SWW_WAF_BYTES_LIMIT_PER_DAY", "100G"),
BytesCounted: env.bytesCounted( BytesCounted: env.bytesCounted("SWW_WAF_BYTES_COUNTED", "both"),
"SWW_WAF_BYTES_COUNTED",
BytesCountedBoth,
),
LookupSource: env.lookupSource( LookupSource: env.lookupSource(
"SWW_WAF_LOOKUP_SOURCE", "SWW_WAF_LOOKUP_SOURCE",
LookupSourceGeoJS, "geojs",
), ),
LookupDBPath: env.value("SWW_WAF_LOOKUP_DB_PATH", ""), LookupDBPath: env.value("SWW_WAF_LOOKUP_DB_PATH", ""),
ASNAndCountryLookupTimeout: env.milliseconds( ASNAndCountryLookupTimeout: env.milliseconds(
@@ -654,20 +587,6 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
"SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS", "SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS",
"2000", "2000",
), ),
AbuseIPDBThreshold: env.abuseIPDBThreshold(
"SWW_WAF_ABUSEIPDB_THRESHOLD",
"90",
),
AbuseIPDBBlacklistRefresh: env.wholeUnits(
"SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS",
"60",
time.Minute,
),
AbuseIPDBBanDuration: env.wholeUnits(
"SWW_WAF_ABUSEIPDB_BAN_DAYS",
"30",
day,
),
BanResponse: env.banResponse("SWW_WAF_BAN_RESPONSE", "403"), BanResponse: env.banResponse("SWW_WAF_BAN_RESPONSE", "403"),
LimitBanDuration: env.durationNotOff( LimitBanDuration: env.durationNotOff(
"SWW_WAF_LIMIT_BAN_DURATION", "SWW_WAF_LIMIT_BAN_DURATION",
@@ -807,9 +726,9 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
cfg.InstanceName, cfg.LogRemoteURL != nil) cfg.InstanceName, cfg.LogRemoteURL != nil)
// The app gets the headers by default, unless no client is looked up. // The app gets the headers by default, unless no client is looked up.
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
strconv.FormatBool(cfg.LookupSource != LookupSourceOff)) strconv.FormatBool(cfg.LookupSource != off))
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action( cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
"SWW_WAF_BLOCKLIST_ACTION", string(ReputationActionDeny)) "SWW_WAF_BLOCKLIST_ACTION", "deny")
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action( cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "limit:25") "SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "limit:25")
@@ -907,12 +826,9 @@ func (e *environment) count(name, defaultValue string) int64 {
// bytesCounted reads the setting that is which body bytes count toward the // bytesCounted reads the setting that is which body bytes count toward the
// byte limits and the byte anomaly thresholds: response, request or both. // byte limits and the byte anomaly thresholds: response, request or both.
func (e *environment) bytesCounted( func (e *environment) bytesCounted(name, defaultValue string) string {
name string, defaultValue BytesCounted, value := e.value(name, defaultValue)
) BytesCounted { if value != "response" && value != "request" && value != "both" {
value := BytesCounted(e.value(name, string(defaultValue)))
if value != BytesCountedResponse && value != BytesCountedRequest &&
value != BytesCountedBoth {
e.check(name, fmt.Errorf("%q %w", value, errNotBytesCounted)) e.check(name, fmt.Errorf("%q %w", value, errNotBytesCounted))
} }
@@ -957,10 +873,8 @@ func (e *environment) trapPaths(name string) []string {
// coreRuleSetMode reads the setting that is what the Core Rule Set does: off, // coreRuleSetMode reads the setting that is what the Core Rule Set does: off,
// detect or block. // detect or block.
func (e *environment) coreRuleSetMode( func (e *environment) coreRuleSetMode(name, defaultValue string) string {
name string, defaultValue CoreRuleSetMode, mode := e.value(name, defaultValue)
) CoreRuleSetMode {
mode := CoreRuleSetMode(e.value(name, string(defaultValue)))
if mode != CoreRuleSetModeOff && mode != CoreRuleSetModeDetect && if mode != CoreRuleSetModeOff && mode != CoreRuleSetModeDetect &&
mode != CoreRuleSetModeBlock { mode != CoreRuleSetModeBlock {
e.check(name, fmt.Errorf("%q %w", mode, errNotCoreRuleSetMode)) e.check(name, fmt.Errorf("%q %w", mode, errNotCoreRuleSetMode))
@@ -1043,35 +957,6 @@ func (e *environment) percent(name, defaultValue string) int64 {
return percent return percent
} }
// abuseIPDBThreshold reads the setting that is the least score of the
// clients on AbuseIPDB's blacklist: a whole number from 25 to 100, the
// range AbuseIPDB takes.
func (e *environment) abuseIPDBThreshold(name, defaultValue string) int64 {
value := e.value(name, defaultValue)
threshold, err := strconv.ParseInt(value, 10, 64)
if err != nil || threshold < minAbuseIPDBThreshold || threshold > 100 {
e.check(name, fmt.Errorf("%q %w", value, errNotAbuseIPDBThreshold))
}
return threshold
}
// wholeUnits reads a setting that is a whole number above zero of unit,
// such as minutes or days, and returns it as a duration.
func (e *environment) wholeUnits(
name, defaultValue string, unit time.Duration,
) time.Duration {
value := e.value(name, defaultValue)
n, err := strconv.ParseInt(value, 10, 64)
if err != nil || n <= 0 || n > math.MaxInt64/int64(unit) {
e.check(name, fmt.Errorf("%q %w", value, errNotWholeUnits))
}
return time.Duration(n) * unit
}
// listURL reads a setting that is the URL a list is fetched from, "" while // listURL reads a setting that is the URL a list is fetched from, "" while
// it is unset or empty. // it is unset or empty.
func (e *environment) listURL(name string) string { func (e *environment) listURL(name string) string {
@@ -1110,24 +995,20 @@ func (e *environment) refresh(name, defaultValue string) time.Duration {
// action reads a setting that is what is done with a client a blocklist // action reads a setting that is what is done with a client a blocklist
// or a DNSBL zone lists: deny, log, or limit:<percent>, which it returns // or a DNSBL zone lists: deny, log, or limit:<percent>, which it returns
// as limit and the percentage. // as limit and the percentage.
func (e *environment) action( func (e *environment) action(name, defaultValue string) (string, int64) {
name, defaultValue string, value := e.value(name, defaultValue)
) (ReputationAction, int64) { if value == "deny" || value == "log" {
value := ReputationAction(e.value(name, defaultValue))
if value == ReputationActionDeny || value == ReputationActionLog {
return value, 0 return value, 0
} }
percentText, isLimit := strings.CutPrefix( percentText, isLimit := strings.CutPrefix(value, "limit:")
string(value), string(ReputationActionLimit)+":",
)
percent, err := ParsePercent(percentText) percent, err := ParsePercent(percentText)
if !isLimit || err != nil { if !isLimit || err != nil {
e.check(name, fmt.Errorf("%q %w", value, errNotAction)) e.check(name, fmt.Errorf("%q %w", value, errNotAction))
} }
return ReputationActionLimit, percent return "limit", percent
} }
// zones reads the setting that is the list of DNSBL zones. It is empty by // zones reads the setting that is the list of DNSBL zones. It is empty by
@@ -1178,12 +1059,9 @@ func (e *environment) routeWatchURL(name string) string {
// lookupSource reads the setting that is where clients are looked up: // lookupSource reads the setting that is where clients are looked up:
// geojs, file, or off. // geojs, file, or off.
func (e *environment) lookupSource( func (e *environment) lookupSource(name, defaultValue string) string {
name string, defaultValue LookupSource, source := e.value(name, defaultValue)
) LookupSource { if source != "geojs" && source != fileSource && source != off {
source := LookupSource(e.value(name, string(defaultValue)))
if source != LookupSourceGeoJS && source != LookupSourceFile &&
source != LookupSourceOff {
e.check(name, fmt.Errorf("%q %w", source, errNotLookupSource)) e.check(name, fmt.Errorf("%q %w", source, errNotLookupSource))
} }
@@ -1195,12 +1073,12 @@ func (e *environment) lookupSource(
// one source at a time. // one source at a time.
func (e *environment) checkLookupDBPath(cfg *WAF) { func (e *environment) checkLookupDBPath(cfg *WAF) {
switch { switch {
case cfg.LookupSource == LookupSourceFile && cfg.LookupDBPath == "": case cfg.LookupSource == fileSource && cfg.LookupDBPath == "":
e.check("SWW_WAF_LOOKUP_SOURCE", fmt.Errorf( e.check("SWW_WAF_LOOKUP_SOURCE", fmt.Errorf(
"is file while SWW_WAF_LOOKUP_DB_PATH is unset; %w", "is file while SWW_WAF_LOOKUP_DB_PATH is unset; %w",
errNeedsDBPath, errNeedsDBPath,
)) ))
case cfg.LookupSource != LookupSourceFile && cfg.LookupDBPath != "": case cfg.LookupSource != fileSource && cfg.LookupDBPath != "":
e.check("SWW_WAF_LOOKUP_DB_PATH", fmt.Errorf( e.check("SWW_WAF_LOOKUP_DB_PATH", fmt.Errorf(
"is set while SWW_WAF_LOOKUP_SOURCE is %s; %w", "is set while SWW_WAF_LOOKUP_SOURCE is %s; %w",
cfg.LookupSource, cfg.LookupSource,
@@ -1225,7 +1103,7 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
} }
} }
if cfg.LookupSource != LookupSourceOff { if cfg.LookupSource != off {
return return
} }
+4 -80
View File
@@ -61,9 +61,6 @@ const (
abuseIPDBKey = "SWW_WAF_ABUSEIPDB_KEY" abuseIPDBKey = "SWW_WAF_ABUSEIPDB_KEY"
abuseIPDBMinScore = "SWW_WAF_ABUSEIPDB_MIN_SCORE" abuseIPDBMinScore = "SWW_WAF_ABUSEIPDB_MIN_SCORE"
abuseIPDBDailyBudget = "SWW_WAF_ABUSEIPDB_DAILY_BUDGET" abuseIPDBDailyBudget = "SWW_WAF_ABUSEIPDB_DAILY_BUDGET"
abuseIPDBThreshold = "SWW_WAF_ABUSEIPDB_THRESHOLD"
abuseIPDBBlacklistRefresh = "SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS"
abuseIPDBBanDays = "SWW_WAF_ABUSEIPDB_BAN_DAYS"
dnsblAbuseIPDBAction = "SWW_WAF_DNSBL_ABUSEIPDB_ACTION" dnsblAbuseIPDBAction = "SWW_WAF_DNSBL_ABUSEIPDB_ACTION"
ipReputationCacheTTL = "SWW_WAF_IP_REPUTATION_CACHE_TTL" ipReputationCacheTTL = "SWW_WAF_IP_REPUTATION_CACHE_TTL"
dnsblAbuseIPDBTimeout = "SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS" dnsblAbuseIPDBTimeout = "SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS"
@@ -614,7 +611,7 @@ func TestCoreRuleSetSettings(t *testing.T) {
}, },
{ {
environment{ environment{
coreRuleSetMode: string(config.CoreRuleSetModeDetect), coreRuleSetMode: config.CoreRuleSetModeDetect,
coreRuleSetParanoiaLevel: "4", coreRuleSetParanoiaLevel: "4",
coreRuleSetScoreThreshold: "10", coreRuleSetScoreThreshold: "10",
coreRuleSetDisabledRules: "942100, 920350", coreRuleSetDisabledRules: "942100, 920350",
@@ -2033,8 +2030,7 @@ func TestReputationSettingsAsSet(t *testing.T) {
} }
for _, tc := range []struct { for _, tc := range []struct {
value string value, action string
action config.ReputationAction
percent int64 percent int64
}{ }{
{actionDeny, actionDeny, 0}, {actionDeny, actionDeny, 0},
@@ -2168,7 +2164,7 @@ func TestDNSBLSettingsAsSet(t *testing.T) {
// want are the zones, resolver, action and percent config.WAF gives. // want are the zones, resolver, action and percent config.WAF gives.
want []string want []string
wantResolver string wantResolver string
wantAction config.ReputationAction wantAction string
wantPercent int64 wantPercent int64
}{ }{
{ {
@@ -2403,75 +2399,6 @@ func TestAbuseIPDBSettingsAsSet(t *testing.T) {
} }
} }
func TestAbuseIPDBBlacklistSettingsAsSet(t *testing.T) {
t.Parallel()
cfg := wafFromEnvironment(t, environment{})
if cfg.AbuseIPDBThreshold != 90 ||
cfg.AbuseIPDBBlacklistRefresh != time.Hour ||
cfg.AbuseIPDBBanDuration != 30*24*time.Hour {
t.Errorf(
"by default, the threshold %d, the refresh %s and the ban %s, "+
"want 90, 1h and 720h",
cfg.AbuseIPDBThreshold,
cfg.AbuseIPDBBlacklistRefresh,
cfg.AbuseIPDBBanDuration,
)
}
cfg = wafFromEnvironment(t, environment{
abuseIPDBThreshold: "25", abuseIPDBBlacklistRefresh: "1440",
abuseIPDBBanDays: "1",
})
if cfg.AbuseIPDBThreshold != 25 ||
cfg.AbuseIPDBBlacklistRefresh != 24*time.Hour ||
cfg.AbuseIPDBBanDuration != 24*time.Hour {
t.Errorf(
"set, the threshold %d, the refresh %s and the ban %s, "+
"want 25, 24h and 24h",
cfg.AbuseIPDBThreshold,
cfg.AbuseIPDBBlacklistRefresh,
cfg.AbuseIPDBBanDuration,
)
}
}
func TestInvalidAbuseIPDBBlacklistSettingStopsTheStartSayingWhatIsWrong(
t *testing.T,
) {
t.Parallel()
const (
notThreshold = " is not a whole number from 25 to 100, the scores " +
"AbuseIPDB's blacklist can be cut at"
notWhole = " is not a whole number above zero, such as 30, that fits " +
"a duration"
)
for _, tc := range []struct{ name, value, want string }{
{abuseIPDBThreshold, "24", `"24"` + notThreshold},
{abuseIPDBThreshold, "101", `"101"` + notThreshold},
{abuseIPDBThreshold, off, `"off"` + notThreshold},
{abuseIPDBBlacklistRefresh, "0", `"0"` + notWhole},
{abuseIPDBBlacklistRefresh, "1h", `"1h"` + notWhole},
{abuseIPDBBanDays, "-1", `"-1"` + notWhole},
{abuseIPDBBanDays, "200000", `"200000"` + notWhole},
} {
t.Run(tc.name+"="+tc.value, func(t *testing.T) {
t.Parallel()
_, err := config.WAFFromEnvironment(
environment{tc.name: tc.value}.viper(),
)
want := tc.name + ": " + tc.want
if err == nil || err.Error() != want {
t.Errorf("error %v, want %s", err, want)
}
})
}
}
func TestInvalidAbuseIPDBSettingStopsTheStartSayingWhatIsWrong( func TestInvalidAbuseIPDBSettingStopsTheStartSayingWhatIsWrong(
t *testing.T, t *testing.T,
) { ) {
@@ -3024,9 +2951,6 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
abuseIPDBKey: "", abuseIPDBKey: "",
abuseIPDBMinScore: "75", abuseIPDBMinScore: "75",
abuseIPDBDailyBudget: "900", abuseIPDBDailyBudget: "900",
abuseIPDBThreshold: "90",
abuseIPDBBlacklistRefresh: "60",
abuseIPDBBanDays: "30",
dnsblAbuseIPDBAction: "limit:25", dnsblAbuseIPDBAction: "limit:25",
ipReputationCacheTTL: defaultIPReputationCacheTTL, ipReputationCacheTTL: defaultIPReputationCacheTTL,
dnsblAbuseIPDBTimeout: "2000", dnsblAbuseIPDBTimeout: "2000",
@@ -3050,7 +2974,7 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
logLevel: "info", logLevel: "info",
ruleFilesDir: "/etc/sww/rules.d", ruleFilesDir: "/etc/sww/rules.d",
ruleFilesEnabled: "true", ruleFilesEnabled: "true",
coreRuleSetMode: string(config.CoreRuleSetModeBlock), coreRuleSetMode: config.CoreRuleSetModeBlock,
coreRuleSetParanoiaLevel: "1", coreRuleSetParanoiaLevel: "1",
coreRuleSetScoreThreshold: "5", coreRuleSetScoreThreshold: "5",
coreRuleSetDisabledRules: defaultCoreRuleSetDisabledRules, coreRuleSetDisabledRules: defaultCoreRuleSetDisabledRules,
+36 -59
View File
@@ -84,7 +84,12 @@ type Params struct {
// one request to GeoJS is under way at a time, and it asks about every // one request to GeoJS is under way at a time, and it asks about every
// client waiting, up to maxPerRequest. It is safe for concurrent use. // client waiting, up to maxPerRequest. It is safe for concurrent use.
type GeoJS struct { type GeoJS struct {
params Params url string
answered func(Answer)
now func() time.Time
asked func()
failed func(err error, askingAgainIn time.Duration)
wentWithoutAnswer func()
// httpClient follows no redirect, so that visitors' addresses go to // httpClient follows no redirect, so that visitors' addresses go to
// GeoJS alone: a redirect is a failure. // GeoJS alone: a redirect is a failure.
httpClient *http.Client httpClient *http.Client
@@ -94,13 +99,8 @@ type GeoJS struct {
// waiting are the clients without an answer: those to ask GeoJS about, // waiting are the clients without an answer: those to ask GeoJS about,
// and those it is being asked about. // and those it is being asked about.
waiting map[netip.Prefix]*wait waiting map[netip.Prefix]*wait
// asking is true while a request to GeoJS is under way, and endAsking // asking is true while a request to GeoJS is under way.
// ends it. stopped is true once StopAsking has been called, and
// askGoroutines waits for the goroutine that makes the requests.
asking bool asking bool
endAsking context.CancelFunc
stopped bool
askGoroutines sync.WaitGroup
// retryDelay is how long GeoJS is left alone after its last failure, // retryDelay is how long GeoJS is left alone after its last failure,
// zero after an answer; retryAt is when it may be asked again. // zero after an answer; retryAt is when it may be asked again.
retryDelay time.Duration retryDelay time.Duration
@@ -140,7 +140,12 @@ func New(params Params) *GeoJS {
} }
return &GeoJS{ return &GeoJS{
params: params, url: params.URL,
answered: params.Answered,
now: params.Now,
asked: params.Asked,
failed: params.Failed,
wentWithoutAnswer: params.WentWithoutAnswer,
httpClient: &http.Client{ httpClient: &http.Client{
CheckRedirect: func(*http.Request, []*http.Request) error { CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse return http.ErrUseLastResponse
@@ -183,7 +188,7 @@ func (g *GeoJS) LookUp(
answer, found := g.kept(client) answer, found := g.kept(client)
if !found { if !found {
g.params.WentWithoutAnswer() g.wentWithoutAnswer()
} }
w, waiting := g.waiting[client] w, waiting := g.waiting[client]
@@ -205,23 +210,6 @@ func (g *GeoJS) LookUpWithoutWaiting(
return answer return answer
} }
// StopAsking ends the request to GeoJS under way, and returns once the
// goroutine that made it has. A request it ends is no failure: Failed is not
// called, and GeoJS is not left alone after it. After StopAsking, GeoJS is
// asked nothing.
func (g *GeoJS) StopAsking() {
g.mu.Lock()
g.stopped = true
if g.asking {
g.endAsking()
}
g.mu.Unlock()
g.askGoroutines.Wait()
}
// Kept returns client's answer, if one is kept, without asking GeoJS. // Kept returns client's answer, if one is kept, without asking GeoJS.
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) { func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
g.mu.Lock() g.mu.Lock()
@@ -263,7 +251,7 @@ func (g *GeoJS) Load(answers []Answer) {
g.answers.Purge() g.answers.Purge()
now := g.params.Now() now := g.now()
for _, answer := range answers { for _, answer := range answers {
if now.Sub(answer.Answered) < keepFor { if now.Sub(answer.Answered) < keepFor {
@@ -300,19 +288,18 @@ func (g *GeoJS) answerOrWait(
} }
if w == nil { if w == nil {
g.params.WentWithoutAnswer() g.wentWithoutAnswer()
return Answer{}, nil // too many clients wait already return Answer{}, nil // too many clients wait already
} }
if !g.asking { if !g.asking {
// GeoJS is left alone after a failure, or asked nothing after // GeoJS is left alone after a failure, so no answer can come.
// StopAsking, so no answer can come.
w.late = true w.late = true
} }
if w.late { if w.late {
g.params.WentWithoutAnswer() g.wentWithoutAnswer()
return Answer{}, nil return Answer{}, nil
} }
@@ -323,7 +310,7 @@ func (g *GeoJS) answerOrWait(
// kept returns client's answer, if GeoJS gave it less than keepFor ago, // kept returns client's answer, if GeoJS gave it less than keepFor ago,
// and notes that it was used. // and notes that it was used.
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) { func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
now := g.params.Now() now := g.now()
kept, found := g.answers.Get(client) kept, found := g.answers.Get(client)
if !found || now.Sub(kept.Answered) >= keepFor { if !found || now.Sub(kept.Answered) >= keepFor {
@@ -336,28 +323,22 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
} }
// ask starts asking GeoJS about the waiting clients, unless a request to // ask starts asking GeoJS about the waiting clients, unless a request to
// it is under way, it is left alone after a failure, or StopAsking has // it is under way or it is left alone after a failure. The requests to
// been called. The requests to GeoJS are for every client waiting, so they // GeoJS are for every client waiting, so they go on when the caller whose
// go on when the caller whose ctx is given stops waiting, until StopAsking. // ctx is given stops waiting.
func (g *GeoJS) ask(ctx context.Context) { func (g *GeoJS) ask(ctx context.Context) {
if g.asking || g.stopped || g.params.Now().Before(g.retryAt) { if g.asking || g.now().Before(g.retryAt) {
return return
} }
askCtx, end := context.WithCancel(context.WithoutCancel(ctx)) g.asking = true
g.asking, g.endAsking = true, end
g.askGoroutines.Go(func() { go g.askAboutWaiting(context.WithoutCancel(ctx))
defer end()
g.askAboutWaiting(askCtx)
})
} }
// askAboutWaiting asks GeoJS about the waiting clients, one request at a // askAboutWaiting asks GeoJS about the waiting clients, one request at a
// time, until none is left, GeoJS fails, or StopAsking ends ctx. Each // time, until none is left or GeoJS fails. Each answer kept is given to
// answer kept is given to Answered, outside the lock, since Answered takes // Answered, outside the lock, since Answered takes locks of its own.
// locks of its own.
func (g *GeoJS) askAboutWaiting(ctx context.Context) { func (g *GeoJS) askAboutWaiting(ctx context.Context) {
for { for {
clients := g.nextClients() clients := g.nextClients()
@@ -366,15 +347,11 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
} }
given, err := g.request(ctx, clients) given, err := g.request(ctx, clients)
if err != nil && ctx.Err() != nil {
continue // ended by StopAsking, which is no failure
}
kept, answered := g.keep(clients, given, err) kept, answered := g.keep(clients, given, err)
if g.params.Answered != nil { if g.answered != nil {
for _, answer := range kept { for _, answer := range kept {
g.params.Answered(answer) g.answered(answer)
} }
} }
@@ -385,13 +362,13 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
} }
// nextClients returns up to maxPerRequest of the waiting clients. When // nextClients returns up to maxPerRequest of the waiting clients. When
// none is waiting, or StopAsking has been called, it returns none and // none is waiting, it returns none and notes that no request to GeoJS is
// notes that no request to GeoJS is under way. // under way.
func (g *GeoJS) nextClients() []netip.Prefix { func (g *GeoJS) nextClients() []netip.Prefix {
g.mu.Lock() g.mu.Lock()
defer g.mu.Unlock() defer g.mu.Unlock()
if len(g.waiting) == 0 || g.stopped { if len(g.waiting) == 0 {
g.asking = false g.asking = false
return nil return nil
@@ -423,7 +400,7 @@ func (g *GeoJS) keep(
g.mu.Lock() g.mu.Lock()
defer g.mu.Unlock() defer g.mu.Unlock()
now := g.params.Now() now := g.now()
kept := make([]Answer, 0, len(clients)) kept := make([]Answer, 0, len(clients))
leftOut := 0 leftOut := 0
@@ -467,7 +444,7 @@ func (g *GeoJS) keep(
w.late = true w.late = true
} }
g.params.Failed(err, g.retryDelay) g.failed(err, g.retryDelay)
return kept, false return kept, false
} }
@@ -497,7 +474,7 @@ func (g *GeoJS) request(
req, err := http.NewRequestWithContext( req, err := http.NewRequestWithContext(
ctx, ctx,
http.MethodGet, http.MethodGet,
g.params.URL, g.url,
http.NoBody, http.NoBody,
) )
if err != nil { if err != nil {
@@ -506,7 +483,7 @@ func (g *GeoJS) request(
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",") req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
g.params.Asked() g.asked()
res, err := g.httpClient.Do(req) res, err := g.httpClient.Do(req)
if err != nil { if err != nil {
-38
View File
@@ -318,44 +318,6 @@ func TestCallerWhoseContextEndsEarlyDoesNotStopTheRequest(t *testing.T) {
}) })
} }
func TestStopAskingEndsTheRequestUnderWayAsNoFailureAndAsksNoMore(
t *testing.T,
) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
service, _, g, reports := startWithReports()
clients := newClients()
service.set(hanging)
g.LookUpWithoutWaiting(t.Context(), clients())
waitForRequests(t, service, 1)
// The request is ended, not waited out, and is no failure.
began := time.Now()
g.StopAsking()
if waited := time.Since(began); waited != 0 {
t.Errorf("StopAsking returned after %s, want at once", waited)
}
reports.wantFailures(t)
// A new client is then not asked about, and its lookup does not
// wait.
service.set(answering)
wantCountry(t, g, clients(), "")
if waited := time.Since(began); waited != 0 {
t.Errorf("the lookup returned after %s, want at once", waited)
}
waitForRequests(t, service, 1)
reports.wantFailures(t)
})
}
func TestRequestOver10SecondsIsAFailure(t *testing.T) { func TestRequestOver10SecondsIsAFailure(t *testing.T) {
t.Parallel() t.Parallel()
+1 -1
View File
@@ -185,7 +185,7 @@ func (m *Metrics) AddBansAndClients(
ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time, ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time,
) { ) {
for _, cause := range []string{ for _, cause := range []string{
bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseAbuseIPDB, bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin,
} { } {
m.registry.MustRegister( m.registry.MustRegister(
prometheus.NewCounterFunc(prometheus.CounterOpts{ prometheus.NewCounterFunc(prometheus.CounterOpts{
-44
View File
@@ -1,44 +0,0 @@
// Package processlog builds the logger for the process's own messages,
// which it writes as JSON lines marked "type":"process". The request log
// is package requestlog's.
package processlog
import (
"io"
"log/slog"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
// Params are what New needs.
type Params struct {
// Output is where the lines are written.
Output io.Writer
// InstanceName, SWW_WAF_INSTANCE_NAME, is each line's instance.
InstanceName string
// Level, SWW_WAF_LOG_LEVEL, is the least severe level written.
Level slog.Level
}
// New returns the logger for the process's own messages: JSON lines on
// params.Output, marked "type":"process", with the time in the same form
// as a request line's. It writes only the messages at params.Level or more
// severe; the request lines requestlog.Write writes are never held back.
func New(params Params) *slog.Logger {
handler := slog.NewJSONHandler(params.Output, &slog.HandlerOptions{
Level: params.Level,
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
if attr.Key == slog.TimeKey && len(groups) == 0 {
return slog.String(
slog.TimeKey,
requestlog.FormatTime(attr.Value.Time()),
)
}
return attr
},
})
return slog.New(handler).
With("type", "process", "instance", params.InstanceName)
}
-105
View File
@@ -1,105 +0,0 @@
package processlog_test
import (
"bytes"
"encoding/json"
"log/slog"
"slices"
"strings"
"testing"
"time"
"sneak.berlin/go/smallwebwaf/internal/processlog"
)
// instanceName is the instance the tests' loggers are built with.
const instanceName = "host1/blog"
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processlog.New(processlog.Params{
Output: &out,
InstanceName: instanceName,
Level: slog.LevelInfo,
}).
Info("starting",
"version", "v1")
var fields map[string]any
err := json.Unmarshal(out.Bytes(), &fields)
if err != nil {
t.Fatalf("decode %q: %v", out.String(), err)
}
if fields["type"] != "process" ||
fields["instance"] != "host1/blog" ||
fields["msg"] != "starting" ||
fields["level"] != "INFO" ||
fields["version"] != "v1" {
t.Errorf("process line %v", fields)
}
timeText, _ := fields["time"].(string)
logged, err := time.Parse(time.RFC3339, timeText)
if err != nil || !strings.HasSuffix(timeText, "Z") ||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
time.Since(logged) > time.Minute {
t.Errorf(
"process line time %q, want now in UTC with milliseconds",
timeText,
)
}
}
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
t.Parallel()
levels := []slog.Level{
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
}
for i, level := range levels {
t.Run(level.String(), func(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processLog := processlog.New(processlog.Params{
Output: &out,
InstanceName: instanceName,
Level: level,
})
for _, at := range levels {
processLog.Log(t.Context(), at, "message")
}
var got, want []string
for line := range strings.Lines(out.String()) {
var fields struct {
Level string `json:"level"`
}
err := json.Unmarshal([]byte(line), &fields)
if err != nil {
t.Fatalf("decode %q: %v", line, err)
}
got = append(got, fields.Level)
}
for _, written := range levels[i:] {
want = append(want, written.String())
}
if !slices.Equal(got, want) {
t.Errorf("lines at %v, want %v", got, want)
}
})
}
}
+4 -7
View File
@@ -11,7 +11,7 @@ import (
func TestHistoryKeepsEveryRequest(t *testing.T) { func TestHistoryKeepsEveryRequest(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -57,7 +57,7 @@ func TestHistoryKeepsEveryRequest(t *testing.T) {
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) { func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
other := netip.MustParsePrefix("198.51.100.7/32") other := netip.MustParsePrefix("198.51.100.7/32")
start := midnight() start := midnight()
@@ -100,10 +100,7 @@ func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
func TestResetKeepsTheHistory(t *testing.T) { func TestResetKeepsTheHistory(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
Limits: ratelimit.Limits{PerMinute: limit},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -126,7 +123,7 @@ func TestResetKeepsTheHistory(t *testing.T) {
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) { func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
for client, requests := range map[string]int{ for client, requests := range map[string]int{
"198.51.100.9/32": 2, "198.51.100.9/32": 2,
+6 -18
View File
@@ -156,28 +156,16 @@ type Request struct {
BlockedByCoreRuleSet bool BlockedByCoreRuleSet bool
} }
// Params are what New needs. // New returns a Limiter for limits, with no client counted yet, whose
type Params struct { // table holds at most maxClients clients (SWW_WAF_MAX_TRACKED_CLIENTS). Past
// Limits are what each client's requests and bytes are counted against. // it, the least recently seen client is dropped, with its history, and
Limits Limits // starts afresh if it comes back.
// MaxClients is the most clients the table holds, func New(limits Limits, maxClients int) *Limiter {
// SWW_WAF_MAX_TRACKED_CLIENTS. Past it, the least recently seen client clients, err := simplelru.NewLRU[netip.Prefix, *Client](maxClients, nil)
// is dropped, with its history, and starts afresh if it comes back.
MaxClients int
}
// New returns a Limiter with no client counted yet.
func New(params Params) *Limiter {
clients, err := simplelru.NewLRU[netip.Prefix, *Client](
params.MaxClients,
nil,
)
if err != nil { if err != nil {
panic(err) // NewLRU fails only for a size below one panic(err) // NewLRU fails only for a size below one
} }
limits := params.Limits
return &Limiter{ return &Limiter{
windows: [3]window{ windows: [3]window{
{ {
+39 -63
View File
@@ -33,10 +33,10 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
now := midnight() now := midnight()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit + 1}, ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
MaxClients: tableSize, tableSize,
}) )
if limiter.OverARateLimit(client, now) { if limiter.OverARateLimit(client, now) {
t.Error("a client never seen is over a rate limit") t.Error("a client never seen is over a rate limit")
@@ -72,7 +72,7 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
wantCount(t, limiter, client, now, minute) wantCount(t, limiter, client, now, minute)
// A limit that is off is never gone over. // A limit that is off is never gone over.
off := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) off := ratelimit.New(ratelimit.Limits{}, tableSize)
off.Count(client, now, whole) off.Count(client, now, whole)
if off.OverARateLimit(client, now) { if off.OverARateLimit(client, now) {
@@ -95,10 +95,7 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
t.Run(tc.window, func(t *testing.T) { t.Run(tc.window, func(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(tc.limits, tableSize)
Limits: tc.limits,
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
quarter := tc.length / 4 quarter := tc.length / 4
@@ -129,10 +126,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) { func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(
Limits: ratelimit.Limits{PerMinute: limit, PerHour: limit}, ratelimit.Limits{PerMinute: limit, PerHour: limit},
MaxClients: tableSize, tableSize,
}) )
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -162,13 +159,10 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) { func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(
Limits: ratelimit.Limits{ ratelimit.Limits{PerMinute: 5, BytesPerDay: math.MaxInt64},
PerMinute: 5, tableSize,
BytesPerDay: math.MaxInt64, )
},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -209,10 +203,10 @@ func TestZeroPercentIsAZeroAllowanceAndALimitOffStaysOff(t *testing.T) {
t.Parallel() t.Parallel()
// Only the hour has limits: the minute's and the day's are off. // Only the hour has limits: the minute's and the day's are off.
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(
Limits: ratelimit.Limits{PerHour: limit, BytesPerHour: 1000}, ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
MaxClients: tableSize, tableSize,
}) )
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -259,10 +253,7 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
t.Run(tc.window, func(t *testing.T) { t.Run(tc.window, func(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(tc.limits, tableSize)
Limits: tc.limits,
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
// 600 bytes are within the limit, 600 more over it. // 600 bytes are within the limit, 600 more over it.
@@ -296,10 +287,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) { func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(
Limits: ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000}, ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
MaxClients: tableSize, tableSize,
}) )
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
other := netip.MustParsePrefix("203.0.113.10/32") other := netip.MustParsePrefix("203.0.113.10/32")
start := midnight() start := midnight()
@@ -323,7 +314,7 @@ func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
func TestCountGivesTheBytesInEachWindow(t *testing.T) { func TestCountGivesTheBytesInEachWindow(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -349,10 +340,7 @@ func TestCountGivesTheBytesInEachWindow(t *testing.T) {
func TestResetSetsTheBytesBackToZero(t *testing.T) { func TestResetSetsTheBytesBackToZero(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(ratelimit.Limits{BytesPerDay: 1000}, tableSize)
Limits: ratelimit.Limits{BytesPerDay: 1000},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -368,7 +356,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
) { ) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -423,7 +411,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
func TestCountGivesTheRequestsInEachWindow(t *testing.T) { func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -450,10 +438,10 @@ func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
func TestResetSetsTheCountsBackToZero(t *testing.T) { func TestResetSetsTheCountsBackToZero(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit}, ratelimit.Limits{PerMinute: limit, PerDay: limit},
MaxClients: tableSize, tableSize,
}) )
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -475,10 +463,7 @@ func TestResetSetsTheCountsBackToZero(t *testing.T) {
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) { func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -497,10 +482,10 @@ func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
func TestRefusedRequestsCount(t *testing.T) { func TestRefusedRequestsCount(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(
Limits: ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit}, ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
MaxClients: tableSize, tableSize,
}) )
refused := netip.MustParsePrefix("203.0.113.9/32") refused := netip.MustParsePrefix("203.0.113.9/32")
within := netip.MustParsePrefix("203.0.113.10/32") within := netip.MustParsePrefix("203.0.113.10/32")
start := midnight() start := midnight()
@@ -533,10 +518,7 @@ func TestRefusedRequestsCount(t *testing.T) {
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) { func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
Limits: ratelimit.Limits{PerMinute: limit},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -552,10 +534,7 @@ func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) { func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -583,10 +562,7 @@ func TestKeepsAtMostMaxClientsDroppingTheLeastRecentlySeen(t *testing.T) {
const maxClients = 3 const maxClients = 3
limiter := ratelimit.New(ratelimit.Params{ limiter := ratelimit.New(ratelimit.Limits{PerMinute: 1}, maxClients)
Limits: ratelimit.Limits{PerMinute: 1},
MaxClients: maxClients,
})
now := midnight() now := midnight()
clients := make([]netip.Prefix, maxClients+1) clients := make([]netip.Prefix, maxClients+1)
+6 -12
View File
@@ -19,7 +19,7 @@ func TestSnapshotListsTheClientsByAddress(t *testing.T) {
"2001:db8::/64", "2001:db8::/64",
} }
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
for _, i := range []int{2, 3, 0, 1} { for _, i := range []int{2, 3, 0, 1} {
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole) limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
} }
@@ -52,10 +52,7 @@ func TestLoadedCountsCarryOn(t *testing.T) {
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
before := ratelimit.New(ratelimit.Params{ before := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
for range limit { for range limit {
wantCount(t, before, client, start, "") wantCount(t, before, client, start, "")
} }
@@ -63,10 +60,7 @@ func TestLoadedCountsCarryOn(t *testing.T) {
// Loaded into a new limiter, as across a restart, the client has no // Loaded into a new limiter, as across a restart, the client has no
// fresh allowance. // fresh allowance.
later := start.Add(time.Minute) later := start.Add(time.Minute)
after := ratelimit.New(ratelimit.Params{ after := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
after.Load(before.Snapshot(), later) after.Load(before.Snapshot(), later)
wantCount(t, after, client, later, hour) wantCount(t, after, client, later, hour)
} }
@@ -77,7 +71,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
limiter.Count(client, start, whole) limiter.Count(client, start, whole)
limiter.CountBytes(client, start, 5, whole) limiter.CountBytes(client, start, 5, whole)
limiter.CountRefusal(client, start, limit) limiter.CountRefusal(client, start, limit)
@@ -86,7 +80,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
loaded := func(now time.Time) ratelimit.Client { loaded := func(now time.Time) ratelimit.Client {
t.Helper() t.Helper()
after := ratelimit.New(ratelimit.Params{MaxClients: tableSize}) after := ratelimit.New(ratelimit.Limits{}, tableSize)
after.Load(limiter.Snapshot(), now) after.Load(limiter.Snapshot(), now)
return after.Snapshot()[0] return after.Snapshot()[0]
@@ -148,7 +142,7 @@ func TestLoadDropsTheLeastRecentlySeenFirst(t *testing.T) {
addr = addr.Next() addr = addr.Next()
} }
limiter := ratelimit.New(ratelimit.Params{MaxClients: maxClients}) limiter := ratelimit.New(ratelimit.Limits{}, maxClients)
limiter.Load(clients, midnight()) limiter.Load(clients, midnight())
got := limiter.Snapshot() got := limiter.Snapshot()
+18 -194
View File
@@ -11,7 +11,6 @@ import (
"net/netip" "net/netip"
"net/url" "net/url"
"slices" "slices"
"strconv"
"sync" "sync"
"time" "time"
@@ -24,9 +23,6 @@ const (
// AbuseIPDBURL is where clients are checked: the check endpoint of // AbuseIPDBURL is where clients are checked: the check endpoint of
// AbuseIPDB's API. // AbuseIPDB's API.
AbuseIPDBURL = "https://api.abuseipdb.com/api/v2/check" AbuseIPDBURL = "https://api.abuseipdb.com/api/v2/check"
// AbuseIPDBBlacklistURL is where AbuseIPDB's blacklist is fetched from:
// the blacklist endpoint of AbuseIPDB's API.
AbuseIPDBBlacklistURL = "https://api.abuseipdb.com/api/v2/blacklist"
// AbuseIPDBSource is how the request log, the alerts and the metrics // AbuseIPDBSource is how the request log, the alerts and the metrics
// name AbuseIPDB. // name AbuseIPDB.
AbuseIPDBSource = "abuseipdb" AbuseIPDBSource = "abuseipdb"
@@ -36,8 +32,6 @@ const (
var ( var (
errNoScore = errors.New("the answer gives no abuseConfidenceScore") errNoScore = errors.New("the answer gives no abuseConfidenceScore")
errNoBlacklist = errors.New("the answer gives no data")
errNotAddress = errors.New("is not an address")
errBudgetUsedUp = errors.New( errBudgetUsedUp = errors.New(
"checks spent; none is made until the day ends at 00:00 UTC") "checks spent; none is made until the day ends at 00:00 UTC")
) )
@@ -59,13 +53,6 @@ type ChecksSpent struct {
Spent int Spent int
} }
// BlacklistEntry is a client on AbuseIPDB's blacklist: its address and its
// abuse confidence score.
type BlacklistEntry struct {
Addr netip.Addr
Score int64
}
// AbuseIPDBParams are what NewAbuseIPDB needs. // AbuseIPDBParams are what NewAbuseIPDB needs.
type AbuseIPDBParams struct { type AbuseIPDBParams struct {
// URL is where clients are checked, normally AbuseIPDBURL, with Key, // URL is where clients are checked, normally AbuseIPDBURL, with Key,
@@ -82,19 +69,10 @@ type AbuseIPDBParams struct {
// take (SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS). // take (SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS).
CacheTTL time.Duration CacheTTL time.Duration
Timeout time.Duration Timeout time.Duration
// BlacklistURL is where the blacklist is fetched from, normally
// AbuseIPDBBlacklistURL, with the clients scored Threshold or more
// (SWW_WAF_ABUSEIPDB_THRESHOLD), every BlacklistRefresh
// (SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS). Blacklisted
// receives the clients on each blacklist fetched.
BlacklistURL string
Threshold int64
BlacklistRefresh time.Duration
Blacklisted func([]BlacklistEntry)
// Now tells the time, normally time.Now in UTC. // Now tells the time, normally time.Now in UTC.
Now func() time.Time Now func() time.Time
// ProcessLog receives each check or fetch of the blacklist that fails, // ProcessLog receives each check that fails, and why, and the day's
// and why, and the day's budget used up. // budget used up.
ProcessLog *slog.Logger ProcessLog *slog.Logger
// Alerts receive a source_failure alert for each. // Alerts receive a source_failure alert for each.
Alerts *alerts.Queue Alerts *alerts.Queue
@@ -110,12 +88,8 @@ type AbuseIPDB struct {
// scores are by client. Each is added as it is fetched and never moved // scores are by client. Each is added as it is fetched and never moved
// up, so that the one fetched longest ago is the first dropped. // up, so that the one fetched longest ago is the first dropped.
scores *simplelru.LRU[netip.Prefix, Score] scores *simplelru.LRU[netip.Prefix, Score]
// checking are the clients whose check is under way, each with the // checking are the clients whose check is under way.
// function that ends it. stopped is true once StopChecking has been checking map[netip.Prefix]bool
// called, and checkGoroutines waits for the goroutine of each check.
checking map[netip.Prefix]context.CancelFunc
stopped bool
checkGoroutines sync.WaitGroup
// day is the day, in UTC, of the checks spent counts. // day is the day, in UTC, of the checks spent counts.
day civil.Date day civil.Date
spent int spent int
@@ -138,7 +112,7 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
params: params, params: params,
httpClient: &http.Client{}, httpClient: &http.Client{},
scores: scores, scores: scores,
checking: map[netip.Prefix]context.CancelFunc{}, checking: map[netip.Prefix]bool{},
} }
} }
@@ -148,11 +122,11 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
// addresses its request comes from. A client without one is checked in // addresses its request comes from. A client without one is checked in
// the background, by addr, the address its request came from, if // the background, by addr, the address its request came from, if
// offender, if it has committed an offence, unless its check is under // offender, if it has committed an offence, unless its check is under
// way, a check failed less than failureDelay ago, the day's checks have // way, a check failed less than failureDelay ago, or the day's checks
// used up DailyBudget, or StopChecking has been called; Hit never waits // have used up DailyBudget; Hit never waits for a check. The check that
// for a check. The check that uses the budget up is logged and raised as a // uses the budget up is logged and raised as a source_failure alert. ctx
// source_failure alert. ctx is the context of the client's request, and a // is the context of the client's request, and a check goes on after the
// check goes on after the request ends, until StopChecking. // request ends.
func (a *AbuseIPDB) Hit( func (a *AbuseIPDB) Hit(
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool, ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
) (int64, bool) { ) (int64, bool) {
@@ -171,21 +145,14 @@ func (a *AbuseIPDB) Hit(
a.day, a.spent = today, 0 a.day, a.spent = today, 0
} }
_, checking := a.checking[client] check := offender && !a.checking[client] && !now.Before(a.retryAt) &&
a.spent < a.params.DailyBudget
check := offender && !checking && !a.stopped &&
!now.Before(a.retryAt) && a.spent < a.params.DailyBudget
if check { if check {
checkCtx, end := context.WithCancel(context.WithoutCancel(ctx)) a.checking[client] = true
a.checking[client] = end
a.checks++ a.checks++
a.spent++ a.spent++
a.checkGoroutines.Go(func() { go a.check(context.WithoutCancel(ctx), client, addr)
defer end()
a.check(checkCtx, client, addr)
})
} }
usedUp := check && a.spent == a.params.DailyBudget usedUp := check && a.spent == a.params.DailyBudget
@@ -200,24 +167,6 @@ func (a *AbuseIPDB) Hit(
return 0, false return 0, false
} }
// StopChecking ends the checks under way, and returns once their goroutines
// have. A check it ends gives no score and is no failure: it is neither
// counted, logged nor raised as an alert, and checks are not left alone
// after it. The ended check stays spent from the day's budget. After
// StopChecking, Hit checks no client.
func (a *AbuseIPDB) StopChecking() {
a.mu.Lock()
a.stopped = true
for _, end := range a.checking {
end()
}
a.mu.Unlock()
a.checkGoroutines.Wait()
}
// Checked returns how many checks were made. // Checked returns how many checks were made.
func (a *AbuseIPDB) Checked() int { func (a *AbuseIPDB) Checked() int {
a.mu.Lock() a.mu.Lock()
@@ -307,45 +256,10 @@ func (a *AbuseIPDB) LoadChecksSpent(checks ChecksSpent) {
a.day, a.spent = checks.Day, checks.Spent a.day, a.spent = checks.Day, checks.Spent
} }
// RunBlacklist fetches AbuseIPDB's blacklist at once, and again every
// BlacklistRefresh, until ctx is done, and gives the clients on each copy
// fetched to Blacklisted. While Key is unset it does nothing. A fetch that
// fails is logged and raised as a source_failure alert, unless it was cut
// off as ctx ended.
func (a *AbuseIPDB) RunBlacklist(ctx context.Context) {
if a.params.Key == "" {
return
}
for {
listed, err := a.fetchBlacklist(ctx)
switch {
case ctx.Err() != nil:
return
case err != nil:
a.alert("fetching AbuseIPDB's blacklist failed", err)
default:
a.params.Blacklisted(listed)
}
timer := time.NewTimer(a.params.BlacklistRefresh)
select {
case <-ctx.Done():
timer.Stop()
return
case <-timer.C:
}
}
}
// check checks client with AbuseIPDB by addr, one of its addresses, keeps // check checks client with AbuseIPDB by addr, one of its addresses, keeps
// the score as client's, and notes the check as no longer under way. A // the score as client's, and notes the check as no longer under way. A
// check that fails gives no score: it is counted, logged and raised as a // check that fails gives no score: it is counted, logged and raised as a
// source_failure alert, and no client is checked for failureDelay. A check // source_failure alert, and no client is checked for failureDelay.
// ended as ctx ends, by StopChecking, is no failure.
func (a *AbuseIPDB) check( func (a *AbuseIPDB) check(
ctx context.Context, ctx context.Context,
client netip.Prefix, client netip.Prefix,
@@ -353,26 +267,24 @@ func (a *AbuseIPDB) check(
) { ) {
score, err := a.ask(ctx, addr) score, err := a.ask(ctx, addr)
now := a.params.Now() now := a.params.Now()
isFailure := err != nil && ctx.Err() == nil
a.mu.Lock() a.mu.Lock()
delete(a.checking, client) delete(a.checking, client)
switch { if err == nil {
case err == nil:
a.scores.Add( a.scores.Add(
client, client,
Score{Client: client, Score: score, Fetched: now}, Score{Client: client, Score: score, Fetched: now},
) )
case isFailure: } else {
a.failures++ a.failures++
a.retryAt = now.Add(failureDelay) a.retryAt = now.Add(failureDelay)
} }
a.mu.Unlock() a.mu.Unlock()
if isFailure { if err != nil {
a.alert("checking a client with AbuseIPDB failed", err) a.alert("checking a client with AbuseIPDB failed", err)
} }
} }
@@ -432,94 +344,6 @@ func (a *AbuseIPDB) ask(
return *answer.Data.AbuseConfidenceScore, nil return *answer.Data.AbuseConfidenceScore, nil
} }
// fetchBlacklist fetches the clients on AbuseIPDB's blacklist scored
// Threshold or more, sending the key in the header Key. An answer other
// than 200, one longer than maxListBytes or without the list, an entry
// whose address does not read, and none within fetchTimeout, fail.
func (a *AbuseIPDB) fetchBlacklist(
ctx context.Context,
) ([]BlacklistEntry, error) {
ctx, cancel := context.WithTimeout(ctx, fetchTimeout)
defer cancel()
query := url.Values{
"confidenceMinimum": {strconv.FormatInt(a.params.Threshold, 10)},
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet,
a.params.BlacklistURL+"?"+query.Encode(), http.NoBody)
if err != nil {
return nil, fmt.Errorf("make the request: %w", err)
}
req.Header.Set("Key", a.params.Key)
req.Header.Set("Accept", "application/json")
res, err := a.httpClient.Do(req)
if err != nil {
return nil, fmt.Errorf("fetch the blacklist: %w", errors.Unwrap(err))
}
defer func() {
_ = res.Body.Close()
}()
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("%w %s", errStatus, res.Status)
}
body, err := io.ReadAll(io.LimitReader(res.Body, maxListBytes+1))
if err != nil {
return nil, fmt.Errorf("read the blacklist: %w", err)
}
if len(body) > maxListBytes {
return nil, errTooLong
}
return parseBlacklist(body)
}
// parseBlacklist reads body, AbuseIPDB's answer with its blacklist.
func parseBlacklist(body []byte) ([]BlacklistEntry, error) {
var answer struct {
Data *[]struct {
IPAddress string `json:"ipAddress"`
AbuseConfidenceScore int64 `json:"abuseConfidenceScore"`
} `json:"data"`
}
err := json.Unmarshal(body, &answer)
if err != nil {
return nil, fmt.Errorf("read the blacklist: %w", err)
}
if answer.Data == nil {
return nil, errNoBlacklist
}
listed := make([]BlacklistEntry, 0, len(*answer.Data))
for i, entry := range *answer.Data {
addr, err := netip.ParseAddr(entry.IPAddress)
if err != nil || addr.Zone() != "" {
// The error leaves out what the entry holds, as it does a
// client's address.
return nil, fmt.Errorf(
"entry %d's ipAddress %w",
i+1,
errNotAddress,
)
}
listed = append(listed, BlacklistEntry{
Addr: addr.Unmap(), Score: entry.AbuseConfidenceScore,
})
}
return listed, nil
}
// alert raises a source_failure alert from AbuseIPDB with reason and err, // alert raises a source_failure alert from AbuseIPDB with reason and err,
// and logs them. // and logs them.
func (a *AbuseIPDB) alert(reason string, err error) { func (a *AbuseIPDB) alert(reason string, err error) {
-37
View File
@@ -394,43 +394,6 @@ func TestCheckNotAnsweredWithinTheTimeoutFailsAndHitNeverWaits(
}) })
} }
func TestStopCheckingEndsTheCheckUnderWayAsNoFailureAndChecksNoMore(
t *testing.T,
) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
queue := newQueue()
p := abuseIPDBParams()
p.Alerts = queue
abuseIPDB := &abuseIPDBStandIn{hanging: true}
checker := newAbuseIPDB(abuseIPDB, p)
wantScore(t, checker, suspect, true, 0, false)
synctest.Wait()
// The check is ended, not waited out, and is no failure.
began := time.Now()
checker.StopChecking()
if waited := time.Since(began); waited != 0 {
t.Errorf("StopChecking returned after %s, want at once", waited)
}
wantFailures(t, checker, 0)
if got := waiting(queue); len(got) != 0 {
t.Errorf("alerts waiting %+v, want none", got)
}
// An offender without a score is then not checked.
wantScore(t, checker, other, true, 0, false)
synctest.Wait()
wantChecked(t, abuseIPDB, suspect)
})
}
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) { func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
t.Parallel() t.Parallel()
-252
View File
@@ -1,252 +0,0 @@
package reputation_test
import (
"context"
"fmt"
"io"
"net/http"
"net/netip"
"slices"
"strings"
"sync"
"testing"
"testing/synctest"
"time"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/reputation"
)
// The tests of the blacklist run in synctest bubbles, as those of the
// checks do, with AbuseIPDB a stand-in reached without the network.
// blacklistURL is where the tests fetch the blacklist from.
const blacklistURL = "https://abuseipdb.example/api/v2/blacklist"
func TestBlacklistIsFetchedAtOnceAndEveryRefresh(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
abuseIPDB := &blacklistStandIn{body: `{"meta":{},"data":[` +
`{"ipAddress":"203.0.113.9","abuseConfidenceScore":100},` +
`{"ipAddress":"::ffff:198.51.100.7","abuseConfidenceScore":95},` +
`{"ipAddress":"2001:db8::9","abuseConfidenceScore":90}]}`}
got := runBlacklist(t, abuseIPDB, key, newQueue())
want := []reputation.BlacklistEntry{
{Addr: netip.MustParseAddr("203.0.113.9"), Score: 100},
{Addr: netip.MustParseAddr("198.51.100.7"), Score: 95},
{Addr: netip.MustParseAddr("2001:db8::9"), Score: 90},
}
synctest.Wait()
got.want(t, want)
time.Sleep(time.Hour - time.Nanosecond)
synctest.Wait()
got.want(t, want)
time.Sleep(time.Nanosecond)
synctest.Wait()
got.want(t, want, want)
// Each fetch asks for the clients scored 90 or more, with the key
// in the header Key alone.
abuseIPDB.mu.Lock()
defer abuseIPDB.mu.Unlock()
for _, req := range abuseIPDB.asked {
if req.URL.String() != blacklistURL+"?confidenceMinimum=90" ||
req.Header.Get("Key") != key {
t.Errorf(
"asked %s with the key %q, want %s?confidenceMinimum=90 "+
"with %q",
req.URL,
req.Header.Get("Key"),
blacklistURL,
key,
)
}
}
})
}
func TestFailedBlacklistFetchIsAlertedAndTriedAgainAtTheNextRefresh(
t *testing.T,
) {
t.Parallel()
for _, tc := range []struct{ name, body, error string }{
{
"a refusal", "",
"the server answered 429 Too Many Requests",
},
{
"an answer without the list", `{"errors":[]}`,
"the answer gives no data",
},
{
"an entry that is not an address",
`{"data":[{"ipAddress":"203.0.113.9"},{"ipAddress":"example"}]}`,
"entry 2's ipAddress is not an address",
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
abuseIPDB := &blacklistStandIn{body: tc.body}
if tc.body == "" {
abuseIPDB.status = http.StatusTooManyRequests
}
queue := newQueue()
got := runBlacklist(t, abuseIPDB, key, queue)
synctest.Wait()
got.want(t)
alerted := waiting(queue)
if len(alerted) != 1 ||
alerted[0].Event != alerts.EventSourceFailure ||
alerted[0].Reason != "fetching AbuseIPDB's blacklist failed" ||
alerted[0].Detail["source"] != reputation.AbuseIPDBSource ||
alerted[0].Detail["error"] != tc.error {
t.Errorf(
"alerts %+v, want the failure %q",
alerted,
tc.error,
)
}
abuseIPDB.answer(`{"data":[]}`)
time.Sleep(time.Hour)
synctest.Wait()
got.want(t, []reputation.BlacklistEntry{})
})
})
}
}
func TestBlacklistIsNotFetchedWithoutAKey(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
abuseIPDB := &blacklistStandIn{body: `{"data":[]}`}
got := runBlacklist(t, abuseIPDB, "", newQueue())
time.Sleep(time.Hour)
synctest.Wait()
got.want(t)
if len(abuseIPDB.asked) != 0 {
t.Errorf("fetched the blacklist %d times, want none",
len(abuseIPDB.asked))
}
})
}
// blacklistStandIn is a stand-in for AbuseIPDB that answers each fetch of
// the blacklist with status, 200 when it is zero, and body, and keeps the
// requests.
type blacklistStandIn struct {
mu sync.Mutex
status int
body string
asked []*http.Request
}
// RoundTrip has the stand-in answer req, in place of the network.
func (s *blacklistStandIn) RoundTrip(
req *http.Request,
) (*http.Response, error) {
s.mu.Lock()
s.asked = append(s.asked, req)
status, body := s.status, s.body
s.mu.Unlock()
if status == 0 {
status = http.StatusOK
}
return &http.Response{
StatusCode: status,
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
Header: http.Header{},
Body: io.NopCloser(strings.NewReader(body)),
Request: req,
}, nil
}
// answer has the stand-in answer each fetch with 200 and body.
func (s *blacklistStandIn) answer(body string) {
s.mu.Lock()
defer s.mu.Unlock()
s.status, s.body = 0, body
}
// blacklisted holds the clients on each blacklist given to Blacklisted.
type blacklisted struct {
mu sync.Mutex
fetched [][]reputation.BlacklistEntry
}
// want checks the clients on each blacklist given so far.
func (b *blacklisted) want(
t *testing.T,
want ...[]reputation.BlacklistEntry,
) {
t.Helper()
b.mu.Lock()
defer b.mu.Unlock()
if !slices.EqualFunc(b.fetched, want, slices.Equal) {
t.Errorf("blacklists %v, want %v", b.fetched, want)
}
}
// runBlacklist runs RunBlacklist, until the test ends, with the key
// accountKey, the threshold 90 and a refresh of an hour, fetching from
// abuseIPDB and raising alerts into queue, and returns what it gives
// Blacklisted.
func runBlacklist(
t *testing.T, abuseIPDB *blacklistStandIn, accountKey string,
queue *alerts.Queue,
) *blacklisted {
t.Helper()
got := &blacklisted{}
p := abuseIPDBParams()
p.Key = accountKey
p.BlacklistURL = blacklistURL
p.Threshold = 90
p.BlacklistRefresh = time.Hour
p.Alerts = queue
p.Blacklisted = func(listed []reputation.BlacklistEntry) {
got.mu.Lock()
defer got.mu.Unlock()
got.fetched = append(got.fetched, listed)
}
checker := reputation.NewAbuseIPDB(p)
checker.SetTransport(abuseIPDB)
ctx, cancel := context.WithCancel(t.Context())
done := make(chan struct{})
go func() {
checker.RunBlacklist(ctx)
close(done)
}()
t.Cleanup(func() {
cancel()
<-done
})
return got
}
+13 -44
View File
@@ -82,12 +82,8 @@ type DNSBL struct {
// verdicts are by query. Each is added as it is fetched and never moved // verdicts are by query. Each is added as it is fetched and never moved
// up, so that the one fetched longest ago is the first dropped. // up, so that the one fetched longest ago is the first dropped.
verdicts *simplelru.LRU[query, Verdict] verdicts *simplelru.LRU[query, Verdict]
// asking are the queries under way, each with the function that ends // asking are the queries under way.
// it. stopped is true once StopAsking has been called, and asking map[query]bool
// queryGoroutines waits for the goroutine of each query.
asking map[query]context.CancelFunc
stopped bool
queryGoroutines sync.WaitGroup
// queries and failures count, by zone, the queries made and those that // queries and failures count, by zone, the queries made and those that
// failed, and retryAt is when a zone whose last query failed may be // failed, and retryAt is when a zone whose last query failed may be
// asked again. // asked again.
@@ -124,7 +120,7 @@ func NewDNSBL(params DNSBLParams) *DNSBL {
params: params, params: params,
resolver: resolver, resolver: resolver,
verdicts: verdicts, verdicts: verdicts,
asking: map[query]context.CancelFunc{}, asking: map[query]bool{},
queries: map[string]int{}, queries: map[string]int{},
failures: map[string]int{}, failures: map[string]int{},
retryAt: map[string]time.Time{}, retryAt: map[string]time.Time{},
@@ -142,10 +138,9 @@ func (d *DNSBL) Zones() []string {
// log, the alerts and the metrics. A verdict is used until CacheTTL has // log, the alerts and the metrics. A verdict is used until CacheTTL has
// passed since it was fetched. Each zone without one is asked about addr // passed since it was fetched. Each zone without one is asked about addr
// in the background, unless a query about addr to it is under way, the // in the background, unless a query about addr to it is under way, the
// zone is left alone after a failure, maxQueries are under way, or // zone is left alone after a failure, or maxQueries are under way;
// StopAsking has been called; ListedBy never waits for a query. ctx is the // ListedBy never waits for a query. ctx is the context of the client's
// context of the client's request, and a query goes on after the request // request, and a query goes on after the request ends.
// ends, until StopAsking.
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string { func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
d.mu.Lock() d.mu.Lock()
defer d.mu.Unlock() defer d.mu.Unlock()
@@ -158,47 +153,23 @@ func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
q := query{zone: zone, client: addr} q := query{zone: zone, client: addr}
kept, found := d.verdicts.Peek(q) kept, found := d.verdicts.Peek(q)
_, asking := d.asking[q]
switch { switch {
case found && now.Sub(kept.Fetched) < d.params.CacheTTL: case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
if kept.Listed { if kept.Listed {
listedBy = append(listedBy, config.MaskZoneKey(zone)) listedBy = append(listedBy, config.MaskZoneKey(zone))
} }
case !asking && !d.stopped && !now.Before(d.retryAt[zone]) && case !d.asking[q] && !now.Before(d.retryAt[zone]) && len(d.asking) < maxQueries:
len(d.asking) < maxQueries: d.asking[q] = true
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
d.asking[q] = end
d.queries[zone]++ d.queries[zone]++
d.queryGoroutines.Go(func() { go d.ask(context.WithoutCancel(ctx), q)
defer end()
d.ask(queryCtx, q)
})
} }
} }
return listedBy return listedBy
} }
// StopAsking ends the queries under way, and returns once their
// goroutines have. A query it ends gives no verdict and is no failure: it
// is neither counted, logged nor raised as an alert, and the zone is not
// left alone after it. After StopAsking, ListedBy asks no zone.
func (d *DNSBL) StopAsking() {
d.mu.Lock()
d.stopped = true
for _, end := range d.asking {
end()
}
d.mu.Unlock()
d.queryGoroutines.Wait()
}
// Queries returns how many queries were made to zone. // Queries returns how many queries were made to zone.
func (d *DNSBL) Queries(zone string) int { func (d *DNSBL) Queries(zone string) int {
d.mu.Lock() d.mu.Lock()
@@ -267,29 +238,27 @@ func (d *DNSBL) Load(verdicts []Verdict) {
// query as no longer under way. A query that fails gives no verdict: it // query as no longer under way. A query that fails gives no verdict: it
// is counted, logged and raised as a source_failure alert, which show the // is counted, logged and raised as a source_failure alert, which show the
// zone with its key masked, and the zone is not asked again for // zone with its key masked, and the zone is not asked again for
// failureDelay. A query ended as ctx ends, by StopAsking, is no failure. // failureDelay.
func (d *DNSBL) ask(ctx context.Context, q query) { func (d *DNSBL) ask(ctx context.Context, q query) {
listed, err := d.lookUp(ctx, q) listed, err := d.lookUp(ctx, q)
now := d.params.Now() now := d.params.Now()
isFailure := err != nil && ctx.Err() == nil
d.mu.Lock() d.mu.Lock()
delete(d.asking, q) delete(d.asking, q)
switch { if err == nil {
case err == nil:
d.verdicts.Add(q, Verdict{ d.verdicts.Add(q, Verdict{
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now, Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
}) })
case isFailure: } else {
d.failures[q.zone]++ d.failures[q.zone]++
d.retryAt[q.zone] = now.Add(failureDelay) d.retryAt[q.zone] = now.Add(failureDelay)
} }
d.mu.Unlock() d.mu.Unlock()
if isFailure { if err != nil {
const failed = "asking a DNSBL zone failed" const failed = "asking a DNSBL zone failed"
shown := config.MaskZoneKey(q.zone) shown := config.MaskZoneKey(q.zone)
-39
View File
@@ -210,45 +210,6 @@ func TestQueryNotAnsweredWithinTheTimeoutFails(t *testing.T) {
}) })
} }
//
//nolint:paralleltest // one at a time, as the comment at the top of this file says
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
t *testing.T,
) {
synctest.Test(t, func(t *testing.T) {
queue := newQueue()
p := dnsblParams(zone)
p.Alerts = queue
resolver := &resolverStandIn{hanging: true}
dnsbl := newDNSBL(resolver, p)
wantZones(t, dnsbl, listed)
synctest.Wait()
// The query is ended, not waited out, and is no failure.
began := time.Now()
dnsbl.StopAsking()
if waited := time.Since(began); waited != 0 {
t.Errorf("StopAsking returned after %s, want at once", waited)
}
wantQueries(t, dnsbl, 1, 0)
if got := waiting(queue); len(got) != 0 {
t.Errorf("alerts waiting %+v, want none", got)
}
// A client without a verdict is then asked about in no zone.
wantZones(t, dnsbl, unlisted)
synctest.Wait()
wantQueries(t, dnsbl, 1, 0)
wantAsked(t, resolver, listedName)
waitForTheResolver()
})
}
// //
//nolint:paralleltest // one at a time, as the comment at the top of this file says //nolint:paralleltest // one at a time, as the comment at the top of this file says
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute( func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(
+1 -2
View File
@@ -7,8 +7,7 @@
// SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks // SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks
// clients with AbuseIPDB, and keeps their scores and the checks spent // clients with AbuseIPDB, and keeps their scores and the checks spent
// today. The state package writes all of these to files and reads them // today. The state package writes all of these to files and reads them
// from there, so that a restart keeps them too. Last, it fetches // from there, so that a restart keeps them too.
// AbuseIPDB's blacklist, which it does not keep.
package reputation package reputation
import ( import (
+33 -4
View File
@@ -1,7 +1,7 @@
// Package requestlog writes the request log sww prints on stdout: JSON // Package requestlog writes the lines sww prints on stdout: one
// lines marked "type":"request", at most two from waf and a short one // JSON object per request, two from waf and a short one from rproxy,
// from rproxy for each request. Package processlog writes the process's // marked "type":"request", and the process's own messages as JSON lines marked
// own messages. // "type":"process".
package requestlog package requestlog
import ( import (
@@ -9,6 +9,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"log/slog"
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/ratelimit"
@@ -299,3 +300,31 @@ func Milliseconds(d time.Duration) float64 {
time.Millisecond/time.Microsecond, time.Millisecond/time.Microsecond,
) )
} }
// NewProcessLogger returns the logger for the process's own messages:
// JSON lines on w, marked "type":"process", with the time in the same form
// as a request line's, and instanceName, SWW_WAF_INSTANCE_NAME, as instance.
// It writes only the messages at level, SWW_WAF_LOG_LEVEL, or more severe;
// the request lines Write writes are never held back.
func NewProcessLogger(
w io.Writer,
instanceName string,
level slog.Level,
) *slog.Logger {
handler := slog.NewJSONHandler(w, &slog.HandlerOptions{
Level: level,
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
if attr.Key == slog.TimeKey && len(groups) == 0 {
return slog.String(
slog.TimeKey,
FormatTime(attr.Value.Time()),
)
}
return attr
},
})
return slog.New(handler).
With("type", "process", "instance", instanceName)
}
+87
View File
@@ -3,6 +3,8 @@ package requestlog_test
import ( import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"log/slog"
"slices"
"strings" "strings"
"testing" "testing"
"time" "time"
@@ -132,3 +134,88 @@ func TestWriteRProxyLineWritesOneJSONLineWithOnlyItsFields(t *testing.T) {
t.Errorf("wrote %s, want %s", out.String(), want) t.Errorf("wrote %s, want %s", out.String(), want)
} }
} }
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
t.Parallel()
var out bytes.Buffer
requestlog.NewProcessLogger(&out, "host1/blog", slog.LevelInfo).
Info("starting",
"version", "v1")
var fields map[string]any
err := json.Unmarshal(out.Bytes(), &fields)
if err != nil {
t.Fatalf("decode %q: %v", out.String(), err)
}
if fields["type"] != "process" ||
fields["instance"] != "host1/blog" ||
fields["msg"] != "starting" ||
fields["level"] != "INFO" ||
fields["version"] != "v1" {
t.Errorf("process line %v", fields)
}
timeText, _ := fields["time"].(string)
logged, err := time.Parse(time.RFC3339, timeText)
if err != nil || !strings.HasSuffix(timeText, "Z") ||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
time.Since(logged) > time.Minute {
t.Errorf(
"process line time %q, want now in UTC with milliseconds",
timeText,
)
}
}
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
t.Parallel()
levels := []slog.Level{
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
}
for i, level := range levels {
t.Run(level.String(), func(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processLog := requestlog.NewProcessLogger(
&out,
"host1/blog",
level,
)
for _, at := range levels {
processLog.Log(t.Context(), at, "message")
}
var got, want []string
for line := range strings.Lines(out.String()) {
var fields struct {
Level string `json:"level"`
}
err := json.Unmarshal([]byte(line), &fields)
if err != nil {
t.Fatalf("decode %q: %v", line, err)
}
got = append(got, fields.Level)
}
for _, written := range levels[i:] {
want = append(want, written.String())
}
if !slices.Equal(got, want) {
t.Errorf("lines at %v, want %v", got, want)
}
})
}
}
+12 -43
View File
@@ -99,12 +99,8 @@ type Client struct {
// answers are by client. Each is added as it comes and never moved up, // answers are by client. Each is added as it comes and never moved up,
// so that the one that came longest ago is the first dropped. // so that the one that came longest ago is the first dropped.
answers *simplelru.LRU[netip.Prefix, Answer] answers *simplelru.LRU[netip.Prefix, Answer]
// asking are the clients whose query is under way, each with the // asking are the clients whose query is under way.
// function that ends it. stopped is true once StopAsking has been asking map[netip.Prefix]bool
// called, and queryGoroutines waits for the goroutine of each query.
asking map[netip.Prefix]context.CancelFunc
stopped bool
queryGoroutines sync.WaitGroup
// retryAt is when routewatch may be asked again after a query failed. // retryAt is when routewatch may be asked again after a query failed.
retryAt time.Time retryAt time.Time
} }
@@ -120,7 +116,7 @@ func New(params Params) *Client {
params: params, params: params,
httpClient: &http.Client{}, httpClient: &http.Client{},
answers: answers, answers: answers,
asking: map[netip.Prefix]context.CancelFunc{}, asking: map[netip.Prefix]bool{},
} }
} }
@@ -128,9 +124,8 @@ func New(params Params) *Client {
// IPv6 group, or why it has no route. It never waits for a query. A client // IPv6 group, or why it has no route. It never waits for a query. A client
// without an answer is asked about in the background, by addr, the address its // without an answer is asked about in the background, by addr, the address its
// request came from, unless its query is under way, a query failed less than // request came from, unless its query is under way, a query failed less than
// failureKept ago, maxQueries are under way, or StopAsking has been called. ctx // failureKept ago, or maxQueries are under way. ctx is the context of the
// is the context of the client's request, and a query goes on after the request // client's request, and a query goes on after the request ends.
// ends, until StopAsking.
func (c *Client) Route( func (c *Client) Route(
ctx context.Context, client netip.Prefix, addr netip.Addr, ctx context.Context, client netip.Prefix, addr netip.Addr,
) (Route, Missing) { ) (Route, Missing) {
@@ -152,38 +147,15 @@ func (c *Client) Route(
return Route{}, RequestFailed return Route{}, RequestFailed
} }
_, asking := c.asking[client] if !c.asking[client] && len(c.asking) < maxQueries {
if !asking && !c.stopped && len(c.asking) < maxQueries { c.asking[client] = true
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
c.asking[client] = end
c.queryGoroutines.Go(func() { go c.ask(context.WithoutCancel(ctx), client, addr)
defer end()
c.ask(queryCtx, client, addr)
})
} }
return Route{}, NoAnswerYet return Route{}, NoAnswerYet
} }
// StopAsking ends the queries under way, and returns once their goroutines
// have. A query it ends gives no answer and is no failure: it is not logged,
// and routewatch is not left alone after it. After StopAsking, Route asks
// routewatch nothing.
func (c *Client) StopAsking() {
c.mu.Lock()
c.stopped = true
for _, end := range c.asking {
end()
}
c.mu.Unlock()
c.queryGoroutines.Wait()
}
// Snapshot returns every answer still in use, sorted by client, as // Snapshot returns every answer still in use, sorted by client, as
// routewatch.json lists them. // routewatch.json lists them.
func (c *Client) Snapshot() []Answer { func (c *Client) Snapshot() []Answer {
@@ -233,8 +205,7 @@ func (c *Client) Load(answers []Answer) {
// ask asks routewatch about addr, keeps the answer as client's, and notes the // ask asks routewatch about addr, keeps the answer as client's, and notes the
// query as no longer under way. A query that fails gives no answer, and // query as no longer under way. A query that fails gives no answer, and
// routewatch is not asked again for failureKept. Only the first failure of // routewatch is not asked again for failureKept. Only the first failure of
// those under way is logged. A query ended as ctx ends, by StopAsking, is no // those under way is logged.
// failure.
func (c *Client) ask( func (c *Client) ask(
ctx context.Context, ctx context.Context,
client netip.Prefix, client netip.Prefix,
@@ -242,20 +213,18 @@ func (c *Client) ask(
) { ) {
route, err := c.query(ctx, addr) route, err := c.query(ctx, addr)
now := c.params.Now() now := c.params.Now()
isFailure := err != nil && ctx.Err() == nil
c.mu.Lock() c.mu.Lock()
delete(c.asking, client) delete(c.asking, client)
firstFailure := isFailure && !now.Before(c.retryAt) firstFailure := err != nil && !now.Before(c.retryAt)
switch { if err == nil {
case err == nil:
c.answers.Add(client, Answer{ c.answers.Add(client, Answer{
Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now, Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
}) })
case isFailure: } else {
c.retryAt = now.Add(failureKept) c.retryAt = now.Add(failureKept)
} }
-48
View File
@@ -361,54 +361,6 @@ func TestHangingRoutewatchNeverDelaysAndFailsTheQuery(t *testing.T) {
}) })
} }
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
t *testing.T,
) {
t.Parallel()
// In a synctest bubble, as TestHangingRoutewatchNeverDelaysAndFailsTheQuery
// says.
synctest.Test(t, func(t *testing.T) {
f := &fixture{t: t, log: &syncBuffer{}}
f.client = routewatch.New(routewatch.Params{
URL: silentURL, Now: time.Now,
ProcessLog: slog.New(slog.NewTextHandler(f.log, nil)),
})
f.route()
synctest.Wait()
// The query is ended, not waited out.
began := time.Now()
f.client.StopAsking()
if waited := time.Since(began); waited != 0 {
t.Errorf("StopAsking returned after %s, want at once", waited)
}
// A query started now would fail within waitLimit. None is, and the
// one ended is no failure: nothing is logged, and routewatch is not
// left alone.
for range 2 {
if _, missing := f.route(); missing != routewatch.NoAnswerYet {
t.Errorf(
"Route gave %q, want %q",
missing,
routewatch.NoAnswerYet,
)
}
time.Sleep(waitLimit)
synctest.Wait()
}
if f.log.String() != "" {
t.Errorf("process log %q, want nothing", f.log.String())
}
})
}
func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) { func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
t.Parallel() t.Parallel()
+6 -59
View File
@@ -18,7 +18,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest" "sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/processlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy" "sneak.berlin/go/smallwebwaf/internal/rproxy"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/waf"
) )
@@ -51,11 +51,11 @@ func TestServerHasTheFixedLimits(t *testing.T) {
server := rproxy.New(rproxy.Params{ server := rproxy.New(rproxy.Params{
Config: cfg, Config: cfg,
RequestLog: io.Discard, RequestLog: io.Discard,
ProcessLog: processlog.New(processlog.Params{ ProcessLog: requestlog.NewProcessLogger(
Output: io.Discard, io.Discard,
InstanceName: "test", "test",
Level: slog.LevelInfo, slog.LevelInfo,
}), ),
}) })
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 || if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
@@ -71,59 +71,6 @@ func TestServerHasTheFixedLimits(t *testing.T) {
} }
} }
// getOfSize is a GET whose request line and headers, the blank line that ends
// them included, are size bytes long.
func getOfSize(size int) string {
const start, end = "GET / HTTP/1.1\r\nHost: site.example\r\nX-Pad: ", "\r\n\r\n"
return start + strings.Repeat("x", size-len(start)-len(end)) + end
}
func TestARequestLineAndHeadersOf32KiBReachTheApp(t *testing.T) {
t.Parallel()
standIn := startWAF(t, allow)
app := startApp(t, greet)
addr, _ := startRProxy(t, standIn.URL, app.URL, nil)
conn := dial(t, addr)
send(t, conn, getOfSize(32<<10))
if status := readStatus(t, conn); status != http.StatusOK {
t.Errorf("status %d, want the app's 200", status)
}
}
func TestARequestLineAndHeadersOver32KiBAreRefusedAndTheConnectionClosed(
t *testing.T,
) {
t.Parallel()
standIn := startWAF(t, allow)
app := startApp(t, func(http.ResponseWriter, *http.Request) {
t.Error("the app was asked")
})
addr, _ := startRProxy(t, standIn.URL, app.URL, nil)
conn := dial(t, addr)
send(t, conn, getOfSize(32<<10+1))
// io.ReadAll returns without an error only once rproxy has closed the
// connection.
text, err := io.ReadAll(conn)
if err != nil || !strings.HasPrefix(string(text), "HTTP/1.1 431 ") {
t.Errorf(
"read %q (%v), want 431 and the connection closed",
text,
err,
)
}
if len(standIn.questions) != 0 {
t.Error("waf was asked")
}
}
func TestABodyAnnouncedOverItsLimitIsRefusedOnceWAFAllowsIt(t *testing.T) { func TestABodyAnnouncedOverItsLimitIsRefusedOnceWAFAllowsIt(t *testing.T) {
t.Parallel() t.Parallel()
+7 -9
View File
@@ -21,7 +21,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest" "sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/processlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy" "sneak.berlin/go/smallwebwaf/internal/rproxy"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/waf"
) )
@@ -324,8 +324,7 @@ func greet(w http.ResponseWriter, r *http.Request) {
// startRProxy starts rproxy with waf at backend and the app at appURL, and // startRProxy starts rproxy with waf at backend and the app at appURL, and
// the settings in env on top of the defaults. It returns where rproxy listens // the settings in env on top of the defaults. It returns where rproxy listens
// and what it writes. SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS is a minute unless env // and what it writes.
// sets it, so that a busy host cannot make rproxy drop a question or a report.
func startRProxy( func startRProxy(
t *testing.T, backend, appURL string, env map[string]string, t *testing.T, backend, appURL string, env map[string]string,
) (string, *output) { ) (string, *output) {
@@ -358,7 +357,6 @@ func startServerLogging(
settings := map[string]string{ settings := map[string]string{
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL, "SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
wafBackendTimeout: "60000",
} }
maps.Copy(settings, env) maps.Copy(settings, env)
@@ -370,11 +368,11 @@ func startServerLogging(
server := rproxy.New(rproxy.Params{ server := rproxy.New(rproxy.Params{
Config: cfg, Config: cfg,
RequestLog: requestLog, RequestLog: requestLog,
ProcessLog: processlog.New(processlog.Params{ ProcessLog: requestlog.NewProcessLogger(
Output: processLog, processLog,
InstanceName: "test", "test",
Level: slog.LevelInfo, slog.LevelInfo,
}), ),
}) })
listener, err := (&net.ListenConfig{}).Listen( listener, err := (&net.ListenConfig{}).Listen(
+3 -6
View File
@@ -104,7 +104,7 @@ var (
errVersion = errors.New("unknown version") errVersion = errors.New("unknown version")
// errMissing is for an entry without a field it needs. // errMissing is for an entry without a field it needs.
errMissing = errors.New("has no") errMissing = errors.New("has no")
errCause = errors.New("is not limit, attack, admin or abuseipdb") errCause = errors.New("is not limit, attack or admin")
errExpires = errors.New("is not after its start") errExpires = errors.New("is not after its start")
errDestination = errors.New("is not webhook, slack or ntfy") errDestination = errors.New("is not webhook, slack or ntfy")
errScope = errors.New( errScope = errors.New(
@@ -638,8 +638,6 @@ func (f *Files) takeInState(
f.params.Ledger.Load(held) f.params.Ledger.Load(held)
} }
f.params.Ledger.ForgetEndedAbuseIPDBBans(f.params.Now())
entries = len(held) entries = len(held)
case clientsJSON: case clientsJSON:
var file clientsFile var file clientsFile
@@ -969,7 +967,7 @@ func (e BanEntry) ban() bans.Ban {
// each expires is read again as written. An expires not after the start // each expires is read again as written. An expires not after the start
// is refused too, since the netblock's next ban for a broken limit, three // is refused too, since the netblock's next ban for a broken limit, three
// times as long as this one, would then refuse nothing; and so is a cause // times as long as this one, would then refuse nothing; and so is a cause
// other than limit, attack, admin or abuseipdb, most likely misspelt. // other than limit, attack or admin, most likely misspelt.
func (f *bansFile) check(data []byte) error { func (f *bansFile) check(data []byte) error {
var written struct { var written struct {
Bans []struct { Bans []struct {
@@ -993,8 +991,7 @@ func (f *bansFile) check(data []byte) error {
case entry.Expires != nil && !entry.Expires.After(entry.Start): case entry.Expires != nil && !entry.Expires.After(entry.Start):
return fmt.Errorf("entry %d's expires %w", i+1, errExpires) return fmt.Errorf("entry %d's expires %w", i+1, errExpires)
case entry.Cause != "" && entry.Cause != bans.CauseLimit && case entry.Cause != "" && entry.Cause != bans.CauseLimit &&
entry.Cause != bans.CauseAttack && entry.Cause != bans.CauseAdmin && entry.Cause != bans.CauseAttack && entry.Cause != bans.CauseAdmin:
entry.Cause != bans.CauseAbuseIPDB:
return fmt.Errorf( return fmt.Errorf(
"entry %d's cause %q %w", "entry %d's cause %q %w",
i+1, i+1,
+3 -37
View File
@@ -114,8 +114,7 @@ const permanentBansJSON = `{
"earlier_bans": { "earlier_bans": {
"limit": 3, "limit": 3,
"attack": 1, "attack": 1,
"admin": 1, "admin": 1
"abuseipdb": 0
} }
} }
} }
@@ -1191,7 +1190,7 @@ func TestBanWithAnotherCauseStopsTheStart(t *testing.T) {
`"expires": "2026-10-06T04:00:00Z", "cause": "limit"}, `+ `"expires": "2026-10-06T04:00:00Z", "cause": "limit"}, `+
`{"netblock": "203.0.113.12/32", "start": "2026-10-06T00:00:00Z", `+ `{"netblock": "203.0.113.12/32", "start": "2026-10-06T00:00:00Z", `+
`"expires": null, "cause": "atack"}]}`, `"expires": null, "cause": "atack"}]}`,
`: entry 4's cause "atack" is not limit, attack, admin or abuseipdb`) `: entry 4's cause "atack" is not limit, attack or admin`)
} }
func TestBanThatDoesNotEndAfterItsStartStopsTheStart(t *testing.T) { func TestBanThatDoesNotEndAfterItsStartStopsTheStart(t *testing.T) {
@@ -2091,39 +2090,6 @@ func TestLiftedBanReadAtTheStart(t *testing.T) {
wantLiftedBanKept(t, load(t, params), dir, params.Ledger) wantLiftedBanKept(t, load(t, params), dir, params.Ledger)
} }
func TestEndedAbuseIPDBBanForgottenWhenBansJSONIsReadAtTheStart(
t *testing.T,
) {
t.Parallel()
ended := netip.MustParsePrefix("203.0.113.9/32")
lasting := netip.MustParsePrefix("203.0.113.10/32")
limited := netip.MustParsePrefix("203.0.113.11/32")
// At midnight, the start's time, the first ban has ended and the second
// lasts. The third has ended too, but is for a limit.
dir := t.TempDir()
edit(t, dir, bansJSON, `{"version": 1, "bans": [`+
`{"netblock": "203.0.113.9/32", "start": "2026-09-06T00:00:00Z", `+
`"expires": "2026-10-06T00:00:00Z", "cause": "abuseipdb"}, `+
`{"netblock": "203.0.113.10/32", "start": "2026-09-07T00:00:00Z", `+
`"expires": "2026-10-07T00:00:00Z", "cause": "abuseipdb"}, `+
`{"netblock": "203.0.113.11/32", "start": "2026-10-05T00:00:00Z", `+
`"expires": "2026-10-05T01:00:00Z", "cause": "limit"}]}`)
params := newParams(dir)
load(t, params)
for netblock, want := range map[netip.Prefix]int{
ended: 0, lasting: 1, limited: 1,
} {
if got := len(params.Ledger.Bans(netblock)); got != want {
t.Errorf("the ledger holds %d bans on %s, want %d",
got, netblock, want)
}
}
}
func TestBanLiftedByAnEditWhileRunning(t *testing.T) { func TestBanLiftedByAnEditWhileRunning(t *testing.T) {
t.Parallel() t.Parallel()
@@ -2448,7 +2414,7 @@ func newParams(dir string) state.Params {
AttackBanDuration: 7 * 24 * time.Hour, AttackBanDuration: 7 * 24 * time.Hour,
MaxBans: 5000, MaxBans: 5000,
}), }),
Limiter: ratelimit.New(ratelimit.Params{MaxClients: 20000}), Limiter: ratelimit.New(ratelimit.Limits{}, 20000),
GeoJS: geojs.New(geojs.Params{Now: midnight}), GeoJS: geojs.New(geojs.Params{Now: midnight}),
Lists: reputation.New(reputation.Params{ Lists: reputation.New(reputation.Params{
BlocklistURLs: []string{ BlocklistURLs: []string{
-198
View File
@@ -1,198 +0,0 @@
package sww
import (
"context"
"errors"
"io"
"log/slog"
"net"
"net/http"
"time"
"go.uber.org/fx"
"go.uber.org/fx/fxevent"
)
// shutdownTimeout is how long requests in progress may take to finish
// once sww is told to stop, before their connections are closed.
// runit and docker wait a little longer before they kill the process.
const shutdownTimeout = 5 * time.Second
// requestLog is where a daemon's server writes the request log: stdout, and
// the remote log sender while there is one. fx tells parts apart by type, so
// it has a type of its own.
type requestLog struct {
io.Writer
}
// servingFailed gets why serving failed, for run, which then stops the
// daemon. fx tells parts apart by type, so it has a type of its own.
type servingFailed chan error
// utcNow is the time the state files, the alerts and the rate limits go by.
func utcNow() time.Time {
return time.Now().UTC()
}
// startError is why a daemon cannot start: the message its process log gives,
// and the error it logs with it.
type startError struct {
message string
err error
}
func (e startError) Error() string {
return e.message + ": " + e.err.Error()
}
func (e startError) Unwrap() error {
return e.err
}
// cannotStart logs why a daemon cannot start, with the message of the
// startError in err, and returns its exit status.
func cannotStart(processLog *slog.Logger, err error) int {
message, cause := "cannot start", err
failed, ok := errors.AsType[startError](err)
if ok {
message, cause = failed.message, failed.err
}
processLog.Error(message, "error", cause.Error())
return 1
}
// listen listens on address, which the setting names.
func listen(address, setting string) (net.Listener, error) {
listener, err := (&net.ListenConfig{}).Listen(
context.Background(), "tcp", address)
if err != nil {
return nil, startError{"cannot listen on " + setting, err}
}
return listener, nil
}
// background returns the hook that runs loop on a goroutine of its own from
// the start, and at the stop ends it and waits for it to return.
func background(loop func(context.Context)) fx.Hook {
running, stop := context.WithCancel(context.Background())
ended := make(chan struct{})
return fx.StartStopHook(
func() {
go func() {
loop(running)
close(ended)
}()
},
func() {
stop()
<-ended
},
)
}
// serve serves requests on listener from the start. At the stop it gives the
// requests in progress shutdownTimeout to finish, then closes their
// connections. When serving fails, it logs why and sends the error to
// failed.
func serve(
lc fx.Lifecycle, server *http.Server, listener net.Listener,
processLog *slog.Logger, failed servingFailed,
) {
served := make(chan error, 1)
lc.Append(fx.Hook{
OnStart: func(context.Context) error {
go func() {
err := server.Serve(listener)
if !errors.Is(err, http.ErrServerClosed) {
processLog.Error("serving failed", "error", err.Error())
failed <- err
}
served <- err
}()
return nil
},
OnStop: func(ctx context.Context) error {
processLog.Info("stopping")
shutdownCtx, cancel := context.WithTimeout(ctx, shutdownTimeout)
defer cancel()
err := server.Shutdown(shutdownCtx)
if err != nil {
processLog.Warn("requests still in progress were cut off",
"error", err.Error())
_ = server.Close()
}
err = <-served
if !errors.Is(err, http.ErrServerClosed) {
return err
}
return nil
},
})
}
// run starts app, which serves until ctx is done or serving fails, as
// failed tells. Then it stops app and takes the daemon's last step. It
// returns the process's exit status, 1 when serving failed, a part could
// not stop in order or the last step failed; each logs why.
func run(
ctx context.Context, app *fx.App, failed servingFailed,
processLog *slog.Logger, lastStep func() error,
) int {
// The hooks only start goroutines, and end them at the stop, so neither
// is cut short by a deadline or by ctx.
err := app.Start(context.WithoutCancel(ctx))
if err != nil {
return cannotStart(processLog, err)
}
status := 0
select {
case <-ctx.Done():
case <-failed:
status = 1
}
err = app.Stop(context.WithoutCancel(ctx))
if err != nil {
status = 1
}
err = lastStep()
if err != nil {
status = 1
}
if status == 0 {
processLog.Info("stopped")
}
return status
}
// fxEventLogger returns what fx.WithLogger takes to write fx's own events to
// processLog at debug, which the daemons' log levels normally hold back. Its
// errors are at debug too: the daemon logs each one itself.
func fxEventLogger(processLog *slog.Logger) func() fxevent.Logger {
return func() fxevent.Logger {
logger := &fxevent.SlogLogger{Logger: processLog}
logger.UseLogLevel(slog.LevelDebug)
logger.UseErrorLevel(slog.LevelDebug)
return logger
}
}
+69 -48
View File
@@ -2,15 +2,14 @@ package sww
import ( import (
"context" "context"
"errors"
"log/slog" "log/slog"
"net" "net"
"net/http" "net/http"
"os" "os"
"go.uber.org/fx"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/processlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy" "sneak.berlin/go/smallwebwaf/internal/rproxy"
) )
@@ -23,39 +22,34 @@ import (
// setting for either. // setting for either.
func RunRProxy(ctx context.Context, params Params) int { func RunRProxy(ctx context.Context, params Params) int {
hostname, _ := os.Hostname() // "" when the host has no name to give hostname, _ := os.Hostname() // "" when the host has no name to give
processLog := processlog.New(processlog.Params{ processLog := requestlog.NewProcessLogger(
Output: params.Stdout, params.Stdout,
InstanceName: hostname, hostname,
Level: slog.LevelInfo, slog.LevelInfo,
}) )
cfg, err := config.RProxyFromEnvironment(params.Viper) cfg, err := config.RProxyFromEnvironment(params.Viper)
if err != nil { if err != nil {
return cannotStart(processLog, startError{"invalid setting", err}) processLog.Error("invalid setting", "error", err.Error())
return 1
} }
var ( server := rproxy.New(rproxy.Params{
server *rproxy.Server Config: cfg,
listener net.Listener RequestLog: params.Stdout,
) ProcessLog: processLog,
})
failed := make(servingFailed, 1) listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", cfg.ListenAddr)
app := fx.New(
fx.Supply(cfg, processLog, requestLog{params.Stdout}, failed),
fx.Provide(
newRProxyServer,
func(server *rproxy.Server) *http.Server { return server.Server },
newRProxyListener,
),
fx.Invoke(serve),
fx.Populate(&server, &listener),
fx.WithLogger(fxEventLogger(processLog)),
)
err = app.Err()
if err != nil { if err != nil {
return cannotStart(processLog, err) processLog.Error(
"cannot listen on SWW_RPROXY_LISTEN_ADDR",
"error",
err.Error(),
)
return 1
} }
processLog.Info("starting", processLog.Info("starting",
@@ -63,25 +57,52 @@ func RunRProxy(ctx context.Context, params Params) int {
"address", listener.Addr().String(), "address", listener.Addr().String(),
"settings", cfg) "settings", cfg)
return run(ctx, app, failed, processLog, func() error { return serveRProxy(ctx, server, listener, processLog)
}
// serveRProxy serves requests on listener until ctx is done, and then stops as
// RunRProxy says.
func serveRProxy(
ctx context.Context, server *rproxy.Server, listener net.Listener,
processLog *slog.Logger,
) int {
served := make(chan error, 1)
go func() {
served <- server.Serve(listener)
}()
select {
case err := <-served:
processLog.Error("serving failed", "error", err.Error())
return 1
case <-ctx.Done():
}
processLog.Info("stopping")
shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx),
shutdownTimeout)
defer cancel()
err := server.Shutdown(shutdownCtx)
if err != nil {
processLog.Warn("requests still in progress were cut off",
"error", err.Error())
_ = server.Close()
}
err = <-served
if !errors.Is(err, http.ErrServerClosed) {
processLog.Error("serving failed", "error", err.Error())
return 1
}
server.WaitForReports() server.WaitForReports()
processLog.Info("stopped")
return nil return 0
})
}
// newRProxyServer returns the server rproxy runs.
func newRProxyServer(
cfg *config.RProxy, out requestLog, processLog *slog.Logger,
) *rproxy.Server {
return rproxy.New(rproxy.Params{
Config: cfg,
RequestLog: out,
ProcessLog: processLog,
})
}
// newRProxyListener listens on SWW_RPROXY_LISTEN_ADDR.
func newRProxyListener(cfg *config.RProxy) (net.Listener, error) {
return listen(cfg.ListenAddr, "SWW_RPROXY_LISTEN_ADDR")
} }
+2 -5
View File
@@ -552,9 +552,7 @@ func TestAStatusWAFDoesNotTakeIsReportedAs599(t *testing.T) {
// startRProxyProcess runs `sww rproxy` in a process of its own, with the // startRProxyProcess runs `sww rproxy` in a process of its own, with the
// settings in env, each as NAME=value. It returns where rproxy listens and a // settings in env, each as NAME=value. It returns where rproxy listens and a
// function that sends rproxy SIGTERM and fails the test unless it exits with // function that sends rproxy SIGTERM and fails the test unless it exits with
// status 0 within waitLimit. It waits a minute for waf's answers unless env // status 0 within waitLimit.
// sets SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS, so that a busy host cannot make it
// drop a question.
func startRProxyProcess(t *testing.T, env ...string) (string, func()) { func startRProxyProcess(t *testing.T, env ...string) (string, func()) {
t.Helper() t.Helper()
@@ -570,8 +568,7 @@ func startRProxyProcess(t *testing.T, env ...string) (string, func()) {
cmd.Env = append(os.Environ(), cmd.Env = append(os.Environ(),
commandForMain+"=rproxy", commandForMain+"=rproxy",
"SWW_RPROXY_LISTEN_ADDR="+localhost+":0", "SWW_RPROXY_LISTEN_ADDR="+localhost+":0")
wafBackendTimeout+"=60000")
cmd.Env = append(cmd.Env, env...) cmd.Env = append(cmd.Env, env...)
exited := startProcess(t, cmd) exited := startProcess(t, cmd)
+397 -2
View File
@@ -4,14 +4,17 @@
// stop, and then stops in an orderly way, writing the state and cache files. // stop, and then stops in an orderly way, writing the state and cache files.
// `sww rproxy` reads rproxy's settings alone, and runs the reverse proxy in // `sww rproxy` reads rproxy's settings alone, and runs the reverse proxy in
// front of one app that asks waf about each request. `sww healthcheck` is the // front of one app that asks waf about each request. `sww healthcheck` is the
// container's health check. fx builds the parts of each daemon, starts their // container's health check.
// background work and stops it.
package sww package sww
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"io" "io"
"log/slog"
"net"
"net/http"
"os" "os"
"os/signal" "os/signal"
"syscall" "syscall"
@@ -19,9 +22,29 @@ import (
"github.com/spf13/viper" "github.com/spf13/viper"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/remotelog"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rules"
"sneak.berlin/go/smallwebwaf/internal/state"
"sneak.berlin/go/smallwebwaf/internal/waf"
) )
// shutdownTimeout is how long requests in progress may take to finish
// once sww is told to stop, before their connections are closed.
// runit and docker wait a little longer before they kill the process.
const shutdownTimeout = 5 * time.Second
// remoteLogStopTimeout is how long, as waf stops, the log lines still
// waiting are sent to SWW_WAF_LOG_REMOTE_URL before they are given up. stdout has
// carried them.
const remoteLogStopTimeout = 2 * time.Second
// usageStatus is the exit status with no command, or one sww does not // usageStatus is the exit status with no command, or one sww does not
// know. // know.
const usageStatus = 2 const usageStatus = 2
@@ -87,3 +110,375 @@ func printUsage(w io.Writer) int {
return usageStatus return usageStatus
} }
// RunWAF is `sww waf`: it reads the settings, the rule files,
// the lookup database and the state and cache files, then serves requests until
// ctx is done. It returns the process's exit status, 1 when waf cannot start.
func RunWAF(ctx context.Context, params Params) int {
// Until the settings are read, the one message is an invalid setting's
// error, which every SWW_WAF_LOG_LEVEL lets through.
processLog := requestlog.NewProcessLogger(params.Stdout,
config.InstanceName(params.Viper), slog.LevelError)
cfg, err := config.WAFFromEnvironment(params.Viper)
if err != nil {
return cannotStart(processLog, "invalid setting", err)
}
// While SWW_WAF_LOG_REMOTE_URL is set, every line on stdout from here on
// is sent there too.
stdout := params.Stdout
var remote *remotelog.Sender
if cfg.LogRemoteURL != nil {
remote = newRemoteLogSender(cfg)
stdout = io.MultiWriter(params.Stdout, remote)
}
processLog = requestlog.NewProcessLogger(
stdout,
cfg.InstanceName,
cfg.LogLevel,
)
if remote != nil {
stopSending := startSending(ctx, remote, processLog)
defer stopSending()
}
return runWAFWithSettings(
ctx,
params,
cfg,
stdout,
remote,
processLog,
)
}
// runWAFWithSettings is the rest of RunWAF once the settings are read and
// the process log goes where they say. remote is nil unless
// SWW_WAF_LOG_REMOTE_URL is set.
func runWAFWithSettings(
ctx context.Context,
params Params,
cfg *config.WAF,
stdout io.Writer,
remote *remotelog.Sender,
processLog *slog.Logger,
) int {
// The state files and the alerts give times in UTC.
now := func() time.Time { return time.Now().UTC() }
alertQueue := newAlertQueue(cfg, now, processLog)
ruleFiles, err := rules.Load(rules.Params{
Dir: cfg.RuleFilesDir,
Enabled: cfg.RuleFilesEnabled,
ProcessLog: processLog,
Alerts: alertQueue,
})
if err != nil {
return cannotStart(processLog, "cannot use the rule files", err)
}
crs, err := waf.NewCoreRuleSet(cfg)
if err != nil {
return cannotStart(processLog, "cannot use the Core Rule Set", err)
}
server, err := newServer(
cfg,
stdout,
processLog,
now,
ruleFiles,
alertQueue,
crs,
params.BesideLookupTimeLimit,
)
if err != nil {
return cannotStart(processLog, "cannot use the lookup database", err)
}
if remote != nil {
server.Metrics.AddRemoteLog(remote)
}
files, err := loadStateFiles(cfg, server, alertQueue, now, processLog)
if err != nil {
return cannotStart(
processLog,
"cannot use the state and cache files",
err,
)
}
listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", cfg.ListenAddr)
if err != nil {
return cannotStart(
processLog,
"cannot listen on SWW_WAF_LISTEN_ADDR",
err,
)
}
processLog.Info("starting",
"version", params.Version,
"address", listener.Addr().String(),
"settings", cfg)
return serve(
ctx,
server,
listener,
files,
ruleFiles,
alertQueue,
processLog,
)
}
// cannotStart logs why waf cannot start, and returns its exit status.
func cannotStart(processLog *slog.Logger, message string, err error) int {
processLog.Error(message, "error", err.Error())
return 1
}
// newServer returns the server waf runs, with the metrics of the alerts,
// after reading the lookup database while SWW_WAF_LOOKUP_SOURCE is file. A lookup
// database that cannot be read is an error.
func newServer(
cfg *config.WAF, stdout io.Writer, processLog *slog.Logger,
now func() time.Time, ruleFiles *rules.Files, alertQueue *alerts.Queue,
crs *coreruleset.CoreRuleSet, besideLookupTimeLimit time.Duration,
) (*waf.Server, error) {
var lookupFile *lookup.File
if cfg.LookupSource == "file" {
var err error
lookupFile, err = lookup.OpenFile(lookup.FileParams{
Path: cfg.LookupDBPath,
Now: now,
ProcessLog: processLog,
Alerts: alertQueue,
})
if err != nil {
return nil, err
}
}
server := waf.New(waf.Params{
Config: cfg,
RequestLog: stdout,
ProcessLog: processLog,
GeoJSURL: geojs.URL,
AbuseIPDBURL: reputation.AbuseIPDBURL,
LookupFile: lookupFile,
CoreRuleSet: crs,
Now: now,
Rules: ruleFiles,
Alerts: alertQueue,
BesideLookupTimeLimit: besideLookupTimeLimit,
})
server.Metrics.AddAlerts(alertQueue)
if lookupFile != nil {
server.Metrics.AddLookupFile(
lookupFile.LastRead,
lookupFile.ReadFailures,
)
}
return server, nil
}
// loadStateFiles reads the state and cache files into the parts of server and
// into alertQueue, as state.Load does.
func loadStateFiles(
cfg *config.WAF, server *waf.Server, alertQueue *alerts.Queue,
now func() time.Time, processLog *slog.Logger,
) (*state.Files, error) {
return state.Load(state.Params{
Dir: cfg.StateDir,
CacheDir: cfg.CacheDir,
BansWriteDelay: cfg.BansWriteDelay,
WriteInterval: cfg.StateWriteInterval,
Ledger: server.Ledger,
Limiter: server.Limiter,
GeoJS: server.GeoJS,
Lists: server.Lists,
DNSBL: server.DNSBL,
AbuseIPDB: server.AbuseIPDB,
RouteWatch: server.RouteWatch,
Alerts: alertQueue,
Anomalies: server.Anomalies,
Now: now,
ProcessLog: processLog,
Metrics: server.Metrics,
})
}
// newAlertQueue returns the queue of the alerts to the webhook, Slack and
// ntfy, with the settings for them.
func newAlertQueue(
cfg *config.WAF, now func() time.Time, processLog *slog.Logger,
) *alerts.Queue {
return alerts.New(alerts.Params{
WebhookURL: cfg.AlertWebhookURL,
WebhookHeaders: cfg.AlertWebhookHeaders,
SlackURL: cfg.AlertSlackWebhookURL,
NtfyURL: cfg.AlertNtfyURL,
NtfyToken: cfg.AlertNtfyToken,
Events: cfg.AlertEvents,
Cooldown: cfg.AlertCooldown,
MaxPerHour: cfg.AlertMaxPerHour,
Instance: cfg.InstanceName,
Now: now,
ProcessLog: processLog,
})
}
// newRemoteLogSender returns a sender of the log lines to
// SWW_WAF_LOG_REMOTE_URL, with the settings for it.
func newRemoteLogSender(cfg *config.WAF) *remotelog.Sender {
return remotelog.New(remotelog.Params{
URL: cfg.LogRemoteURL,
RootCAs: cfg.LogRemoteTLSCAs,
BufferLines: cfg.LogRemoteBufferLines,
Facility: cfg.LogRemoteFacility,
AppName: cfg.LogRemoteAppName,
})
}
// startSending runs remote until the function it returns is called, which
// then waits at most remoteLogStopTimeout for the lines still waiting to
// be sent. Sending goes on after ctx is done, so that the lines written
// while sww stops are sent too.
func startSending(
ctx context.Context, remote *remotelog.Sender, processLog *slog.Logger,
) func() {
sending, stop := context.WithCancel(context.WithoutCancel(ctx))
sent := make(chan struct{})
go func() {
remote.Run(sending, processLog)
close(sent)
}()
return func() {
stop()
select {
case <-sent:
case <-time.After(remoteLogStopTimeout):
}
}
}
// serve serves requests on listener, writes the state and cache files as they
// are due, takes in an admin's edits of them, reads the rule files again as
// they change, and the lookup database when it is replaced, fetches the
// lists the settings name by URL as they are due, and sends the alerts,
// until ctx is done. Then it gives the requests in progress
// shutdownTimeout to finish, and writes every state and cache file,
// alerts.json with the alerts still waiting.
func serve(
ctx context.Context, server *waf.Server, listener net.Listener,
files *state.Files, ruleFiles *rules.Files, alertQueue *alerts.Queue,
processLog *slog.Logger,
) int {
served := make(chan error, 1)
go func() {
served <- server.Serve(listener)
}()
writing, stopWriting := context.WithCancel(ctx)
defer stopWriting()
written := inBackground(func() { files.Run(writing) })
watched := inBackground(func() { files.Watch(writing) })
rulesWatched := inBackground(func() { ruleFiles.Watch(writing) })
lookupFileWatched := inBackground(func() {
if server.LookupFile != nil {
server.LookupFile.Watch(writing)
}
})
listsFetched := inBackground(func() { server.Lists.Run(writing) })
alertsSent := inBackground(func() { alertQueue.Run(writing) })
select {
case err := <-served:
processLog.Error("serving failed", "error", err.Error())
return 1
case <-ctx.Done():
}
processLog.Info("stopping")
shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx),
shutdownTimeout)
defer cancel()
err := server.Shutdown(shutdownCtx)
if err != nil {
processLog.Warn("requests still in progress were cut off",
"error", err.Error())
_ = server.Close()
}
err = <-served
if !errors.Is(err, http.ErrServerClosed) {
processLog.Error("serving failed", "error", err.Error())
return 1
}
// Run and Watch have ended, so nothing else reads or writes the
// files, and no alert is being sent, so that alerts.json keeps every
// alert not yet sent. Every request has ended too, but for one cut off
// because Shutdown timed out, which adds to its client's history only as
// it ends. That can be after this write, and then that request is missing
// from clients.json.
<-written
<-watched
<-rulesWatched
<-lookupFileWatched
<-listsFetched
<-alertsSent
err = files.WriteAll()
if err != nil {
processLog.Error(
"writing the state and cache files failed",
"error",
err.Error(),
)
return 1
}
processLog.Info("stopped")
return 0
}
// inBackground runs task on a goroutine of its own, and returns a channel
// that is closed once task has returned.
func inBackground(task func()) <-chan struct{} {
done := make(chan struct{})
go func() {
task()
close(done)
}()
return done
}
+1 -3
View File
@@ -660,9 +660,7 @@ func TestCoreRuleSetExclusionsFileThatCannotBeUsedStopsTheStartWhileTheCoreRuleS
status := run(t.Context(), map[string]string{ status := run(t.Context(), map[string]string{
listenAddr: localhost + ":0", stateDir: t.TempDir(), ruleFilesDir: t.TempDir(), listenAddr: localhost + ":0", stateDir: t.TempDir(), ruleFilesDir: t.TempDir(),
"SWW_WAF_CORE_RULE_SET_MODE": string( "SWW_WAF_CORE_RULE_SET_MODE": config.CoreRuleSetModeOff,
config.CoreRuleSetModeOff,
),
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE": path, "SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE": path,
}, out) }, out)
if status != 1 { if status != 1 {
-560
View File
@@ -1,560 +0,0 @@
package sww
import (
"context"
"fmt"
"io"
"log/slog"
"net"
"net/http"
"time"
"go.uber.org/fx"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/anomaly"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/remotelog"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/routewatch"
"sneak.berlin/go/smallwebwaf/internal/rules"
"sneak.berlin/go/smallwebwaf/internal/state"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
// remoteLogStopTimeout is how long, as waf stops, the log lines still
// waiting are sent to SWW_WAF_LOG_REMOTE_URL before they are given up. stdout has
// carried them.
const remoteLogStopTimeout = 2 * time.Second
// RunWAF is `sww waf`: it reads the settings, the rule files,
// the lookup database and the state and cache files, then serves requests until
// ctx is done. Then it gives the requests in progress shutdownTimeout to
// finish, ends its background work and writes every state and cache file,
// alerts.json with the alerts still waiting. It returns the process's exit
// status, 1 when waf cannot start or stop in order.
func RunWAF(ctx context.Context, params Params) int {
// Until the settings are read, the one message is an invalid setting's
// error, which every SWW_WAF_LOG_LEVEL lets through.
processLog := processlog.New(processlog.Params{
Output: params.Stdout,
InstanceName: config.InstanceName(params.Viper),
Level: slog.LevelError,
})
cfg, err := config.WAFFromEnvironment(params.Viper)
if err != nil {
return cannotStart(processLog, startError{"invalid setting", err})
}
// While SWW_WAF_LOG_REMOTE_URL is set, every line on stdout from here on
// is sent there too. The sender runs before any part is built, so that it
// sends why waf cannot start.
stdout := params.Stdout
var remote *remotelog.Sender
if cfg.LogRemoteURL != nil {
remote = newRemoteLogSender(cfg)
stdout = io.MultiWriter(params.Stdout, remote)
}
processLog = processlog.New(processlog.Params{
Output: stdout,
InstanceName: cfg.InstanceName,
Level: cfg.LogLevel,
})
if remote != nil {
stopSending := startSending(ctx, remote, processLog)
defer stopSending()
}
var (
files *state.Files
listener net.Listener
)
failed := make(servingFailed, 1)
app := fx.New(
fx.Supply(
params,
cfg,
processLog,
requestLog{stdout},
remote,
failed,
),
fx.Provide(
newAlertQueue, newRuleFiles, newCoreRuleSet, newLookupFile,
newLimiter, newLedger, newAnomalyCounters,
newLists, newDNSBL, newAbuseIPDB, newRouteWatch,
newMetrics, newGeoJS, newWAFServer,
func(server *waf.Server) *http.Server { return server.Server },
newWAFListener, newStateFiles,
),
// fx stops hooks in the reverse of the order they were added in. The
// state files are read before serving is set up, so that serving
// stops before their loops do.
fx.Populate(&files),
fx.Invoke(serve),
fx.Populate(&listener),
fx.WithLogger(fxEventLogger(processLog)),
)
err = app.Err()
if err != nil {
return cannotStart(processLog, err)
}
processLog.Info("starting",
"version", params.Version,
"address", listener.Addr().String(),
"settings", cfg)
// The last write follows the stop of every hook, and so the end of every
// loop that changes the state or raises an alert, and of every ask of
// GeoJS, the DNSBL zones, AbuseIPDB and routewatch. A hook of the state
// files would stop before the loops of the parts built ahead of them.
return run(ctx, app, failed, processLog, func() error {
err := files.WriteAll()
if err != nil {
processLog.Error("writing the state and cache files failed",
"error", err.Error())
}
return err
})
}
// newAlertQueue returns the queue of the alerts to the webhook, Slack and
// ntfy, with the settings for them. Until waf stops, it sends them.
func newAlertQueue(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
) *alerts.Queue {
alertQueue := alerts.New(alerts.Params{
WebhookURL: cfg.AlertWebhookURL,
WebhookHeaders: cfg.AlertWebhookHeaders,
SlackURL: cfg.AlertSlackWebhookURL,
NtfyURL: cfg.AlertNtfyURL,
NtfyToken: cfg.AlertNtfyToken,
Events: cfg.AlertEvents,
Cooldown: cfg.AlertCooldown,
MaxPerHour: cfg.AlertMaxPerHour,
Instance: cfg.InstanceName,
Now: utcNow,
ProcessLog: processLog,
})
lc.Append(background(alertQueue.Run))
return alertQueue
}
// newRuleFiles reads the rule files, and reads them again as they change
// until waf stops.
func newRuleFiles(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
alertQueue *alerts.Queue,
) (*rules.Files, error) {
ruleFiles, err := rules.Load(rules.Params{
Dir: cfg.RuleFilesDir,
Enabled: cfg.RuleFilesEnabled,
ProcessLog: processLog,
Alerts: alertQueue,
})
if err != nil {
return nil, startError{"cannot use the rule files", err}
}
lc.Append(background(ruleFiles.Watch))
return ruleFiles, nil
}
// newCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
// coreruleset.New refuses is an error in every mode. While the mode is off, a
// set file is loaded with the Core Rule Set only to check it.
func newCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
cfg.CoreRuleSetExclusionsFile == "" {
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
}
crs, err := coreruleset.New(coreruleset.Params{
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
DisabledRules: cfg.CoreRuleSetDisabledRules,
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
Apps: cfg.Apps,
})
if err != nil {
return nil, startError{
"cannot use the Core Rule Set",
fmt.Errorf("SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w", err),
}
}
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
}
return crs, nil
}
// newLookupFile reads the lookup database while SWW_WAF_LOOKUP_SOURCE is
// file, and reads it again when it is replaced until waf stops. It returns
// nil while the source is anything else.
func newLookupFile(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
alertQueue *alerts.Queue,
) (*lookup.File, error) {
if cfg.LookupSource != config.LookupSourceFile {
return nil, nil //nolint:nilnil // no lookup database, and no error
}
lookupFile, err := lookup.OpenFile(lookup.FileParams{
Path: cfg.LookupDBPath,
Now: utcNow,
ProcessLog: processLog,
Alerts: alertQueue,
})
if err != nil {
return nil, startError{"cannot use the lookup database", err}
}
lc.Append(background(lookupFile.Watch))
return lookupFile, nil
}
// newLimiter returns the table of clients, with the rate and byte limits.
func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
return ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{
PerMinute: cfg.RateLimitPerMinute,
PerHour: cfg.RateLimitPerHour,
PerDay: cfg.RateLimitPerDay,
BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerHour: cfg.BytesLimitPerHour,
BytesPerDay: cfg.BytesLimitPerDay,
},
MaxClients: cfg.MaxTrackedClients,
})
}
// newLedger returns the ledger of the bans, with the settings for them.
func newLedger(cfg *config.WAF) *bans.Ledger {
return bans.New(bans.Params{
LimitBanDuration: cfg.LimitBanDuration,
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
LimitBanPermanentAbove: cfg.LimitBanPermanentAbove,
AttackBanDuration: cfg.AttackBanDuration,
MaxBans: cfg.MaxBans,
})
}
// newAnomalyCounters returns the counters of the anomaly thresholds.
func newAnomalyCounters(
cfg *config.WAF, alertQueue *alerts.Queue,
) *anomaly.Counters {
return anomaly.New(anomaly.Params{
Client: cfg.AnomalyClient,
Net: cfg.AnomalyNet,
ASN: cfg.AnomalyASN,
Total: cfg.AnomalyTotal,
WatchedNet: cfg.AnomalyWatchedNet,
NetV4Prefix: cfg.AnomalyNetV4Prefix,
NetV6Prefix: cfg.AnomalyNetV6Prefix,
NamedNetblocks: cfg.AnomalyWatchedNets,
Alerts: alertQueue,
})
}
// newLists returns the lists the settings name by URL, none fetched yet.
// Until waf stops, it fetches them as they are due.
func newLists(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
alertQueue *alerts.Queue,
) *reputation.Lists {
lists := reputation.New(reputation.Params{
BlocklistURLs: cfg.BlocklistURLs,
Refresh: cfg.BlocklistRefresh,
ASNLimitPercentURL: cfg.ASNLimitPercentURL,
TagLists: cfg.TagLists,
Now: utcNow,
ProcessLog: processLog,
Alerts: alertQueue,
})
lc.Append(background(lists.Run))
return lists
}
// newDNSBL returns what asks the DNSBL zones, none asked yet. At the stop it
// ends the queries under way, as reputation.DNSBL.StopAsking says.
func newDNSBL(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
alertQueue *alerts.Queue,
) *reputation.DNSBL {
dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{
Zones: cfg.DNSBLZones,
Resolver: cfg.DNSBLResolver,
CacheTTL: cfg.IPReputationCacheTTL,
Timeout: cfg.DNSBLAbuseIPDBTimeout,
Now: utcNow,
ProcessLog: processLog,
Alerts: alertQueue,
})
lc.Append(fx.StopHook(dnsbl.StopAsking))
return dnsbl
}
// newAbuseIPDB returns what checks clients with AbuseIPDB while
// SWW_WAF_ABUSEIPDB_KEY is set, none checked yet. Until waf stops, it
// fetches AbuseIPDB's blacklist as it is due, and bans the clients on it in
// ledger, as waf.BanAbuseIPDBBlacklist says. At the stop it ends the checks
// under way, as reputation.AbuseIPDB.StopChecking says.
func newAbuseIPDB(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
alertQueue *alerts.Queue, ledger *bans.Ledger,
) *reputation.AbuseIPDB {
abuseIPDB := reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
URL: reputation.AbuseIPDBURL,
Key: cfg.AbuseIPDBKey,
MinScore: cfg.AbuseIPDBMinScore,
DailyBudget: cfg.AbuseIPDBDailyBudget,
CacheTTL: cfg.IPReputationCacheTTL,
Timeout: cfg.DNSBLAbuseIPDBTimeout,
BlacklistURL: reputation.AbuseIPDBBlacklistURL,
Threshold: cfg.AbuseIPDBThreshold,
BlacklistRefresh: cfg.AbuseIPDBBlacklistRefresh,
Blacklisted: func(listed []reputation.BlacklistEntry) {
waf.BanAbuseIPDBBlacklist(
cfg, ledger, processLog, utcNow(), listed,
)
},
Now: utcNow,
ProcessLog: processLog,
Alerts: alertQueue,
})
lc.Append(background(abuseIPDB.RunBlacklist))
lc.Append(fx.StopHook(abuseIPDB.StopChecking))
return abuseIPDB
}
// newRouteWatch returns what asks routewatch about clients, or nil while
// SWW_WAF_ROUTEWATCH_URL is unset. At the stop it ends the queries under
// way, as routewatch.Client.StopAsking says.
func newRouteWatch(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
) *routewatch.Client {
if cfg.RouteWatchURL == "" {
return nil
}
routeWatch := routewatch.New(routewatch.Params{
URL: cfg.RouteWatchURL,
Now: utcNow,
ProcessLog: processLog,
})
lc.Append(fx.StopHook(routeWatch.StopAsking))
return routeWatch
}
// newMetrics returns the metrics, with those read from the other parts as
// the metrics are asked for: AbuseIPDB's while SWW_WAF_ABUSEIPDB_KEY is set,
// the lookup database's while there is one, and the remote log sender's
// while there is one.
func newMetrics(
cfg *config.WAF, ledger *bans.Ledger, limiter *ratelimit.Limiter,
ruleFiles *rules.Files, lists *reputation.Lists,
dnsbl *reputation.DNSBL, abuseIPDB *reputation.AbuseIPDB,
alertQueue *alerts.Queue, lookupFile *lookup.File,
remote *remotelog.Sender,
) *metrics.Metrics {
m := metrics.New(cfg.MetricsBusiestCountriesAndASNs, cfg.InstanceName)
m.AddBansAndClients(ledger, limiter, utcNow)
m.AddRules(ruleFiles)
m.AddReputation(lists, dnsbl)
if cfg.AbuseIPDBKey != "" {
m.AddAbuseIPDB(abuseIPDB)
}
m.AddAlerts(alertQueue)
if lookupFile != nil {
m.AddLookupFile(lookupFile.LastRead, lookupFile.ReadFailures)
}
if remote != nil {
m.AddRemoteLog(remote)
}
return m
}
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
// answer goes to the limiter and the ledger, as waf.AddLookup says, and each
// failure is handled as waf.GeoJSFailed says. At the stop it ends the request
// under way, as geojs.GeoJS.StopAsking says.
func newGeoJS(
lc fx.Lifecycle,
cfg *config.WAF,
m *metrics.Metrics,
limiter *ratelimit.Limiter,
ledger *bans.Ledger,
alertQueue *alerts.Queue,
processLog *slog.Logger,
) *geojs.GeoJS {
geoJS := geojs.New(geojs.Params{
URL: geojs.URL,
Answered: func(answer geojs.Answer) {
waf.AddLookup(cfg, limiter, ledger, answer)
},
Now: utcNow,
Asked: m.GeoJSRequests.Inc,
Failed: func(err error, askingAgainIn time.Duration) {
waf.GeoJSFailed(processLog, alertQueue, m, err, askingAgainIn)
},
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
})
lc.Append(fx.StopHook(geoJS.StopAsking))
return geoJS
}
// newWAFServer returns the server waf runs, with its parts and the time
// limit params gives.
func newWAFServer(
params Params, cfg *config.WAF, out requestLog, processLog *slog.Logger,
m *metrics.Metrics, limiter *ratelimit.Limiter, ledger *bans.Ledger,
anomalies *anomaly.Counters, lists *reputation.Lists,
dnsbl *reputation.DNSBL, abuseIPDB *reputation.AbuseIPDB,
routeWatch *routewatch.Client, geoJS *geojs.GeoJS,
lookupFile *lookup.File, crs *coreruleset.CoreRuleSet,
ruleFiles *rules.Files, alertQueue *alerts.Queue,
) *waf.Server {
return waf.New(waf.Params{
Config: cfg,
RequestLog: out,
ProcessLog: processLog,
Metrics: m,
Limiter: limiter,
Ledger: ledger,
Anomalies: anomalies,
Lists: lists,
DNSBL: dnsbl,
AbuseIPDB: abuseIPDB,
RouteWatch: routeWatch,
GeoJS: geoJS,
LookupFile: lookupFile,
CoreRuleSet: crs,
Now: utcNow,
Rules: ruleFiles,
Alerts: alertQueue,
BesideLookupTimeLimit: params.BesideLookupTimeLimit,
})
}
// newWAFListener listens on SWW_WAF_LISTEN_ADDR.
func newWAFListener(cfg *config.WAF) (net.Listener, error) {
return listen(cfg.ListenAddr, "SWW_WAF_LISTEN_ADDR")
}
// newStateFiles reads the state and cache files into the parts and the
// alert queue, as state.Load does. Until waf stops, it writes them as they
// are due and takes in an admin's edits of them.
func newStateFiles(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
m *metrics.Metrics, ledger *bans.Ledger, limiter *ratelimit.Limiter,
geoJS *geojs.GeoJS, lists *reputation.Lists, dnsbl *reputation.DNSBL,
abuseIPDB *reputation.AbuseIPDB, routeWatch *routewatch.Client,
alertQueue *alerts.Queue, anomalies *anomaly.Counters,
) (*state.Files, error) {
files, err := state.Load(state.Params{
Dir: cfg.StateDir,
CacheDir: cfg.CacheDir,
BansWriteDelay: cfg.BansWriteDelay,
WriteInterval: cfg.StateWriteInterval,
Ledger: ledger,
Limiter: limiter,
GeoJS: geoJS,
Lists: lists,
DNSBL: dnsbl,
AbuseIPDB: abuseIPDB,
RouteWatch: routeWatch,
Alerts: alertQueue,
Anomalies: anomalies,
Now: utcNow,
ProcessLog: processLog,
Metrics: m,
})
if err != nil {
return nil, startError{"cannot use the state and cache files", err}
}
lc.Append(background(files.Run))
lc.Append(background(files.Watch))
return files, nil
}
// newRemoteLogSender returns a sender of the log lines to
// SWW_WAF_LOG_REMOTE_URL, with the settings for it.
func newRemoteLogSender(cfg *config.WAF) *remotelog.Sender {
return remotelog.New(remotelog.Params{
URL: cfg.LogRemoteURL,
RootCAs: cfg.LogRemoteTLSCAs,
BufferLines: cfg.LogRemoteBufferLines,
Facility: cfg.LogRemoteFacility,
AppName: cfg.LogRemoteAppName,
})
}
// startSending runs remote until the function it returns is called, which
// then waits at most remoteLogStopTimeout for the lines still waiting to
// be sent. Sending goes on after ctx is done, so that the lines written
// while sww stops are sent too.
func startSending(
ctx context.Context, remote *remotelog.Sender, processLog *slog.Logger,
) func() {
sending, stop := context.WithCancel(context.WithoutCancel(ctx))
sent := make(chan struct{})
go func() {
remote.Run(sending, processLog)
close(sent)
}()
return func() {
stop()
select {
case <-sent:
case <-time.After(remoteLogStopTimeout):
}
}
}
+1 -4
View File
@@ -90,8 +90,6 @@ func TestMainRunsWAF(t *testing.T) {
cmd := exec.CommandContext(t.Context(), binary) cmd := exec.CommandContext(t.Context(), binary)
cmd.Stdout = out cmd.Stdout = out
// The Core Rule Set is off. Main keeps waf's own limit for its checks,
// and on a busy host the Core Rule Set's inspection can pass it.
cmd.Env = append(os.Environ(), cmd.Env = append(os.Environ(),
"TZ="+zone, "TZ="+zone,
commandForMain+"="+wafCommand, commandForMain+"="+wafCommand,
@@ -99,8 +97,7 @@ func TestMainRunsWAF(t *testing.T) {
stateDir+"="+dir, stateDir+"="+dir,
cacheDir+"="+dir, cacheDir+"="+dir,
ruleFilesDir+"="+t.TempDir(), ruleFilesDir+"="+t.TempDir(),
lookupSource+"=off", lookupSource+"=off")
"SWW_WAF_CORE_RULE_SET_MODE=off")
exited := startProcess(t, cmd) exited := startProcess(t, cmd)
+2 -2
View File
@@ -35,7 +35,7 @@ func (abuseIPDB) check(
return nil return nil
} }
client := clientGroup(rq.h.config, rq.client) client := rq.h.clientGroup(rq.client)
held, _ := rq.h.limiter.Client(client) held, _ := rq.h.limiter.Client(client)
offender := held.History.Offences != ratelimit.Offences{} offender := held.History.Offences != ratelimit.Offences{}
@@ -54,7 +54,7 @@ func (abuseIPDB) check(
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE", "scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
) )
if rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny { if rq.h.config.DNSBLAbuseIPDBAction != deny {
return nil return nil
} }
-46
View File
@@ -1,46 +0,0 @@
package waf
// The bans for AbuseIPDB's blacklist. They read SWW_WAF_ABUSEIPDB_BAN_DAYS and
// SWW_WAF_MODE; the blacklist is fetched by the reputation package.
import (
"log/slog"
"net/netip"
"time"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/reputation"
)
// BanAbuseIPDBBlacklist bans in ledger the netblock of each client on
// AbuseIPDB's blacklist, listed, for SWW_WAF_ABUSEIPDB_BAN_DAYS from now, as
// the ledger's BanForAbuseIPDB describes. It logs to processLog how many
// clients were listed and how many bans were made. Clients in one netblock
// make one ban, with the highest of their scores. In observe mode it makes
// no ban.
func BanAbuseIPDBBlacklist(
cfg *config.WAF, ledger *bans.Ledger, processLog *slog.Logger,
now time.Time, listed []reputation.BlacklistEntry,
) {
scores := make(map[netip.Prefix]int64, len(listed))
for _, client := range listed {
netblock := banNetblock(cfg, client.Addr)
scores[netblock] = max(scores[netblock], client.Score)
}
made := 0
if !cfg.Observe {
made = ledger.BanForAbuseIPDB(
now, now.Add(cfg.AbuseIPDBBanDuration), scores,
)
}
processLog.Info(
"fetched AbuseIPDB's blacklist",
"clients", len(listed),
"bans_made", made,
)
}
-100
View File
@@ -1,100 +0,0 @@
package waf_test
import (
"context"
"net/http"
"net/netip"
"testing"
"time"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
// abuseIPDBBanDays is the setting of how long a ban for AbuseIPDB's
// blacklist lasts, in days.
const abuseIPDBBanDays = "SWW_WAF_ABUSEIPDB_BAN_DAYS"
func TestClientOnTheFetchedAbuseIPDBBlacklistIsRefused(t *testing.T) {
t.Parallel()
s, clk, server := startWithClock(t, "", map[string]string{
abuseIPDBKey: accountKey,
})
ctx, cancel := context.WithCancel(t.Context())
stopped := make(chan struct{})
go func() {
server.AbuseIPDB.RunBlacklist(ctx)
close(stopped)
}()
t.Cleanup(func() {
cancel()
<-stopped
})
waitUntil(func() bool {
active, _ := server.Ledger.Count(clk.Now())
return active == 1
})
s.get(client, http.StatusForbidden, requestlog.ActionTemporarilyBanned)
s.get(otherClient, http.StatusOK, requestlog.ActionForward)
}
func TestClientsOnTheAbuseIPDBBlacklistAreBannedForTheBanDays(t *testing.T) {
t.Parallel()
s, clk, server := startWithClock(t, "", map[string]string{
abuseIPDBBanDays: "2", banScopeV4Prefix: "24",
})
// Two clients in one netblock make one ban, with the higher score.
server.BanAbuseIPDBBlacklist([]reputation.BlacklistEntry{
{Addr: netip.MustParseAddr(client), Score: 95},
{Addr: netip.MustParseAddr(otherClient), Score: 100},
{Addr: netip.MustParseAddr("2001:db8:5::1"), Score: 90},
})
held := server.Ledger.Bans(netip.MustParsePrefix(clientsNet))
if len(held) != 1 || held[0].Cause != bans.CauseAbuseIPDB ||
*held[0].Notes.Reputation[0].Score != 100 {
t.Errorf("bans on %s %+v, want one for the blacklist, with 100",
clientsNet, held)
}
// Every client of a banned netblock is refused, the IPv6 address's /64
// included.
for _, from := range []string{"203.0.113.200", "2001:db8:5::ffff:1"} {
s.get(from, http.StatusForbidden, requestlog.ActionTemporarilyBanned)
}
s.get("2001:db8:5:1::1", http.StatusOK, requestlog.ActionForward)
clk.advance(2 * 24 * time.Hour)
s.get(client, http.StatusOK, requestlog.ActionForward)
}
func TestAbuseIPDBBlacklistMakesNoBanInObserveMode(t *testing.T) {
t.Parallel()
s, clk, server := startWithClock(
t,
"",
map[string]string{mode: "observe"},
)
server.BanAbuseIPDBBlacklist([]reputation.BlacklistEntry{
{Addr: netip.MustParseAddr(client), Score: 100},
})
if active, _ := server.Ledger.Count(clk.Now()); active != 0 {
t.Errorf("%d bans active, want none", active)
}
s.get(client, http.StatusOK, requestlog.ActionForward)
}
+5 -5
View File
@@ -139,7 +139,7 @@ func (rq *request) addBan() {
) )
if err == nil { if err == nil {
netblock, err = rq.h.parseBanNetblock(toAdd.Netblock) netblock, err = rq.h.banNetblock(toAdd.Netblock)
} }
if err == nil { if err == nil {
@@ -190,12 +190,12 @@ func (rq *request) readBanToAdd() (banToAdd, error) {
return toAdd, nil return toAdd, nil
} }
// parseBanNetblock reads value, a netblock such as 203.0.113.0/24, or a // banNetblock reads value, a netblock such as 203.0.113.0/24, or a
// client's address, which stands for the netblock a ban on that client // client's address, which stands for the netblock a ban on that client
// covers. An IPv4-mapped netblock, such as ::ffff:203.0.113.0/120, is // covers. An IPv4-mapped netblock, such as ::ffff:203.0.113.0/120, is
// refused, since a client's address is looked up as IPv4 and a ban on it // refused, since a client's address is looked up as IPv4 and a ban on it
// would refuse nothing, and so is a value with a zone. // would refuse nothing, and so is a value with a zone.
func (h *handler) parseBanNetblock(value string) (netip.Prefix, error) { func (h *handler) banNetblock(value string) (netip.Prefix, error) {
netblock, err := netip.ParsePrefix(value) netblock, err := netip.ParsePrefix(value)
if err == nil { if err == nil {
if netblock.Addr().Is4In6() { if netblock.Addr().Is4In6() {
@@ -224,7 +224,7 @@ func (h *handler) parseBanNetblock(value string) (netip.Prefix, error) {
return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errZone) return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errZone)
} }
return banNetblock(h.config, addr), nil return h.netblock(addr), nil
} }
// expiry returns when a ban made at now for duration ends: duration // expiry returns when a ban made at now for duration ends: duration
@@ -295,7 +295,7 @@ func (rq *request) showClient() {
Bans: state.BanEntries(rq.h.ledger.Covering(addr)), Bans: state.BanEntries(rq.h.ledger.Covering(addr)),
} }
client, seen := rq.h.limiter.Client(clientGroup(rq.h.config, addr)) client, seen := rq.h.limiter.Client(rq.h.clientGroup(addr))
if seen { if seen {
answer.Client = &client answer.Client = &client
} }
+7 -2
View File
@@ -877,8 +877,13 @@ func TestWAFAnswersWithinTheTimeLimitWhileEveryOutsideSourceHangs(
held, _, _ := startHeldGeoJS(t) held, _, _ := startHeldGeoJS(t)
resolver, releaseResolver := startHeldResolver(t) resolver, releaseResolver := startHeldResolver(t)
server, out, _ := newWAFWith(t, waf.New, server, out, _ := newWAFWith(t, func(
silentGeoJSURL, silentAbuseIPDBURL, time.Now, map[string]string{ params waf.Params,
) *waf.Server {
params.AbuseIPDBURL = silentAbuseIPDBURL
return waf.New(params)
}, silentGeoJSURL, time.Now, map[string]string{
tagDeny: "country=KP", coreRuleSetMode: block, tagDeny: "country=KP", coreRuleSetMode: block,
dnsblZones: dnsblZone, dnsblResolver: resolver, dnsblZones: dnsblZone, dnsblResolver: resolver,
abuseIPDBKey: accountKey, abuseIPDBKey: accountKey,
+11 -13
View File
@@ -10,7 +10,6 @@ import (
"sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
) )
@@ -86,15 +85,14 @@ func (rq *request) countedBytes() int64 {
response, request := rq.out.bytes, rq.requestBytes response, request := rq.out.bytes, rq.requestBytes
switch rq.h.config.BytesCounted { switch rq.h.config.BytesCounted {
case config.BytesCountedResponse: case "response":
return response return response
case config.BytesCountedRequest: case "request":
return request return request
case config.BytesCountedBoth: default: // both
}
return response + request return response + request
} }
}
// banForLimit bans the client's netblock at now for a broken limit, the // banForLimit bans the client's netblock at now for a broken limit, the
// one hit names, notes the offence for the log line and counts the hit in // one hit names, notes the offence for the log line and counts the hit in
@@ -123,7 +121,7 @@ func (rq *request) banForLimit(
rq.line.Offence = requestlog.OffenceLimit rq.line.Offence = requestlog.OffenceLimit
rq.h.metrics.LimitHit(hit) rq.h.metrics.LimitHit(hit)
netblock := banNetblock(rq.h.config, rq.client) netblock := rq.h.netblock(rq.client)
if rq.h.config.Observe && if rq.h.config.Observe &&
!rq.wouldAlertBan(netblock, now, bans.CauseLimit) { !rq.wouldAlertBan(netblock, now, bans.CauseLimit) {
return return
@@ -159,7 +157,7 @@ func (rq *request) banForLimit(
} }
ban, made := rq.h.ledger.BanForLimit(netblock, now, notes) ban, made := rq.h.ledger.BanForLimit(netblock, now, notes)
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client)) rq.h.limiter.Reset(rq.h.clientGroup(rq.client))
rq.line.BanExpires = banExpires(ban) rq.line.BanExpires = banExpires(ban)
if made { if made {
@@ -173,7 +171,7 @@ func (rq *request) banForLimit(
// no ban, and raises the alert for the ban it would have made, if that // no ban, and raises the alert for the ban it would have made, if that
// alert would be sent. It returns the action for that ban. // alert would be sent. It returns the action for that ban.
func (rq *request) banForAttack(now time.Time, notes bans.Notes) string { func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
netblock := banNetblock(rq.h.config, rq.client) netblock := rq.h.netblock(rq.client)
if rq.h.config.Observe && if rq.h.config.Observe &&
!rq.wouldAlertBan(netblock, now, bans.CauseAttack) { !rq.wouldAlertBan(netblock, now, bans.CauseAttack) {
return banAction( return banAction(
@@ -275,16 +273,16 @@ func (rq *request) netblockRequests(netblock netip.Prefix) int64 {
return rq.h.limiter.Requests(netblock) + 1 return rq.h.limiter.Requests(netblock) + 1
} }
// banNetblock is the netblock a ban on client covers: its IPv4 address, // netblock is the netblock a ban on client covers: its IPv4 address,
// widened to SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group clientGroup // widened to SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group clientGroup
// counts it in. // counts it in.
func banNetblock(cfg *config.WAF, client netip.Addr) netip.Prefix { func (h *handler) netblock(client netip.Addr) netip.Prefix {
addr := client.Unmap() addr := client.Unmap()
if addr.Is4() { if addr.Is4() {
return netip.PrefixFrom(addr, cfg.BanScopeV4Prefix).Masked() return netip.PrefixFrom(addr, h.config.BanScopeV4Prefix).Masked()
} }
return clientGroup(cfg, addr) return h.clientGroup(addr)
} }
// banExpires is when ban ends, as the log line gives it: a time, or // banExpires is when ban ends, as the log line gives it: a time, or
+1 -1
View File
@@ -76,7 +76,7 @@ func newWAFHoldingTheCoreRuleSet(
params.BesideLookupTimeLimit = waf.BesideLookupTimeLimit params.BesideLookupTimeLimit = waf.BesideLookupTimeLimit
return waf.New(params) return waf.New(params)
}, silentGeoJSURL, abuseIPDBURL, time.Now, settings) }, silentGeoJSURL, time.Now, settings)
return server, out, queue return server, out, queue
} }
+2 -3
View File
@@ -63,8 +63,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
} }
blocklisted := percentage{percent: whole} blocklisted := percentage{percent: whole}
if rq.blocklisted && if rq.blocklisted && cfg.BlocklistAction == "limit" {
cfg.BlocklistAction == config.ReputationActionLimit {
blocklisted = percentage{ blocklisted = percentage{
cfg.BlocklistLimitPercent, cfg.BlocklistLimitPercent,
"SWW_WAF_BLOCKLIST_ACTION", "SWW_WAF_BLOCKLIST_ACTION",
@@ -73,7 +72,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
dnsblAbuseIPDBListed := percentage{percent: whole} dnsblAbuseIPDBListed := percentage{percent: whole}
if (rq.dnsblListed || rq.abuseIPDBHit) && if (rq.dnsblListed || rq.abuseIPDBHit) &&
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit { cfg.DNSBLAbuseIPDBAction == "limit" {
dnsblAbuseIPDBListed = percentage{ dnsblAbuseIPDBListed = percentage{
cfg.DNSBLAbuseIPDBLimitPercent, cfg.DNSBLAbuseIPDBLimitPercent,
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
+1 -3
View File
@@ -7,7 +7,6 @@ import (
"context" "context"
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
) )
@@ -42,6 +41,5 @@ func (blocklists) check(
// client, given listedBy, the blocklists that list it: it does when one lists // client, given listedBy, the blocklists that list it: it does when one lists
// it and SWW_WAF_BLOCKLIST_ACTION is deny. // it and SWW_WAF_BLOCKLIST_ACTION is deny.
func (rq *request) refusedByBlocklists(listedBy []string) bool { func (rq *request) refusedByBlocklists(listedBy []string) bool {
return len(listedBy) > 0 && rq.h.config.BlocklistAction == return len(listedBy) > 0 && rq.h.config.BlocklistAction == deny
config.ReputationActionDeny
} }
+1 -1
View File
@@ -30,7 +30,7 @@ func (byteLimits) check(
} }
counts, hit, over := rq.h.limiter.CountBytes( counts, hit, over := rq.h.limiter.CountBytes(
clientGroup(rq.h.config, rq.client), rq.h.clientGroup(rq.client),
now, now,
rq.countedBytes(), rq.countedBytes(),
rq.bytesLimitPercent.percent, rq.bytesLimitPercent.percent,
+2 -4
View File
@@ -5,8 +5,6 @@ import (
"net/http" "net/http"
"net/netip" "net/netip"
"slices" "slices"
"sneak.berlin/go/smallwebwaf/internal/config"
) )
// peerAddress is the address of the request's TCP peer. // peerAddress is the address of the request's TCP peer.
@@ -47,10 +45,10 @@ func scheme(r *http.Request, isQuestion bool) string {
// address, or its IPv6 group, the netblock its IPv6 address is in of the // address, or its IPv6 group, the netblock its IPv6 address is in of the
// length SWW_WAF_IPV6_GROUP_PREFIX sets, a /64 by default, since one abuser // length SWW_WAF_IPV6_GROUP_PREFIX sets, a /64 by default, since one abuser
// usually holds a whole /64. An IPv4 address in IPv6 form counts as IPv4. // usually holds a whole /64. An IPv4 address in IPv6 form counts as IPv4.
func clientGroup(cfg *config.WAF, addr netip.Addr) netip.Prefix { func (h *handler) clientGroup(addr netip.Addr) netip.Prefix {
addr = addr.Unmap() addr = addr.Unmap()
if addr.Is6() { if addr.Is6() {
return netip.PrefixFrom(addr, cfg.IPv6GroupPrefix).Masked() return netip.PrefixFrom(addr, h.config.IPv6GroupPrefix).Masked()
} }
return netip.PrefixFrom(addr, addr.BitLen()) return netip.PrefixFrom(addr, addr.BitLen())
+3 -3
View File
@@ -47,7 +47,7 @@ func (coreRuleSet) check(
rq.line.CoreRuleSetScore = &result.Score rq.line.CoreRuleSetScore = &result.Score
for _, id := range result.RuleIDs { for _, id := range result.RuleIDs {
rq.h.metrics.CoreRuleSetMatched(string(cfg.CoreRuleSetMode), id) rq.h.metrics.CoreRuleSetMatched(cfg.CoreRuleSetMode, id)
} }
threshold := cfg.CoreRuleSetScoreThreshold threshold := cfg.CoreRuleSetScoreThreshold
@@ -99,7 +99,7 @@ func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
switch { switch {
case rq.h.config.CoreRuleSetMode == config.CoreRuleSetModeDetect: case rq.h.config.CoreRuleSetMode == config.CoreRuleSetModeDetect:
detail["mode"] = string(config.CoreRuleSetModeDetect) detail["mode"] = config.CoreRuleSetModeDetect
case rq.h.config.Observe: case rq.h.config.Observe:
detail["mode"] = "observe" detail["mode"] = "observe"
} }
@@ -108,7 +108,7 @@ func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
Event: alerts.EventCoreRuleSetMatch, Event: alerts.EventCoreRuleSetMatch,
Host: rq.site, Host: rq.site,
Client: rq.client, Client: rq.client,
Netblock: clientGroup(rq.h.config, rq.client), Netblock: rq.h.clientGroup(rq.client),
ASN: rq.line.ASN, ASN: rq.line.ASN,
ASName: rq.line.ASName, ASName: rq.line.ASName,
Country: rq.line.Country, Country: rq.line.Country,
+1 -2
View File
@@ -32,8 +32,7 @@ func (dnsbl) check(ctx context.Context, rq *request, _ time.Time) *verdict {
rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone)) rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone))
} }
if !rq.dnsblListed || if !rq.dnsblListed || rq.h.config.DNSBLAbuseIPDBAction != deny {
rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
return nil return nil
} }
-9
View File
@@ -1,13 +1,4 @@
package waf package waf
import "sneak.berlin/go/smallwebwaf/internal/reputation"
// BesideLookupTimeLimit is besideLookupTimeLimit. // BesideLookupTimeLimit is besideLookupTimeLimit.
const BesideLookupTimeLimit = besideLookupTimeLimit const BesideLookupTimeLimit = besideLookupTimeLimit
// BanAbuseIPDBBlacklist does on s what a fetch of AbuseIPDB's blacklist with
// the clients on listed does.
func (s *Server) BanAbuseIPDBBlacklist(listed []reputation.BlacklistEntry) {
h := s.Handler.(*handler) //nolint:forcetypeassert // New makes it one
BanAbuseIPDBBlacklist(h.config, h.ledger, h.processLog, h.now(), listed)
}
+15 -28
View File
@@ -2,18 +2,14 @@ package waf
import ( import (
"context" "context"
"log/slog"
"net/http" "net/http"
"net/netip" "net/netip"
"strings" "strings"
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/geojs" "sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
) )
// The headers in which an answer of allow gives rproxy the client's AS number // The headers in which an answer of allow gives rproxy the client's AS number
@@ -49,16 +45,15 @@ func (rq *request) startLookUp(
) <-chan lookupResult { ) <-chan lookupResult {
result := make(chan lookupResult, 1) result := make(chan lookupResult, 1)
if rq.h.config.LookupSource == config.LookupSourceOff || if rq.h.config.LookupSource == "off" || !canBePlaced(rq.client) {
!canBePlaced(rq.client) {
result <- lookupResult{} result <- lookupResult{}
return result return result
} }
client := clientGroup(rq.h.config, rq.client) client := rq.h.clientGroup(rq.client)
if rq.h.config.LookupSource == config.LookupSourceFile { if rq.h.config.LookupSource == "file" {
result <- lookupResult{ result <- lookupResult{
lookedUp: true, answer: rq.h.lookupFile.LookUp(client), lookedUp: true, answer: rq.h.lookupFile.LookUp(client),
} }
@@ -133,30 +128,22 @@ func countryTag(country string) string {
return config.CountryTagPrefix + country return config.CountryTagPrefix + country
} }
// AddLookup adds answer, an answer about a client from the lookup database // addLookup adds answer, an answer about a client from the lookup
// or GeoJS, to the client's history in limiter, and to the notes of the bans // database or GeoJS, to the client's history, and to the notes of the bans
// in ledger on its netblock that have no AS number, AS name or country yet. // on its netblock that have no AS number, AS name or country yet.
// cfg sets the netblock's size. func (h *handler) addLookup(answer geojs.Answer) {
func AddLookup( h.limiter.AddLookup(answer.Client, answer.Answered,
cfg *config.WAF, limiter *ratelimit.Limiter, ledger *bans.Ledger,
answer geojs.Answer,
) {
limiter.AddLookup(answer.Client, answer.Answered,
answer.ASN, answer.ASName, answer.Country) answer.ASN, answer.ASName, answer.Country)
ledger.AddLookup(banNetblock(cfg, answer.Client.Addr()), h.ledger.AddLookup(h.netblock(answer.Client.Addr()),
answer.ASN, answer.ASName, answer.Country) answer.ASN, answer.ASName, answer.Country)
} }
// GeoJSFailed logs a failure of GeoJS to processLog, raises a // geojsFailed logs a failure of GeoJS, raises a source_failure alert for it
// source_failure alert for it on alertQueue and counts it in m. // and counts it. askingAgainIn is how long GeoJS is left alone after it.
// askingAgainIn is how long GeoJS is left alone after it. func (h *handler) geojsFailed(err error, askingAgainIn time.Duration) {
func GeoJSFailed( h.processLog.Warn("asking GeoJS failed",
processLog *slog.Logger, alertQueue *alerts.Queue, m *metrics.Metrics,
err error, askingAgainIn time.Duration,
) {
processLog.Warn("asking GeoJS failed",
"error", err.Error(), "asking_again_in", askingAgainIn.String()) "error", err.Error(), "asking_again_in", askingAgainIn.String())
alertQueue.Raise(alerts.Alert{ h.alerts.Raise(alerts.Alert{
Event: alerts.EventSourceFailure, Event: alerts.EventSourceFailure,
Reason: "asking GeoJS failed", Reason: "asking GeoJS failed",
Detail: map[string]any{ Detail: map[string]any{
@@ -164,7 +151,7 @@ func GeoJSFailed(
"asking_again_in": askingAgainIn.String(), "asking_again_in": askingAgainIn.String(),
}, },
}) })
m.GeoJSFailures.Inc() h.metrics.GeoJSFailures.Inc()
} }
// setLookupHeaders sets the headers that give the client's AS number and // setLookupHeaders sets the headers that give the client's AS number and
+152 -47
View File
@@ -1,11 +1,11 @@
// Package waf is the server the waf command runs. It answers // Package waf is the server the waf command runs. It answers
// rproxy's questions, takes rproxy's reports and answers its own endpoints, and // rproxy's questions, takes rproxy's reports and answers its own endpoints, and
// writes a log line for each. It also bans the clients on AbuseIPDB's // writes a log line for each.
// blacklist each time the reputation package fetches it.
package waf package waf
import ( import (
"cmp" "cmp"
"fmt"
"io" "io"
"log/slog" "log/slog"
"net/http" "net/http"
@@ -57,33 +57,17 @@ type Params struct {
RequestLog io.Writer RequestLog io.Writer
// ProcessLog receives the process's own messages. // ProcessLog receives the process's own messages.
ProcessLog *slog.Logger ProcessLog *slog.Logger
// Metrics count what waf does. Whoever builds them adds the metrics // GeoJSURL is where clients' AS numbers and countries are looked up
// read from the other parts. // while SWW_WAF_LOOKUP_SOURCE is geojs, normally geojs.URL.
Metrics *metrics.Metrics GeoJSURL string
// Limiter is the table of clients the rate limits count, with each // AbuseIPDBURL is where clients are checked with AbuseIPDB while
// client's history. // SWW_WAF_ABUSEIPDB_KEY is set, normally reputation.AbuseIPDBURL.
Limiter *ratelimit.Limiter AbuseIPDBURL string
// Ledger holds the bans.
Ledger *bans.Ledger
// Anomalies are the counters of the anomaly thresholds.
Anomalies *anomaly.Counters
// Lists are the lists fetched from URLs, DNSBL the DNSBL zones'
// verdicts, and AbuseIPDB AbuseIPDB's scores and the checks spent.
Lists *reputation.Lists
DNSBL *reputation.DNSBL
AbuseIPDB *reputation.AbuseIPDB
// RouteWatch asks routewatch about clients, and is nil while
// SWW_WAF_ROUTEWATCH_URL is unset.
RouteWatch *routewatch.Client
// GeoJS looks clients' AS numbers and countries up while
// SWW_WAF_LOOKUP_SOURCE is geojs. It is given AddLookup as its Answered
// and GeoJSFailed as its Failed.
GeoJS *geojs.GeoJS
// LookupFile is the lookup database they are looked up in while // LookupFile is the lookup database they are looked up in while
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise. // SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
LookupFile *lookup.File LookupFile *lookup.File
// CoreRuleSet is what inspects requests while // CoreRuleSet is what NewCoreRuleSet returns: what inspects requests
// SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is. // while SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
CoreRuleSet CoreRuleSetInspector CoreRuleSet CoreRuleSetInspector
// Now tells the time by which requests are counted for the rate // Now tells the time by which requests are counted for the rate
// limits, bans are made and run out, and GeoJS's answers are kept, // limits, bans are made and run out, and GeoJS's answers are kept,
@@ -115,8 +99,11 @@ type CoreRuleSetInspector interface {
) (coreruleset.Result, []byte, error) ) (coreruleset.Result, []byte, error)
} }
// Server is the server waf runs, with the parts of its Params. The tests // Server is the server waf runs, with the parts whose state the state
// read them. // files keep, the lookup database, nil unless SWW_WAF_LOOKUP_SOURCE is file, the
// lists fetched from URLs, which its Run fetches, the DNSBL zones' verdicts,
// AbuseIPDB's scores and checks spent, routewatch's answers, and the metrics.
// RouteWatch is nil while SWW_WAF_ROUTEWATCH_URL is unset.
type Server struct { type Server struct {
*http.Server *http.Server
@@ -132,45 +119,163 @@ type Server struct {
Metrics *metrics.Metrics Metrics *metrics.Metrics
} }
// newHandler returns the handler with what every request shares, the parts // newHandler returns the handler with what every request shares, as the
// in params among it. // settings in params make it.
func newHandler(params Params) *handler { func newHandler(params Params) *handler {
m := metrics.New(
params.Config.MetricsBusiestCountriesAndASNs,
params.Config.InstanceName,
)
lists, dnsbl, abuseIPDB := newReputation(params, m)
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport) awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
if err != nil { if err != nil {
panic(err) // New fails only for a size below one panic(err) // New fails only for a size below one
} }
return &handler{ h := &handler{
config: params.Config, config: params.Config,
requestLog: params.RequestLog, requestLog: params.RequestLog,
processLog: params.ProcessLog, processLog: params.ProcessLog,
now: params.Now, now: params.Now,
metrics: params.Metrics, metrics: m,
awaitingReport: awaitingReport, awaitingReport: awaitingReport,
limiter: params.Limiter, limiter: ratelimit.New(ratelimit.Limits{
ledger: params.Ledger, PerMinute: params.Config.RateLimitPerMinute,
geojs: params.GeoJS, PerHour: params.Config.RateLimitPerHour,
anomalies: params.Anomalies, PerDay: params.Config.RateLimitPerDay,
BytesPerMinute: params.Config.BytesLimitPerMinute,
BytesPerHour: params.Config.BytesLimitPerHour,
BytesPerDay: params.Config.BytesLimitPerDay,
}, params.Config.MaxTrackedClients),
ledger: bans.New(bans.Params{
LimitBanDuration: params.Config.LimitBanDuration,
LimitBanRepeatWindow: params.Config.LimitBanRepeatWindow,
LimitBanPermanentAbove: params.Config.LimitBanPermanentAbove,
AttackBanDuration: params.Config.AttackBanDuration,
MaxBans: params.Config.MaxBans,
}),
anomalies: anomaly.New(anomaly.Params{
Client: params.Config.AnomalyClient,
Net: params.Config.AnomalyNet,
ASN: params.Config.AnomalyASN,
Total: params.Config.AnomalyTotal,
WatchedNet: params.Config.AnomalyWatchedNet,
NetV4Prefix: params.Config.AnomalyNetV4Prefix,
NetV6Prefix: params.Config.AnomalyNetV6Prefix,
NamedNetblocks: params.Config.AnomalyWatchedNets,
Alerts: params.Alerts,
}),
lookupFile: params.LookupFile, lookupFile: params.LookupFile,
lists: params.Lists, lists: lists,
dnsbl: params.DNSBL, dnsbl: dnsbl,
abuseIPDB: params.AbuseIPDB, abuseIPDB: abuseIPDB,
routeWatch: params.RouteWatch,
rules: params.Rules, rules: params.Rules,
coreRuleSet: params.CoreRuleSet, coreRuleSet: params.CoreRuleSet,
alerts: params.Alerts, alerts: params.Alerts,
// The country list, the headers, the tag settings that name an asn=
// or country= tag and the biased thresholds act on the answer before
// the request goes on.
waitForGeoJS: len(params.Config.ExclusivelyAllowedCountries) > 0 ||
params.Config.AddLookupHeaders ||
params.Config.TagsNeedLookup() ||
biasedThresholdsSet(params.Config),
besideLookupLimit: cmp.Or( besideLookupLimit: cmp.Or(
params.BesideLookupTimeLimit, params.BesideLookupTimeLimit,
besideLookupTimeLimit, besideLookupTimeLimit,
), ),
} }
if params.Config.RouteWatchURL != "" {
h.routeWatch = routewatch.New(routewatch.Params{
URL: params.Config.RouteWatchURL, Now: params.Now, ProcessLog: params.ProcessLog,
})
}
// The country list, the headers, the tag settings that name an asn= or
// country= tag and the biased thresholds act on the answer before the
// request goes on.
h.waitForGeoJS = len(params.Config.ExclusivelyAllowedCountries) > 0 ||
params.Config.AddLookupHeaders || params.Config.TagsNeedLookup() ||
biasedThresholdsSet(params.Config)
h.geojs = geojs.New(geojs.Params{
URL: params.GeoJSURL,
Answered: h.addLookup,
Now: params.Now,
Asked: m.GeoJSRequests.Inc,
Failed: h.geojsFailed,
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
})
m.AddBansAndClients(h.ledger, h.limiter, params.Now)
m.AddRules(params.Rules)
return h
}
// newReputation returns the lists fetched from URLs, the DNSBL zones'
// verdicts and AbuseIPDB's scores, as the settings in params name them,
// with none fetched, asked for or checked yet, and adds their metrics to
// m, AbuseIPDB's while SWW_WAF_ABUSEIPDB_KEY is set.
func newReputation(
params Params, m *metrics.Metrics,
) (*reputation.Lists, *reputation.DNSBL, *reputation.AbuseIPDB) {
cfg := params.Config
lists := reputation.New(reputation.Params{
BlocklistURLs: cfg.BlocklistURLs,
Refresh: cfg.BlocklistRefresh,
ASNLimitPercentURL: cfg.ASNLimitPercentURL,
TagLists: cfg.TagLists,
Now: params.Now,
ProcessLog: params.ProcessLog,
Alerts: params.Alerts,
})
dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{
Zones: cfg.DNSBLZones, Resolver: cfg.DNSBLResolver,
CacheTTL: cfg.IPReputationCacheTTL,
Timeout: cfg.DNSBLAbuseIPDBTimeout, Now: params.Now, ProcessLog: params.ProcessLog,
Alerts: params.Alerts,
})
abuseIPDB := reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
URL: params.AbuseIPDBURL, Key: cfg.AbuseIPDBKey, MinScore: cfg.AbuseIPDBMinScore,
DailyBudget: cfg.AbuseIPDBDailyBudget, CacheTTL: cfg.IPReputationCacheTTL,
Timeout: cfg.DNSBLAbuseIPDBTimeout, Now: params.Now, ProcessLog: params.ProcessLog,
Alerts: params.Alerts,
})
m.AddReputation(lists, dnsbl)
if cfg.AbuseIPDBKey != "" {
m.AddAbuseIPDB(abuseIPDB)
}
return lists, dnsbl, abuseIPDB
}
// NewCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
// coreruleset.New refuses is an error in every mode. While the mode is off, a
// set file is loaded with the Core Rule Set only to check it.
func NewCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
cfg.CoreRuleSetExclusionsFile == "" {
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
}
coreRuleSet, err := coreruleset.New(coreruleset.Params{
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
DisabledRules: cfg.CoreRuleSetDisabledRules,
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
Apps: cfg.Apps,
})
if err != nil {
return nil, fmt.Errorf(
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w",
err,
)
}
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
}
return coreRuleSet, nil
} }
// handler is waf's handler. It holds what every request shares; what // handler is waf's handler. It holds what every request shares; what
+24 -185
View File
@@ -16,19 +16,10 @@ import (
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/anomaly"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest" "sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/routewatch"
"sneak.berlin/go/smallwebwaf/internal/rproxy" "sneak.berlin/go/smallwebwaf/internal/rproxy"
"sneak.berlin/go/smallwebwaf/internal/rules" "sneak.berlin/go/smallwebwaf/internal/rules"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/waf"
@@ -399,11 +390,11 @@ func startRProxy(
server := rproxy.New(rproxy.Params{ server := rproxy.New(rproxy.Params{
Config: cfg, Config: cfg,
RequestLog: io.Discard, RequestLog: io.Discard,
ProcessLog: processlog.New(processlog.Params{ ProcessLog: requestlog.NewProcessLogger(
Output: io.Discard, io.Discard,
InstanceName: "rproxy", "rproxy",
Level: slog.LevelInfo, slog.LevelInfo,
}), ),
}) })
listener, err := (&net.ListenConfig{}).Listen( listener, err := (&net.ListenConfig{}).Listen(
@@ -433,8 +424,7 @@ func startRProxy(
// With no geojsURL, there is no stand-in for GeoJS to look clients up at, and // With no geojsURL, there is no stand-in for GeoJS to look clients up at, and
// SWW_WAF_LOOKUP_SOURCE is off unless env sets it. While it is file, the lookup // SWW_WAF_LOOKUP_SOURCE is off unless env sets it. While it is file, the lookup
// database SWW_WAF_LOOKUP_DB_PATH names is read. Clients are checked with AbuseIPDB // database SWW_WAF_LOOKUP_DB_PATH names is read. Clients are checked with AbuseIPDB
// at abuseIPDBURL while env sets SWW_WAF_ABUSEIPDB_KEY, and its blacklist is // at abuseIPDBURL while env sets SWW_WAF_ABUSEIPDB_KEY.
// fetched from abuseIPDBBlacklistURL.
// SWW_WAF_CORE_RULE_SET_MODE is off unless env sets it, so that only the tests of the // SWW_WAF_CORE_RULE_SET_MODE is off unless env sets it, so that only the tests of the
// Core Rule Set have their requests inspected by it. // Core Rule Set have their requests inspected by it.
func newWAF( func newWAF(
@@ -445,15 +435,13 @@ func newWAF(
) (*waf.Server, *output, *alerts.Queue) { ) (*waf.Server, *output, *alerts.Queue) {
t.Helper() t.Helper()
return newWAFWith(t, waf.New, geojsURL, abuseIPDBURL, now, env) return newWAFWith(t, waf.New, geojsURL, now, env)
} }
// newWAFWith is newWAF for the server build makes, with clients checked with // newWAFWith is newWAF for the server build makes.
// AbuseIPDB at abuseIPDBCheckURL.
func newWAFWith( func newWAFWith(
t *testing.T, build func(waf.Params) *waf.Server, t *testing.T, build func(waf.Params) *waf.Server,
geojsURL, abuseIPDBCheckURL string, now func() time.Time, geojsURL string, now func() time.Time, env map[string]string,
env map[string]string,
) (*waf.Server, *output, *alerts.Queue) { ) (*waf.Server, *output, *alerts.Queue) {
t.Helper() t.Helper()
@@ -472,11 +460,11 @@ func newWAFWith(
} }
out := &output{} out := &output{}
processLog := processlog.New(processlog.Params{ processLog := requestlog.NewProcessLogger(
Output: out, out,
InstanceName: cfg.InstanceName, cfg.InstanceName,
Level: cfg.LogLevel, cfg.LogLevel,
}) )
ruleFiles, err := rules.Load(rules.Params{ ruleFiles, err := rules.Load(rules.Params{
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog, Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
@@ -506,179 +494,30 @@ func newWAFWith(
} }
} }
params := waf.Params{ coreRuleSet, err := waf.NewCoreRuleSet(cfg)
if err != nil {
t.Fatalf("Core Rule Set: %v", err)
}
server := build(waf.Params{
Config: cfg, Config: cfg,
RequestLog: out, RequestLog: out,
ProcessLog: processLog, ProcessLog: processLog,
GeoJSURL: geojsURL,
AbuseIPDBURL: abuseIPDBURL,
LookupFile: lookupFile, LookupFile: lookupFile,
CoreRuleSet: coreRuleSet,
Now: now, Now: now,
Rules: ruleFiles, Rules: ruleFiles,
Alerts: alertQueue, Alerts: alertQueue,
// The tests of the limit set it back, and hold the Core Rule Set // The tests of the limit set it back, and hold the Core Rule Set
// with a stand-in. // with a stand-in.
BesideLookupTimeLimit: time.Hour, BesideLookupTimeLimit: time.Hour,
} })
addCoreRuleSet(t, &params)
addCounters(&params)
addOutsideSources(&params, geojsURL, abuseIPDBCheckURL)
server := build(params)
return server, out, alertQueue return server, out, alertQueue
} }
// addCoreRuleSet adds to params the Core Rule Set, with the settings for it,
// while SWW_WAF_CORE_RULE_SET_MODE is not off.
func addCoreRuleSet(t *testing.T, params *waf.Params) {
t.Helper()
cfg := params.Config
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
return
}
coreRuleSet, err := coreruleset.New(coreruleset.Params{
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
DisabledRules: cfg.CoreRuleSetDisabledRules,
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
Apps: cfg.Apps,
})
if err != nil {
t.Fatalf("Core Rule Set: %v", err)
}
params.CoreRuleSet = coreRuleSet
}
// addCounters adds to params the metrics, the table of clients, the ledger
// and the anomaly counters, as sww builds them from params' settings. The
// metrics read the ledger, the table of clients and the rules.
func addCounters(params *waf.Params) {
cfg := params.Config
params.Metrics = metrics.New(
cfg.MetricsBusiestCountriesAndASNs,
cfg.InstanceName,
)
params.Limiter = ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{
PerMinute: cfg.RateLimitPerMinute,
PerHour: cfg.RateLimitPerHour,
PerDay: cfg.RateLimitPerDay,
BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerHour: cfg.BytesLimitPerHour,
BytesPerDay: cfg.BytesLimitPerDay,
},
MaxClients: cfg.MaxTrackedClients,
})
params.Ledger = bans.New(bans.Params{
LimitBanDuration: cfg.LimitBanDuration,
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
LimitBanPermanentAbove: cfg.LimitBanPermanentAbove,
AttackBanDuration: cfg.AttackBanDuration,
MaxBans: cfg.MaxBans,
})
params.Anomalies = anomaly.New(anomaly.Params{
Client: cfg.AnomalyClient,
Net: cfg.AnomalyNet,
ASN: cfg.AnomalyASN,
Total: cfg.AnomalyTotal,
WatchedNet: cfg.AnomalyWatchedNet,
NetV4Prefix: cfg.AnomalyNetV4Prefix,
NetV6Prefix: cfg.AnomalyNetV6Prefix,
NamedNetblocks: cfg.AnomalyWatchedNets,
Alerts: params.Alerts,
})
params.Metrics.AddBansAndClients(
params.Ledger,
params.Limiter,
params.Now,
)
params.Metrics.AddRules(params.Rules)
}
// addOutsideSources adds to params the lists fetched from URLs, the DNSBL
// zones' verdicts, AbuseIPDB's scores from abuseIPDBCheckURL and its
// blacklist from abuseIPDBBlacklistURL, routewatch's answers while its
// setting is set, and GeoJS at geojsURL, as sww builds
// them from params' settings and the parts addCounters added. The metrics
// read the lists, the zones' verdicts, and AbuseIPDB's while its key is set.
func addOutsideSources(
params *waf.Params, geojsURL, abuseIPDBCheckURL string,
) {
cfg := params.Config
m, limiter, ledger := params.Metrics, params.Limiter, params.Ledger
processLog, alertQueue := params.ProcessLog, params.Alerts
params.Lists = reputation.New(reputation.Params{
BlocklistURLs: cfg.BlocklistURLs,
Refresh: cfg.BlocklistRefresh,
ASNLimitPercentURL: cfg.ASNLimitPercentURL,
TagLists: cfg.TagLists,
Now: params.Now,
ProcessLog: processLog,
Alerts: alertQueue,
})
params.DNSBL = reputation.NewDNSBL(reputation.DNSBLParams{
Zones: cfg.DNSBLZones,
Resolver: cfg.DNSBLResolver,
CacheTTL: cfg.IPReputationCacheTTL,
Timeout: cfg.DNSBLAbuseIPDBTimeout,
Now: params.Now,
ProcessLog: processLog,
Alerts: alertQueue,
})
params.AbuseIPDB = reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
URL: abuseIPDBCheckURL,
Key: cfg.AbuseIPDBKey,
MinScore: cfg.AbuseIPDBMinScore,
DailyBudget: cfg.AbuseIPDBDailyBudget,
CacheTTL: cfg.IPReputationCacheTTL,
Timeout: cfg.DNSBLAbuseIPDBTimeout,
BlacklistURL: abuseIPDBBlacklistURL,
Threshold: cfg.AbuseIPDBThreshold,
BlacklistRefresh: cfg.AbuseIPDBBlacklistRefresh,
Blacklisted: func(listed []reputation.BlacklistEntry) {
waf.BanAbuseIPDBBlacklist(
cfg, ledger, processLog, params.Now(), listed,
)
},
Now: params.Now,
ProcessLog: processLog,
Alerts: alertQueue,
})
if cfg.RouteWatchURL != "" {
params.RouteWatch = routewatch.New(routewatch.Params{
URL: cfg.RouteWatchURL,
Now: params.Now,
ProcessLog: processLog,
})
}
params.GeoJS = geojs.New(geojs.Params{
URL: geojsURL,
Answered: func(answer geojs.Answer) {
waf.AddLookup(cfg, limiter, ledger, answer)
},
Now: params.Now,
Asked: m.GeoJSRequests.Inc,
Failed: func(err error, askingAgainIn time.Duration) {
waf.GeoJSFailed(processLog, alertQueue, m, err, askingAgainIn)
},
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
})
m.AddReputation(params.Lists, params.DNSBL)
if cfg.AbuseIPDBKey != "" {
m.AddAbuseIPDB(params.AbuseIPDB)
}
}
// newClient returns an HTTP client that sends requests as they are made, // newClient returns an HTTP client that sends requests as they are made,
// with no compression of its own. // with no compression of its own.
func newClient(t *testing.T) *http.Client { func newClient(t *testing.T) *http.Client {
+3 -6
View File
@@ -38,11 +38,8 @@ func (rateLimits) check(
rq.line.BytesLimitPercent, rq.line.BytesLimitPercentSetting = rq.line.BytesLimitPercent, rq.line.BytesLimitPercentSetting =
rq.bytesLimitPercent.logged() rq.bytesLimitPercent.logged()
counts, hit, over := rq.h.limiter.Count( counts, hit, over := rq.h.limiter.Count(rq.h.clientGroup(rq.client), now,
clientGroup(rq.h.config, rq.client), rq.rateLimitPercent.percent)
now,
rq.rateLimitPercent.percent,
)
rq.line.Counts = counts rq.line.Counts = counts
if !over { if !over {
@@ -62,7 +59,7 @@ func (rateLimits) check(
// limit. It counts nothing. // limit. It counts nothing.
func (rq *request) overAWholeRateLimit(now time.Time) bool { func (rq *request) overAWholeRateLimit(now time.Time) bool {
return rq.countedByRateLimits() && return rq.countedByRateLimits() &&
rq.h.limiter.OverARateLimit(clientGroup(rq.h.config, rq.client), now) rq.h.limiter.OverARateLimit(rq.h.clientGroup(rq.client), now)
} }
// countedByRateLimits reports whether the rate limits count the request: its // countedByRateLimits reports whether the rate limits count the request: its
+2 -5
View File
@@ -36,11 +36,8 @@ func (refusalLimit) check(
return nil return nil
} }
hit, over := rq.h.limiter.CountRefusal( hit, over := rq.h.limiter.CountRefusal(rq.h.clientGroup(rq.client), now,
clientGroup(rq.h.config, rq.client), cfg.RefusalsLimitPerMinute)
now,
cfg.RefusalsLimitPerMinute,
)
if !over { if !over {
return nil return nil
} }
+5 -1
View File
@@ -7,6 +7,10 @@ import (
"sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/bans"
) )
// deny is the SWW_WAF_BLOCKLIST_ACTION and the SWW_WAF_DNSBL_ABUSEIPDB_ACTION that
// refuses the requests of a client a source lists.
const deny = "deny"
// noteListed notes each of sources, the URLs of the blocklists or the // noteListed notes each of sources, the URLs of the blocklists or the
// DNSBL zones, their keys masked, that list the client, as noteHit does, // DNSBL zones, their keys masked, that list the client, as noteHit does,
// with reason. // with reason.
@@ -33,7 +37,7 @@ func (rq *request) noteHit(hit bans.ReputationHit, reason string) {
Event: alerts.EventReputationHit, Event: alerts.EventReputationHit,
Host: rq.site, Host: rq.site,
Client: rq.client, Client: rq.client,
Netblock: clientGroup(rq.h.config, rq.client), Netblock: rq.h.clientGroup(rq.client),
ASN: rq.line.ASN, ASN: rq.line.ASN,
ASName: rq.line.ASName, ASName: rq.line.ASName,
Country: rq.line.Country, Country: rq.line.Country,
+7 -24
View File
@@ -822,38 +822,21 @@ const abuseipdb = reputation.AbuseIPDBSource
// through which AbuseIPDB is asked. // through which AbuseIPDB is asked.
const abuseIPDBURL = "abuseipdb://stand-in/api/v2/check" const abuseIPDBURL = "abuseipdb://stand-in/api/v2/check"
// abuseIPDBBlacklistURL is where newWAF has AbuseIPDB's blacklist fetched
// from: at abuseIPDBStandIn too.
const abuseIPDBBlacklistURL = "abuseipdb://stand-in/api/v2/blacklist"
// abuseIPDBStandIn is a stand-in for AbuseIPDB that gives every client the // abuseIPDBStandIn is a stand-in for AbuseIPDB that gives every client the
// score 100, at once and without the network. Its blacklist lists client // score 100, at once and without the network.
// alone, with the score 100, to a fetch with accountKey that asks for the
// clients scored 90 or more, the default threshold; it refuses any other
// fetch of it with 400.
type abuseIPDBStandIn struct{} type abuseIPDBStandIn struct{}
// RoundTrip answers req with the score 100, or with the blacklist. // RoundTrip answers req with the score 100.
func (abuseIPDBStandIn) RoundTrip( func (abuseIPDBStandIn) RoundTrip(
req *http.Request, req *http.Request,
) (*http.Response, error) { ) (*http.Response, error) {
status, body := http.StatusOK, `{"data":{"abuseConfidenceScore":100}}`
if req.URL.Path == "/api/v2/blacklist" {
body = `{"data":[{"ipAddress":"` + client +
`","abuseConfidenceScore":100}]}`
if req.Header.Get("Key") != accountKey ||
req.URL.RawQuery != "confidenceMinimum=90" {
status, body = http.StatusBadRequest, `{"errors":[]}`
}
}
return &http.Response{ return &http.Response{
StatusCode: status, StatusCode: http.StatusOK,
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)), Status: "200 OK",
Header: http.Header{}, Header: http.Header{},
Body: io.NopCloser(strings.NewReader(body)), Body: io.NopCloser(
strings.NewReader(`{"data":{"abuseConfidenceScore":100}}`),
),
Request: req, Request: req,
}, nil }, nil
} }
+6 -6
View File
@@ -122,7 +122,7 @@ func (h *handler) startRequest(
RequestID: requestID(r, isQuestion), RequestID: requestID(r, isQuestion),
PeerIP: peer.String(), PeerIP: peer.String(),
ForwardedFor: strings.Join(forwardedFor, ", "), ForwardedFor: strings.Join(forwardedFor, ", "),
ClientGroup: clientGroup(h.config, client).String(), ClientGroup: h.clientGroup(client).String(),
ContentType: r.Header.Get("Content-Type"), ContentType: r.Header.Get("Content-Type"),
RequestHeaders: requestHeaders( RequestHeaders: requestHeaders(
r, r,
@@ -365,7 +365,7 @@ func (rq *request) addToHistory() {
} }
rq.h.limiter.AddToHistory( rq.h.limiter.AddToHistory(
clientGroup(rq.h.config, rq.client), rq.h.clientGroup(rq.client),
rq.h.now(), rq.h.now(),
request, request,
) )
@@ -373,7 +373,7 @@ func (rq *request) addToHistory() {
answer, found := rq.answerAtTheEnd() answer, found := rq.answerAtTheEnd()
if found { if found {
AddLookup(rq.h.config, rq.h.limiter, rq.h.ledger, answer) rq.h.addLookup(answer)
} }
} }
@@ -394,7 +394,7 @@ func (rq *request) countAnomalies() {
rq.h.anomalies.Count(rq.h.now(), anomaly.Request{ rq.h.anomalies.Count(rq.h.now(), anomaly.Request{
Host: rq.site, Host: rq.site,
Client: rq.client, Client: rq.client,
ClientGroup: clientGroup(rq.h.config, rq.client), ClientGroup: rq.h.clientGroup(rq.client),
ASN: answer.ASN, ASN: answer.ASN,
ASName: answer.ASName, ASName: answer.ASName,
Country: answer.Country, Country: answer.Country,
@@ -421,11 +421,11 @@ func (rq *request) answerAtTheEnd() (geojs.Answer, bool) {
return geojs.Answer{}, false return geojs.Answer{}, false
} }
if rq.h.config.LookupSource == config.LookupSourceFile { if rq.h.config.LookupSource == "file" {
return rq.lookupAnswer, true return rq.lookupAnswer, true
} }
return rq.h.geojs.Kept(clientGroup(rq.h.config, rq.client)) return rq.h.geojs.Kept(rq.h.clientGroup(rq.client))
} }
// responseWriter is waf's answer to a request. It notes the status and // responseWriter is waf's answer to a request. It notes the status and
+1 -1
View File
@@ -32,7 +32,7 @@ func (routeWatchTags) check(
route, missing := rq.h.routeWatch.Route( route, missing := rq.h.routeWatch.Route(
ctx, ctx,
clientGroup(rq.h.config, rq.client), rq.h.clientGroup(rq.client),
rq.client, rq.client,
) )
if missing != "" { if missing != "" {