Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5a73af61b0 |
No files matched your search
@@ -75,8 +75,7 @@ Together, `rproxy` and `waf` protect against:
|
|||||||
- give the clients of chosen networks and countries a fraction of every
|
- give the clients of chosen networks and countries a fraction of every
|
||||||
limit, so that they are banned sooner;
|
limit, so that they are banned sooner;
|
||||||
- refuse or limit clients on a blocklist, a DNS blocklist, or with a high
|
- refuse or limit clients on a blocklist, a DNS blocklist, or with a high
|
||||||
AbuseIPDB score;
|
AbuseIPDB score.
|
||||||
- ban the clients on AbuseIPDB's blacklist before they send a request.
|
|
||||||
|
|
||||||
For the operator, it:
|
For the operator, it:
|
||||||
|
|
||||||
@@ -295,126 +294,123 @@ account, a key, a token or an alert destination stays off until you give one.
|
|||||||
|
|
||||||
Every setting the code reads, with its default:
|
Every setting the code reads, with its default:
|
||||||
|
|
||||||
| Variable | Default | What it does |
|
| Variable | Default | What it does |
|
||||||
| --------------------------------------------------- | ---------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
| ------------------------------------------------- | ---------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
| `SWW_COMMAND` | `rproxy` | Which command the image runs, and its health check checks: `rproxy` or `waf`. |
|
| `SWW_COMMAND` | `rproxy` | Which command the image runs, and its health check checks: `rproxy` or `waf`. |
|
||||||
| `SWW_RPROXY_WAF_BACKEND` | required | `waf`'s URL, such as `http://waf:8080`. |
|
| `SWW_RPROXY_WAF_BACKEND` | required | `waf`'s URL, such as `http://waf:8080`. |
|
||||||
| `SWW_RPROXY_UPSTREAM_URL` | `http://127.0.0.1:8081` | The app's URL: a scheme, a host and an optional port, nothing more. |
|
| `SWW_RPROXY_UPSTREAM_URL` | `http://127.0.0.1:8081` | The app's URL: a scheme, a host and an optional port, nothing more. |
|
||||||
| `SWW_RPROXY_LISTEN_ADDR` | `:8080` | Where `rproxy` listens; leave the host part empty. |
|
| `SWW_RPROXY_LISTEN_ADDR` | `:8080` | Where `rproxy` listens; leave the host part empty. |
|
||||||
| `SWW_RPROXY_TRUSTED_PROXIES` | `10.0.0.0/8,172.16.0.0/12,192.168.0.0/16` | The netblocks whose `X-Forwarded-For` `rproxy` believes; set but empty, it believes none. |
|
| `SWW_RPROXY_TRUSTED_PROXIES` | `10.0.0.0/8,172.16.0.0/12,192.168.0.0/16` | The netblocks whose `X-Forwarded-For` `rproxy` believes; set but empty, it believes none. |
|
||||||
| `SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS` | `2000` | How long `rproxy` waits for `waf`'s answer to a question and to a report, in milliseconds. |
|
| `SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS` | `2000` | How long `rproxy` waits for `waf`'s answer to a question and to a report, in milliseconds. |
|
||||||
| `SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION` | `allow` | What `rproxy` does with a request when `waf` gives no answer in time, and while the circuit breaker for asks to `waf` is open: `allow` passes it to the app, `deny` answers `503`. |
|
| `SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION` | `allow` | What `rproxy` does with a request when `waf` gives no answer in time, and while the circuit breaker for asks to `waf` is open: `allow` passes it to the app, `deny` answers `503`. |
|
||||||
| `SWW_RPROXY_MAX_REQUEST_SIZE` | `100M` | The largest request body, a size or `off`; a larger one is answered `413`. |
|
| `SWW_RPROXY_MAX_REQUEST_SIZE` | `100M` | The largest request body, a size or `off`; a larger one is answered `413`. |
|
||||||
| `SWW_RPROXY_MAX_RESPONSE_SIZE` | `1G` | The largest response body, a size or `off`. The visitor gets `502` in place of a larger one, or it is cut off if it has already started. |
|
| `SWW_RPROXY_MAX_RESPONSE_SIZE` | `1G` | The largest response body, a size or `off`. The visitor gets `502` in place of a larger one, or it is cut off if it has already started. |
|
||||||
| `SWW_RPROXY_MAX_REQUEST_TIME_MS` | `60000` | Two time limits, each this long, in milliseconds or `off`: one for the visitor to send its request line and headers, then one for the whole request to reach the app. |
|
| `SWW_RPROXY_MAX_REQUEST_TIME_MS` | `60000` | Two time limits, each this long, in milliseconds or `off`: one for the visitor to send its request line and headers, then one for the whole request to reach the app. |
|
||||||
| `SWW_RPROXY_MAX_RESPONSE_TIME_MS` | `1800000` | How long the answer may take, from the end of the request to its last byte reaching the visitor, in milliseconds or `off`. |
|
| `SWW_RPROXY_MAX_RESPONSE_TIME_MS` | `1800000` | How long the answer may take, from the end of the request to its last byte reaching the visitor, in milliseconds or `off`. |
|
||||||
| `SWW_WAF_LISTEN_ADDR` | `:8080` | Where `waf` listens; leave the host part empty. |
|
| `SWW_WAF_LISTEN_ADDR` | `:8080` | Where `waf` listens; leave the host part empty. |
|
||||||
| `SWW_WAF_INSTANCE_NAME` | the container's host name | The name in every line `waf` logs, and in every alert and metric, such as `host1/waf`. |
|
| `SWW_WAF_INSTANCE_NAME` | the container's host name | The name in every line `waf` logs, and in every alert and metric, such as `host1/waf`. |
|
||||||
| `SWW_WAF_MODE` | `enforce` | `observe` refuses nothing and makes no ban; each log line says what `enforce` would have done. |
|
| `SWW_WAF_MODE` | `enforce` | `observe` refuses nothing and makes no ban; each log line says what `enforce` would have done. |
|
||||||
| `SWW_WAF_IPV6_GROUP_PREFIX` | `64` | The prefix length of the IPv6 netblock that counts as one client, from 32 to 128. |
|
| `SWW_WAF_IPV6_GROUP_PREFIX` | `64` | The prefix length of the IPv6 netblock that counts as one client, from 32 to 128. |
|
||||||
| `SWW_WAF_MAX_TRACKED_CLIENTS` | `20000` | The most clients kept, in memory and in `clients.json`; the least recently seen is dropped first. |
|
| `SWW_WAF_MAX_TRACKED_CLIENTS` | `20000` | The most clients kept, in memory and in `clients.json`; the least recently seen is dropped first. |
|
||||||
| `SWW_WAF_MAX_BANS` | `5000` | The most bans `waf` keeps for broken limits and attacks; bans an admin made and bans for AbuseIPDB's blacklist do not count. |
|
| `SWW_WAF_MAX_BANS` | `5000` | The most bans `waf` keeps of its own; bans an admin made do not count. |
|
||||||
| `SWW_WAF_ALLOW_NETS` | empty | Netblocks whose clients skip every check, such as your monitoring. |
|
| `SWW_WAF_ALLOW_NETS` | empty | Netblocks whose clients skip every check, such as your monitoring. |
|
||||||
| `SWW_WAF_RATE_LIMIT_EXEMPT_NETS` | empty | Netblocks whose clients are exempt from rate limiting and the byte limits. |
|
| `SWW_WAF_RATE_LIMIT_EXEMPT_NETS` | empty | Netblocks whose clients are exempt from rate limiting and the byte limits. |
|
||||||
| `SWW_WAF_DENY_NETS` | empty | Netblocks whose clients are refused. |
|
| `SWW_WAF_DENY_NETS` | empty | Netblocks whose clients are refused. |
|
||||||
| `SWW_WAF_RATE_LIMIT_PER_MINUTE` | `120` | The most requests a client may make in a minute, a count or `off`; the next request is refused and bans the client. |
|
| `SWW_WAF_RATE_LIMIT_PER_MINUTE` | `120` | The most requests a client may make in a minute, a count or `off`; the next request is refused and bans the client. |
|
||||||
| `SWW_WAF_RATE_LIMIT_PER_HOUR` | `1500` | The most requests a client may make in an hour, a count or `off`; the next request is refused and bans the client. |
|
| `SWW_WAF_RATE_LIMIT_PER_HOUR` | `1500` | The most requests a client may make in an hour, a count or `off`; the next request is refused and bans the client. |
|
||||||
| `SWW_WAF_RATE_LIMIT_PER_DAY` | `6000` | The most requests a client may make in a day, a count or `off`; the next request is refused and bans the client. |
|
| `SWW_WAF_RATE_LIMIT_PER_DAY` | `6000` | The most requests a client may make in a day, a count or `off`; the next request is refused and bans the client. |
|
||||||
| `SWW_WAF_RATE_LIMIT_EXEMPT_PATHS` | empty | Path prefixes exempt from rate limiting and the byte limits, such as `/assets/`. |
|
| `SWW_WAF_RATE_LIMIT_EXEMPT_PATHS` | empty | Path prefixes exempt from rate limiting and the byte limits, such as `/assets/`. |
|
||||||
| `SWW_WAF_BYTES_LIMIT_PER_MINUTE` | `off` | The most bytes counted for a client in a minute, a size or `off`. `SWW_WAF_BYTES_COUNTED` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. |
|
| `SWW_WAF_BYTES_LIMIT_PER_MINUTE` | `off` | The most bytes counted for a client in a minute, a size or `off`. `SWW_WAF_BYTES_COUNTED` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. |
|
||||||
| `SWW_WAF_BYTES_LIMIT_PER_HOUR` | `50G` | The most bytes counted for a client in an hour, a size or `off`. `SWW_WAF_BYTES_COUNTED` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. |
|
| `SWW_WAF_BYTES_LIMIT_PER_HOUR` | `50G` | The most bytes counted for a client in an hour, a size or `off`. `SWW_WAF_BYTES_COUNTED` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. |
|
||||||
| `SWW_WAF_BYTES_LIMIT_PER_DAY` | `100G` | The most bytes counted for a client in a day, a size or `off`. `SWW_WAF_BYTES_COUNTED` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. |
|
| `SWW_WAF_BYTES_LIMIT_PER_DAY` | `100G` | The most bytes counted for a client in a day, a size or `off`. `SWW_WAF_BYTES_COUNTED` says which bytes count: response bodies, request bodies or both. The request that passes the limit is not cut off, and the client is banned when its response ends. |
|
||||||
| `SWW_WAF_BYTES_COUNTED` | `both` | What the byte limits and the byte anomaly thresholds count: `response`, the app's response bodies; `request`, the visitors' request bodies; or `both`. What the visitor sent over a WebSocket counts as request bytes, and what the app sent as response bytes. |
|
| `SWW_WAF_BYTES_COUNTED` | `both` | What the byte limits and the byte anomaly thresholds count: `response`, the app's response bodies; `request`, the visitors' request bodies; or `both`. What the visitor sent over a WebSocket counts as request bytes, and what the app sent as response bytes. |
|
||||||
| `SWW_WAF_REFUSALS_LIMIT_PER_MINUTE` | `30` | The most refusals a client may get in a minute from a rule file, a trap path or the Core Rule Set, a count or `off`. The next one bans the client. |
|
| `SWW_WAF_REFUSALS_LIMIT_PER_MINUTE` | `30` | The most refusals a client may get in a minute from a rule file, a trap path or the Core Rule Set, a count or `off`. The next one bans the client. |
|
||||||
| `SWW_WAF_BAN_RESPONSE` | `403` | How a refused client is answered: `403`, `429` or `close`, which drops the connection. |
|
| `SWW_WAF_BAN_RESPONSE` | `403` | How a refused client is answered: `403`, `429` or `close`, which drops the connection. |
|
||||||
| `SWW_WAF_LIMIT_BAN_DURATION` | `1h` | The ban for a first broken limit. |
|
| `SWW_WAF_LIMIT_BAN_DURATION` | `1h` | The ban for a first broken limit. |
|
||||||
| `SWW_WAF_LIMIT_BAN_REPEAT_WINDOW` | `24h` | A client that breaks a limit again within this time after its last such ban ended gets a ban three times as long. |
|
| `SWW_WAF_LIMIT_BAN_REPEAT_WINDOW` | `24h` | A client that breaks a limit again within this time after its last such ban ended gets a ban three times as long. |
|
||||||
| `SWW_WAF_LIMIT_BAN_PERMANENT_ABOVE` | `7d` | A ban for a broken limit that would be longer than this is permanent instead. |
|
| `SWW_WAF_LIMIT_BAN_PERMANENT_ABOVE` | `7d` | A ban for a broken limit that would be longer than this is permanent instead. |
|
||||||
| `SWW_WAF_ATTACK_BAN_DURATION` | `7d` | The ban for a first clear sign of attack: a `ban` rule or a trap path. |
|
| `SWW_WAF_ATTACK_BAN_DURATION` | `7d` | The ban for a first clear sign of attack: a `ban` rule or a trap path. |
|
||||||
| `SWW_WAF_BAN_SCOPE_V4_PREFIX` | `32` | The prefix length of the netblock a ban on an IPv4 client covers, from 0 to 32. |
|
| `SWW_WAF_BAN_SCOPE_V4_PREFIX` | `32` | The prefix length of the netblock a ban on an IPv4 client covers, from 0 to 32. |
|
||||||
| `SWW_WAF_LOOKUP_SOURCE` | `geojs` | Where each client's AS number and country come from: `geojs` (a web service), `file` (a local database) or `off`. |
|
| `SWW_WAF_LOOKUP_SOURCE` | `geojs` | Where each client's AS number and country come from: `geojs` (a web service), `file` (a local database) or `off`. |
|
||||||
| `SWW_WAF_LOOKUP_DB_PATH` | empty | The IPinfo Lite database file, in its `.mmdb` form, read while the source is `file`. |
|
| `SWW_WAF_LOOKUP_DB_PATH` | empty | The IPinfo Lite database file, in its `.mmdb` form, read while the source is `file`. |
|
||||||
| `SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS` | `800` | How long a request waits for its client's first GeoJS answer when a setting needs it, in milliseconds. The request to GeoJS goes on, and a later answer is kept. |
|
| `SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS` | `800` | How long a request waits for its client's first GeoJS answer when a setting needs it, in milliseconds. The request to GeoJS goes on, and a later answer is kept. |
|
||||||
| `SWW_WAF_ADD_LOOKUP_HEADERS` | `true`, or `false` while the lookup source is `off` | `true` passes the app the client's AS number and country in `X-Client-ASN` and `X-Client-Country`. |
|
| `SWW_WAF_ADD_LOOKUP_HEADERS` | `true`, or `false` while the lookup source is `off` | `true` passes the app the client's AS number and country in `X-Client-ASN` and `X-Client-Country`. |
|
||||||
| `SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES` | empty | When set, the only countries whose clients get through; a client the lookup cannot place is refused too. |
|
| `SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES` | empty | When set, the only countries whose clients get through; a client the lookup cannot place is refused too. |
|
||||||
| `SWW_WAF_TAG_LISTS` | empty | Address lists that each add a tag to their clients' requests, as `name=URL` pairs, such as `torexit=https://example.com/tor.txt`. |
|
| `SWW_WAF_TAG_LISTS` | empty | Address lists that each add a tag to their clients' requests, as `name=URL` pairs, such as `torexit=https://example.com/tor.txt`. |
|
||||||
| `SWW_WAF_TAG_DENY` | empty | Tags whose requests are refused, such as `torexit,country=CN`. |
|
| `SWW_WAF_TAG_DENY` | empty | Tags whose requests are refused, such as `torexit,country=CN`. |
|
||||||
| `SWW_WAF_TAG_LIMIT_PERCENT` | empty | Tags whose clients get a percentage, from 1 to 99, of every request and byte limit, such as `vpn:50,asn=14061:25`; the lowest applies. |
|
| `SWW_WAF_TAG_LIMIT_PERCENT` | empty | Tags whose clients get a percentage, from 1 to 99, of every request and byte limit, such as `vpn:50,asn=14061:25`; the lowest applies. |
|
||||||
| `SWW_WAF_ROUTEWATCH_URL` | unset | The URL of a routewatch instance, which gives each client's `asn=` and `prefix=` tags. Unset, nothing asks routewatch. |
|
| `SWW_WAF_ROUTEWATCH_URL` | unset | The URL of a routewatch instance, which gives each client's `asn=` and `prefix=` tags. Unset, nothing asks routewatch. |
|
||||||
| `SWW_WAF_ASN_BYTES_LIMIT_PERCENT` | empty | A percentage of the byte limits alone for the AS numbers listed, in place of their `asn=` tag's percentage. |
|
| `SWW_WAF_ASN_BYTES_LIMIT_PERCENT` | empty | A percentage of the byte limits alone for the AS numbers listed, in place of their `asn=` tag's percentage. |
|
||||||
| `SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT` | empty | A percentage of the byte limits alone for the countries listed, in place of their `country=` tag's percentage. |
|
| `SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT` | empty | A percentage of the byte limits alone for the countries listed, in place of their `country=` tag's percentage. |
|
||||||
| `SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT` | `100` | The percentage of every limit a client without a country gets. |
|
| `SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT` | `100` | The percentage of every limit a client without a country gets. |
|
||||||
| `SWW_WAF_ASN_LIMIT_PERCENT_URL` | unset | The URL of a text file of `AS<number>:<percent>` lines, fetched like a blocklist; each line counts as an `asn=` tag in `SWW_WAF_TAG_LIMIT_PERCENT` (the lowest wins). |
|
| `SWW_WAF_ASN_LIMIT_PERCENT_URL` | unset | The URL of a text file of `AS<number>:<percent>` lines, fetched like a blocklist; each line counts as an `asn=` tag in `SWW_WAF_TAG_LIMIT_PERCENT` (the lowest wins). |
|
||||||
| `SWW_WAF_RULE_FILES_DIR` | `/etc/sww/rules.d` | The directory of the rule files, read at start and again when a file there changes. |
|
| `SWW_WAF_RULE_FILES_DIR` | `/etc/sww/rules.d` | The directory of the rule files, read at start and again when a file there changes. |
|
||||||
| `SWW_WAF_RULE_FILES_ENABLED` | `true` | `false` reads no rule file. |
|
| `SWW_WAF_RULE_FILES_ENABLED` | `true` | `false` reads no rule file. |
|
||||||
| `SWW_WAF_TRAP_PATHS` | empty | Paths only scanners ask for, such as `/wp-login.php,/xmlrpc.php`; a request for one bans the client. |
|
| `SWW_WAF_TRAP_PATHS` | empty | Paths only scanners ask for, such as `/wp-login.php,/xmlrpc.php`; a request for one bans the client. |
|
||||||
| `SWW_WAF_CORE_RULE_SET_MODE` | `block` | What the Core Rule Set does with a match: `block` refuses it with `403`, `detect` only logs and alerts it, `off` runs no rule. |
|
| `SWW_WAF_CORE_RULE_SET_MODE` | `block` | What the Core Rule Set does with a match: `block` refuses it with `403`, `detect` only logs and alerts it, `off` runs no rule. |
|
||||||
| `SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL` | `1` | The Core Rule Set's paranoia level, from 1 to 4; each level up finds more attacks and takes more ordinary requests for attacks. |
|
| `SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL` | `1` | The Core Rule Set's paranoia level, from 1 to 4; each level up finds more attacks and takes more ordinary requests for attacks. |
|
||||||
| `SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD` | `5` | The Core Rule Set score at which a request is a match, a whole number or `off`; one critical rule scores 5. |
|
| `SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD` | `5` | The Core Rule Set score at which a request is a match, a whole number or `off`; one critical rule scores 5. |
|
||||||
| `SWW_WAF_CORE_RULE_SET_DISABLED_RULES` | `920340,920420,920440,920640,930130,930140` | Core Rule Set rule ids switched off; a list you give replaces the default, so repeat these six to keep them off. |
|
| `SWW_WAF_CORE_RULE_SET_DISABLED_RULES` | `920340,920420,920440,920640,930130,930140` | Core Rule Set rule ids switched off; a list you give replaces the default, so repeat these six to keep them off. |
|
||||||
| `SWW_WAF_CORE_RULE_SET_EXEMPT_PATHS` | empty | Path prefixes the Core Rule Set does not inspect. |
|
| `SWW_WAF_CORE_RULE_SET_EXEMPT_PATHS` | empty | Path prefixes the Core Rule Set does not inspect. |
|
||||||
| `SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE` | `off` | How much of a request body the Core Rule Set reads, a size up to `8K`; `off` reads none. |
|
| `SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE` | `off` | How much of a request body the Core Rule Set reads, a size up to `8K`; `off` reads none. |
|
||||||
| `SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE` | unset | A file of Core Rule Set exclusions, as Coraza directives, that switch rules off for some fields or paths only. |
|
| `SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE` | unset | A file of Core Rule Set exclusions, as Coraza directives, that switch rules off for some fields or paths only. |
|
||||||
| `SWW_WAF_APPS` | unset | The apps each hostname serves, such as `git.example.com:gitea;blog.example.com:wordpress`; only those apps' fixes apply there. "Attack detection" lists the apps. Listing `forgejo` brings gitea's fixes too. Unset, every app's fixes apply to every hostname. |
|
| `SWW_WAF_APPS` | unset | The apps each hostname serves, such as `git.example.com:gitea;blog.example.com:wordpress`; only those apps' fixes apply there. "Attack detection" lists the apps. Listing `forgejo` brings gitea's fixes too. Unset, every app's fixes apply to every hostname. |
|
||||||
| `SWW_WAF_BLOCKLIST_URLS` | empty | Blocklists to fetch, text files of addresses and netblocks one to a line, such as `https://www.spamhaus.org/drop/drop.txt`. |
|
| `SWW_WAF_BLOCKLIST_URLS` | empty | Blocklists to fetch, text files of addresses and netblocks one to a line, such as `https://www.spamhaus.org/drop/drop.txt`. |
|
||||||
| `SWW_WAF_BLOCKLIST_REFRESH` | `24h` | How long after a list was last tried it is fetched again, `1h` or more. |
|
| `SWW_WAF_BLOCKLIST_REFRESH` | `24h` | How long after a list was last tried it is fetched again, `1h` or more. |
|
||||||
| `SWW_WAF_BLOCKLIST_ACTION` | `deny` | What a client on a blocklist gets: `deny`, `limit:<percent>` of every limit, or `log`. |
|
| `SWW_WAF_BLOCKLIST_ACTION` | `deny` | What a client on a blocklist gets: `deny`, `limit:<percent>` of every limit, or `log`. |
|
||||||
| `SWW_WAF_DNSBL_ZONES` | empty | DNS blocklists asked about each new client in the background, such as `dnsbl.dronebl.org`. |
|
| `SWW_WAF_DNSBL_ZONES` | empty | DNS blocklists asked about each new client in the background, such as `dnsbl.dronebl.org`. |
|
||||||
| `SWW_WAF_DNSBL_RESOLVER` | unset | The resolver the zones are asked through, an IP address with an optional port; unset, the host's resolver is used. |
|
| `SWW_WAF_DNSBL_RESOLVER` | unset | The resolver the zones are asked through, an IP address with an optional port; unset, the host's resolver is used. |
|
||||||
| `SWW_WAF_ABUSEIPDB_KEY` | unset | The key of your AbuseIPDB account. Set, a client that has committed an offence is checked in the background, and the clients on AbuseIPDB's blacklist are banned. |
|
| `SWW_WAF_ABUSEIPDB_KEY` | unset | The key of your AbuseIPDB account; set, a client that has committed an offence is checked in the background. |
|
||||||
| `SWW_WAF_ABUSEIPDB_MIN_SCORE` | `75` | The least AbuseIPDB abuse confidence score, from 0 to 100, that is a hit. |
|
| `SWW_WAF_ABUSEIPDB_MIN_SCORE` | `75` | The least AbuseIPDB abuse confidence score, from 0 to 100, that is a hit. |
|
||||||
| `SWW_WAF_ABUSEIPDB_DAILY_BUDGET` | `900` | The most AbuseIPDB checks made in a day. |
|
| `SWW_WAF_ABUSEIPDB_DAILY_BUDGET` | `900` | The most AbuseIPDB checks made in a day. |
|
||||||
| `SWW_WAF_DNSBL_ABUSEIPDB_ACTION` | `limit:25` | What a client listed by a DNS blocklist, or scored a hit by AbuseIPDB, gets: `deny`, `limit:<percent>` or `log`. |
|
| `SWW_WAF_DNSBL_ABUSEIPDB_ACTION` | `limit:25` | What a client listed by a DNS blocklist, or scored a hit by AbuseIPDB, gets: `deny`, `limit:<percent>` or `log`. |
|
||||||
| `SWW_WAF_IP_REPUTATION_CACHE_TTL` | `24h` | How long a DNS blocklist verdict or an AbuseIPDB score is used. |
|
| `SWW_WAF_IP_REPUTATION_CACHE_TTL` | `24h` | How long a DNS blocklist verdict or an AbuseIPDB score is used. |
|
||||||
| `SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS` | `2000` | How long a DNS blocklist query or an AbuseIPDB check may take, in milliseconds. |
|
| `SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS` | `2000` | How long a DNS blocklist query or an AbuseIPDB check may take, in milliseconds. |
|
||||||
| `SWW_WAF_ABUSEIPDB_THRESHOLD` | `90` | The least AbuseIPDB score, from 25 to 100, at which a client on its blacklist is banned. Do not set it below 50 to 75: lower scores list ordinary visitors. |
|
| `SWW_WAF_STATE_DIR` | `/var/lib/sww` | The directory of the state files, an absolute path. |
|
||||||
| `SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS` | `60` | How often AbuseIPDB's blacklist is fetched again, in minutes. |
|
| `SWW_WAF_BANS_WRITE_DELAY` | `10s` | How long `bans.json` waits to be written after a ban is made, lifted or made permanent. |
|
||||||
| `SWW_WAF_ABUSEIPDB_BAN_DAYS` | `30` | How long a client on AbuseIPDB's blacklist is banned, in days. |
|
| `SWW_WAF_STATE_WRITE_INTERVAL` | `15m` | How often every state and cache file is written. |
|
||||||
| `SWW_WAF_STATE_DIR` | `/var/lib/sww` | The directory of the state files, an absolute path. |
|
| `SWW_WAF_CACHE_DIR` | `/var/cache/sww` | The directory of the cache files, an absolute path. |
|
||||||
| `SWW_WAF_BANS_WRITE_DELAY` | `10s` | How long `bans.json` waits to be written after a ban is made, lifted or made permanent. |
|
| `SWW_WAF_LOG_LEVEL` | `info` | The least severe of `waf`'s own messages that are written: `debug`, `info`, `warn` or `error`; request lines are always written. |
|
||||||
| `SWW_WAF_STATE_WRITE_INTERVAL` | `15m` | How often every state and cache file is written. |
|
| `SWW_WAF_LOG_REQUEST_HEADERS` | `accept,accept-language,accept-encoding,content-type,origin,range` | The request headers whose values the request log records; `Authorization` and `Cookie` are never recorded. |
|
||||||
| `SWW_WAF_CACHE_DIR` | `/var/cache/sww` | The directory of the cache files, an absolute path. |
|
| `SWW_WAF_LOG_REMOTE_URL` | unset | A syslog server that also gets every line `waf` writes: `syslog+udp://`, `syslog+tcp://` or `syslog+tls://` with a host and a port. |
|
||||||
| `SWW_WAF_LOG_LEVEL` | `info` | The least severe of `waf`'s own messages that are written: `debug`, `info`, `warn` or `error`; request lines are always written. |
|
| `SWW_WAF_LOG_REMOTE_TLS_CA_FILE` | unset | A file of PEM certificates a `syslog+tls` server's certificate must chain to, instead of the host's own. |
|
||||||
| `SWW_WAF_LOG_REQUEST_HEADERS` | `accept,accept-language,accept-encoding,content-type,origin,range` | The request headers whose values the request log records; `Authorization` and `Cookie` are never recorded. |
|
| `SWW_WAF_LOG_REMOTE_BUFFER_LINES` | `10000` | The most lines held while they wait to be sent; past it, the oldest is dropped. |
|
||||||
| `SWW_WAF_LOG_REMOTE_URL` | unset | A syslog server that also gets every line `waf` writes: `syslog+udp://`, `syslog+tcp://` or `syslog+tls://` with a host and a port. |
|
| `SWW_WAF_LOG_REMOTE_FACILITY` | `local0` | The syslog facility of each record sent. |
|
||||||
| `SWW_WAF_LOG_REMOTE_TLS_CA_FILE` | unset | A file of PEM certificates a `syslog+tls` server's certificate must chain to, instead of the host's own. |
|
| `SWW_WAF_LOG_REMOTE_APP_NAME` | `SWW_WAF_INSTANCE_NAME` | The app name in each syslog record, 1 to 48 printable ASCII characters without a space. |
|
||||||
| `SWW_WAF_LOG_REMOTE_BUFFER_LINES` | `10000` | The most lines held while they wait to be sent; past it, the oldest is dropped. |
|
| `SWW_WAF_ADMIN_TOKEN` | unset | The bearer token for the ban and client endpoints, at least 32 characters; unset, they answer `404`. |
|
||||||
| `SWW_WAF_LOG_REMOTE_FACILITY` | `local0` | The syslog facility of each record sent. |
|
| `SWW_WAF_METRICS_TOKEN` | unset | The bearer token for `/_sww/metrics`, at least 32 characters; unset, the metrics are off. |
|
||||||
| `SWW_WAF_LOG_REMOTE_APP_NAME` | `SWW_WAF_INSTANCE_NAME` | The app name in each syslog record, 1 to 48 printable ASCII characters without a space. |
|
| `SWW_WAF_METRICS_BUSIEST_COUNTRIES_AND_ASNS` | `50` | How many AS numbers and how many countries get metric series of their own; the rest are summed as `other`. |
|
||||||
| `SWW_WAF_ADMIN_TOKEN` | unset | The bearer token for the ban and client endpoints, at least 32 characters; unset, they answer `404`. |
|
| `SWW_WAF_ALERT_WEBHOOK_URL` | unset | A webhook each alert is posted to as a JSON object. |
|
||||||
| `SWW_WAF_METRICS_TOKEN` | unset | The bearer token for `/_sww/metrics`, at least 32 characters; unset, the metrics are off. |
|
| `SWW_WAF_ALERT_WEBHOOK_HEADERS` | empty | Headers sent with each post, a list of `Name:value`, such as `Authorization:Bearer <token>`. |
|
||||||
| `SWW_WAF_METRICS_BUSIEST_COUNTRIES_AND_ASNS` | `50` | How many AS numbers and how many countries get metric series of their own; the rest are summed as `other`. |
|
| `SWW_WAF_ALERT_SLACK_WEBHOOK_URL` | unset | A Slack incoming webhook each alert is posted to as a message. |
|
||||||
| `SWW_WAF_ALERT_WEBHOOK_URL` | unset | A webhook each alert is posted to as a JSON object. |
|
| `SWW_WAF_ALERT_NTFY_URL` | unset | An ntfy topic each alert is published to, as its full URL, such as `https://ntfy.sh/my-alerts`. |
|
||||||
| `SWW_WAF_ALERT_WEBHOOK_HEADERS` | empty | Headers sent with each post, a list of `Name:value`, such as `Authorization:Bearer <token>`. |
|
| `SWW_WAF_ALERT_NTFY_TOKEN` | unset | An ntfy access token sent with each alert. |
|
||||||
| `SWW_WAF_ALERT_SLACK_WEBHOOK_URL` | unset | A Slack incoming webhook each alert is posted to as a message. |
|
| `SWW_WAF_ALERT_EVENTS` | `ban,permanent_ban,core_rule_set_match,anomaly,reputation_hit,source_failure,file_error` | The events alerts are sent for; the hourly `summary` is sent whatever it says. |
|
||||||
| `SWW_WAF_ALERT_NTFY_URL` | unset | An ntfy topic each alert is published to, as its full URL, such as `https://ntfy.sh/my-alerts`. |
|
| `SWW_WAF_ALERT_COOLDOWN` | `15m` | How long after an alert its repeats are held back and counted, a duration or `off`. |
|
||||||
| `SWW_WAF_ALERT_NTFY_TOKEN` | unset | An ntfy access token sent with each alert. |
|
| `SWW_WAF_ALERT_MAX_PER_HOUR` | `60` | The most alerts sent in an hour, a number or `off`; the rest are counted in the hour's `summary`. |
|
||||||
| `SWW_WAF_ALERT_EVENTS` | `ban,permanent_ban,core_rule_set_match,anomaly,reputation_hit,source_failure,file_error` | The events alerts are sent for; the hourly `summary` is sent whatever it says. |
|
| `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when one client makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ALERT_COOLDOWN` | `15m` | How long after an alert its repeats are held back and counted, a duration or `off`. |
|
| `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when one client makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ALERT_MAX_PER_HOUR` | `60` | The most alerts sent in an hour, a number or `off`; the rest are counted in the hour's `summary`. |
|
| `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one client in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when one client makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one client in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_CLIENT_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when one client makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_NET_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when the netblock around a client makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one client in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_NET_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when the netblock around a client makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_CLIENT_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one client in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_NET_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the netblock around a client in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_NET_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when the netblock around a client makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_NET_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the netblock around a client in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_NET_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when the netblock around a client makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_NET_V4_PREFIX` | `24` | The prefix length of the netblock around an IPv4 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 32. Those thresholds only send alerts. |
|
||||||
| `SWW_WAF_ANOMALY_NET_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the netblock around a client in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_NET_V6_PREFIX` | `48` | The prefix length of the netblock around an IPv6 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 128. Those thresholds only send alerts. |
|
||||||
| `SWW_WAF_ANOMALY_NET_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the netblock around a client in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when one AS number makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_NET_V4_PREFIX` | `24` | The prefix length of the netblock around an IPv4 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 32. Those thresholds only send alerts. |
|
| `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when one AS number makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_NET_V6_PREFIX` | `48` | The prefix length of the netblock around an IPv6 client that the `SWW_WAF_ANOMALY_NET_` thresholds count, from 0 to 128. Those thresholds only send alerts. |
|
| `SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one AS number in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when one AS number makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_ASN_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one AS number in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when one AS number makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when the whole service gets more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one AS number in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when the whole service gets more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_ASN_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for one AS number in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_TOTAL_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the whole service in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when the whole service gets more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_TOTAL_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the whole service in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_TOTAL_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when the whole service gets more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_WATCHED_NETS` | empty | Named netblocks the `SWW_WAF_ANOMALY_WATCHED_NET_` thresholds count each as a whole, such as `office=203.0.113.0/24,scraper-x=198.51.100.0/22`. Those thresholds only send alerts. |
|
||||||
| `SWW_WAF_ANOMALY_TOTAL_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the whole service in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_WATCHED_NET_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when a watched netblock makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_TOTAL_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for the whole service in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_WATCHED_NET_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when a watched netblock makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_WATCHED_NETS` | empty | Named netblocks the `SWW_WAF_ANOMALY_WATCHED_NET_` thresholds count each as a whole, such as `office=203.0.113.0/24,scraper-x=198.51.100.0/22`. Those thresholds only send alerts. |
|
| `SWW_WAF_ANOMALY_WATCHED_NET_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for a watched netblock in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_WATCHED_NET_REQUESTS_PER_MINUTE` | `off` | Sends one alert to the alert destinations when a watched netblock makes more requests than this in a minute, a count. No client is banned, rate limited or blocked. |
|
| `SWW_WAF_ANOMALY_WATCHED_NET_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for a watched netblock in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
||||||
| `SWW_WAF_ANOMALY_WATCHED_NET_REQUESTS_PER_HOUR` | `off` | Sends one alert to the alert destinations when a watched netblock makes more requests than this in an hour, a count. No client is banned, rate limited or blocked. |
|
|
||||||
| `SWW_WAF_ANOMALY_WATCHED_NET_BYTES_PER_MINUTE` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for a watched netblock in a minute, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
|
||||||
| `SWW_WAF_ANOMALY_WATCHED_NET_BYTES_PER_HOUR` | `off` | Sends one alert to the alert destinations when more bytes than this are counted for a watched netblock in an hour, a size. `SWW_WAF_BYTES_COUNTED` says which bytes count. No client is banned, rate limited or blocked. |
|
|
||||||
|
|
||||||
[`docs/settings.md`](docs/settings.md) gives the values each setting takes in
|
[`docs/settings.md`](docs/settings.md) gives the values each setting takes in
|
||||||
full, and the limits that are fixed rather than settings.
|
full, and the limits that are fixed rather than settings.
|
||||||
@@ -424,8 +420,8 @@ full, and the limits that are fixed rather than settings.
|
|||||||
Each example is a complete `compose.yaml`. `smallwebwaf` is the tag
|
Each example is a complete `compose.yaml`. `smallwebwaf` is the tag
|
||||||
`make docker` gives the image; on another host, use the name you pushed it
|
`make docker` gives the image; on another host, use the name you pushed it
|
||||||
under, by digest. Docker Compose puts the services on a private network of its
|
under, by digest. Docker Compose puts the services on a private network of its
|
||||||
own, where each reaches the others by service name. Each example but "On the
|
own, where each reaches the others by service name. Each example publishes
|
||||||
open internet" publishes `rproxy`'s port on the host, for trying it out.
|
`rproxy`'s port on the host, for trying it out.
|
||||||
|
|
||||||
### Smallest setup
|
### Smallest setup
|
||||||
|
|
||||||
@@ -457,142 +453,6 @@ volumes:
|
|||||||
waf-state:
|
waf-state:
|
||||||
```
|
```
|
||||||
|
|
||||||
### On the open internet
|
|
||||||
|
|
||||||
One gitea at `git.example.com`, with traefik in front of `rproxy`. traefik gets
|
|
||||||
the certificate from Let's Encrypt and is the only service with published ports.
|
|
||||||
`waf` sets what most sites set. More `waf` settings follow, commented out, each
|
|
||||||
with what it does and when to set it.
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
services:
|
|
||||||
traefik:
|
|
||||||
# traefik v3.7.19, 2026-11-13
|
|
||||||
image: traefik@sha256:a6c0194dda77bcb804824580f36fa7edc56e315b1acfe933366c8b52844a78a4
|
|
||||||
restart: unless-stopped
|
|
||||||
command:
|
|
||||||
- --providers.docker=true
|
|
||||||
- --providers.docker.exposedByDefault=false
|
|
||||||
- --entryPoints.web.address=:80
|
|
||||||
- --entryPoints.websecure.address=:443
|
|
||||||
# Redirect plain HTTP to HTTPS.
|
|
||||||
- --entryPoints.web.http.redirections.entryPoint.to=websecure
|
|
||||||
- --entryPoints.web.http.redirections.entryPoint.scheme=https
|
|
||||||
# A Let's Encrypt certificate for each site, by the TLS-ALPN
|
|
||||||
# challenge on port 443.
|
|
||||||
- --entryPoints.websecure.http.tls.certResolver=le
|
|
||||||
- --certificatesResolvers.le.acme.email=hostmaster@example.com
|
|
||||||
- --certificatesResolvers.le.acme.storage=/letsencrypt/acme.json
|
|
||||||
- --certificatesResolvers.le.acme.tlsChallenge=true
|
|
||||||
# Give a request 10 minutes to arrive, rather than 1, for the
|
|
||||||
# large git pushes the rproxy lines below accept.
|
|
||||||
# - --entryPoints.websecure.transport.respondingTimeouts.readTimeout=600s
|
|
||||||
ports:
|
|
||||||
- "80:80"
|
|
||||||
- "443:443"
|
|
||||||
volumes:
|
|
||||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
|
||||||
- acme:/letsencrypt
|
|
||||||
|
|
||||||
app:
|
|
||||||
# Your app: here a gitea, which listens on port 3000.
|
|
||||||
image: registry.example/app@sha256:0000000000000000000000000000000000000000000000000000000000000000
|
|
||||||
restart: unless-stopped
|
|
||||||
|
|
||||||
rproxy:
|
|
||||||
image: smallwebwaf
|
|
||||||
restart: unless-stopped
|
|
||||||
environment:
|
|
||||||
SWW_RPROXY_WAF_BACKEND: http://waf:8080
|
|
||||||
SWW_RPROXY_UPSTREAM_URL: http://app:3000
|
|
||||||
# Accept git pushes up to 1 GiB, rather than 100 MiB, sent in
|
|
||||||
# up to 10 minutes, rather than 1, with traefik's readTimeout
|
|
||||||
# line above.
|
|
||||||
# SWW_RPROXY_MAX_REQUEST_SIZE: 1G
|
|
||||||
# SWW_RPROXY_MAX_REQUEST_TIME_MS: "600000"
|
|
||||||
labels:
|
|
||||||
- traefik.enable=true
|
|
||||||
- traefik.http.routers.git.rule=Host(`git.example.com`)
|
|
||||||
- traefik.http.routers.git.entryPoints=websecure
|
|
||||||
- traefik.http.services.git.loadBalancer.server.port=8080
|
|
||||||
|
|
||||||
waf:
|
|
||||||
image: smallwebwaf
|
|
||||||
restart: unless-stopped
|
|
||||||
environment:
|
|
||||||
SWW_COMMAND: waf
|
|
||||||
# The name in every log line, alert and metric. The default,
|
|
||||||
# the container's host name, is a random id under Compose.
|
|
||||||
SWW_WAF_INSTANCE_NAME: host1/waf
|
|
||||||
# The app each hostname serves. Only that app's fixes to the
|
|
||||||
# Core Rule Set apply there.
|
|
||||||
SWW_WAF_APPS: git.example.com:gitea
|
|
||||||
# Monitoring skips every check.
|
|
||||||
SWW_WAF_ALLOW_NETS: 192.0.2.10
|
|
||||||
# The office and the CI runner, which would exceed the rate
|
|
||||||
# limits, are never rate limited.
|
|
||||||
SWW_WAF_RATE_LIMIT_EXEMPT_NETS: 203.0.113.0/24,198.51.100.7
|
|
||||||
# WordPress paths, which only scanners ask a gitea for. One
|
|
||||||
# request for one bans the client.
|
|
||||||
SWW_WAF_TRAP_PATHS: /wp-login.php,/xmlrpc.php
|
|
||||||
# Spamhaus's list of netblocks hijacked or leased by
|
|
||||||
# criminals. Their clients are refused.
|
|
||||||
SWW_WAF_BLOCKLIST_URLS: https://www.spamhaus.org/drop/drop.txt
|
|
||||||
# The token for the ban endpoints, to lift a ban without
|
|
||||||
# editing bans.json.
|
|
||||||
SWW_WAF_ADMIN_TOKEN_FILE: /etc/sww/tokens/admin
|
|
||||||
# Alerts, such as one for each ban, to an ntfy topic.
|
|
||||||
SWW_WAF_ALERT_NTFY_URL: https://ntfy.example.com/smallwebwaf
|
|
||||||
|
|
||||||
# Refuse nothing and ban no one, for the first days. Each log
|
|
||||||
# line says what enforce mode would have done.
|
|
||||||
# SWW_WAF_MODE: observe
|
|
||||||
# The access token for an ntfy topic that is not public.
|
|
||||||
# SWW_WAF_ALERT_NTFY_TOKEN_FILE: /etc/sww/tokens/ntfy
|
|
||||||
# The token for /_sww/metrics, for Prometheus. Unset, the
|
|
||||||
# metrics are off.
|
|
||||||
# SWW_WAF_METRICS_TOKEN_FILE: /etc/sww/tokens/metrics
|
|
||||||
# 600 requests an hour rather than 1500, to ban slower
|
|
||||||
# scrapers. A fast human reader can reach it too.
|
|
||||||
# SWW_WAF_RATE_LIMIT_PER_HOUR: "600"
|
|
||||||
# A quarter of every limit for clients in two hosting
|
|
||||||
# networks, where scrapers run rather than people.
|
|
||||||
# SWW_WAF_TAG_LIMIT_PERCENT: asn=14061:25,asn=16276:25
|
|
||||||
# For a site only for people in a few countries. Clients
|
|
||||||
# elsewhere are refused, and so are any the lookup cannot place.
|
|
||||||
# SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES: de,at,ch
|
|
||||||
# Close a refused request's connection rather than answer
|
|
||||||
# 403. traefik then answers 502, so a scraper cannot tell a
|
|
||||||
# ban from a broken site.
|
|
||||||
# SWW_WAF_BAN_RESPONSE: close
|
|
||||||
# A copy of every log line to a syslog server.
|
|
||||||
# SWW_WAF_LOG_REMOTE_URL: syslog+tls://logs.example.com:6514
|
|
||||||
volumes:
|
|
||||||
- waf-state:/var/lib/sww
|
|
||||||
- /srv/sww/tokens:/etc/sww/tokens:ro
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
acme:
|
|
||||||
waf-state:
|
|
||||||
```
|
|
||||||
|
|
||||||
Change these before the first start:
|
|
||||||
|
|
||||||
- `git.example.com`, in the router rule and in `SWW_WAF_APPS`;
|
|
||||||
- the email address for Let's Encrypt;
|
|
||||||
- the ntfy topic's URL;
|
|
||||||
- the app's image and port;
|
|
||||||
- the netblocks, which are documentation addresses.
|
|
||||||
|
|
||||||
Make the admin token file first, as for any token file:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
mkdir -p /srv/sww/tokens
|
|
||||||
openssl rand -hex 32 > /srv/sww/tokens/admin
|
|
||||||
chown 65532:65532 /srv/sww/tokens/admin
|
|
||||||
chmod 0400 /srv/sww/tokens/admin
|
|
||||||
```
|
|
||||||
|
|
||||||
### Custom per-address limits
|
### Custom per-address limits
|
||||||
|
|
||||||
For a site with no API:
|
For a site with no API:
|
||||||
@@ -770,10 +630,8 @@ status for a size or time limit, such as `413` for a body too large.
|
|||||||
- A client is one IPv4 address or one IPv6 netblock of
|
- A client is one IPv4 address or one IPv6 netblock of
|
||||||
`SWW_WAF_IPV6_GROUP_PREFIX`, since one abuser usually controls a whole /64.
|
`SWW_WAF_IPV6_GROUP_PREFIX`, since one abuser usually controls a whole /64.
|
||||||
- Each limit is counted over a sliding window with a few integers per client.
|
- Each limit is counted over a sliding window with a few integers per client.
|
||||||
The table of clients and the bans for limits and attacks each have a bound
|
The table of clients and the table of bans each have a bound
|
||||||
(`SWW_WAF_MAX_TRACKED_CLIENTS`, `SWW_WAF_MAX_BANS`). The bans for AbuseIPDB's
|
(`SWW_WAF_MAX_TRACKED_CLIENTS`, `SWW_WAF_MAX_BANS`), so memory is bounded too.
|
||||||
blacklist are bounded by the clients it listed over the last
|
|
||||||
`SWW_WAF_ABUSEIPDB_BAN_DAYS` days. So memory is bounded too.
|
|
||||||
- A ban refuses every request from its netblock until it ends. Requests refused
|
- A ban refuses every request from its netblock until it ends. Requests refused
|
||||||
under a ban count toward no limit. A permanent ban ends only when an admin
|
under a ban count toward no limit. A permanent ban ends only when an admin
|
||||||
lifts it.
|
lifts it.
|
||||||
@@ -906,11 +764,9 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
|
|||||||
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
||||||
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
||||||
`internal/alerts`: one check or source each.
|
`internal/alerts`: one check or source each.
|
||||||
- `internal/state`: the state and cache files.
|
- `internal/state`, `internal/requestlog`, `internal/remotelog` and
|
||||||
- `internal/requestlog`: the request log.
|
`internal/metrics`: the state and cache files, the log lines, the syslog
|
||||||
- `internal/processlog`: the process's own messages.
|
sender and the metrics.
|
||||||
- `internal/remotelog`: the syslog sender.
|
|
||||||
- `internal/metrics`: the metrics.
|
|
||||||
|
|
||||||
## Rationale
|
## Rationale
|
||||||
|
|
||||||
|
|||||||
@@ -94,8 +94,7 @@ never as a time.
|
|||||||
to count and report failures.
|
to count and report failures.
|
||||||
- `internal/lookup`: looks clients up in the lookup database.
|
- `internal/lookup`: looks clients up in the lookup database.
|
||||||
- `internal/reputation`: fetches the lists, asks the DNS blocklists and
|
- `internal/reputation`: fetches the lists, asks the DNS blocklists and
|
||||||
AbuseIPDB in the background, and keeps their answers. It also fetches
|
AbuseIPDB in the background, and keeps their answers.
|
||||||
AbuseIPDB's blacklist, which `internal/waf` bans.
|
|
||||||
- `internal/routewatch`: asks a routewatch instance about clients in the
|
- `internal/routewatch`: asks a routewatch instance about clients in the
|
||||||
background, and keeps its answers.
|
background, and keeps its answers.
|
||||||
- `internal/anomaly`: the anomaly counters and their alerts.
|
- `internal/anomaly`: the anomaly counters and their alerts.
|
||||||
@@ -104,10 +103,8 @@ never as a time.
|
|||||||
its own.
|
its own.
|
||||||
- `internal/state`: reads the state and cache files at start, takes in edits
|
- `internal/state`: reads the state and cache files at start, takes in edits
|
||||||
while running, and writes the files when due and at the stop.
|
while running, and writes the files when due and at the stop.
|
||||||
- `internal/requestlog`: the request log, as JSON lines on stdout.
|
- `internal/requestlog` and `internal/remotelog`: the lines on stdout, and
|
||||||
- `internal/processlog`: the process lines on stdout, which hold the
|
sending them to the syslog server.
|
||||||
process's own messages.
|
|
||||||
- `internal/remotelog`: sends the lines on stdout to the syslog server.
|
|
||||||
- `internal/metrics`: the metrics, in the Prometheus text format.
|
- `internal/metrics`: the metrics, in the Prometheus text format.
|
||||||
- `Dockerfile`: the lint and test phases, then the image.
|
- `Dockerfile`: the lint and test phases, then the image.
|
||||||
`deploy/example-app` is an app built on the image, with `rproxy` in its
|
`deploy/example-app` is an app built on the image, with `rproxy` in its
|
||||||
@@ -243,15 +240,13 @@ apply.
|
|||||||
load.
|
load.
|
||||||
- Memory is bounded. When the table of clients holds
|
- Memory is bounded. When the table of clients holds
|
||||||
`SWW_WAF_MAX_TRACKED_CLIENTS`, the least recently seen client is dropped
|
`SWW_WAF_MAX_TRACKED_CLIENTS`, the least recently seen client is dropped
|
||||||
first, with its history. The bans for broken limits and clear signs of attack
|
first, with its history. The bans are bounded by `SWW_WAF_MAX_BANS` (see
|
||||||
are bounded by `SWW_WAF_MAX_BANS` (see "Bans"). Those for AbuseIPDB's
|
"Bans"). The GeoJS answers, the reputation verdicts and scores, the DNS
|
||||||
blacklist are bounded by the clients it listed over the last
|
queries under way and the anomaly counters have fixed bounds, listed under
|
||||||
`SWW_WAF_ABUSEIPDB_BAN_DAYS` days (see "Reputation"). The GeoJS answers, the
|
"Fixed limits" in `docs/settings.md`. Past its bound, the GeoJS answer or
|
||||||
reputation verdicts and scores, the DNS queries under way and the anomaly
|
anomaly counter used longest ago goes first, and the verdict or score fetched
|
||||||
counters have fixed bounds, listed under "Fixed limits" in `docs/settings.md`.
|
longest ago. Past the bound on DNS queries under way, a zone is asked at the
|
||||||
Past its bound, the GeoJS answer or anomaly counter used longest ago goes
|
client's next request.
|
||||||
first, and the verdict or score fetched longest ago. Past the bound on DNS
|
|
||||||
queries under way, a zone is asked at the client's next request.
|
|
||||||
|
|
||||||
## Bans
|
## Bans
|
||||||
|
|
||||||
@@ -306,9 +301,6 @@ to a ban, each by its own rule:
|
|||||||
Once it ends, only new requests can break a limit again. The client's
|
Once it ends, only new requests can break a limit again. The client's
|
||||||
history keeps its totals.
|
history keeps its totals.
|
||||||
|
|
||||||
A client on AbuseIPDB's blacklist is banned before it sends a request, with the
|
|
||||||
cause `abuseipdb` (see "Reputation").
|
|
||||||
|
|
||||||
A Core Rule Set match refuses only the request it matched, with `403` and
|
A Core Rule Set match refuses only the request it matched, with `403` and
|
||||||
without a ban, because the Core Rule Set has false positives. A `block` rule
|
without a ban, because the Core Rule Set has false positives. A `block` rule
|
||||||
does the same. Those refusals count toward the refusal limit, so a client that
|
does the same. Those refusals count toward the refusal limit, so a client that
|
||||||
@@ -335,12 +327,11 @@ does not count toward a longer ban; it refuses nothing. Deleting its entry from
|
|||||||
|
|
||||||
A permanent ban does not run out. It ends when an admin lifts it, or when `waf`
|
A permanent ban does not run out. It ends when an admin lifts it, or when `waf`
|
||||||
drops it from a full table, as below. When `waf` holds `SWW_WAF_MAX_BANS` bans
|
drops it from a full table, as below. When `waf` holds `SWW_WAF_MAX_BANS` bans
|
||||||
for broken limits and clear signs of attack, it drops the earliest such ban of
|
of its own, it drops the earliest ban of one netblock: the netblock that has
|
||||||
one netblock: the netblock that has gone longest without a request. A past,
|
gone longest without a request. A past, active or permanent ban is dropped
|
||||||
active or permanent ban is dropped alike. A scanner whose ban was dropped that
|
alike. A scanner whose ban was dropped that way is refused and banned again by
|
||||||
way is refused and banned again by its next probe. Bans whose cause is `admin`
|
its next probe. A ban whose cause is `admin` is never dropped and does not count
|
||||||
or `abuseipdb` do not count toward `SWW_WAF_MAX_BANS` and are never dropped to
|
toward `SWW_WAF_MAX_BANS`, so such bans can never fill the table.
|
||||||
make room, so they can never push out the others.
|
|
||||||
|
|
||||||
A request refused under a ban that runs out is logged with the action
|
A request refused under a ban that runs out is logged with the action
|
||||||
`temporarily_banned`. Under a permanent ban the action is `permanently_banned`.
|
`temporarily_banned`. Under a permanent ban the action is `permanently_banned`.
|
||||||
@@ -2047,26 +2038,6 @@ host, sharing one address, would often fetch too close together.
|
|||||||
- AbuseIPDB answers other than `200`;
|
- AbuseIPDB answers other than `200`;
|
||||||
- the answer gives no score;
|
- the answer gives no score;
|
||||||
- no answer comes within `SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS`.
|
- no answer comes within `SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS`.
|
||||||
- While `SWW_WAF_ABUSEIPDB_KEY` is set, AbuseIPDB's blacklist is fetched at the
|
|
||||||
start and every `SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS` minutes
|
|
||||||
(default `60`). It lists the clients AbuseIPDB scores at
|
|
||||||
`SWW_WAF_ABUSEIPDB_THRESHOLD` (default `90`) or more. The blacklist itself is
|
|
||||||
not kept.
|
|
||||||
- Each client on it is banned for `SWW_WAF_ABUSEIPDB_BAN_DAYS` days (default
|
|
||||||
`30`), with the cause `abuseipdb`. The ban covers the client's netblock,
|
|
||||||
as any ban does.
|
|
||||||
- A netblock that already has an active ban is left as it is. So is one
|
|
||||||
whose `abuseipdb` ban an admin lifted, until that ban would have ended.
|
|
||||||
- An `abuseipdb` ban that has ended is forgotten at the next of these:
|
|
||||||
- a fetch of the blacklist in `enforce` mode;
|
|
||||||
- a read of `bans.json`, at the start or after an admin's edit.
|
|
||||||
- In `observe` mode the blacklist makes no ban.
|
|
||||||
- The threshold takes 25 to 100. It should not be set below 50 to 75, since
|
|
||||||
lower scores list ordinary visitors. AbuseIPDB serves free accounts the
|
|
||||||
blacklist at the threshold 100, whatever is asked for.
|
|
||||||
- AbuseIPDB limits the fetches of the blacklist a day: 5 for a free account.
|
|
||||||
A fetch over the limit fails, and is logged and alerted. The default
|
|
||||||
refresh fetches it 24 times a day.
|
|
||||||
- `SWW_WAF_DNSBL_ABUSEIPDB_ACTION`, for DNS blocklist and AbuseIPDB hits, is
|
- `SWW_WAF_DNSBL_ABUSEIPDB_ACTION`, for DNS blocklist and AbuseIPDB hits, is
|
||||||
`limit:25` by default. Such verdicts are less certain than a blocklist, so a
|
`limit:25` by default. Such verdicts are less certain than a blocklist, so a
|
||||||
listed client gets a quarter of every request and byte limit rather than a
|
listed client gets a quarter of every request and byte limit rather than a
|
||||||
@@ -2081,10 +2052,9 @@ host, sharing one address, would often fetch too close together.
|
|||||||
- Alerts go to a webhook as JSON, and to Slack and ntfy as formatted messages.
|
- Alerts go to a webhook as JSON, and to Slack and ntfy as formatted messages.
|
||||||
`docs/alerts.md` lists the events, the JSON object and the messages.
|
`docs/alerts.md` lists the events, the JSON object and the messages.
|
||||||
`SWW_WAF_ALERT_EVENTS` chooses which of the seven event types are sent; the
|
`SWW_WAF_ALERT_EVENTS` chooses which of the seven event types are sent; the
|
||||||
hourly `summary` is sent whatever it says. The bans an admin makes, and the
|
hourly `summary` is sent whatever it says. The bans an admin makes raise no
|
||||||
bans for AbuseIPDB's blacklist, raise no alert. In `observe` mode, a request
|
alert. In `observe` mode, a request that would have made a ban still raises
|
||||||
that would have made a ban still raises the ban's alert, with `mode` set to
|
the ban's alert, with `mode` set to `observe` in its `detail`.
|
||||||
`observe` in its `detail`.
|
|
||||||
- A repeat is an alert with the same event, and the same netblock, source, file
|
- A repeat is an alert with the same event, and the same netblock, source, file
|
||||||
or anomaly scope, as one sent less than `SWW_WAF_ALERT_COOLDOWN` before. An
|
or anomaly scope, as one sent less than `SWW_WAF_ALERT_COOLDOWN` before. An
|
||||||
anomaly scope is one of the things counted under "Anomaly thresholds". A
|
anomaly scope is one of the things counted under "Anomaly thresholds". A
|
||||||
@@ -2232,7 +2202,7 @@ lists the files, what each holds, and their sizes.
|
|||||||
these stop the process with a message naming the file and position:
|
these stop the process with a message naming the file and position:
|
||||||
- a file that does not parse, or has an unknown `version`;
|
- a file that does not parse, or has an unknown `version`;
|
||||||
- an entry without a field it needs;
|
- an entry without a field it needs;
|
||||||
- a ban whose cause is not `limit`, `attack`, `admin` or `abuseipdb`;
|
- a ban whose cause is not one of the four;
|
||||||
- a ban whose `expires` is not after its `start`.
|
- a ban whose `expires` is not after its `start`.
|
||||||
|
|
||||||
Starting with empty state would silently forgive every repeat offender.
|
Starting with empty state would silently forgive every repeat offender.
|
||||||
|
|||||||
+2
-3
@@ -6,7 +6,7 @@ back or dropped.
|
|||||||
|
|
||||||
## Events
|
## Events
|
||||||
|
|
||||||
- `ban`: a ban `waf` made for a broken limit or a clear sign of attack.
|
- `ban`: a ban `waf` made.
|
||||||
- `permanent_ban`: a permanent ban it made, or an existing ban made permanent by
|
- `permanent_ban`: a permanent ban it made, or an existing ban made permanent by
|
||||||
a request.
|
a request.
|
||||||
- `core_rule_set_match`: the Core Rule Set scored a request at or over the
|
- `core_rule_set_match`: the Core Rule Set scored a request at or over the
|
||||||
@@ -68,8 +68,7 @@ on one line:
|
|||||||
"earlier_bans": {
|
"earlier_bans": {
|
||||||
"limit": 0,
|
"limit": 0,
|
||||||
"attack": 0,
|
"attack": 0,
|
||||||
"admin": 0,
|
"admin": 0
|
||||||
"abuseipdb": 0
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|||||||
+2
-9
@@ -42,8 +42,8 @@ chooses between the two (see "The image's command" below).
|
|||||||
client's group, from 32 to 128.
|
client's group, from 32 to 128.
|
||||||
- `SWW_WAF_MAX_TRACKED_CLIENTS` (default `20000`): the most clients kept, a
|
- `SWW_WAF_MAX_TRACKED_CLIENTS` (default `20000`): the most clients kept, a
|
||||||
whole number above zero.
|
whole number above zero.
|
||||||
- `SWW_WAF_MAX_BANS` (default `5000`): the most bans `waf` keeps for broken
|
- `SWW_WAF_MAX_BANS` (default `5000`): how many of its own bans `waf` keeps, a
|
||||||
limits and attacks, a whole number above zero.
|
whole number above zero.
|
||||||
|
|
||||||
## Tokens
|
## Tokens
|
||||||
|
|
||||||
@@ -338,13 +338,6 @@ tag that no request can get stops the start. A request can get only these:
|
|||||||
or AbuseIPDB's score is used. A duration.
|
or AbuseIPDB's score is used. A duration.
|
||||||
- `SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS` (default `2000`): how long a query to a
|
- `SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS` (default `2000`): how long a query to a
|
||||||
zone or a check with AbuseIPDB may take. A time in milliseconds.
|
zone or a check with AbuseIPDB may take. A time in milliseconds.
|
||||||
- `SWW_WAF_ABUSEIPDB_THRESHOLD` (default `90`): the least abuse confidence score
|
|
||||||
of the clients on AbuseIPDB's blacklist, a whole number from 25 to 100. It
|
|
||||||
should not be set below 50 to 75, since lower scores list ordinary visitors.
|
|
||||||
- `SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS` (default `60`): how often
|
|
||||||
the blacklist is fetched again, in minutes. A whole number above zero.
|
|
||||||
- `SWW_WAF_ABUSEIPDB_BAN_DAYS` (default `30`): how long the ban of a client on
|
|
||||||
the blacklist lasts, in days. A whole number above zero.
|
|
||||||
|
|
||||||
## Alerting
|
## Alerting
|
||||||
|
|
||||||
|
|||||||
+5
-19
@@ -21,22 +21,11 @@ top-level `version` of 1.
|
|||||||
|
|
||||||
## `bans.json`
|
## `bans.json`
|
||||||
|
|
||||||
Active and past bans:
|
Active and past bans: every ban an admin made, and up to `SWW_WAF_MAX_BANS` of
|
||||||
|
those `waf` made. Per entry:
|
||||||
- every ban an admin made;
|
|
||||||
- up to `SWW_WAF_MAX_BANS` of those `waf` made for a broken limit or an attack;
|
|
||||||
- the bans for AbuseIPDB's blacklist.
|
|
||||||
|
|
||||||
A ban for AbuseIPDB's blacklist that has ended is forgotten at the next of
|
|
||||||
these:
|
|
||||||
|
|
||||||
- a fetch of the blacklist in `enforce` mode;
|
|
||||||
- a read of `bans.json`, at the start or after an admin's edit.
|
|
||||||
|
|
||||||
Per entry:
|
|
||||||
|
|
||||||
- `netblock`, `start` and `expires` (`null` for a permanent ban);
|
- `netblock`, `start` and `expires` (`null` for a permanent ban);
|
||||||
- `cause`: `attack`, `limit`, `admin` or `abuseipdb`;
|
- `cause`: `attack`, `limit` or `admin`;
|
||||||
- `reason`: a short reason;
|
- `reason`: a short reason;
|
||||||
- `lifted`: when an admin lifted it, if one did;
|
- `lifted`: when an admin lifted it, if one did;
|
||||||
- `notes`: the notes below.
|
- `notes`: the notes below.
|
||||||
@@ -71,8 +60,7 @@ ban. The notes:
|
|||||||
- for a clear sign of attack, `rule_id` and `target`, or `trap_path`;
|
- for a clear sign of attack, `rule_id` and `target`, or `trap_path`;
|
||||||
- `reputation`: the sources the client was on, each with its `source` and, for
|
- `reputation`: the sources the client was on, each with its `source` and, for
|
||||||
AbuseIPDB, its `score`;
|
AbuseIPDB, its `score`;
|
||||||
- `request`: the request that caused the ban, each text cut to 256 bytes, and
|
- `request`: the request that caused the ban, each text cut to 256 bytes:
|
||||||
empty for a ban for AbuseIPDB's blacklist:
|
|
||||||
- `time`, `method` and `host`;
|
- `time`, `method` and `host`;
|
||||||
- `path`, with its query string;
|
- `path`, with its query string;
|
||||||
- `status` and `user_agent`;
|
- `status` and `user_agent`;
|
||||||
@@ -163,9 +151,7 @@ A cache file. routewatch's answers from the last hour, one per line:
|
|||||||
- `bans.json`: about 1.2 KiB per ban, at most about 2.5 KiB, so about 6 MiB at
|
- `bans.json`: about 1.2 KiB per ban, at most about 2.5 KiB, so about 6 MiB at
|
||||||
the default `SWW_WAF_MAX_BANS`. Written every 15 minutes, a full file comes to
|
the default `SWW_WAF_MAX_BANS`. Written every 15 minutes, a full file comes to
|
||||||
about 600 MiB of writes a day, and a hundred new bans a day add about 600 MiB
|
about 600 MiB of writes a day, and a hundred new bans a day add about 600 MiB
|
||||||
more. A ban for AbuseIPDB's blacklist takes about 1 KiB. A free account's
|
more.
|
||||||
blacklist lists 10,000 clients, so it adds at least 10 MiB, and 1 GiB of
|
|
||||||
writes a day.
|
|
||||||
- `lookups.json`: about 150 bytes per answer, under 1.5 GiB of writes a day when
|
- `lookups.json`: about 150 bytes per answer, under 1.5 GiB of writes a day when
|
||||||
full.
|
full.
|
||||||
- `routewatch.json`: about 120 bytes per answer, at most about 12 MiB.
|
- `routewatch.json`: about 120 bytes per answer, at most about 12 MiB.
|
||||||
|
|||||||
@@ -1,165 +0,0 @@
|
|||||||
package bans_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/netip"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestBlacklistedNetblockIsBannedUntilTheGivenEnd(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
|
||||||
now := midnight()
|
|
||||||
expires := now.Add(30 * day)
|
|
||||||
|
|
||||||
made := ledger.BanForAbuseIPDB(
|
|
||||||
now, expires, map[netip.Prefix]int64{netblock: 95},
|
|
||||||
)
|
|
||||||
if made != 1 || ledger.Made(bans.CauseAbuseIPDB) != 1 {
|
|
||||||
t.Fatalf("made %d bans, %d counted, want 1 and 1",
|
|
||||||
made, ledger.Made(bans.CauseAbuseIPDB))
|
|
||||||
}
|
|
||||||
|
|
||||||
ban, banned, madePermanent := ledger.Check(netblock.Addr(), now)
|
|
||||||
if !banned || madePermanent || !ban.Expires.Equal(expires) ||
|
|
||||||
ban.Cause != bans.CauseAbuseIPDB ||
|
|
||||||
ban.Reason != "on AbuseIPDB's blacklist with a score of 95" ||
|
|
||||||
len(ban.Notes.Reputation) != 1 ||
|
|
||||||
ban.Notes.Reputation[0].Source != "abuseipdb" ||
|
|
||||||
*ban.Notes.Reputation[0].Score != 95 {
|
|
||||||
t.Errorf(
|
|
||||||
"ban %+v, banned %t, made permanent %t, want the blacklist's "+
|
|
||||||
"ban until %s",
|
|
||||||
ban,
|
|
||||||
banned,
|
|
||||||
madePermanent,
|
|
||||||
expires,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, banned, _ := ledger.Check(netblock.Addr(), expires); banned {
|
|
||||||
t.Error("the ban refuses a request at its end")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBlacklistLeavesANetblockWithAnActiveBanAsItIs(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
|
||||||
now := midnight()
|
|
||||||
scores := map[netip.Prefix]int64{netblock: 100}
|
|
||||||
|
|
||||||
limitBan, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
|
||||||
|
|
||||||
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 0 {
|
|
||||||
t.Errorf("made %d bans over a ban for a limit, want none", made)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Once that ban has ended the netblock is banned, and the blacklist
|
|
||||||
// fetched again makes no second ban while the first lasts.
|
|
||||||
later := limitBan.Expires
|
|
||||||
if made := ledger.BanForAbuseIPDB(later, later.Add(30*day), scores); made != 1 {
|
|
||||||
t.Errorf("made %d bans after the ban for a limit, want 1", made)
|
|
||||||
}
|
|
||||||
|
|
||||||
later = later.Add(time.Hour)
|
|
||||||
if made := ledger.BanForAbuseIPDB(later, later.Add(30*day), scores); made != 0 {
|
|
||||||
t.Errorf("made %d bans while the blacklist's lasts, want none", made)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{netblock: 2})
|
|
||||||
|
|
||||||
if got := ledger.Bans(netblock)[1].Notes.EarlierBans; got != (bans.EarlierBans{
|
|
||||||
Limit: 1,
|
|
||||||
}) {
|
|
||||||
t.Errorf("earlier bans %+v, want 1 for a limit", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLiftedBlacklistBanHoldsUntilItsEnd(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
|
||||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
|
||||||
now := midnight()
|
|
||||||
expires := now.Add(30 * day)
|
|
||||||
scores := map[netip.Prefix]int64{netblock: 100}
|
|
||||||
|
|
||||||
ledger.BanForAbuseIPDB(now, expires, scores)
|
|
||||||
ledger.Lift(netblock.Addr(), now)
|
|
||||||
|
|
||||||
// The admin lifted it: the blacklist fetched again until its end bans
|
|
||||||
// nothing.
|
|
||||||
later := expires.Add(-time.Hour)
|
|
||||||
if made := ledger.BanForAbuseIPDB(later, later.Add(30*day), scores); made != 0 {
|
|
||||||
t.Errorf("made %d bans before the lifted ban's end, want none", made)
|
|
||||||
}
|
|
||||||
|
|
||||||
// From its end on, the blacklist bans the netblock again, and the
|
|
||||||
// lifted ban, ended, is forgotten.
|
|
||||||
if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), scores); made != 1 {
|
|
||||||
t.Errorf("made %d bans at the lifted ban's end, want 1", made)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{netblock: 1})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEndedBlacklistBansAreForgottenAtTheNextFetch(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
ledger := bans.New(defaultParams())
|
|
||||||
listed := netip.MustParsePrefix("203.0.113.9/32")
|
|
||||||
limited := netip.MustParsePrefix("203.0.113.10/32")
|
|
||||||
now := midnight()
|
|
||||||
expires := now.Add(30 * day)
|
|
||||||
|
|
||||||
ledger.BanForAbuseIPDB(now, expires, map[netip.Prefix]int64{listed: 100})
|
|
||||||
ledger.BanForLimit(limited, now, bans.Notes{})
|
|
||||||
|
|
||||||
// listed is off the blacklist when it is fetched after the ban's end.
|
|
||||||
if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), nil); made != 0 {
|
|
||||||
t.Errorf("made %d bans for an empty blacklist, want none", made)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{listed: 0, limited: 1})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
params := defaultParams()
|
|
||||||
params.MaxBans = 1
|
|
||||||
ledger := bans.New(params)
|
|
||||||
limited := netip.MustParsePrefix("203.0.113.1/32")
|
|
||||||
attacker := netip.MustParsePrefix("203.0.113.2/32")
|
|
||||||
now := midnight()
|
|
||||||
|
|
||||||
scores := map[netip.Prefix]int64{}
|
|
||||||
for i := range 3 {
|
|
||||||
scores[netip.PrefixFrom(netip.AddrFrom4([4]byte{198, 51, 100, byte(i)}), 32)] = 100
|
|
||||||
}
|
|
||||||
|
|
||||||
ledger.BanForLimit(limited, now, bans.Notes{})
|
|
||||||
|
|
||||||
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 3 {
|
|
||||||
t.Errorf("made %d bans for 3 netblocks, want 3", made)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantBans(t, ledger, map[netip.Prefix]int{limited: 1})
|
|
||||||
|
|
||||||
// The ban for an attack takes the place of the one for a limit, and no
|
|
||||||
// blacklist ban goes.
|
|
||||||
ledger.BanForAttack(attacker, now, bans.Notes{})
|
|
||||||
|
|
||||||
want := map[netip.Prefix]int{limited: 0, attacker: 1}
|
|
||||||
for netblock := range scores {
|
|
||||||
want[netblock] = 1
|
|
||||||
}
|
|
||||||
|
|
||||||
wantBans(t, ledger, want)
|
|
||||||
}
|
|
||||||
+34
-161
@@ -1,9 +1,8 @@
|
|||||||
// Package bans is the ban ledger: the bans sww makes on the
|
// Package bans is the ban ledger: the bans sww makes on the
|
||||||
// netblocks of clients that break a rate limit, a byte limit or the refusal
|
// netblocks of clients that break a rate limit, a byte limit or the refusal
|
||||||
// limit or show a clear sign of attack, or are on AbuseIPDB's blacklist, and
|
// limit or show a clear sign of attack, and those an admin makes, with their
|
||||||
// those an admin makes, with their notes, as the "Bans" section of SPEC.md
|
// notes, as the "Bans" section of SPEC.md describes. The bans are kept in
|
||||||
// describes. The bans are kept in memory, and written to bans.json and read
|
// memory, and written to bans.json and read from it by the state package.
|
||||||
// from it by the state package.
|
|
||||||
package bans
|
package bans
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -27,9 +26,6 @@ const (
|
|||||||
// CauseAdmin is a ban an admin made, or one sww made that an
|
// CauseAdmin is a ban an admin made, or one sww made that an
|
||||||
// admin keeps. It is never dropped.
|
// admin keeps. It is never dropped.
|
||||||
CauseAdmin = "admin"
|
CauseAdmin = "admin"
|
||||||
// CauseAbuseIPDB is a ban sww made for a client on AbuseIPDB's
|
|
||||||
// blacklist. It is forgotten once it has ended.
|
|
||||||
CauseAbuseIPDB = "abuseipdb"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// repeatFactor is how many times as long as the netblock's last ban a ban
|
// repeatFactor is how many times as long as the netblock's last ban a ban
|
||||||
@@ -54,11 +50,10 @@ type Params struct {
|
|||||||
// AttackBanDuration is how long a first ban for a clear sign of attack
|
// AttackBanDuration is how long a first ban for a clear sign of attack
|
||||||
// lasts.
|
// lasts.
|
||||||
AttackBanDuration time.Duration
|
AttackBanDuration time.Duration
|
||||||
// MaxBans is the most bans held whose cause is CauseLimit or
|
// MaxBans is the most bans held whose cause is not CauseAdmin, at
|
||||||
// CauseAttack, at least one. Past it, the earliest such ban of the
|
// least one. Past it, the earliest such ban of the netblock that has
|
||||||
// netblock that has gone longest without a request is dropped. Bans
|
// gone longest without a request is dropped. Bans whose cause is
|
||||||
// whose cause is CauseAdmin or CauseAbuseIPDB are held besides, and
|
// CauseAdmin are held besides, and never dropped.
|
||||||
// never dropped this way.
|
|
||||||
MaxBans int
|
MaxBans int
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -68,11 +63,10 @@ type Ban struct {
|
|||||||
Start time.Time
|
Start time.Time
|
||||||
// Expires is when the ban ends, zero for a permanent ban.
|
// Expires is when the ban ends, zero for a permanent ban.
|
||||||
Expires time.Time
|
Expires time.Time
|
||||||
// Cause is CauseLimit, CauseAttack, CauseAdmin or CauseAbuseIPDB.
|
// Cause is CauseLimit, CauseAttack or CauseAdmin.
|
||||||
Cause string
|
Cause string
|
||||||
// Reason is a short text: for a ban sww made, the limit broken, the
|
// Reason is a short text: for a ban sww made, the limit broken or
|
||||||
// rule that matched or AbuseIPDB's score; for an admin's, what the
|
// the rule that matched; for an admin's, what the admin wrote.
|
||||||
// admin wrote.
|
|
||||||
Reason string
|
Reason string
|
||||||
// Lifted is when an admin lifted the ban, zero while no admin has. A
|
// Lifted is when an admin lifted the ban, zero while no admin has. A
|
||||||
// lifted ban refuses nothing, and does not make the netblock's next
|
// lifted ban refuses nothing, and does not make the netblock's next
|
||||||
@@ -133,8 +127,7 @@ type Notes struct {
|
|||||||
// log's reputation names them. It is left out when none did.
|
// log's reputation names them. It is left out when none did.
|
||||||
Reputation []ReputationHit `json:"reputation,omitempty"`
|
Reputation []ReputationHit `json:"reputation,omitempty"`
|
||||||
// Request is the request that broke the limit, or whose bytes broke
|
// Request is the request that broke the limit, or whose bytes broke
|
||||||
// it, or that was the clear sign of attack. A ban for AbuseIPDB's
|
// it, or that was the clear sign of attack.
|
||||||
// blacklist has none, and leaves it zero.
|
|
||||||
Request Request `json:"request"`
|
Request Request `json:"request"`
|
||||||
// Requests is how many requests the netblock has sent since it was
|
// Requests is how many requests the netblock has sent since it was
|
||||||
// first seen, and Refused how many of them the ban has refused so
|
// first seen, and Refused how many of them the ban has refused so
|
||||||
@@ -157,10 +150,9 @@ type ReputationHit struct {
|
|||||||
|
|
||||||
// EarlierBans counts a netblock's bans before a ban, by cause.
|
// EarlierBans counts a netblock's bans before a ban, by cause.
|
||||||
type EarlierBans struct {
|
type EarlierBans struct {
|
||||||
Limit int `json:"limit"`
|
Limit int `json:"limit"`
|
||||||
Attack int `json:"attack"`
|
Attack int `json:"attack"`
|
||||||
Admin int `json:"admin"`
|
Admin int `json:"admin"`
|
||||||
AbuseIPDB int `json:"abuseipdb"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Request is a request in a ban's notes. Each text is cut to 256 bytes.
|
// Request is a request in a ban's notes. Each text is cut to 256 bytes.
|
||||||
@@ -188,8 +180,8 @@ type Ledger struct {
|
|||||||
// netblocks holds each banned netblock's bans, oldest first. Check and
|
// netblocks holds each banned netblock's bans, oldest first. Check and
|
||||||
// Find make each netblock they find the most recently seen.
|
// Find make each netblock they find the most recently seen.
|
||||||
netblocks *simplelru.LRU[netip.Prefix, *[]Ban]
|
netblocks *simplelru.LRU[netip.Prefix, *[]Ban]
|
||||||
// held is how many bans netblocks holds whose cause is CauseLimit or
|
// held is how many bans netblocks holds whose cause is not CauseAdmin,
|
||||||
// CauseAttack, at most params.MaxBans.
|
// at most params.MaxBans.
|
||||||
held int
|
held int
|
||||||
// made is how many bans have been made since the start, by cause: by
|
// made is how many bans have been made since the start, by cause: by
|
||||||
// the ledger, and by an admin, through BanForAdmin or in an edit of
|
// the ledger, and by an admin, through BanForAdmin or in an edit of
|
||||||
@@ -206,9 +198,8 @@ type Ledger struct {
|
|||||||
// New returns a Ledger with no ban yet.
|
// New returns a Ledger with no ban yet.
|
||||||
func New(params Params) *Ledger {
|
func New(params Params) *Ledger {
|
||||||
// The ledger drops bans itself, and never those whose cause is
|
// The ledger drops bans itself, and never those whose cause is
|
||||||
// CauseAdmin or CauseAbuseIPDB to make room, however many there are, so
|
// CauseAdmin, however many there are, so the LRU has no limit of its
|
||||||
// the LRU has no limit of its own: it keeps the netblocks in the order
|
// own: it keeps the netblocks in the order they were last seen.
|
||||||
// they were last seen.
|
|
||||||
netblocks, err := simplelru.NewLRU[netip.Prefix, *[]Ban](
|
netblocks, err := simplelru.NewLRU[netip.Prefix, *[]Ban](
|
||||||
math.MaxInt,
|
math.MaxInt,
|
||||||
nil,
|
nil,
|
||||||
@@ -438,88 +429,6 @@ func (l *Ledger) BanForAdmin(
|
|||||||
return ban
|
return ban
|
||||||
}
|
}
|
||||||
|
|
||||||
// BanForAbuseIPDB bans each netblock in scores at now until expires, and
|
|
||||||
// returns how many bans it made. scores are the netblocks of the clients
|
|
||||||
// on AbuseIPDB's blacklist, each with the highest score of its clients.
|
|
||||||
// A netblock is left as it is while a ban on it is active, or while a ban
|
|
||||||
// for the blacklist that an admin lifted has not reached its end: the
|
|
||||||
// admin's lift holds that long. Each ban whose cause is CauseAbuseIPDB
|
|
||||||
// and that has ended is forgotten first, so that the bans held for the
|
|
||||||
// blacklist are those of the clients listed within one ban's length.
|
|
||||||
func (l *Ledger) BanForAbuseIPDB(
|
|
||||||
now, expires time.Time, scores map[netip.Prefix]int64,
|
|
||||||
) int {
|
|
||||||
l.mu.Lock()
|
|
||||||
defer l.mu.Unlock()
|
|
||||||
|
|
||||||
changed := l.forgetEndedAbuseIPDBBans(now)
|
|
||||||
made := 0
|
|
||||||
|
|
||||||
for netblock, score := range scores {
|
|
||||||
netblock = netblock.Masked()
|
|
||||||
|
|
||||||
var notes Notes
|
|
||||||
|
|
||||||
if held, found := l.netblocks.Peek(netblock); found {
|
|
||||||
if activeBan(*held, now) != nil ||
|
|
||||||
liftedAbuseIPDBBan(*held, now) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
notes.EarlierBans = earlierBans(*held)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The source is named as the request log names AbuseIPDB.
|
|
||||||
notes.Reputation = []ReputationHit{
|
|
||||||
{Source: CauseAbuseIPDB, Score: &score},
|
|
||||||
}
|
|
||||||
|
|
||||||
l.add(Ban{
|
|
||||||
Netblock: netblock,
|
|
||||||
Start: now,
|
|
||||||
Expires: expires,
|
|
||||||
Cause: CauseAbuseIPDB,
|
|
||||||
Reason: fmt.Sprintf(
|
|
||||||
"on AbuseIPDB's blacklist with a score of %d", score,
|
|
||||||
),
|
|
||||||
Notes: notes,
|
|
||||||
})
|
|
||||||
|
|
||||||
made++
|
|
||||||
}
|
|
||||||
|
|
||||||
l.made[CauseAbuseIPDB] += made
|
|
||||||
|
|
||||||
if changed || made > 0 {
|
|
||||||
l.markChanged()
|
|
||||||
}
|
|
||||||
|
|
||||||
return made
|
|
||||||
}
|
|
||||||
|
|
||||||
// ForgetEndedAbuseIPDBBans forgets the bans whose cause is CauseAbuseIPDB
|
|
||||||
// and that have ended at now, as BanForAbuseIPDB does first. The state
|
|
||||||
// package calls it each time it reads bans.json, so that these bans are
|
|
||||||
// also forgotten while no fetch of the blacklist reaches the ledger.
|
|
||||||
func (l *Ledger) ForgetEndedAbuseIPDBBans(now time.Time) {
|
|
||||||
l.mu.Lock()
|
|
||||||
defer l.mu.Unlock()
|
|
||||||
|
|
||||||
if l.forgetEndedAbuseIPDBBans(now) {
|
|
||||||
l.markChanged()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// liftedAbuseIPDBBan reports whether bans, a netblock's, hold a ban whose
|
|
||||||
// cause is CauseAbuseIPDB that an admin lifted, and whose end is not
|
|
||||||
// reached at now.
|
|
||||||
func liftedAbuseIPDBBan(bans []Ban, now time.Time) bool {
|
|
||||||
return slices.ContainsFunc(bans, func(ban Ban) bool {
|
|
||||||
return ban.Cause == CauseAbuseIPDB && !ban.Lifted.IsZero() &&
|
|
||||||
(ban.Permanent() || now.Before(ban.Expires))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Lift lifts, at now, every ban active then on a netblock client is in,
|
// Lift lifts, at now, every ban active then on a netblock client is in,
|
||||||
// as an admin does, and returns those bans. A lifted ban is kept, refuses
|
// as an admin does, and returns those bans. A lifted ban is kept, refuses
|
||||||
// nothing, and does not make the netblock's next ban longer.
|
// nothing, and does not make the netblock's next ban longer.
|
||||||
@@ -606,9 +515,9 @@ func (l *Ledger) AddLookup(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Made returns how many bans for cause have been made since the start:
|
// Made returns how many bans for cause have been made since the start:
|
||||||
// for CauseLimit, CauseAttack and CauseAbuseIPDB, by the ledger; for
|
// for CauseLimit and CauseAttack, by the ledger; for CauseAdmin, by an
|
||||||
// CauseAdmin, by an admin, with BanForAdmin or in an edit of bans.json, as
|
// admin, with BanForAdmin or in an edit of bans.json, as LoadEdit counts
|
||||||
// LoadEdit counts them. The bans read from bans.json at the start are not among them.
|
// them. The bans read from bans.json at the start are not among them.
|
||||||
func (l *Ledger) Made(cause string) int {
|
func (l *Ledger) Made(cause string) int {
|
||||||
l.mu.Lock()
|
l.mu.Lock()
|
||||||
defer l.mu.Unlock()
|
defer l.mu.Unlock()
|
||||||
@@ -665,8 +574,8 @@ func (l *Ledger) Snapshot() []Ban {
|
|||||||
// seen. A ban without a cause is an admin's, and gets CauseAdmin. Each
|
// seen. A ban without a cause is an admin's, and gets CauseAdmin. Each
|
||||||
// netblock is masked to its length, so that 203.0.113.9/24 is
|
// netblock is masked to its length, so that 203.0.113.9/24 is
|
||||||
// 203.0.113.0/24, and each text in the notes is cut to 256 bytes. Past
|
// 203.0.113.0/24, and each text in the notes is cut to 256 bytes. Past
|
||||||
// MaxBans the earliest bans that count toward it are dropped, as when they
|
// MaxBans the earliest bans whose cause is not CauseAdmin are dropped, as
|
||||||
// are made.
|
// when they are made.
|
||||||
func (l *Ledger) Load(bans []Ban) {
|
func (l *Ledger) Load(bans []Ban) {
|
||||||
l.mu.Lock()
|
l.mu.Lock()
|
||||||
defer l.mu.Unlock()
|
defer l.mu.Unlock()
|
||||||
@@ -730,35 +639,6 @@ func (l *Ledger) holds(netblock netip.Prefix, start time.Time) bool {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// forgetEndedAbuseIPDBBans forgets each ban whose cause is CauseAbuseIPDB
|
|
||||||
// and whose end is reached at now, lifted or not, and each netblock left
|
|
||||||
// without a ban. It reports whether it forgot one.
|
|
||||||
func (l *Ledger) forgetEndedAbuseIPDBBans(now time.Time) bool {
|
|
||||||
forgot := false
|
|
||||||
|
|
||||||
for _, netblock := range l.netblocks.Keys() {
|
|
||||||
bans, _ := l.netblocks.Peek(netblock)
|
|
||||||
|
|
||||||
kept := slices.DeleteFunc(*bans, func(ban Ban) bool {
|
|
||||||
return ban.Cause == CauseAbuseIPDB && !ban.Permanent() &&
|
|
||||||
!now.Before(ban.Expires)
|
|
||||||
})
|
|
||||||
if len(kept) == len(*bans) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
forgot = true
|
|
||||||
|
|
||||||
if len(kept) == 0 {
|
|
||||||
l.netblocks.Remove(netblock)
|
|
||||||
} else {
|
|
||||||
*bans = kept
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return forgot
|
|
||||||
}
|
|
||||||
|
|
||||||
// ban bans netblock at now for cause, CauseLimit or CauseAttack, with
|
// ban bans netblock at now for cause, CauseLimit or CauseAttack, with
|
||||||
// reason and notes, as BanForLimit and BanForAttack describe, and returns
|
// reason and notes, as BanForLimit and BanForAttack describe, and returns
|
||||||
// the ban, and whether it made it. Unless keep is true, the ban is not
|
// the ban, and whether it made it. Unless keep is true, the ban is not
|
||||||
@@ -834,16 +714,13 @@ func earlierBans(held []Ban) EarlierBans {
|
|||||||
fromHere.Attack++
|
fromHere.Attack++
|
||||||
case CauseAdmin:
|
case CauseAdmin:
|
||||||
fromHere.Admin++
|
fromHere.Admin++
|
||||||
case CauseAbuseIPDB:
|
|
||||||
fromHere.AbuseIPDB++
|
|
||||||
}
|
}
|
||||||
|
|
||||||
notes := ban.Notes.EarlierBans
|
notes := ban.Notes.EarlierBans
|
||||||
count := EarlierBans{
|
count := EarlierBans{
|
||||||
Limit: notes.Limit + fromHere.Limit,
|
Limit: notes.Limit + fromHere.Limit,
|
||||||
Attack: notes.Attack + fromHere.Attack,
|
Attack: notes.Attack + fromHere.Attack,
|
||||||
Admin: notes.Admin + fromHere.Admin,
|
Admin: notes.Admin + fromHere.Admin,
|
||||||
AbuseIPDB: notes.AbuseIPDB + fromHere.AbuseIPDB,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Of counts that name as many bans, the earlier ban's is taken: it
|
// Of counts that name as many bans, the earlier ban's is taken: it
|
||||||
@@ -858,7 +735,7 @@ func earlierBans(held []Ban) EarlierBans {
|
|||||||
|
|
||||||
// total returns how many bans e counts, of all causes.
|
// total returns how many bans e counts, of all causes.
|
||||||
func (e EarlierBans) total() int {
|
func (e EarlierBans) total() int {
|
||||||
return e.Limit + e.Attack + e.Admin + e.AbuseIPDB
|
return e.Limit + e.Attack + e.Admin
|
||||||
}
|
}
|
||||||
|
|
||||||
// markChanged has Changed receive a value, unless one is waiting already.
|
// markChanged has Changed receive a value, unless one is waiting already.
|
||||||
@@ -929,9 +806,9 @@ func (l *Ledger) covering(client netip.Addr) []*[]Ban {
|
|||||||
|
|
||||||
// add adds ban to its netblock's bans, after the last, and makes its
|
// add adds ban to its netblock's bans, after the last, and makes its
|
||||||
// netblock the most recently seen. With MaxBans held, it drops one first,
|
// netblock the most recently seen. With MaxBans held, it drops one first,
|
||||||
// unless ban does not count toward MaxBans.
|
// unless ban's cause is CauseAdmin, which does not count toward MaxBans.
|
||||||
func (l *Ledger) add(ban Ban) {
|
func (l *Ledger) add(ban Ban) {
|
||||||
counted := countsTowardMaxBans(ban)
|
counted := ban.Cause != CauseAdmin
|
||||||
if counted && l.held == l.params.MaxBans {
|
if counted && l.held == l.params.MaxBans {
|
||||||
l.dropOne()
|
l.dropOne()
|
||||||
}
|
}
|
||||||
@@ -1012,13 +889,7 @@ func (l *Ledger) attackExpiry(held []Ban, now time.Time) time.Time {
|
|||||||
return now.Add(l.params.AttackBanDuration)
|
return now.Add(l.params.AttackBanDuration)
|
||||||
}
|
}
|
||||||
|
|
||||||
// countsTowardMaxBans reports whether ban counts toward MaxBans: its cause
|
// dropOne drops the earliest ban whose cause is not CauseAdmin of the
|
||||||
// is CauseLimit or CauseAttack.
|
|
||||||
func countsTowardMaxBans(ban Ban) bool {
|
|
||||||
return ban.Cause == CauseLimit || ban.Cause == CauseAttack
|
|
||||||
}
|
|
||||||
|
|
||||||
// dropOne drops the earliest ban that counts toward MaxBans of the
|
|
||||||
// netblock that has gone longest without a request, of those that hold
|
// netblock that has gone longest without a request, of those that hold
|
||||||
// such a ban, and the netblock with it if that was its only ban. It is
|
// such a ban, and the netblock with it if that was its only ban. It is
|
||||||
// called with at least one such ban held. It looks at each netblock once
|
// called with at least one such ban held. It looks at each netblock once
|
||||||
@@ -1027,9 +898,11 @@ func (l *Ledger) dropOne() {
|
|||||||
for range l.netblocks.Len() {
|
for range l.netblocks.Len() {
|
||||||
netblock, bans, _ := l.netblocks.GetOldest()
|
netblock, bans, _ := l.netblocks.GetOldest()
|
||||||
|
|
||||||
i := slices.IndexFunc(*bans, countsTowardMaxBans)
|
i := slices.IndexFunc(*bans, func(ban Ban) bool {
|
||||||
|
return ban.Cause != CauseAdmin
|
||||||
|
})
|
||||||
if i < 0 {
|
if i < 0 {
|
||||||
// None of its bans is dropped to make room. Get makes
|
// Its bans are all an admin's, and never dropped. Get makes
|
||||||
// it the most recently seen, so that the next netblock is
|
// it the most recently seen, so that the next netblock is
|
||||||
// looked at; when it was seen matters only for dropping a
|
// looked at; when it was seen matters only for dropping a
|
||||||
// ban, and a ban added to it makes it the most recently seen
|
// ban, and a ban added to it makes it the most recently seen
|
||||||
|
|||||||
+32
-154
@@ -75,7 +75,7 @@ type WAF struct {
|
|||||||
BytesLimitPerMinute int64
|
BytesLimitPerMinute int64
|
||||||
BytesLimitPerHour int64
|
BytesLimitPerHour int64
|
||||||
BytesLimitPerDay int64
|
BytesLimitPerDay int64
|
||||||
BytesCounted BytesCounted
|
BytesCounted string
|
||||||
// LookupSource is where each client's AS number and country are looked
|
// LookupSource is where each client's AS number and country are looked
|
||||||
// up (SWW_WAF_LOOKUP_SOURCE): geojs, file, or off for nowhere.
|
// up (SWW_WAF_LOOKUP_SOURCE): geojs, file, or off for nowhere.
|
||||||
// LookupDBPath is the lookup database, the IPinfo Lite file looked up in
|
// LookupDBPath is the lookup database, the IPinfo Lite file looked up in
|
||||||
@@ -86,7 +86,7 @@ type WAF struct {
|
|||||||
// AddLookupHeaders is true when the app is passed the client's AS number
|
// AddLookupHeaders is true when the app is passed the client's AS number
|
||||||
// and country in headers (SWW_WAF_ADD_LOOKUP_HEADERS), by default unless
|
// and country in headers (SWW_WAF_ADD_LOOKUP_HEADERS), by default unless
|
||||||
// LookupSource is off.
|
// LookupSource is off.
|
||||||
LookupSource LookupSource
|
LookupSource string
|
||||||
LookupDBPath string
|
LookupDBPath string
|
||||||
ASNAndCountryLookupTimeout time.Duration
|
ASNAndCountryLookupTimeout time.Duration
|
||||||
AddLookupHeaders bool
|
AddLookupHeaders bool
|
||||||
@@ -131,7 +131,7 @@ type WAF struct {
|
|||||||
// BlocklistLimitPercent is the percentage of every limit it gets.
|
// BlocklistLimitPercent is the percentage of every limit it gets.
|
||||||
BlocklistURLs []string
|
BlocklistURLs []string
|
||||||
BlocklistRefresh time.Duration
|
BlocklistRefresh time.Duration
|
||||||
BlocklistAction ReputationAction
|
BlocklistAction string
|
||||||
BlocklistLimitPercent int64
|
BlocklistLimitPercent int64
|
||||||
// DNSBLZones are the DNSBL zones clients are asked about
|
// DNSBLZones are the DNSBL zones clients are asked about
|
||||||
// (SWW_WAF_DNSBL_ZONES), through DNSBLResolver (SWW_WAF_DNSBL_RESOLVER), or
|
// (SWW_WAF_DNSBL_ZONES), through DNSBLResolver (SWW_WAF_DNSBL_RESOLVER), or
|
||||||
@@ -154,18 +154,10 @@ type WAF struct {
|
|||||||
AbuseIPDBKey string
|
AbuseIPDBKey string
|
||||||
AbuseIPDBMinScore int64
|
AbuseIPDBMinScore int64
|
||||||
AbuseIPDBDailyBudget int
|
AbuseIPDBDailyBudget int
|
||||||
DNSBLAbuseIPDBAction ReputationAction
|
DNSBLAbuseIPDBAction string
|
||||||
DNSBLAbuseIPDBLimitPercent int64
|
DNSBLAbuseIPDBLimitPercent int64
|
||||||
IPReputationCacheTTL time.Duration
|
IPReputationCacheTTL time.Duration
|
||||||
DNSBLAbuseIPDBTimeout time.Duration
|
DNSBLAbuseIPDBTimeout time.Duration
|
||||||
// While AbuseIPDBKey is set, AbuseIPDB's blacklist of the clients it
|
|
||||||
// scores AbuseIPDBThreshold or more (SWW_WAF_ABUSEIPDB_THRESHOLD), from
|
|
||||||
// 25 to 100, is fetched at the start and every AbuseIPDBBlacklistRefresh
|
|
||||||
// (SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS). Each client on it
|
|
||||||
// is banned for AbuseIPDBBanDuration (SWW_WAF_ABUSEIPDB_BAN_DAYS).
|
|
||||||
AbuseIPDBThreshold int64
|
|
||||||
AbuseIPDBBlacklistRefresh time.Duration
|
|
||||||
AbuseIPDBBanDuration time.Duration
|
|
||||||
// BanResponse is the status a refused client is answered with, 403
|
// BanResponse is the status a refused client is answered with, 403
|
||||||
// or 429, or 0 to close the connection without an answer
|
// or 429, or 0 to close the connection without an answer
|
||||||
// (SWW_WAF_BAN_RESPONSE). It answers a banned client, a request that
|
// (SWW_WAF_BAN_RESPONSE). It answers a banned client, a request that
|
||||||
@@ -237,7 +229,7 @@ type WAF struct {
|
|||||||
// reads none.
|
// reads none.
|
||||||
// CoreRuleSetExclusionsFile is the path of the file of its exclusions
|
// CoreRuleSetExclusionsFile is the path of the file of its exclusions
|
||||||
// (SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE), "" while there is none.
|
// (SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE), "" while there is none.
|
||||||
CoreRuleSetMode CoreRuleSetMode
|
CoreRuleSetMode string
|
||||||
CoreRuleSetParanoiaLevel int
|
CoreRuleSetParanoiaLevel int
|
||||||
CoreRuleSetScoreThreshold int
|
CoreRuleSetScoreThreshold int
|
||||||
CoreRuleSetDisabledRules []int
|
CoreRuleSetDisabledRules []int
|
||||||
@@ -316,66 +308,20 @@ type WAF struct {
|
|||||||
settings []slog.Attr
|
settings []slog.Attr
|
||||||
}
|
}
|
||||||
|
|
||||||
// BytesCounted is which body bytes count toward the byte limits and the
|
|
||||||
// byte anomaly thresholds, a value of SWW_WAF_BYTES_COUNTED.
|
|
||||||
type BytesCounted string
|
|
||||||
|
|
||||||
// The values of SWW_WAF_BYTES_COUNTED.
|
|
||||||
const (
|
|
||||||
// BytesCountedResponse counts the response's body bytes.
|
|
||||||
BytesCountedResponse BytesCounted = "response"
|
|
||||||
// BytesCountedRequest counts the request's body bytes.
|
|
||||||
BytesCountedRequest BytesCounted = "request"
|
|
||||||
// BytesCountedBoth counts the body bytes of both.
|
|
||||||
BytesCountedBoth BytesCounted = "both"
|
|
||||||
)
|
|
||||||
|
|
||||||
// LookupSource is where each client's AS number and country are looked up,
|
|
||||||
// a value of SWW_WAF_LOOKUP_SOURCE.
|
|
||||||
type LookupSource string
|
|
||||||
|
|
||||||
// The values of SWW_WAF_LOOKUP_SOURCE.
|
|
||||||
const (
|
|
||||||
// LookupSourceGeoJS asks GeoJS.
|
|
||||||
LookupSourceGeoJS LookupSource = "geojs"
|
|
||||||
// LookupSourceFile looks clients up in the lookup database, the file
|
|
||||||
// SWW_WAF_LOOKUP_DB_PATH names.
|
|
||||||
LookupSourceFile LookupSource = "file"
|
|
||||||
// LookupSourceOff looks no client up.
|
|
||||||
LookupSourceOff LookupSource = "off"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ReputationAction is what is done with a client a reputation source
|
|
||||||
// lists, a value of SWW_WAF_BLOCKLIST_ACTION and of
|
|
||||||
// SWW_WAF_DNSBL_ABUSEIPDB_ACTION.
|
|
||||||
type ReputationAction string
|
|
||||||
|
|
||||||
// The values of SWW_WAF_BLOCKLIST_ACTION and SWW_WAF_DNSBL_ABUSEIPDB_ACTION.
|
|
||||||
const (
|
|
||||||
// ReputationActionDeny refuses the client's requests.
|
|
||||||
ReputationActionDeny ReputationAction = "deny"
|
|
||||||
// ReputationActionLimit gives the client a percentage of every limit,
|
|
||||||
// written limit:<percent> in the setting.
|
|
||||||
ReputationActionLimit ReputationAction = "limit"
|
|
||||||
// ReputationActionLog only notes the hit in the log line and raises an
|
|
||||||
// alert.
|
|
||||||
ReputationActionLog ReputationAction = "log"
|
|
||||||
)
|
|
||||||
|
|
||||||
// CoreRuleSetMode is what the Core Rule Set does, a value of
|
|
||||||
// SWW_WAF_CORE_RULE_SET_MODE.
|
|
||||||
type CoreRuleSetMode string
|
|
||||||
|
|
||||||
// The values of SWW_WAF_CORE_RULE_SET_MODE.
|
// The values of SWW_WAF_CORE_RULE_SET_MODE.
|
||||||
const (
|
const (
|
||||||
// CoreRuleSetModeOff runs no request through the Core Rule Set.
|
// CoreRuleSetModeOff runs no request through the Core Rule Set.
|
||||||
CoreRuleSetModeOff CoreRuleSetMode = "off"
|
CoreRuleSetModeOff = off
|
||||||
// CoreRuleSetModeDetect logs and alerts a match, and refuses nothing.
|
// CoreRuleSetModeDetect logs and alerts a match, and refuses nothing.
|
||||||
CoreRuleSetModeDetect CoreRuleSetMode = "detect"
|
CoreRuleSetModeDetect = "detect"
|
||||||
// CoreRuleSetModeBlock refuses a match with 403.
|
// CoreRuleSetModeBlock refuses a match with 403.
|
||||||
CoreRuleSetModeBlock CoreRuleSetMode = "block"
|
CoreRuleSetModeBlock = "block"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// fileSource is the SWW_WAF_LOOKUP_SOURCE that looks clients up in the
|
||||||
|
// lookup database, the file SWW_WAF_LOOKUP_DB_PATH names.
|
||||||
|
const fileSource = "file"
|
||||||
|
|
||||||
const (
|
const (
|
||||||
day = 24 * time.Hour
|
day = 24 * time.Hour
|
||||||
ipv4Bits = 32
|
ipv4Bits = 32
|
||||||
@@ -384,9 +330,6 @@ const (
|
|||||||
// netblock a provider is usually given: a shorter one would make one
|
// netblock a provider is usually given: a shorter one would make one
|
||||||
// client of the customers of several providers.
|
// client of the customers of several providers.
|
||||||
minIPv6GroupPrefix = 32
|
minIPv6GroupPrefix = 32
|
||||||
// minAbuseIPDBThreshold is the lowest score AbuseIPDB's blacklist can
|
|
||||||
// be cut at.
|
|
||||||
minAbuseIPDBThreshold = 25
|
|
||||||
// minTokenLength is the fewest characters a token may have.
|
// minTokenLength is the fewest characters a token may have.
|
||||||
minTokenLength = 32
|
minTokenLength = 32
|
||||||
// maxParanoiaLevel is the Core Rule Set's highest paranoia level.
|
// maxParanoiaLevel is the Core Rule Set's highest paranoia level.
|
||||||
@@ -503,13 +446,6 @@ var (
|
|||||||
errNotPercent = errors.New(
|
errNotPercent = errors.New(
|
||||||
"is not a percentage, a whole number from 0 to 100",
|
"is not a percentage, a whole number from 0 to 100",
|
||||||
)
|
)
|
||||||
errNotAbuseIPDBThreshold = errors.New(
|
|
||||||
"is not a whole number from 25 to 100, the scores AbuseIPDB's " +
|
|
||||||
"blacklist can be cut at",
|
|
||||||
)
|
|
||||||
errNotWholeUnits = errors.New(
|
|
||||||
"is not a whole number above zero, such as 30, that fits a duration",
|
|
||||||
)
|
|
||||||
errListedTwice = errors.New("is listed twice")
|
errListedTwice = errors.New("is listed twice")
|
||||||
errNotListURL = errors.New(
|
errNotListURL = errors.New(
|
||||||
"is not an http or https URL without a user or a fragment, " +
|
"is not an http or https URL without a user or a fragment, " +
|
||||||
@@ -604,13 +540,10 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
|||||||
),
|
),
|
||||||
BytesLimitPerHour: env.size("SWW_WAF_BYTES_LIMIT_PER_HOUR", "50G"),
|
BytesLimitPerHour: env.size("SWW_WAF_BYTES_LIMIT_PER_HOUR", "50G"),
|
||||||
BytesLimitPerDay: env.size("SWW_WAF_BYTES_LIMIT_PER_DAY", "100G"),
|
BytesLimitPerDay: env.size("SWW_WAF_BYTES_LIMIT_PER_DAY", "100G"),
|
||||||
BytesCounted: env.bytesCounted(
|
BytesCounted: env.bytesCounted("SWW_WAF_BYTES_COUNTED", "both"),
|
||||||
"SWW_WAF_BYTES_COUNTED",
|
|
||||||
BytesCountedBoth,
|
|
||||||
),
|
|
||||||
LookupSource: env.lookupSource(
|
LookupSource: env.lookupSource(
|
||||||
"SWW_WAF_LOOKUP_SOURCE",
|
"SWW_WAF_LOOKUP_SOURCE",
|
||||||
LookupSourceGeoJS,
|
"geojs",
|
||||||
),
|
),
|
||||||
LookupDBPath: env.value("SWW_WAF_LOOKUP_DB_PATH", ""),
|
LookupDBPath: env.value("SWW_WAF_LOOKUP_DB_PATH", ""),
|
||||||
ASNAndCountryLookupTimeout: env.milliseconds(
|
ASNAndCountryLookupTimeout: env.milliseconds(
|
||||||
@@ -654,20 +587,6 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
|||||||
"SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS",
|
"SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS",
|
||||||
"2000",
|
"2000",
|
||||||
),
|
),
|
||||||
AbuseIPDBThreshold: env.abuseIPDBThreshold(
|
|
||||||
"SWW_WAF_ABUSEIPDB_THRESHOLD",
|
|
||||||
"90",
|
|
||||||
),
|
|
||||||
AbuseIPDBBlacklistRefresh: env.wholeUnits(
|
|
||||||
"SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS",
|
|
||||||
"60",
|
|
||||||
time.Minute,
|
|
||||||
),
|
|
||||||
AbuseIPDBBanDuration: env.wholeUnits(
|
|
||||||
"SWW_WAF_ABUSEIPDB_BAN_DAYS",
|
|
||||||
"30",
|
|
||||||
day,
|
|
||||||
),
|
|
||||||
BanResponse: env.banResponse("SWW_WAF_BAN_RESPONSE", "403"),
|
BanResponse: env.banResponse("SWW_WAF_BAN_RESPONSE", "403"),
|
||||||
LimitBanDuration: env.durationNotOff(
|
LimitBanDuration: env.durationNotOff(
|
||||||
"SWW_WAF_LIMIT_BAN_DURATION",
|
"SWW_WAF_LIMIT_BAN_DURATION",
|
||||||
@@ -807,9 +726,9 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
|||||||
cfg.InstanceName, cfg.LogRemoteURL != nil)
|
cfg.InstanceName, cfg.LogRemoteURL != nil)
|
||||||
// The app gets the headers by default, unless no client is looked up.
|
// The app gets the headers by default, unless no client is looked up.
|
||||||
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
|
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
|
||||||
strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
|
strconv.FormatBool(cfg.LookupSource != off))
|
||||||
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
|
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
|
||||||
"SWW_WAF_BLOCKLIST_ACTION", string(ReputationActionDeny))
|
"SWW_WAF_BLOCKLIST_ACTION", "deny")
|
||||||
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
|
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
|
||||||
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "limit:25")
|
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "limit:25")
|
||||||
|
|
||||||
@@ -907,12 +826,9 @@ func (e *environment) count(name, defaultValue string) int64 {
|
|||||||
|
|
||||||
// bytesCounted reads the setting that is which body bytes count toward the
|
// bytesCounted reads the setting that is which body bytes count toward the
|
||||||
// byte limits and the byte anomaly thresholds: response, request or both.
|
// byte limits and the byte anomaly thresholds: response, request or both.
|
||||||
func (e *environment) bytesCounted(
|
func (e *environment) bytesCounted(name, defaultValue string) string {
|
||||||
name string, defaultValue BytesCounted,
|
value := e.value(name, defaultValue)
|
||||||
) BytesCounted {
|
if value != "response" && value != "request" && value != "both" {
|
||||||
value := BytesCounted(e.value(name, string(defaultValue)))
|
|
||||||
if value != BytesCountedResponse && value != BytesCountedRequest &&
|
|
||||||
value != BytesCountedBoth {
|
|
||||||
e.check(name, fmt.Errorf("%q %w", value, errNotBytesCounted))
|
e.check(name, fmt.Errorf("%q %w", value, errNotBytesCounted))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -957,10 +873,8 @@ func (e *environment) trapPaths(name string) []string {
|
|||||||
|
|
||||||
// coreRuleSetMode reads the setting that is what the Core Rule Set does: off,
|
// coreRuleSetMode reads the setting that is what the Core Rule Set does: off,
|
||||||
// detect or block.
|
// detect or block.
|
||||||
func (e *environment) coreRuleSetMode(
|
func (e *environment) coreRuleSetMode(name, defaultValue string) string {
|
||||||
name string, defaultValue CoreRuleSetMode,
|
mode := e.value(name, defaultValue)
|
||||||
) CoreRuleSetMode {
|
|
||||||
mode := CoreRuleSetMode(e.value(name, string(defaultValue)))
|
|
||||||
if mode != CoreRuleSetModeOff && mode != CoreRuleSetModeDetect &&
|
if mode != CoreRuleSetModeOff && mode != CoreRuleSetModeDetect &&
|
||||||
mode != CoreRuleSetModeBlock {
|
mode != CoreRuleSetModeBlock {
|
||||||
e.check(name, fmt.Errorf("%q %w", mode, errNotCoreRuleSetMode))
|
e.check(name, fmt.Errorf("%q %w", mode, errNotCoreRuleSetMode))
|
||||||
@@ -1043,35 +957,6 @@ func (e *environment) percent(name, defaultValue string) int64 {
|
|||||||
return percent
|
return percent
|
||||||
}
|
}
|
||||||
|
|
||||||
// abuseIPDBThreshold reads the setting that is the least score of the
|
|
||||||
// clients on AbuseIPDB's blacklist: a whole number from 25 to 100, the
|
|
||||||
// range AbuseIPDB takes.
|
|
||||||
func (e *environment) abuseIPDBThreshold(name, defaultValue string) int64 {
|
|
||||||
value := e.value(name, defaultValue)
|
|
||||||
|
|
||||||
threshold, err := strconv.ParseInt(value, 10, 64)
|
|
||||||
if err != nil || threshold < minAbuseIPDBThreshold || threshold > 100 {
|
|
||||||
e.check(name, fmt.Errorf("%q %w", value, errNotAbuseIPDBThreshold))
|
|
||||||
}
|
|
||||||
|
|
||||||
return threshold
|
|
||||||
}
|
|
||||||
|
|
||||||
// wholeUnits reads a setting that is a whole number above zero of unit,
|
|
||||||
// such as minutes or days, and returns it as a duration.
|
|
||||||
func (e *environment) wholeUnits(
|
|
||||||
name, defaultValue string, unit time.Duration,
|
|
||||||
) time.Duration {
|
|
||||||
value := e.value(name, defaultValue)
|
|
||||||
|
|
||||||
n, err := strconv.ParseInt(value, 10, 64)
|
|
||||||
if err != nil || n <= 0 || n > math.MaxInt64/int64(unit) {
|
|
||||||
e.check(name, fmt.Errorf("%q %w", value, errNotWholeUnits))
|
|
||||||
}
|
|
||||||
|
|
||||||
return time.Duration(n) * unit
|
|
||||||
}
|
|
||||||
|
|
||||||
// listURL reads a setting that is the URL a list is fetched from, "" while
|
// listURL reads a setting that is the URL a list is fetched from, "" while
|
||||||
// it is unset or empty.
|
// it is unset or empty.
|
||||||
func (e *environment) listURL(name string) string {
|
func (e *environment) listURL(name string) string {
|
||||||
@@ -1110,24 +995,20 @@ func (e *environment) refresh(name, defaultValue string) time.Duration {
|
|||||||
// action reads a setting that is what is done with a client a blocklist
|
// action reads a setting that is what is done with a client a blocklist
|
||||||
// or a DNSBL zone lists: deny, log, or limit:<percent>, which it returns
|
// or a DNSBL zone lists: deny, log, or limit:<percent>, which it returns
|
||||||
// as limit and the percentage.
|
// as limit and the percentage.
|
||||||
func (e *environment) action(
|
func (e *environment) action(name, defaultValue string) (string, int64) {
|
||||||
name, defaultValue string,
|
value := e.value(name, defaultValue)
|
||||||
) (ReputationAction, int64) {
|
if value == "deny" || value == "log" {
|
||||||
value := ReputationAction(e.value(name, defaultValue))
|
|
||||||
if value == ReputationActionDeny || value == ReputationActionLog {
|
|
||||||
return value, 0
|
return value, 0
|
||||||
}
|
}
|
||||||
|
|
||||||
percentText, isLimit := strings.CutPrefix(
|
percentText, isLimit := strings.CutPrefix(value, "limit:")
|
||||||
string(value), string(ReputationActionLimit)+":",
|
|
||||||
)
|
|
||||||
|
|
||||||
percent, err := ParsePercent(percentText)
|
percent, err := ParsePercent(percentText)
|
||||||
if !isLimit || err != nil {
|
if !isLimit || err != nil {
|
||||||
e.check(name, fmt.Errorf("%q %w", value, errNotAction))
|
e.check(name, fmt.Errorf("%q %w", value, errNotAction))
|
||||||
}
|
}
|
||||||
|
|
||||||
return ReputationActionLimit, percent
|
return "limit", percent
|
||||||
}
|
}
|
||||||
|
|
||||||
// zones reads the setting that is the list of DNSBL zones. It is empty by
|
// zones reads the setting that is the list of DNSBL zones. It is empty by
|
||||||
@@ -1178,12 +1059,9 @@ func (e *environment) routeWatchURL(name string) string {
|
|||||||
|
|
||||||
// lookupSource reads the setting that is where clients are looked up:
|
// lookupSource reads the setting that is where clients are looked up:
|
||||||
// geojs, file, or off.
|
// geojs, file, or off.
|
||||||
func (e *environment) lookupSource(
|
func (e *environment) lookupSource(name, defaultValue string) string {
|
||||||
name string, defaultValue LookupSource,
|
source := e.value(name, defaultValue)
|
||||||
) LookupSource {
|
if source != "geojs" && source != fileSource && source != off {
|
||||||
source := LookupSource(e.value(name, string(defaultValue)))
|
|
||||||
if source != LookupSourceGeoJS && source != LookupSourceFile &&
|
|
||||||
source != LookupSourceOff {
|
|
||||||
e.check(name, fmt.Errorf("%q %w", source, errNotLookupSource))
|
e.check(name, fmt.Errorf("%q %w", source, errNotLookupSource))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1195,12 +1073,12 @@ func (e *environment) lookupSource(
|
|||||||
// one source at a time.
|
// one source at a time.
|
||||||
func (e *environment) checkLookupDBPath(cfg *WAF) {
|
func (e *environment) checkLookupDBPath(cfg *WAF) {
|
||||||
switch {
|
switch {
|
||||||
case cfg.LookupSource == LookupSourceFile && cfg.LookupDBPath == "":
|
case cfg.LookupSource == fileSource && cfg.LookupDBPath == "":
|
||||||
e.check("SWW_WAF_LOOKUP_SOURCE", fmt.Errorf(
|
e.check("SWW_WAF_LOOKUP_SOURCE", fmt.Errorf(
|
||||||
"is file while SWW_WAF_LOOKUP_DB_PATH is unset; %w",
|
"is file while SWW_WAF_LOOKUP_DB_PATH is unset; %w",
|
||||||
errNeedsDBPath,
|
errNeedsDBPath,
|
||||||
))
|
))
|
||||||
case cfg.LookupSource != LookupSourceFile && cfg.LookupDBPath != "":
|
case cfg.LookupSource != fileSource && cfg.LookupDBPath != "":
|
||||||
e.check("SWW_WAF_LOOKUP_DB_PATH", fmt.Errorf(
|
e.check("SWW_WAF_LOOKUP_DB_PATH", fmt.Errorf(
|
||||||
"is set while SWW_WAF_LOOKUP_SOURCE is %s; %w",
|
"is set while SWW_WAF_LOOKUP_SOURCE is %s; %w",
|
||||||
cfg.LookupSource,
|
cfg.LookupSource,
|
||||||
@@ -1225,7 +1103,7 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.LookupSource != LookupSourceOff {
|
if cfg.LookupSource != off {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -61,9 +61,6 @@ const (
|
|||||||
abuseIPDBKey = "SWW_WAF_ABUSEIPDB_KEY"
|
abuseIPDBKey = "SWW_WAF_ABUSEIPDB_KEY"
|
||||||
abuseIPDBMinScore = "SWW_WAF_ABUSEIPDB_MIN_SCORE"
|
abuseIPDBMinScore = "SWW_WAF_ABUSEIPDB_MIN_SCORE"
|
||||||
abuseIPDBDailyBudget = "SWW_WAF_ABUSEIPDB_DAILY_BUDGET"
|
abuseIPDBDailyBudget = "SWW_WAF_ABUSEIPDB_DAILY_BUDGET"
|
||||||
abuseIPDBThreshold = "SWW_WAF_ABUSEIPDB_THRESHOLD"
|
|
||||||
abuseIPDBBlacklistRefresh = "SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS"
|
|
||||||
abuseIPDBBanDays = "SWW_WAF_ABUSEIPDB_BAN_DAYS"
|
|
||||||
dnsblAbuseIPDBAction = "SWW_WAF_DNSBL_ABUSEIPDB_ACTION"
|
dnsblAbuseIPDBAction = "SWW_WAF_DNSBL_ABUSEIPDB_ACTION"
|
||||||
ipReputationCacheTTL = "SWW_WAF_IP_REPUTATION_CACHE_TTL"
|
ipReputationCacheTTL = "SWW_WAF_IP_REPUTATION_CACHE_TTL"
|
||||||
dnsblAbuseIPDBTimeout = "SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS"
|
dnsblAbuseIPDBTimeout = "SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS"
|
||||||
@@ -614,7 +611,7 @@ func TestCoreRuleSetSettings(t *testing.T) {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
environment{
|
environment{
|
||||||
coreRuleSetMode: string(config.CoreRuleSetModeDetect),
|
coreRuleSetMode: config.CoreRuleSetModeDetect,
|
||||||
coreRuleSetParanoiaLevel: "4",
|
coreRuleSetParanoiaLevel: "4",
|
||||||
coreRuleSetScoreThreshold: "10",
|
coreRuleSetScoreThreshold: "10",
|
||||||
coreRuleSetDisabledRules: "942100, 920350",
|
coreRuleSetDisabledRules: "942100, 920350",
|
||||||
@@ -2033,9 +2030,8 @@ func TestReputationSettingsAsSet(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
value string
|
value, action string
|
||||||
action config.ReputationAction
|
percent int64
|
||||||
percent int64
|
|
||||||
}{
|
}{
|
||||||
{actionDeny, actionDeny, 0},
|
{actionDeny, actionDeny, 0},
|
||||||
{actionLog, actionLog, 0},
|
{actionLog, actionLog, 0},
|
||||||
@@ -2168,7 +2164,7 @@ func TestDNSBLSettingsAsSet(t *testing.T) {
|
|||||||
// want are the zones, resolver, action and percent config.WAF gives.
|
// want are the zones, resolver, action and percent config.WAF gives.
|
||||||
want []string
|
want []string
|
||||||
wantResolver string
|
wantResolver string
|
||||||
wantAction config.ReputationAction
|
wantAction string
|
||||||
wantPercent int64
|
wantPercent int64
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
@@ -2403,75 +2399,6 @@ func TestAbuseIPDBSettingsAsSet(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAbuseIPDBBlacklistSettingsAsSet(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cfg := wafFromEnvironment(t, environment{})
|
|
||||||
if cfg.AbuseIPDBThreshold != 90 ||
|
|
||||||
cfg.AbuseIPDBBlacklistRefresh != time.Hour ||
|
|
||||||
cfg.AbuseIPDBBanDuration != 30*24*time.Hour {
|
|
||||||
t.Errorf(
|
|
||||||
"by default, the threshold %d, the refresh %s and the ban %s, "+
|
|
||||||
"want 90, 1h and 720h",
|
|
||||||
cfg.AbuseIPDBThreshold,
|
|
||||||
cfg.AbuseIPDBBlacklistRefresh,
|
|
||||||
cfg.AbuseIPDBBanDuration,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg = wafFromEnvironment(t, environment{
|
|
||||||
abuseIPDBThreshold: "25", abuseIPDBBlacklistRefresh: "1440",
|
|
||||||
abuseIPDBBanDays: "1",
|
|
||||||
})
|
|
||||||
if cfg.AbuseIPDBThreshold != 25 ||
|
|
||||||
cfg.AbuseIPDBBlacklistRefresh != 24*time.Hour ||
|
|
||||||
cfg.AbuseIPDBBanDuration != 24*time.Hour {
|
|
||||||
t.Errorf(
|
|
||||||
"set, the threshold %d, the refresh %s and the ban %s, "+
|
|
||||||
"want 25, 24h and 24h",
|
|
||||||
cfg.AbuseIPDBThreshold,
|
|
||||||
cfg.AbuseIPDBBlacklistRefresh,
|
|
||||||
cfg.AbuseIPDBBanDuration,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestInvalidAbuseIPDBBlacklistSettingStopsTheStartSayingWhatIsWrong(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
notThreshold = " is not a whole number from 25 to 100, the scores " +
|
|
||||||
"AbuseIPDB's blacklist can be cut at"
|
|
||||||
notWhole = " is not a whole number above zero, such as 30, that fits " +
|
|
||||||
"a duration"
|
|
||||||
)
|
|
||||||
|
|
||||||
for _, tc := range []struct{ name, value, want string }{
|
|
||||||
{abuseIPDBThreshold, "24", `"24"` + notThreshold},
|
|
||||||
{abuseIPDBThreshold, "101", `"101"` + notThreshold},
|
|
||||||
{abuseIPDBThreshold, off, `"off"` + notThreshold},
|
|
||||||
{abuseIPDBBlacklistRefresh, "0", `"0"` + notWhole},
|
|
||||||
{abuseIPDBBlacklistRefresh, "1h", `"1h"` + notWhole},
|
|
||||||
{abuseIPDBBanDays, "-1", `"-1"` + notWhole},
|
|
||||||
{abuseIPDBBanDays, "200000", `"200000"` + notWhole},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name+"="+tc.value, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, err := config.WAFFromEnvironment(
|
|
||||||
environment{tc.name: tc.value}.viper(),
|
|
||||||
)
|
|
||||||
|
|
||||||
want := tc.name + ": " + tc.want
|
|
||||||
if err == nil || err.Error() != want {
|
|
||||||
t.Errorf("error %v, want %s", err, want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestInvalidAbuseIPDBSettingStopsTheStartSayingWhatIsWrong(
|
func TestInvalidAbuseIPDBSettingStopsTheStartSayingWhatIsWrong(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
@@ -3024,9 +2951,6 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
|
|||||||
abuseIPDBKey: "",
|
abuseIPDBKey: "",
|
||||||
abuseIPDBMinScore: "75",
|
abuseIPDBMinScore: "75",
|
||||||
abuseIPDBDailyBudget: "900",
|
abuseIPDBDailyBudget: "900",
|
||||||
abuseIPDBThreshold: "90",
|
|
||||||
abuseIPDBBlacklistRefresh: "60",
|
|
||||||
abuseIPDBBanDays: "30",
|
|
||||||
dnsblAbuseIPDBAction: "limit:25",
|
dnsblAbuseIPDBAction: "limit:25",
|
||||||
ipReputationCacheTTL: defaultIPReputationCacheTTL,
|
ipReputationCacheTTL: defaultIPReputationCacheTTL,
|
||||||
dnsblAbuseIPDBTimeout: "2000",
|
dnsblAbuseIPDBTimeout: "2000",
|
||||||
@@ -3050,7 +2974,7 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
|
|||||||
logLevel: "info",
|
logLevel: "info",
|
||||||
ruleFilesDir: "/etc/sww/rules.d",
|
ruleFilesDir: "/etc/sww/rules.d",
|
||||||
ruleFilesEnabled: "true",
|
ruleFilesEnabled: "true",
|
||||||
coreRuleSetMode: string(config.CoreRuleSetModeBlock),
|
coreRuleSetMode: config.CoreRuleSetModeBlock,
|
||||||
coreRuleSetParanoiaLevel: "1",
|
coreRuleSetParanoiaLevel: "1",
|
||||||
coreRuleSetScoreThreshold: "5",
|
coreRuleSetScoreThreshold: "5",
|
||||||
coreRuleSetDisabledRules: defaultCoreRuleSetDisabledRules,
|
coreRuleSetDisabledRules: defaultCoreRuleSetDisabledRules,
|
||||||
|
|||||||
+37
-60
@@ -84,7 +84,12 @@ type Params struct {
|
|||||||
// one request to GeoJS is under way at a time, and it asks about every
|
// one request to GeoJS is under way at a time, and it asks about every
|
||||||
// client waiting, up to maxPerRequest. It is safe for concurrent use.
|
// client waiting, up to maxPerRequest. It is safe for concurrent use.
|
||||||
type GeoJS struct {
|
type GeoJS struct {
|
||||||
params Params
|
url string
|
||||||
|
answered func(Answer)
|
||||||
|
now func() time.Time
|
||||||
|
asked func()
|
||||||
|
failed func(err error, askingAgainIn time.Duration)
|
||||||
|
wentWithoutAnswer func()
|
||||||
// httpClient follows no redirect, so that visitors' addresses go to
|
// httpClient follows no redirect, so that visitors' addresses go to
|
||||||
// GeoJS alone: a redirect is a failure.
|
// GeoJS alone: a redirect is a failure.
|
||||||
httpClient *http.Client
|
httpClient *http.Client
|
||||||
@@ -94,13 +99,8 @@ type GeoJS struct {
|
|||||||
// waiting are the clients without an answer: those to ask GeoJS about,
|
// waiting are the clients without an answer: those to ask GeoJS about,
|
||||||
// and those it is being asked about.
|
// and those it is being asked about.
|
||||||
waiting map[netip.Prefix]*wait
|
waiting map[netip.Prefix]*wait
|
||||||
// asking is true while a request to GeoJS is under way, and endAsking
|
// asking is true while a request to GeoJS is under way.
|
||||||
// ends it. stopped is true once StopAsking has been called, and
|
asking bool
|
||||||
// askGoroutines waits for the goroutine that makes the requests.
|
|
||||||
asking bool
|
|
||||||
endAsking context.CancelFunc
|
|
||||||
stopped bool
|
|
||||||
askGoroutines sync.WaitGroup
|
|
||||||
// retryDelay is how long GeoJS is left alone after its last failure,
|
// retryDelay is how long GeoJS is left alone after its last failure,
|
||||||
// zero after an answer; retryAt is when it may be asked again.
|
// zero after an answer; retryAt is when it may be asked again.
|
||||||
retryDelay time.Duration
|
retryDelay time.Duration
|
||||||
@@ -140,7 +140,12 @@ func New(params Params) *GeoJS {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &GeoJS{
|
return &GeoJS{
|
||||||
params: params,
|
url: params.URL,
|
||||||
|
answered: params.Answered,
|
||||||
|
now: params.Now,
|
||||||
|
asked: params.Asked,
|
||||||
|
failed: params.Failed,
|
||||||
|
wentWithoutAnswer: params.WentWithoutAnswer,
|
||||||
httpClient: &http.Client{
|
httpClient: &http.Client{
|
||||||
CheckRedirect: func(*http.Request, []*http.Request) error {
|
CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||||
return http.ErrUseLastResponse
|
return http.ErrUseLastResponse
|
||||||
@@ -183,7 +188,7 @@ func (g *GeoJS) LookUp(
|
|||||||
|
|
||||||
answer, found := g.kept(client)
|
answer, found := g.kept(client)
|
||||||
if !found {
|
if !found {
|
||||||
g.params.WentWithoutAnswer()
|
g.wentWithoutAnswer()
|
||||||
}
|
}
|
||||||
|
|
||||||
w, waiting := g.waiting[client]
|
w, waiting := g.waiting[client]
|
||||||
@@ -205,23 +210,6 @@ func (g *GeoJS) LookUpWithoutWaiting(
|
|||||||
return answer
|
return answer
|
||||||
}
|
}
|
||||||
|
|
||||||
// StopAsking ends the request to GeoJS under way, and returns once the
|
|
||||||
// goroutine that made it has. A request it ends is no failure: Failed is not
|
|
||||||
// called, and GeoJS is not left alone after it. After StopAsking, GeoJS is
|
|
||||||
// asked nothing.
|
|
||||||
func (g *GeoJS) StopAsking() {
|
|
||||||
g.mu.Lock()
|
|
||||||
|
|
||||||
g.stopped = true
|
|
||||||
if g.asking {
|
|
||||||
g.endAsking()
|
|
||||||
}
|
|
||||||
|
|
||||||
g.mu.Unlock()
|
|
||||||
|
|
||||||
g.askGoroutines.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Kept returns client's answer, if one is kept, without asking GeoJS.
|
// Kept returns client's answer, if one is kept, without asking GeoJS.
|
||||||
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
|
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
|
||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
@@ -263,7 +251,7 @@ func (g *GeoJS) Load(answers []Answer) {
|
|||||||
|
|
||||||
g.answers.Purge()
|
g.answers.Purge()
|
||||||
|
|
||||||
now := g.params.Now()
|
now := g.now()
|
||||||
|
|
||||||
for _, answer := range answers {
|
for _, answer := range answers {
|
||||||
if now.Sub(answer.Answered) < keepFor {
|
if now.Sub(answer.Answered) < keepFor {
|
||||||
@@ -300,19 +288,18 @@ func (g *GeoJS) answerOrWait(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if w == nil {
|
if w == nil {
|
||||||
g.params.WentWithoutAnswer()
|
g.wentWithoutAnswer()
|
||||||
|
|
||||||
return Answer{}, nil // too many clients wait already
|
return Answer{}, nil // too many clients wait already
|
||||||
}
|
}
|
||||||
|
|
||||||
if !g.asking {
|
if !g.asking {
|
||||||
// GeoJS is left alone after a failure, or asked nothing after
|
// GeoJS is left alone after a failure, so no answer can come.
|
||||||
// StopAsking, so no answer can come.
|
|
||||||
w.late = true
|
w.late = true
|
||||||
}
|
}
|
||||||
|
|
||||||
if w.late {
|
if w.late {
|
||||||
g.params.WentWithoutAnswer()
|
g.wentWithoutAnswer()
|
||||||
|
|
||||||
return Answer{}, nil
|
return Answer{}, nil
|
||||||
}
|
}
|
||||||
@@ -323,7 +310,7 @@ func (g *GeoJS) answerOrWait(
|
|||||||
// kept returns client's answer, if GeoJS gave it less than keepFor ago,
|
// kept returns client's answer, if GeoJS gave it less than keepFor ago,
|
||||||
// and notes that it was used.
|
// and notes that it was used.
|
||||||
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
||||||
now := g.params.Now()
|
now := g.now()
|
||||||
|
|
||||||
kept, found := g.answers.Get(client)
|
kept, found := g.answers.Get(client)
|
||||||
if !found || now.Sub(kept.Answered) >= keepFor {
|
if !found || now.Sub(kept.Answered) >= keepFor {
|
||||||
@@ -336,28 +323,22 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ask starts asking GeoJS about the waiting clients, unless a request to
|
// ask starts asking GeoJS about the waiting clients, unless a request to
|
||||||
// it is under way, it is left alone after a failure, or StopAsking has
|
// it is under way or it is left alone after a failure. The requests to
|
||||||
// been called. The requests to GeoJS are for every client waiting, so they
|
// GeoJS are for every client waiting, so they go on when the caller whose
|
||||||
// go on when the caller whose ctx is given stops waiting, until StopAsking.
|
// ctx is given stops waiting.
|
||||||
func (g *GeoJS) ask(ctx context.Context) {
|
func (g *GeoJS) ask(ctx context.Context) {
|
||||||
if g.asking || g.stopped || g.params.Now().Before(g.retryAt) {
|
if g.asking || g.now().Before(g.retryAt) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
askCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
g.asking = true
|
||||||
g.asking, g.endAsking = true, end
|
|
||||||
|
|
||||||
g.askGoroutines.Go(func() {
|
go g.askAboutWaiting(context.WithoutCancel(ctx))
|
||||||
defer end()
|
|
||||||
|
|
||||||
g.askAboutWaiting(askCtx)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// askAboutWaiting asks GeoJS about the waiting clients, one request at a
|
// askAboutWaiting asks GeoJS about the waiting clients, one request at a
|
||||||
// time, until none is left, GeoJS fails, or StopAsking ends ctx. Each
|
// time, until none is left or GeoJS fails. Each answer kept is given to
|
||||||
// answer kept is given to Answered, outside the lock, since Answered takes
|
// Answered, outside the lock, since Answered takes locks of its own.
|
||||||
// locks of its own.
|
|
||||||
func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
||||||
for {
|
for {
|
||||||
clients := g.nextClients()
|
clients := g.nextClients()
|
||||||
@@ -366,15 +347,11 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
given, err := g.request(ctx, clients)
|
given, err := g.request(ctx, clients)
|
||||||
if err != nil && ctx.Err() != nil {
|
|
||||||
continue // ended by StopAsking, which is no failure
|
|
||||||
}
|
|
||||||
|
|
||||||
kept, answered := g.keep(clients, given, err)
|
kept, answered := g.keep(clients, given, err)
|
||||||
|
|
||||||
if g.params.Answered != nil {
|
if g.answered != nil {
|
||||||
for _, answer := range kept {
|
for _, answer := range kept {
|
||||||
g.params.Answered(answer)
|
g.answered(answer)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -385,13 +362,13 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// nextClients returns up to maxPerRequest of the waiting clients. When
|
// nextClients returns up to maxPerRequest of the waiting clients. When
|
||||||
// none is waiting, or StopAsking has been called, it returns none and
|
// none is waiting, it returns none and notes that no request to GeoJS is
|
||||||
// notes that no request to GeoJS is under way.
|
// under way.
|
||||||
func (g *GeoJS) nextClients() []netip.Prefix {
|
func (g *GeoJS) nextClients() []netip.Prefix {
|
||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
defer g.mu.Unlock()
|
defer g.mu.Unlock()
|
||||||
|
|
||||||
if len(g.waiting) == 0 || g.stopped {
|
if len(g.waiting) == 0 {
|
||||||
g.asking = false
|
g.asking = false
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -423,7 +400,7 @@ func (g *GeoJS) keep(
|
|||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
defer g.mu.Unlock()
|
defer g.mu.Unlock()
|
||||||
|
|
||||||
now := g.params.Now()
|
now := g.now()
|
||||||
kept := make([]Answer, 0, len(clients))
|
kept := make([]Answer, 0, len(clients))
|
||||||
leftOut := 0
|
leftOut := 0
|
||||||
|
|
||||||
@@ -467,7 +444,7 @@ func (g *GeoJS) keep(
|
|||||||
w.late = true
|
w.late = true
|
||||||
}
|
}
|
||||||
|
|
||||||
g.params.Failed(err, g.retryDelay)
|
g.failed(err, g.retryDelay)
|
||||||
|
|
||||||
return kept, false
|
return kept, false
|
||||||
}
|
}
|
||||||
@@ -497,7 +474,7 @@ func (g *GeoJS) request(
|
|||||||
req, err := http.NewRequestWithContext(
|
req, err := http.NewRequestWithContext(
|
||||||
ctx,
|
ctx,
|
||||||
http.MethodGet,
|
http.MethodGet,
|
||||||
g.params.URL,
|
g.url,
|
||||||
http.NoBody,
|
http.NoBody,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -506,7 +483,7 @@ func (g *GeoJS) request(
|
|||||||
|
|
||||||
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
|
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
|
||||||
|
|
||||||
g.params.Asked()
|
g.asked()
|
||||||
|
|
||||||
res, err := g.httpClient.Do(req)
|
res, err := g.httpClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -318,44 +318,6 @@ func TestCallerWhoseContextEndsEarlyDoesNotStopTheRequest(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestStopAskingEndsTheRequestUnderWayAsNoFailureAndAsksNoMore(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
service, _, g, reports := startWithReports()
|
|
||||||
clients := newClients()
|
|
||||||
|
|
||||||
service.set(hanging)
|
|
||||||
g.LookUpWithoutWaiting(t.Context(), clients())
|
|
||||||
waitForRequests(t, service, 1)
|
|
||||||
|
|
||||||
// The request is ended, not waited out, and is no failure.
|
|
||||||
began := time.Now()
|
|
||||||
|
|
||||||
g.StopAsking()
|
|
||||||
|
|
||||||
if waited := time.Since(began); waited != 0 {
|
|
||||||
t.Errorf("StopAsking returned after %s, want at once", waited)
|
|
||||||
}
|
|
||||||
|
|
||||||
reports.wantFailures(t)
|
|
||||||
|
|
||||||
// A new client is then not asked about, and its lookup does not
|
|
||||||
// wait.
|
|
||||||
service.set(answering)
|
|
||||||
wantCountry(t, g, clients(), "")
|
|
||||||
|
|
||||||
if waited := time.Since(began); waited != 0 {
|
|
||||||
t.Errorf("the lookup returned after %s, want at once", waited)
|
|
||||||
}
|
|
||||||
|
|
||||||
waitForRequests(t, service, 1)
|
|
||||||
reports.wantFailures(t)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRequestOver10SecondsIsAFailure(t *testing.T) {
|
func TestRequestOver10SecondsIsAFailure(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -185,7 +185,7 @@ func (m *Metrics) AddBansAndClients(
|
|||||||
ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time,
|
ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time,
|
||||||
) {
|
) {
|
||||||
for _, cause := range []string{
|
for _, cause := range []string{
|
||||||
bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseAbuseIPDB,
|
bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin,
|
||||||
} {
|
} {
|
||||||
m.registry.MustRegister(
|
m.registry.MustRegister(
|
||||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||||
|
|||||||
@@ -1,44 +0,0 @@
|
|||||||
// Package processlog builds the logger for the process's own messages,
|
|
||||||
// which it writes as JSON lines marked "type":"process". The request log
|
|
||||||
// is package requestlog's.
|
|
||||||
package processlog
|
|
||||||
|
|
||||||
import (
|
|
||||||
"io"
|
|
||||||
"log/slog"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Params are what New needs.
|
|
||||||
type Params struct {
|
|
||||||
// Output is where the lines are written.
|
|
||||||
Output io.Writer
|
|
||||||
// InstanceName, SWW_WAF_INSTANCE_NAME, is each line's instance.
|
|
||||||
InstanceName string
|
|
||||||
// Level, SWW_WAF_LOG_LEVEL, is the least severe level written.
|
|
||||||
Level slog.Level
|
|
||||||
}
|
|
||||||
|
|
||||||
// New returns the logger for the process's own messages: JSON lines on
|
|
||||||
// params.Output, marked "type":"process", with the time in the same form
|
|
||||||
// as a request line's. It writes only the messages at params.Level or more
|
|
||||||
// severe; the request lines requestlog.Write writes are never held back.
|
|
||||||
func New(params Params) *slog.Logger {
|
|
||||||
handler := slog.NewJSONHandler(params.Output, &slog.HandlerOptions{
|
|
||||||
Level: params.Level,
|
|
||||||
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
|
|
||||||
if attr.Key == slog.TimeKey && len(groups) == 0 {
|
|
||||||
return slog.String(
|
|
||||||
slog.TimeKey,
|
|
||||||
requestlog.FormatTime(attr.Value.Time()),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return attr
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
return slog.New(handler).
|
|
||||||
With("type", "process", "instance", params.InstanceName)
|
|
||||||
}
|
|
||||||
@@ -1,105 +0,0 @@
|
|||||||
package processlog_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"encoding/json"
|
|
||||||
"log/slog"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
|
||||||
)
|
|
||||||
|
|
||||||
// instanceName is the instance the tests' loggers are built with.
|
|
||||||
const instanceName = "host1/blog"
|
|
||||||
|
|
||||||
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
processlog.New(processlog.Params{
|
|
||||||
Output: &out,
|
|
||||||
InstanceName: instanceName,
|
|
||||||
Level: slog.LevelInfo,
|
|
||||||
}).
|
|
||||||
Info("starting",
|
|
||||||
"version", "v1")
|
|
||||||
|
|
||||||
var fields map[string]any
|
|
||||||
|
|
||||||
err := json.Unmarshal(out.Bytes(), &fields)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("decode %q: %v", out.String(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if fields["type"] != "process" ||
|
|
||||||
fields["instance"] != "host1/blog" ||
|
|
||||||
fields["msg"] != "starting" ||
|
|
||||||
fields["level"] != "INFO" ||
|
|
||||||
fields["version"] != "v1" {
|
|
||||||
t.Errorf("process line %v", fields)
|
|
||||||
}
|
|
||||||
|
|
||||||
timeText, _ := fields["time"].(string)
|
|
||||||
|
|
||||||
logged, err := time.Parse(time.RFC3339, timeText)
|
|
||||||
if err != nil || !strings.HasSuffix(timeText, "Z") ||
|
|
||||||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
|
|
||||||
time.Since(logged) > time.Minute {
|
|
||||||
t.Errorf(
|
|
||||||
"process line time %q, want now in UTC with milliseconds",
|
|
||||||
timeText,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
levels := []slog.Level{
|
|
||||||
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, level := range levels {
|
|
||||||
t.Run(level.String(), func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
processLog := processlog.New(processlog.Params{
|
|
||||||
Output: &out,
|
|
||||||
InstanceName: instanceName,
|
|
||||||
Level: level,
|
|
||||||
})
|
|
||||||
for _, at := range levels {
|
|
||||||
processLog.Log(t.Context(), at, "message")
|
|
||||||
}
|
|
||||||
|
|
||||||
var got, want []string
|
|
||||||
|
|
||||||
for line := range strings.Lines(out.String()) {
|
|
||||||
var fields struct {
|
|
||||||
Level string `json:"level"`
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(line), &fields)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("decode %q: %v", line, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
got = append(got, fields.Level)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, written := range levels[i:] {
|
|
||||||
want = append(want, written.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(got, want) {
|
|
||||||
t.Errorf("lines at %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
func TestHistoryKeepsEveryRequest(t *testing.T) {
|
func TestHistoryKeepsEveryRequest(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -57,7 +57,7 @@ func TestHistoryKeepsEveryRequest(t *testing.T) {
|
|||||||
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
other := netip.MustParsePrefix("198.51.100.7/32")
|
other := netip.MustParsePrefix("198.51.100.7/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -100,10 +100,7 @@ func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
|||||||
func TestResetKeepsTheHistory(t *testing.T) {
|
func TestResetKeepsTheHistory(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerMinute: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -126,7 +123,7 @@ func TestResetKeepsTheHistory(t *testing.T) {
|
|||||||
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
|
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
|
|
||||||
for client, requests := range map[string]int{
|
for client, requests := range map[string]int{
|
||||||
"198.51.100.9/32": 2,
|
"198.51.100.9/32": 2,
|
||||||
|
|||||||
@@ -156,28 +156,16 @@ type Request struct {
|
|||||||
BlockedByCoreRuleSet bool
|
BlockedByCoreRuleSet bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// Params are what New needs.
|
// New returns a Limiter for limits, with no client counted yet, whose
|
||||||
type Params struct {
|
// table holds at most maxClients clients (SWW_WAF_MAX_TRACKED_CLIENTS). Past
|
||||||
// Limits are what each client's requests and bytes are counted against.
|
// it, the least recently seen client is dropped, with its history, and
|
||||||
Limits Limits
|
// starts afresh if it comes back.
|
||||||
// MaxClients is the most clients the table holds,
|
func New(limits Limits, maxClients int) *Limiter {
|
||||||
// SWW_WAF_MAX_TRACKED_CLIENTS. Past it, the least recently seen client
|
clients, err := simplelru.NewLRU[netip.Prefix, *Client](maxClients, nil)
|
||||||
// is dropped, with its history, and starts afresh if it comes back.
|
|
||||||
MaxClients int
|
|
||||||
}
|
|
||||||
|
|
||||||
// New returns a Limiter with no client counted yet.
|
|
||||||
func New(params Params) *Limiter {
|
|
||||||
clients, err := simplelru.NewLRU[netip.Prefix, *Client](
|
|
||||||
params.MaxClients,
|
|
||||||
nil,
|
|
||||||
)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err) // NewLRU fails only for a size below one
|
panic(err) // NewLRU fails only for a size below one
|
||||||
}
|
}
|
||||||
|
|
||||||
limits := params.Limits
|
|
||||||
|
|
||||||
return &Limiter{
|
return &Limiter{
|
||||||
windows: [3]window{
|
windows: [3]window{
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -33,10 +33,10 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
|
|||||||
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
now := midnight()
|
now := midnight()
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
|
ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
|
|
||||||
if limiter.OverARateLimit(client, now) {
|
if limiter.OverARateLimit(client, now) {
|
||||||
t.Error("a client never seen is over a rate limit")
|
t.Error("a client never seen is over a rate limit")
|
||||||
@@ -72,7 +72,7 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
|
|||||||
wantCount(t, limiter, client, now, minute)
|
wantCount(t, limiter, client, now, minute)
|
||||||
|
|
||||||
// A limit that is off is never gone over.
|
// A limit that is off is never gone over.
|
||||||
off := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
off := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
off.Count(client, now, whole)
|
off.Count(client, now, whole)
|
||||||
|
|
||||||
if off.OverARateLimit(client, now) {
|
if off.OverARateLimit(client, now) {
|
||||||
@@ -95,10 +95,7 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
|
|||||||
t.Run(tc.window, func(t *testing.T) {
|
t.Run(tc.window, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(tc.limits, tableSize)
|
||||||
Limits: tc.limits,
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
quarter := tc.length / 4
|
quarter := tc.length / 4
|
||||||
@@ -129,10 +126,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
|
|||||||
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerMinute: limit, PerHour: limit},
|
ratelimit.Limits{PerMinute: limit, PerHour: limit},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -162,13 +159,10 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
|||||||
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
|
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{
|
ratelimit.Limits{PerMinute: 5, BytesPerDay: math.MaxInt64},
|
||||||
PerMinute: 5,
|
tableSize,
|
||||||
BytesPerDay: math.MaxInt64,
|
)
|
||||||
},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -209,10 +203,10 @@ func TestZeroPercentIsAZeroAllowanceAndALimitOffStaysOff(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// Only the hour has limits: the minute's and the day's are off.
|
// Only the hour has limits: the minute's and the day's are off.
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
|
ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -259,10 +253,7 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
|
|||||||
t.Run(tc.window, func(t *testing.T) {
|
t.Run(tc.window, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(tc.limits, tableSize)
|
||||||
Limits: tc.limits,
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
// 600 bytes are within the limit, 600 more over it.
|
// 600 bytes are within the limit, 600 more over it.
|
||||||
@@ -296,10 +287,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
|
|||||||
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
|
ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
other := netip.MustParsePrefix("203.0.113.10/32")
|
other := netip.MustParsePrefix("203.0.113.10/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -323,7 +314,7 @@ func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
|||||||
func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -349,10 +340,7 @@ func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
|||||||
func TestResetSetsTheBytesBackToZero(t *testing.T) {
|
func TestResetSetsTheBytesBackToZero(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{BytesPerDay: 1000}, tableSize)
|
||||||
Limits: ratelimit.Limits{BytesPerDay: 1000},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -368,7 +356,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -423,7 +411,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
|
|||||||
func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -450,10 +438,10 @@ func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
|||||||
func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit},
|
ratelimit.Limits{PerMinute: limit, PerDay: limit},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -475,10 +463,7 @@ func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
|||||||
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerHour: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -497,10 +482,10 @@ func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
|||||||
func TestRefusedRequestsCount(t *testing.T) {
|
func TestRefusedRequestsCount(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
|
ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
refused := netip.MustParsePrefix("203.0.113.9/32")
|
refused := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
within := netip.MustParsePrefix("203.0.113.10/32")
|
within := netip.MustParsePrefix("203.0.113.10/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -533,10 +518,7 @@ func TestRefusedRequestsCount(t *testing.T) {
|
|||||||
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerMinute: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -552,10 +534,7 @@ func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
|||||||
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
|
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerHour: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -583,10 +562,7 @@ func TestKeepsAtMostMaxClientsDroppingTheLeastRecentlySeen(t *testing.T) {
|
|||||||
|
|
||||||
const maxClients = 3
|
const maxClients = 3
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{PerMinute: 1}, maxClients)
|
||||||
Limits: ratelimit.Limits{PerMinute: 1},
|
|
||||||
MaxClients: maxClients,
|
|
||||||
})
|
|
||||||
now := midnight()
|
now := midnight()
|
||||||
|
|
||||||
clients := make([]netip.Prefix, maxClients+1)
|
clients := make([]netip.Prefix, maxClients+1)
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ func TestSnapshotListsTheClientsByAddress(t *testing.T) {
|
|||||||
"2001:db8::/64",
|
"2001:db8::/64",
|
||||||
}
|
}
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
for _, i := range []int{2, 3, 0, 1} {
|
for _, i := range []int{2, 3, 0, 1} {
|
||||||
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
|
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
|
||||||
}
|
}
|
||||||
@@ -52,10 +52,7 @@ func TestLoadedCountsCarryOn(t *testing.T) {
|
|||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
before := ratelimit.New(ratelimit.Params{
|
before := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerHour: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
for range limit {
|
for range limit {
|
||||||
wantCount(t, before, client, start, "")
|
wantCount(t, before, client, start, "")
|
||||||
}
|
}
|
||||||
@@ -63,10 +60,7 @@ func TestLoadedCountsCarryOn(t *testing.T) {
|
|||||||
// Loaded into a new limiter, as across a restart, the client has no
|
// Loaded into a new limiter, as across a restart, the client has no
|
||||||
// fresh allowance.
|
// fresh allowance.
|
||||||
later := start.Add(time.Minute)
|
later := start.Add(time.Minute)
|
||||||
after := ratelimit.New(ratelimit.Params{
|
after := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerHour: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
after.Load(before.Snapshot(), later)
|
after.Load(before.Snapshot(), later)
|
||||||
wantCount(t, after, client, later, hour)
|
wantCount(t, after, client, later, hour)
|
||||||
}
|
}
|
||||||
@@ -77,7 +71,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
|
|||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
limiter.Count(client, start, whole)
|
limiter.Count(client, start, whole)
|
||||||
limiter.CountBytes(client, start, 5, whole)
|
limiter.CountBytes(client, start, 5, whole)
|
||||||
limiter.CountRefusal(client, start, limit)
|
limiter.CountRefusal(client, start, limit)
|
||||||
@@ -86,7 +80,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
|
|||||||
loaded := func(now time.Time) ratelimit.Client {
|
loaded := func(now time.Time) ratelimit.Client {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
after := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
after := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
after.Load(limiter.Snapshot(), now)
|
after.Load(limiter.Snapshot(), now)
|
||||||
|
|
||||||
return after.Snapshot()[0]
|
return after.Snapshot()[0]
|
||||||
@@ -148,7 +142,7 @@ func TestLoadDropsTheLeastRecentlySeenFirst(t *testing.T) {
|
|||||||
addr = addr.Next()
|
addr = addr.Next()
|
||||||
}
|
}
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: maxClients})
|
limiter := ratelimit.New(ratelimit.Limits{}, maxClients)
|
||||||
limiter.Load(clients, midnight())
|
limiter.Load(clients, midnight())
|
||||||
|
|
||||||
got := limiter.Snapshot()
|
got := limiter.Snapshot()
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ import (
|
|||||||
"net/netip"
|
"net/netip"
|
||||||
"net/url"
|
"net/url"
|
||||||
"slices"
|
"slices"
|
||||||
"strconv"
|
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -24,9 +23,6 @@ const (
|
|||||||
// AbuseIPDBURL is where clients are checked: the check endpoint of
|
// AbuseIPDBURL is where clients are checked: the check endpoint of
|
||||||
// AbuseIPDB's API.
|
// AbuseIPDB's API.
|
||||||
AbuseIPDBURL = "https://api.abuseipdb.com/api/v2/check"
|
AbuseIPDBURL = "https://api.abuseipdb.com/api/v2/check"
|
||||||
// AbuseIPDBBlacklistURL is where AbuseIPDB's blacklist is fetched from:
|
|
||||||
// the blacklist endpoint of AbuseIPDB's API.
|
|
||||||
AbuseIPDBBlacklistURL = "https://api.abuseipdb.com/api/v2/blacklist"
|
|
||||||
// AbuseIPDBSource is how the request log, the alerts and the metrics
|
// AbuseIPDBSource is how the request log, the alerts and the metrics
|
||||||
// name AbuseIPDB.
|
// name AbuseIPDB.
|
||||||
AbuseIPDBSource = "abuseipdb"
|
AbuseIPDBSource = "abuseipdb"
|
||||||
@@ -36,8 +32,6 @@ const (
|
|||||||
|
|
||||||
var (
|
var (
|
||||||
errNoScore = errors.New("the answer gives no abuseConfidenceScore")
|
errNoScore = errors.New("the answer gives no abuseConfidenceScore")
|
||||||
errNoBlacklist = errors.New("the answer gives no data")
|
|
||||||
errNotAddress = errors.New("is not an address")
|
|
||||||
errBudgetUsedUp = errors.New(
|
errBudgetUsedUp = errors.New(
|
||||||
"checks spent; none is made until the day ends at 00:00 UTC")
|
"checks spent; none is made until the day ends at 00:00 UTC")
|
||||||
)
|
)
|
||||||
@@ -59,13 +53,6 @@ type ChecksSpent struct {
|
|||||||
Spent int
|
Spent int
|
||||||
}
|
}
|
||||||
|
|
||||||
// BlacklistEntry is a client on AbuseIPDB's blacklist: its address and its
|
|
||||||
// abuse confidence score.
|
|
||||||
type BlacklistEntry struct {
|
|
||||||
Addr netip.Addr
|
|
||||||
Score int64
|
|
||||||
}
|
|
||||||
|
|
||||||
// AbuseIPDBParams are what NewAbuseIPDB needs.
|
// AbuseIPDBParams are what NewAbuseIPDB needs.
|
||||||
type AbuseIPDBParams struct {
|
type AbuseIPDBParams struct {
|
||||||
// URL is where clients are checked, normally AbuseIPDBURL, with Key,
|
// URL is where clients are checked, normally AbuseIPDBURL, with Key,
|
||||||
@@ -82,19 +69,10 @@ type AbuseIPDBParams struct {
|
|||||||
// take (SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS).
|
// take (SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS).
|
||||||
CacheTTL time.Duration
|
CacheTTL time.Duration
|
||||||
Timeout time.Duration
|
Timeout time.Duration
|
||||||
// BlacklistURL is where the blacklist is fetched from, normally
|
|
||||||
// AbuseIPDBBlacklistURL, with the clients scored Threshold or more
|
|
||||||
// (SWW_WAF_ABUSEIPDB_THRESHOLD), every BlacklistRefresh
|
|
||||||
// (SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS). Blacklisted
|
|
||||||
// receives the clients on each blacklist fetched.
|
|
||||||
BlacklistURL string
|
|
||||||
Threshold int64
|
|
||||||
BlacklistRefresh time.Duration
|
|
||||||
Blacklisted func([]BlacklistEntry)
|
|
||||||
// Now tells the time, normally time.Now in UTC.
|
// Now tells the time, normally time.Now in UTC.
|
||||||
Now func() time.Time
|
Now func() time.Time
|
||||||
// ProcessLog receives each check or fetch of the blacklist that fails,
|
// ProcessLog receives each check that fails, and why, and the day's
|
||||||
// and why, and the day's budget used up.
|
// budget used up.
|
||||||
ProcessLog *slog.Logger
|
ProcessLog *slog.Logger
|
||||||
// Alerts receive a source_failure alert for each.
|
// Alerts receive a source_failure alert for each.
|
||||||
Alerts *alerts.Queue
|
Alerts *alerts.Queue
|
||||||
@@ -110,12 +88,8 @@ type AbuseIPDB struct {
|
|||||||
// scores are by client. Each is added as it is fetched and never moved
|
// scores are by client. Each is added as it is fetched and never moved
|
||||||
// up, so that the one fetched longest ago is the first dropped.
|
// up, so that the one fetched longest ago is the first dropped.
|
||||||
scores *simplelru.LRU[netip.Prefix, Score]
|
scores *simplelru.LRU[netip.Prefix, Score]
|
||||||
// checking are the clients whose check is under way, each with the
|
// checking are the clients whose check is under way.
|
||||||
// function that ends it. stopped is true once StopChecking has been
|
checking map[netip.Prefix]bool
|
||||||
// called, and checkGoroutines waits for the goroutine of each check.
|
|
||||||
checking map[netip.Prefix]context.CancelFunc
|
|
||||||
stopped bool
|
|
||||||
checkGoroutines sync.WaitGroup
|
|
||||||
// day is the day, in UTC, of the checks spent counts.
|
// day is the day, in UTC, of the checks spent counts.
|
||||||
day civil.Date
|
day civil.Date
|
||||||
spent int
|
spent int
|
||||||
@@ -138,7 +112,7 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
|
|||||||
params: params,
|
params: params,
|
||||||
httpClient: &http.Client{},
|
httpClient: &http.Client{},
|
||||||
scores: scores,
|
scores: scores,
|
||||||
checking: map[netip.Prefix]context.CancelFunc{},
|
checking: map[netip.Prefix]bool{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -148,11 +122,11 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
|
|||||||
// addresses its request comes from. A client without one is checked in
|
// addresses its request comes from. A client without one is checked in
|
||||||
// the background, by addr, the address its request came from, if
|
// the background, by addr, the address its request came from, if
|
||||||
// offender, if it has committed an offence, unless its check is under
|
// offender, if it has committed an offence, unless its check is under
|
||||||
// way, a check failed less than failureDelay ago, the day's checks have
|
// way, a check failed less than failureDelay ago, or the day's checks
|
||||||
// used up DailyBudget, or StopChecking has been called; Hit never waits
|
// have used up DailyBudget; Hit never waits for a check. The check that
|
||||||
// for a check. The check that uses the budget up is logged and raised as a
|
// uses the budget up is logged and raised as a source_failure alert. ctx
|
||||||
// source_failure alert. ctx is the context of the client's request, and a
|
// is the context of the client's request, and a check goes on after the
|
||||||
// check goes on after the request ends, until StopChecking.
|
// request ends.
|
||||||
func (a *AbuseIPDB) Hit(
|
func (a *AbuseIPDB) Hit(
|
||||||
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
|
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
|
||||||
) (int64, bool) {
|
) (int64, bool) {
|
||||||
@@ -171,21 +145,14 @@ func (a *AbuseIPDB) Hit(
|
|||||||
a.day, a.spent = today, 0
|
a.day, a.spent = today, 0
|
||||||
}
|
}
|
||||||
|
|
||||||
_, checking := a.checking[client]
|
check := offender && !a.checking[client] && !now.Before(a.retryAt) &&
|
||||||
|
a.spent < a.params.DailyBudget
|
||||||
check := offender && !checking && !a.stopped &&
|
|
||||||
!now.Before(a.retryAt) && a.spent < a.params.DailyBudget
|
|
||||||
if check {
|
if check {
|
||||||
checkCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
a.checking[client] = true
|
||||||
a.checking[client] = end
|
|
||||||
a.checks++
|
a.checks++
|
||||||
a.spent++
|
a.spent++
|
||||||
|
|
||||||
a.checkGoroutines.Go(func() {
|
go a.check(context.WithoutCancel(ctx), client, addr)
|
||||||
defer end()
|
|
||||||
|
|
||||||
a.check(checkCtx, client, addr)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
usedUp := check && a.spent == a.params.DailyBudget
|
usedUp := check && a.spent == a.params.DailyBudget
|
||||||
@@ -200,24 +167,6 @@ func (a *AbuseIPDB) Hit(
|
|||||||
return 0, false
|
return 0, false
|
||||||
}
|
}
|
||||||
|
|
||||||
// StopChecking ends the checks under way, and returns once their goroutines
|
|
||||||
// have. A check it ends gives no score and is no failure: it is neither
|
|
||||||
// counted, logged nor raised as an alert, and checks are not left alone
|
|
||||||
// after it. The ended check stays spent from the day's budget. After
|
|
||||||
// StopChecking, Hit checks no client.
|
|
||||||
func (a *AbuseIPDB) StopChecking() {
|
|
||||||
a.mu.Lock()
|
|
||||||
|
|
||||||
a.stopped = true
|
|
||||||
for _, end := range a.checking {
|
|
||||||
end()
|
|
||||||
}
|
|
||||||
|
|
||||||
a.mu.Unlock()
|
|
||||||
|
|
||||||
a.checkGoroutines.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Checked returns how many checks were made.
|
// Checked returns how many checks were made.
|
||||||
func (a *AbuseIPDB) Checked() int {
|
func (a *AbuseIPDB) Checked() int {
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
@@ -307,45 +256,10 @@ func (a *AbuseIPDB) LoadChecksSpent(checks ChecksSpent) {
|
|||||||
a.day, a.spent = checks.Day, checks.Spent
|
a.day, a.spent = checks.Day, checks.Spent
|
||||||
}
|
}
|
||||||
|
|
||||||
// RunBlacklist fetches AbuseIPDB's blacklist at once, and again every
|
|
||||||
// BlacklistRefresh, until ctx is done, and gives the clients on each copy
|
|
||||||
// fetched to Blacklisted. While Key is unset it does nothing. A fetch that
|
|
||||||
// fails is logged and raised as a source_failure alert, unless it was cut
|
|
||||||
// off as ctx ended.
|
|
||||||
func (a *AbuseIPDB) RunBlacklist(ctx context.Context) {
|
|
||||||
if a.params.Key == "" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
for {
|
|
||||||
listed, err := a.fetchBlacklist(ctx)
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case ctx.Err() != nil:
|
|
||||||
return
|
|
||||||
case err != nil:
|
|
||||||
a.alert("fetching AbuseIPDB's blacklist failed", err)
|
|
||||||
default:
|
|
||||||
a.params.Blacklisted(listed)
|
|
||||||
}
|
|
||||||
|
|
||||||
timer := time.NewTimer(a.params.BlacklistRefresh)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
timer.Stop()
|
|
||||||
|
|
||||||
return
|
|
||||||
case <-timer.C:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
|
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
|
||||||
// the score as client's, and notes the check as no longer under way. A
|
// the score as client's, and notes the check as no longer under way. A
|
||||||
// check that fails gives no score: it is counted, logged and raised as a
|
// check that fails gives no score: it is counted, logged and raised as a
|
||||||
// source_failure alert, and no client is checked for failureDelay. A check
|
// source_failure alert, and no client is checked for failureDelay.
|
||||||
// ended as ctx ends, by StopChecking, is no failure.
|
|
||||||
func (a *AbuseIPDB) check(
|
func (a *AbuseIPDB) check(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
client netip.Prefix,
|
client netip.Prefix,
|
||||||
@@ -353,26 +267,24 @@ func (a *AbuseIPDB) check(
|
|||||||
) {
|
) {
|
||||||
score, err := a.ask(ctx, addr)
|
score, err := a.ask(ctx, addr)
|
||||||
now := a.params.Now()
|
now := a.params.Now()
|
||||||
isFailure := err != nil && ctx.Err() == nil
|
|
||||||
|
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
|
|
||||||
delete(a.checking, client)
|
delete(a.checking, client)
|
||||||
|
|
||||||
switch {
|
if err == nil {
|
||||||
case err == nil:
|
|
||||||
a.scores.Add(
|
a.scores.Add(
|
||||||
client,
|
client,
|
||||||
Score{Client: client, Score: score, Fetched: now},
|
Score{Client: client, Score: score, Fetched: now},
|
||||||
)
|
)
|
||||||
case isFailure:
|
} else {
|
||||||
a.failures++
|
a.failures++
|
||||||
a.retryAt = now.Add(failureDelay)
|
a.retryAt = now.Add(failureDelay)
|
||||||
}
|
}
|
||||||
|
|
||||||
a.mu.Unlock()
|
a.mu.Unlock()
|
||||||
|
|
||||||
if isFailure {
|
if err != nil {
|
||||||
a.alert("checking a client with AbuseIPDB failed", err)
|
a.alert("checking a client with AbuseIPDB failed", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -432,94 +344,6 @@ func (a *AbuseIPDB) ask(
|
|||||||
return *answer.Data.AbuseConfidenceScore, nil
|
return *answer.Data.AbuseConfidenceScore, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// fetchBlacklist fetches the clients on AbuseIPDB's blacklist scored
|
|
||||||
// Threshold or more, sending the key in the header Key. An answer other
|
|
||||||
// than 200, one longer than maxListBytes or without the list, an entry
|
|
||||||
// whose address does not read, and none within fetchTimeout, fail.
|
|
||||||
func (a *AbuseIPDB) fetchBlacklist(
|
|
||||||
ctx context.Context,
|
|
||||||
) ([]BlacklistEntry, error) {
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, fetchTimeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
query := url.Values{
|
|
||||||
"confidenceMinimum": {strconv.FormatInt(a.params.Threshold, 10)},
|
|
||||||
}
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet,
|
|
||||||
a.params.BlacklistURL+"?"+query.Encode(), http.NoBody)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("make the request: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
req.Header.Set("Key", a.params.Key)
|
|
||||||
req.Header.Set("Accept", "application/json")
|
|
||||||
|
|
||||||
res, err := a.httpClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("fetch the blacklist: %w", errors.Unwrap(err))
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
_ = res.Body.Close()
|
|
||||||
}()
|
|
||||||
|
|
||||||
if res.StatusCode != http.StatusOK {
|
|
||||||
return nil, fmt.Errorf("%w %s", errStatus, res.Status)
|
|
||||||
}
|
|
||||||
|
|
||||||
body, err := io.ReadAll(io.LimitReader(res.Body, maxListBytes+1))
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("read the blacklist: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(body) > maxListBytes {
|
|
||||||
return nil, errTooLong
|
|
||||||
}
|
|
||||||
|
|
||||||
return parseBlacklist(body)
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseBlacklist reads body, AbuseIPDB's answer with its blacklist.
|
|
||||||
func parseBlacklist(body []byte) ([]BlacklistEntry, error) {
|
|
||||||
var answer struct {
|
|
||||||
Data *[]struct {
|
|
||||||
IPAddress string `json:"ipAddress"`
|
|
||||||
AbuseConfidenceScore int64 `json:"abuseConfidenceScore"`
|
|
||||||
} `json:"data"`
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal(body, &answer)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("read the blacklist: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if answer.Data == nil {
|
|
||||||
return nil, errNoBlacklist
|
|
||||||
}
|
|
||||||
|
|
||||||
listed := make([]BlacklistEntry, 0, len(*answer.Data))
|
|
||||||
|
|
||||||
for i, entry := range *answer.Data {
|
|
||||||
addr, err := netip.ParseAddr(entry.IPAddress)
|
|
||||||
if err != nil || addr.Zone() != "" {
|
|
||||||
// The error leaves out what the entry holds, as it does a
|
|
||||||
// client's address.
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"entry %d's ipAddress %w",
|
|
||||||
i+1,
|
|
||||||
errNotAddress,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
listed = append(listed, BlacklistEntry{
|
|
||||||
Addr: addr.Unmap(), Score: entry.AbuseConfidenceScore,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return listed, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// alert raises a source_failure alert from AbuseIPDB with reason and err,
|
// alert raises a source_failure alert from AbuseIPDB with reason and err,
|
||||||
// and logs them.
|
// and logs them.
|
||||||
func (a *AbuseIPDB) alert(reason string, err error) {
|
func (a *AbuseIPDB) alert(reason string, err error) {
|
||||||
|
|||||||
@@ -394,43 +394,6 @@ func TestCheckNotAnsweredWithinTheTimeoutFailsAndHitNeverWaits(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestStopCheckingEndsTheCheckUnderWayAsNoFailureAndChecksNoMore(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
queue := newQueue()
|
|
||||||
p := abuseIPDBParams()
|
|
||||||
p.Alerts = queue
|
|
||||||
abuseIPDB := &abuseIPDBStandIn{hanging: true}
|
|
||||||
checker := newAbuseIPDB(abuseIPDB, p)
|
|
||||||
|
|
||||||
wantScore(t, checker, suspect, true, 0, false)
|
|
||||||
synctest.Wait()
|
|
||||||
|
|
||||||
// The check is ended, not waited out, and is no failure.
|
|
||||||
began := time.Now()
|
|
||||||
|
|
||||||
checker.StopChecking()
|
|
||||||
|
|
||||||
if waited := time.Since(began); waited != 0 {
|
|
||||||
t.Errorf("StopChecking returned after %s, want at once", waited)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantFailures(t, checker, 0)
|
|
||||||
|
|
||||||
if got := waiting(queue); len(got) != 0 {
|
|
||||||
t.Errorf("alerts waiting %+v, want none", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// An offender without a score is then not checked.
|
|
||||||
wantScore(t, checker, other, true, 0, false)
|
|
||||||
synctest.Wait()
|
|
||||||
wantChecked(t, abuseIPDB, suspect)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
|
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -1,252 +0,0 @@
|
|||||||
package reputation_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/netip"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"testing/synctest"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The tests of the blacklist run in synctest bubbles, as those of the
|
|
||||||
// checks do, with AbuseIPDB a stand-in reached without the network.
|
|
||||||
|
|
||||||
// blacklistURL is where the tests fetch the blacklist from.
|
|
||||||
const blacklistURL = "https://abuseipdb.example/api/v2/blacklist"
|
|
||||||
|
|
||||||
func TestBlacklistIsFetchedAtOnceAndEveryRefresh(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
abuseIPDB := &blacklistStandIn{body: `{"meta":{},"data":[` +
|
|
||||||
`{"ipAddress":"203.0.113.9","abuseConfidenceScore":100},` +
|
|
||||||
`{"ipAddress":"::ffff:198.51.100.7","abuseConfidenceScore":95},` +
|
|
||||||
`{"ipAddress":"2001:db8::9","abuseConfidenceScore":90}]}`}
|
|
||||||
got := runBlacklist(t, abuseIPDB, key, newQueue())
|
|
||||||
|
|
||||||
want := []reputation.BlacklistEntry{
|
|
||||||
{Addr: netip.MustParseAddr("203.0.113.9"), Score: 100},
|
|
||||||
{Addr: netip.MustParseAddr("198.51.100.7"), Score: 95},
|
|
||||||
{Addr: netip.MustParseAddr("2001:db8::9"), Score: 90},
|
|
||||||
}
|
|
||||||
|
|
||||||
synctest.Wait()
|
|
||||||
got.want(t, want)
|
|
||||||
|
|
||||||
time.Sleep(time.Hour - time.Nanosecond)
|
|
||||||
synctest.Wait()
|
|
||||||
got.want(t, want)
|
|
||||||
|
|
||||||
time.Sleep(time.Nanosecond)
|
|
||||||
synctest.Wait()
|
|
||||||
got.want(t, want, want)
|
|
||||||
|
|
||||||
// Each fetch asks for the clients scored 90 or more, with the key
|
|
||||||
// in the header Key alone.
|
|
||||||
abuseIPDB.mu.Lock()
|
|
||||||
defer abuseIPDB.mu.Unlock()
|
|
||||||
|
|
||||||
for _, req := range abuseIPDB.asked {
|
|
||||||
if req.URL.String() != blacklistURL+"?confidenceMinimum=90" ||
|
|
||||||
req.Header.Get("Key") != key {
|
|
||||||
t.Errorf(
|
|
||||||
"asked %s with the key %q, want %s?confidenceMinimum=90 "+
|
|
||||||
"with %q",
|
|
||||||
req.URL,
|
|
||||||
req.Header.Get("Key"),
|
|
||||||
blacklistURL,
|
|
||||||
key,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFailedBlacklistFetchIsAlertedAndTriedAgainAtTheNextRefresh(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, tc := range []struct{ name, body, error string }{
|
|
||||||
{
|
|
||||||
"a refusal", "",
|
|
||||||
"the server answered 429 Too Many Requests",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"an answer without the list", `{"errors":[]}`,
|
|
||||||
"the answer gives no data",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"an entry that is not an address",
|
|
||||||
`{"data":[{"ipAddress":"203.0.113.9"},{"ipAddress":"example"}]}`,
|
|
||||||
"entry 2's ipAddress is not an address",
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
abuseIPDB := &blacklistStandIn{body: tc.body}
|
|
||||||
if tc.body == "" {
|
|
||||||
abuseIPDB.status = http.StatusTooManyRequests
|
|
||||||
}
|
|
||||||
|
|
||||||
queue := newQueue()
|
|
||||||
got := runBlacklist(t, abuseIPDB, key, queue)
|
|
||||||
|
|
||||||
synctest.Wait()
|
|
||||||
|
|
||||||
got.want(t)
|
|
||||||
|
|
||||||
alerted := waiting(queue)
|
|
||||||
if len(alerted) != 1 ||
|
|
||||||
alerted[0].Event != alerts.EventSourceFailure ||
|
|
||||||
alerted[0].Reason != "fetching AbuseIPDB's blacklist failed" ||
|
|
||||||
alerted[0].Detail["source"] != reputation.AbuseIPDBSource ||
|
|
||||||
alerted[0].Detail["error"] != tc.error {
|
|
||||||
t.Errorf(
|
|
||||||
"alerts %+v, want the failure %q",
|
|
||||||
alerted,
|
|
||||||
tc.error,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
abuseIPDB.answer(`{"data":[]}`)
|
|
||||||
time.Sleep(time.Hour)
|
|
||||||
synctest.Wait()
|
|
||||||
got.want(t, []reputation.BlacklistEntry{})
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBlacklistIsNotFetchedWithoutAKey(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
abuseIPDB := &blacklistStandIn{body: `{"data":[]}`}
|
|
||||||
got := runBlacklist(t, abuseIPDB, "", newQueue())
|
|
||||||
|
|
||||||
time.Sleep(time.Hour)
|
|
||||||
synctest.Wait()
|
|
||||||
got.want(t)
|
|
||||||
|
|
||||||
if len(abuseIPDB.asked) != 0 {
|
|
||||||
t.Errorf("fetched the blacklist %d times, want none",
|
|
||||||
len(abuseIPDB.asked))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// blacklistStandIn is a stand-in for AbuseIPDB that answers each fetch of
|
|
||||||
// the blacklist with status, 200 when it is zero, and body, and keeps the
|
|
||||||
// requests.
|
|
||||||
type blacklistStandIn struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
status int
|
|
||||||
body string
|
|
||||||
asked []*http.Request
|
|
||||||
}
|
|
||||||
|
|
||||||
// RoundTrip has the stand-in answer req, in place of the network.
|
|
||||||
func (s *blacklistStandIn) RoundTrip(
|
|
||||||
req *http.Request,
|
|
||||||
) (*http.Response, error) {
|
|
||||||
s.mu.Lock()
|
|
||||||
s.asked = append(s.asked, req)
|
|
||||||
status, body := s.status, s.body
|
|
||||||
s.mu.Unlock()
|
|
||||||
|
|
||||||
if status == 0 {
|
|
||||||
status = http.StatusOK
|
|
||||||
}
|
|
||||||
|
|
||||||
return &http.Response{
|
|
||||||
StatusCode: status,
|
|
||||||
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
|
|
||||||
Header: http.Header{},
|
|
||||||
Body: io.NopCloser(strings.NewReader(body)),
|
|
||||||
Request: req,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// answer has the stand-in answer each fetch with 200 and body.
|
|
||||||
func (s *blacklistStandIn) answer(body string) {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
|
|
||||||
s.status, s.body = 0, body
|
|
||||||
}
|
|
||||||
|
|
||||||
// blacklisted holds the clients on each blacklist given to Blacklisted.
|
|
||||||
type blacklisted struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
fetched [][]reputation.BlacklistEntry
|
|
||||||
}
|
|
||||||
|
|
||||||
// want checks the clients on each blacklist given so far.
|
|
||||||
func (b *blacklisted) want(
|
|
||||||
t *testing.T,
|
|
||||||
want ...[]reputation.BlacklistEntry,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
b.mu.Lock()
|
|
||||||
defer b.mu.Unlock()
|
|
||||||
|
|
||||||
if !slices.EqualFunc(b.fetched, want, slices.Equal) {
|
|
||||||
t.Errorf("blacklists %v, want %v", b.fetched, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// runBlacklist runs RunBlacklist, until the test ends, with the key
|
|
||||||
// accountKey, the threshold 90 and a refresh of an hour, fetching from
|
|
||||||
// abuseIPDB and raising alerts into queue, and returns what it gives
|
|
||||||
// Blacklisted.
|
|
||||||
func runBlacklist(
|
|
||||||
t *testing.T, abuseIPDB *blacklistStandIn, accountKey string,
|
|
||||||
queue *alerts.Queue,
|
|
||||||
) *blacklisted {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
got := &blacklisted{}
|
|
||||||
p := abuseIPDBParams()
|
|
||||||
p.Key = accountKey
|
|
||||||
p.BlacklistURL = blacklistURL
|
|
||||||
p.Threshold = 90
|
|
||||||
p.BlacklistRefresh = time.Hour
|
|
||||||
p.Alerts = queue
|
|
||||||
p.Blacklisted = func(listed []reputation.BlacklistEntry) {
|
|
||||||
got.mu.Lock()
|
|
||||||
defer got.mu.Unlock()
|
|
||||||
|
|
||||||
got.fetched = append(got.fetched, listed)
|
|
||||||
}
|
|
||||||
|
|
||||||
checker := reputation.NewAbuseIPDB(p)
|
|
||||||
checker.SetTransport(abuseIPDB)
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(t.Context())
|
|
||||||
done := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
checker.RunBlacklist(ctx)
|
|
||||||
close(done)
|
|
||||||
}()
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
cancel()
|
|
||||||
<-done
|
|
||||||
})
|
|
||||||
|
|
||||||
return got
|
|
||||||
}
|
|
||||||
@@ -82,12 +82,8 @@ type DNSBL struct {
|
|||||||
// verdicts are by query. Each is added as it is fetched and never moved
|
// verdicts are by query. Each is added as it is fetched and never moved
|
||||||
// up, so that the one fetched longest ago is the first dropped.
|
// up, so that the one fetched longest ago is the first dropped.
|
||||||
verdicts *simplelru.LRU[query, Verdict]
|
verdicts *simplelru.LRU[query, Verdict]
|
||||||
// asking are the queries under way, each with the function that ends
|
// asking are the queries under way.
|
||||||
// it. stopped is true once StopAsking has been called, and
|
asking map[query]bool
|
||||||
// queryGoroutines waits for the goroutine of each query.
|
|
||||||
asking map[query]context.CancelFunc
|
|
||||||
stopped bool
|
|
||||||
queryGoroutines sync.WaitGroup
|
|
||||||
// queries and failures count, by zone, the queries made and those that
|
// queries and failures count, by zone, the queries made and those that
|
||||||
// failed, and retryAt is when a zone whose last query failed may be
|
// failed, and retryAt is when a zone whose last query failed may be
|
||||||
// asked again.
|
// asked again.
|
||||||
@@ -124,7 +120,7 @@ func NewDNSBL(params DNSBLParams) *DNSBL {
|
|||||||
params: params,
|
params: params,
|
||||||
resolver: resolver,
|
resolver: resolver,
|
||||||
verdicts: verdicts,
|
verdicts: verdicts,
|
||||||
asking: map[query]context.CancelFunc{},
|
asking: map[query]bool{},
|
||||||
queries: map[string]int{},
|
queries: map[string]int{},
|
||||||
failures: map[string]int{},
|
failures: map[string]int{},
|
||||||
retryAt: map[string]time.Time{},
|
retryAt: map[string]time.Time{},
|
||||||
@@ -142,10 +138,9 @@ func (d *DNSBL) Zones() []string {
|
|||||||
// log, the alerts and the metrics. A verdict is used until CacheTTL has
|
// log, the alerts and the metrics. A verdict is used until CacheTTL has
|
||||||
// passed since it was fetched. Each zone without one is asked about addr
|
// passed since it was fetched. Each zone without one is asked about addr
|
||||||
// in the background, unless a query about addr to it is under way, the
|
// in the background, unless a query about addr to it is under way, the
|
||||||
// zone is left alone after a failure, maxQueries are under way, or
|
// zone is left alone after a failure, or maxQueries are under way;
|
||||||
// StopAsking has been called; ListedBy never waits for a query. ctx is the
|
// ListedBy never waits for a query. ctx is the context of the client's
|
||||||
// context of the client's request, and a query goes on after the request
|
// request, and a query goes on after the request ends.
|
||||||
// ends, until StopAsking.
|
|
||||||
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
defer d.mu.Unlock()
|
defer d.mu.Unlock()
|
||||||
@@ -158,47 +153,23 @@ func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
|||||||
q := query{zone: zone, client: addr}
|
q := query{zone: zone, client: addr}
|
||||||
|
|
||||||
kept, found := d.verdicts.Peek(q)
|
kept, found := d.verdicts.Peek(q)
|
||||||
_, asking := d.asking[q]
|
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
|
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
|
||||||
if kept.Listed {
|
if kept.Listed {
|
||||||
listedBy = append(listedBy, config.MaskZoneKey(zone))
|
listedBy = append(listedBy, config.MaskZoneKey(zone))
|
||||||
}
|
}
|
||||||
case !asking && !d.stopped && !now.Before(d.retryAt[zone]) &&
|
case !d.asking[q] && !now.Before(d.retryAt[zone]) && len(d.asking) < maxQueries:
|
||||||
len(d.asking) < maxQueries:
|
d.asking[q] = true
|
||||||
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
|
||||||
d.asking[q] = end
|
|
||||||
d.queries[zone]++
|
d.queries[zone]++
|
||||||
|
|
||||||
d.queryGoroutines.Go(func() {
|
go d.ask(context.WithoutCancel(ctx), q)
|
||||||
defer end()
|
|
||||||
|
|
||||||
d.ask(queryCtx, q)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return listedBy
|
return listedBy
|
||||||
}
|
}
|
||||||
|
|
||||||
// StopAsking ends the queries under way, and returns once their
|
|
||||||
// goroutines have. A query it ends gives no verdict and is no failure: it
|
|
||||||
// is neither counted, logged nor raised as an alert, and the zone is not
|
|
||||||
// left alone after it. After StopAsking, ListedBy asks no zone.
|
|
||||||
func (d *DNSBL) StopAsking() {
|
|
||||||
d.mu.Lock()
|
|
||||||
|
|
||||||
d.stopped = true
|
|
||||||
for _, end := range d.asking {
|
|
||||||
end()
|
|
||||||
}
|
|
||||||
|
|
||||||
d.mu.Unlock()
|
|
||||||
|
|
||||||
d.queryGoroutines.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Queries returns how many queries were made to zone.
|
// Queries returns how many queries were made to zone.
|
||||||
func (d *DNSBL) Queries(zone string) int {
|
func (d *DNSBL) Queries(zone string) int {
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
@@ -267,29 +238,27 @@ func (d *DNSBL) Load(verdicts []Verdict) {
|
|||||||
// query as no longer under way. A query that fails gives no verdict: it
|
// query as no longer under way. A query that fails gives no verdict: it
|
||||||
// is counted, logged and raised as a source_failure alert, which show the
|
// is counted, logged and raised as a source_failure alert, which show the
|
||||||
// zone with its key masked, and the zone is not asked again for
|
// zone with its key masked, and the zone is not asked again for
|
||||||
// failureDelay. A query ended as ctx ends, by StopAsking, is no failure.
|
// failureDelay.
|
||||||
func (d *DNSBL) ask(ctx context.Context, q query) {
|
func (d *DNSBL) ask(ctx context.Context, q query) {
|
||||||
listed, err := d.lookUp(ctx, q)
|
listed, err := d.lookUp(ctx, q)
|
||||||
now := d.params.Now()
|
now := d.params.Now()
|
||||||
isFailure := err != nil && ctx.Err() == nil
|
|
||||||
|
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
|
|
||||||
delete(d.asking, q)
|
delete(d.asking, q)
|
||||||
|
|
||||||
switch {
|
if err == nil {
|
||||||
case err == nil:
|
|
||||||
d.verdicts.Add(q, Verdict{
|
d.verdicts.Add(q, Verdict{
|
||||||
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
|
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
|
||||||
})
|
})
|
||||||
case isFailure:
|
} else {
|
||||||
d.failures[q.zone]++
|
d.failures[q.zone]++
|
||||||
d.retryAt[q.zone] = now.Add(failureDelay)
|
d.retryAt[q.zone] = now.Add(failureDelay)
|
||||||
}
|
}
|
||||||
|
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
if isFailure {
|
if err != nil {
|
||||||
const failed = "asking a DNSBL zone failed"
|
const failed = "asking a DNSBL zone failed"
|
||||||
|
|
||||||
shown := config.MaskZoneKey(q.zone)
|
shown := config.MaskZoneKey(q.zone)
|
||||||
|
|||||||
@@ -210,45 +210,6 @@ func TestQueryNotAnsweredWithinTheTimeoutFails(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
|
||||||
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
queue := newQueue()
|
|
||||||
p := dnsblParams(zone)
|
|
||||||
p.Alerts = queue
|
|
||||||
resolver := &resolverStandIn{hanging: true}
|
|
||||||
dnsbl := newDNSBL(resolver, p)
|
|
||||||
|
|
||||||
wantZones(t, dnsbl, listed)
|
|
||||||
synctest.Wait()
|
|
||||||
|
|
||||||
// The query is ended, not waited out, and is no failure.
|
|
||||||
began := time.Now()
|
|
||||||
|
|
||||||
dnsbl.StopAsking()
|
|
||||||
|
|
||||||
if waited := time.Since(began); waited != 0 {
|
|
||||||
t.Errorf("StopAsking returned after %s, want at once", waited)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantQueries(t, dnsbl, 1, 0)
|
|
||||||
|
|
||||||
if got := waiting(queue); len(got) != 0 {
|
|
||||||
t.Errorf("alerts waiting %+v, want none", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A client without a verdict is then asked about in no zone.
|
|
||||||
wantZones(t, dnsbl, unlisted)
|
|
||||||
synctest.Wait()
|
|
||||||
wantQueries(t, dnsbl, 1, 0)
|
|
||||||
wantAsked(t, resolver, listedName)
|
|
||||||
waitForTheResolver()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||||
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(
|
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(
|
||||||
|
|||||||
@@ -7,8 +7,7 @@
|
|||||||
// SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks
|
// SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks
|
||||||
// clients with AbuseIPDB, and keeps their scores and the checks spent
|
// clients with AbuseIPDB, and keeps their scores and the checks spent
|
||||||
// today. The state package writes all of these to files and reads them
|
// today. The state package writes all of these to files and reads them
|
||||||
// from there, so that a restart keeps them too. Last, it fetches
|
// from there, so that a restart keeps them too.
|
||||||
// AbuseIPDB's blacklist, which it does not keep.
|
|
||||||
package reputation
|
package reputation
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
// Package requestlog writes the request log sww prints on stdout: JSON
|
// Package requestlog writes the lines sww prints on stdout: one
|
||||||
// lines marked "type":"request", at most two from waf and a short one
|
// JSON object per request, two from waf and a short one from rproxy,
|
||||||
// from rproxy for each request. Package processlog writes the process's
|
// marked "type":"request", and the process's own messages as JSON lines marked
|
||||||
// own messages.
|
// "type":"process".
|
||||||
package requestlog
|
package requestlog
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"log/slog"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
@@ -299,3 +300,31 @@ func Milliseconds(d time.Duration) float64 {
|
|||||||
time.Millisecond/time.Microsecond,
|
time.Millisecond/time.Microsecond,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewProcessLogger returns the logger for the process's own messages:
|
||||||
|
// JSON lines on w, marked "type":"process", with the time in the same form
|
||||||
|
// as a request line's, and instanceName, SWW_WAF_INSTANCE_NAME, as instance.
|
||||||
|
// It writes only the messages at level, SWW_WAF_LOG_LEVEL, or more severe;
|
||||||
|
// the request lines Write writes are never held back.
|
||||||
|
func NewProcessLogger(
|
||||||
|
w io.Writer,
|
||||||
|
instanceName string,
|
||||||
|
level slog.Level,
|
||||||
|
) *slog.Logger {
|
||||||
|
handler := slog.NewJSONHandler(w, &slog.HandlerOptions{
|
||||||
|
Level: level,
|
||||||
|
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
|
||||||
|
if attr.Key == slog.TimeKey && len(groups) == 0 {
|
||||||
|
return slog.String(
|
||||||
|
slog.TimeKey,
|
||||||
|
FormatTime(attr.Value.Time()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return attr
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
return slog.New(handler).
|
||||||
|
With("type", "process", "instance", instanceName)
|
||||||
|
}
|
||||||
@@ -3,6 +3,8 @@ package requestlog_test
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"log/slog"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -132,3 +134,88 @@ func TestWriteRProxyLineWritesOneJSONLineWithOnlyItsFields(t *testing.T) {
|
|||||||
t.Errorf("wrote %s, want %s", out.String(), want)
|
t.Errorf("wrote %s, want %s", out.String(), want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
|
||||||
|
requestlog.NewProcessLogger(&out, "host1/blog", slog.LevelInfo).
|
||||||
|
Info("starting",
|
||||||
|
"version", "v1")
|
||||||
|
|
||||||
|
var fields map[string]any
|
||||||
|
|
||||||
|
err := json.Unmarshal(out.Bytes(), &fields)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decode %q: %v", out.String(), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if fields["type"] != "process" ||
|
||||||
|
fields["instance"] != "host1/blog" ||
|
||||||
|
fields["msg"] != "starting" ||
|
||||||
|
fields["level"] != "INFO" ||
|
||||||
|
fields["version"] != "v1" {
|
||||||
|
t.Errorf("process line %v", fields)
|
||||||
|
}
|
||||||
|
|
||||||
|
timeText, _ := fields["time"].(string)
|
||||||
|
|
||||||
|
logged, err := time.Parse(time.RFC3339, timeText)
|
||||||
|
if err != nil || !strings.HasSuffix(timeText, "Z") ||
|
||||||
|
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
|
||||||
|
time.Since(logged) > time.Minute {
|
||||||
|
t.Errorf(
|
||||||
|
"process line time %q, want now in UTC with milliseconds",
|
||||||
|
timeText,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
levels := []slog.Level{
|
||||||
|
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, level := range levels {
|
||||||
|
t.Run(level.String(), func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
|
||||||
|
processLog := requestlog.NewProcessLogger(
|
||||||
|
&out,
|
||||||
|
"host1/blog",
|
||||||
|
level,
|
||||||
|
)
|
||||||
|
for _, at := range levels {
|
||||||
|
processLog.Log(t.Context(), at, "message")
|
||||||
|
}
|
||||||
|
|
||||||
|
var got, want []string
|
||||||
|
|
||||||
|
for line := range strings.Lines(out.String()) {
|
||||||
|
var fields struct {
|
||||||
|
Level string `json:"level"`
|
||||||
|
}
|
||||||
|
|
||||||
|
err := json.Unmarshal([]byte(line), &fields)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decode %q: %v", line, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got = append(got, fields.Level)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, written := range levels[i:] {
|
||||||
|
want = append(want, written.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
if !slices.Equal(got, want) {
|
||||||
|
t.Errorf("lines at %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -99,12 +99,8 @@ type Client struct {
|
|||||||
// answers are by client. Each is added as it comes and never moved up,
|
// answers are by client. Each is added as it comes and never moved up,
|
||||||
// so that the one that came longest ago is the first dropped.
|
// so that the one that came longest ago is the first dropped.
|
||||||
answers *simplelru.LRU[netip.Prefix, Answer]
|
answers *simplelru.LRU[netip.Prefix, Answer]
|
||||||
// asking are the clients whose query is under way, each with the
|
// asking are the clients whose query is under way.
|
||||||
// function that ends it. stopped is true once StopAsking has been
|
asking map[netip.Prefix]bool
|
||||||
// called, and queryGoroutines waits for the goroutine of each query.
|
|
||||||
asking map[netip.Prefix]context.CancelFunc
|
|
||||||
stopped bool
|
|
||||||
queryGoroutines sync.WaitGroup
|
|
||||||
// retryAt is when routewatch may be asked again after a query failed.
|
// retryAt is when routewatch may be asked again after a query failed.
|
||||||
retryAt time.Time
|
retryAt time.Time
|
||||||
}
|
}
|
||||||
@@ -120,7 +116,7 @@ func New(params Params) *Client {
|
|||||||
params: params,
|
params: params,
|
||||||
httpClient: &http.Client{},
|
httpClient: &http.Client{},
|
||||||
answers: answers,
|
answers: answers,
|
||||||
asking: map[netip.Prefix]context.CancelFunc{},
|
asking: map[netip.Prefix]bool{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -128,9 +124,8 @@ func New(params Params) *Client {
|
|||||||
// IPv6 group, or why it has no route. It never waits for a query. A client
|
// IPv6 group, or why it has no route. It never waits for a query. A client
|
||||||
// without an answer is asked about in the background, by addr, the address its
|
// without an answer is asked about in the background, by addr, the address its
|
||||||
// request came from, unless its query is under way, a query failed less than
|
// request came from, unless its query is under way, a query failed less than
|
||||||
// failureKept ago, maxQueries are under way, or StopAsking has been called. ctx
|
// failureKept ago, or maxQueries are under way. ctx is the context of the
|
||||||
// is the context of the client's request, and a query goes on after the request
|
// client's request, and a query goes on after the request ends.
|
||||||
// ends, until StopAsking.
|
|
||||||
func (c *Client) Route(
|
func (c *Client) Route(
|
||||||
ctx context.Context, client netip.Prefix, addr netip.Addr,
|
ctx context.Context, client netip.Prefix, addr netip.Addr,
|
||||||
) (Route, Missing) {
|
) (Route, Missing) {
|
||||||
@@ -152,38 +147,15 @@ func (c *Client) Route(
|
|||||||
return Route{}, RequestFailed
|
return Route{}, RequestFailed
|
||||||
}
|
}
|
||||||
|
|
||||||
_, asking := c.asking[client]
|
if !c.asking[client] && len(c.asking) < maxQueries {
|
||||||
if !asking && !c.stopped && len(c.asking) < maxQueries {
|
c.asking[client] = true
|
||||||
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
|
||||||
c.asking[client] = end
|
|
||||||
|
|
||||||
c.queryGoroutines.Go(func() {
|
go c.ask(context.WithoutCancel(ctx), client, addr)
|
||||||
defer end()
|
|
||||||
|
|
||||||
c.ask(queryCtx, client, addr)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return Route{}, NoAnswerYet
|
return Route{}, NoAnswerYet
|
||||||
}
|
}
|
||||||
|
|
||||||
// StopAsking ends the queries under way, and returns once their goroutines
|
|
||||||
// have. A query it ends gives no answer and is no failure: it is not logged,
|
|
||||||
// and routewatch is not left alone after it. After StopAsking, Route asks
|
|
||||||
// routewatch nothing.
|
|
||||||
func (c *Client) StopAsking() {
|
|
||||||
c.mu.Lock()
|
|
||||||
|
|
||||||
c.stopped = true
|
|
||||||
for _, end := range c.asking {
|
|
||||||
end()
|
|
||||||
}
|
|
||||||
|
|
||||||
c.mu.Unlock()
|
|
||||||
|
|
||||||
c.queryGoroutines.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Snapshot returns every answer still in use, sorted by client, as
|
// Snapshot returns every answer still in use, sorted by client, as
|
||||||
// routewatch.json lists them.
|
// routewatch.json lists them.
|
||||||
func (c *Client) Snapshot() []Answer {
|
func (c *Client) Snapshot() []Answer {
|
||||||
@@ -233,8 +205,7 @@ func (c *Client) Load(answers []Answer) {
|
|||||||
// ask asks routewatch about addr, keeps the answer as client's, and notes the
|
// ask asks routewatch about addr, keeps the answer as client's, and notes the
|
||||||
// query as no longer under way. A query that fails gives no answer, and
|
// query as no longer under way. A query that fails gives no answer, and
|
||||||
// routewatch is not asked again for failureKept. Only the first failure of
|
// routewatch is not asked again for failureKept. Only the first failure of
|
||||||
// those under way is logged. A query ended as ctx ends, by StopAsking, is no
|
// those under way is logged.
|
||||||
// failure.
|
|
||||||
func (c *Client) ask(
|
func (c *Client) ask(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
client netip.Prefix,
|
client netip.Prefix,
|
||||||
@@ -242,20 +213,18 @@ func (c *Client) ask(
|
|||||||
) {
|
) {
|
||||||
route, err := c.query(ctx, addr)
|
route, err := c.query(ctx, addr)
|
||||||
now := c.params.Now()
|
now := c.params.Now()
|
||||||
isFailure := err != nil && ctx.Err() == nil
|
|
||||||
|
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
|
|
||||||
delete(c.asking, client)
|
delete(c.asking, client)
|
||||||
|
|
||||||
firstFailure := isFailure && !now.Before(c.retryAt)
|
firstFailure := err != nil && !now.Before(c.retryAt)
|
||||||
|
|
||||||
switch {
|
if err == nil {
|
||||||
case err == nil:
|
|
||||||
c.answers.Add(client, Answer{
|
c.answers.Add(client, Answer{
|
||||||
Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
|
Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
|
||||||
})
|
})
|
||||||
case isFailure:
|
} else {
|
||||||
c.retryAt = now.Add(failureKept)
|
c.retryAt = now.Add(failureKept)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -361,54 +361,6 @@ func TestHangingRoutewatchNeverDelaysAndFailsTheQuery(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// In a synctest bubble, as TestHangingRoutewatchNeverDelaysAndFailsTheQuery
|
|
||||||
// says.
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
f := &fixture{t: t, log: &syncBuffer{}}
|
|
||||||
f.client = routewatch.New(routewatch.Params{
|
|
||||||
URL: silentURL, Now: time.Now,
|
|
||||||
ProcessLog: slog.New(slog.NewTextHandler(f.log, nil)),
|
|
||||||
})
|
|
||||||
|
|
||||||
f.route()
|
|
||||||
synctest.Wait()
|
|
||||||
|
|
||||||
// The query is ended, not waited out.
|
|
||||||
began := time.Now()
|
|
||||||
|
|
||||||
f.client.StopAsking()
|
|
||||||
|
|
||||||
if waited := time.Since(began); waited != 0 {
|
|
||||||
t.Errorf("StopAsking returned after %s, want at once", waited)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A query started now would fail within waitLimit. None is, and the
|
|
||||||
// one ended is no failure: nothing is logged, and routewatch is not
|
|
||||||
// left alone.
|
|
||||||
for range 2 {
|
|
||||||
if _, missing := f.route(); missing != routewatch.NoAnswerYet {
|
|
||||||
t.Errorf(
|
|
||||||
"Route gave %q, want %q",
|
|
||||||
missing,
|
|
||||||
routewatch.NoAnswerYet,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
time.Sleep(waitLimit)
|
|
||||||
synctest.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
if f.log.String() != "" {
|
|
||||||
t.Errorf("process log %q, want nothing", f.log.String())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
|
func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
@@ -51,11 +51,11 @@ func TestServerHasTheFixedLimits(t *testing.T) {
|
|||||||
server := rproxy.New(rproxy.Params{
|
server := rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: io.Discard,
|
RequestLog: io.Discard,
|
||||||
ProcessLog: processlog.New(processlog.Params{
|
ProcessLog: requestlog.NewProcessLogger(
|
||||||
Output: io.Discard,
|
io.Discard,
|
||||||
InstanceName: "test",
|
"test",
|
||||||
Level: slog.LevelInfo,
|
slog.LevelInfo,
|
||||||
}),
|
),
|
||||||
})
|
})
|
||||||
|
|
||||||
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
|
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
@@ -324,8 +324,7 @@ func greet(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
// startRProxy starts rproxy with waf at backend and the app at appURL, and
|
// startRProxy starts rproxy with waf at backend and the app at appURL, and
|
||||||
// the settings in env on top of the defaults. It returns where rproxy listens
|
// the settings in env on top of the defaults. It returns where rproxy listens
|
||||||
// and what it writes. SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS is a minute unless env
|
// and what it writes.
|
||||||
// sets it, so that a busy host cannot make rproxy drop a question or a report.
|
|
||||||
func startRProxy(
|
func startRProxy(
|
||||||
t *testing.T, backend, appURL string, env map[string]string,
|
t *testing.T, backend, appURL string, env map[string]string,
|
||||||
) (string, *output) {
|
) (string, *output) {
|
||||||
@@ -358,7 +357,6 @@ func startServerLogging(
|
|||||||
|
|
||||||
settings := map[string]string{
|
settings := map[string]string{
|
||||||
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
|
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
|
||||||
wafBackendTimeout: "60000",
|
|
||||||
}
|
}
|
||||||
maps.Copy(settings, env)
|
maps.Copy(settings, env)
|
||||||
|
|
||||||
@@ -370,11 +368,11 @@ func startServerLogging(
|
|||||||
server := rproxy.New(rproxy.Params{
|
server := rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: requestLog,
|
RequestLog: requestLog,
|
||||||
ProcessLog: processlog.New(processlog.Params{
|
ProcessLog: requestlog.NewProcessLogger(
|
||||||
Output: processLog,
|
processLog,
|
||||||
InstanceName: "test",
|
"test",
|
||||||
Level: slog.LevelInfo,
|
slog.LevelInfo,
|
||||||
}),
|
),
|
||||||
})
|
})
|
||||||
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(
|
listener, err := (&net.ListenConfig{}).Listen(
|
||||||
|
|||||||
@@ -104,7 +104,7 @@ var (
|
|||||||
errVersion = errors.New("unknown version")
|
errVersion = errors.New("unknown version")
|
||||||
// errMissing is for an entry without a field it needs.
|
// errMissing is for an entry without a field it needs.
|
||||||
errMissing = errors.New("has no")
|
errMissing = errors.New("has no")
|
||||||
errCause = errors.New("is not limit, attack, admin or abuseipdb")
|
errCause = errors.New("is not limit, attack or admin")
|
||||||
errExpires = errors.New("is not after its start")
|
errExpires = errors.New("is not after its start")
|
||||||
errDestination = errors.New("is not webhook, slack or ntfy")
|
errDestination = errors.New("is not webhook, slack or ntfy")
|
||||||
errScope = errors.New(
|
errScope = errors.New(
|
||||||
@@ -638,8 +638,6 @@ func (f *Files) takeInState(
|
|||||||
f.params.Ledger.Load(held)
|
f.params.Ledger.Load(held)
|
||||||
}
|
}
|
||||||
|
|
||||||
f.params.Ledger.ForgetEndedAbuseIPDBBans(f.params.Now())
|
|
||||||
|
|
||||||
entries = len(held)
|
entries = len(held)
|
||||||
case clientsJSON:
|
case clientsJSON:
|
||||||
var file clientsFile
|
var file clientsFile
|
||||||
@@ -969,7 +967,7 @@ func (e BanEntry) ban() bans.Ban {
|
|||||||
// each expires is read again as written. An expires not after the start
|
// each expires is read again as written. An expires not after the start
|
||||||
// is refused too, since the netblock's next ban for a broken limit, three
|
// is refused too, since the netblock's next ban for a broken limit, three
|
||||||
// times as long as this one, would then refuse nothing; and so is a cause
|
// times as long as this one, would then refuse nothing; and so is a cause
|
||||||
// other than limit, attack, admin or abuseipdb, most likely misspelt.
|
// other than limit, attack or admin, most likely misspelt.
|
||||||
func (f *bansFile) check(data []byte) error {
|
func (f *bansFile) check(data []byte) error {
|
||||||
var written struct {
|
var written struct {
|
||||||
Bans []struct {
|
Bans []struct {
|
||||||
@@ -993,8 +991,7 @@ func (f *bansFile) check(data []byte) error {
|
|||||||
case entry.Expires != nil && !entry.Expires.After(entry.Start):
|
case entry.Expires != nil && !entry.Expires.After(entry.Start):
|
||||||
return fmt.Errorf("entry %d's expires %w", i+1, errExpires)
|
return fmt.Errorf("entry %d's expires %w", i+1, errExpires)
|
||||||
case entry.Cause != "" && entry.Cause != bans.CauseLimit &&
|
case entry.Cause != "" && entry.Cause != bans.CauseLimit &&
|
||||||
entry.Cause != bans.CauseAttack && entry.Cause != bans.CauseAdmin &&
|
entry.Cause != bans.CauseAttack && entry.Cause != bans.CauseAdmin:
|
||||||
entry.Cause != bans.CauseAbuseIPDB:
|
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"entry %d's cause %q %w",
|
"entry %d's cause %q %w",
|
||||||
i+1,
|
i+1,
|
||||||
|
|||||||
@@ -114,8 +114,7 @@ const permanentBansJSON = `{
|
|||||||
"earlier_bans": {
|
"earlier_bans": {
|
||||||
"limit": 3,
|
"limit": 3,
|
||||||
"attack": 1,
|
"attack": 1,
|
||||||
"admin": 1,
|
"admin": 1
|
||||||
"abuseipdb": 0
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1191,7 +1190,7 @@ func TestBanWithAnotherCauseStopsTheStart(t *testing.T) {
|
|||||||
`"expires": "2026-10-06T04:00:00Z", "cause": "limit"}, `+
|
`"expires": "2026-10-06T04:00:00Z", "cause": "limit"}, `+
|
||||||
`{"netblock": "203.0.113.12/32", "start": "2026-10-06T00:00:00Z", `+
|
`{"netblock": "203.0.113.12/32", "start": "2026-10-06T00:00:00Z", `+
|
||||||
`"expires": null, "cause": "atack"}]}`,
|
`"expires": null, "cause": "atack"}]}`,
|
||||||
`: entry 4's cause "atack" is not limit, attack, admin or abuseipdb`)
|
`: entry 4's cause "atack" is not limit, attack or admin`)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBanThatDoesNotEndAfterItsStartStopsTheStart(t *testing.T) {
|
func TestBanThatDoesNotEndAfterItsStartStopsTheStart(t *testing.T) {
|
||||||
@@ -2091,39 +2090,6 @@ func TestLiftedBanReadAtTheStart(t *testing.T) {
|
|||||||
wantLiftedBanKept(t, load(t, params), dir, params.Ledger)
|
wantLiftedBanKept(t, load(t, params), dir, params.Ledger)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEndedAbuseIPDBBanForgottenWhenBansJSONIsReadAtTheStart(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
ended := netip.MustParsePrefix("203.0.113.9/32")
|
|
||||||
lasting := netip.MustParsePrefix("203.0.113.10/32")
|
|
||||||
limited := netip.MustParsePrefix("203.0.113.11/32")
|
|
||||||
|
|
||||||
// At midnight, the start's time, the first ban has ended and the second
|
|
||||||
// lasts. The third has ended too, but is for a limit.
|
|
||||||
dir := t.TempDir()
|
|
||||||
edit(t, dir, bansJSON, `{"version": 1, "bans": [`+
|
|
||||||
`{"netblock": "203.0.113.9/32", "start": "2026-09-06T00:00:00Z", `+
|
|
||||||
`"expires": "2026-10-06T00:00:00Z", "cause": "abuseipdb"}, `+
|
|
||||||
`{"netblock": "203.0.113.10/32", "start": "2026-09-07T00:00:00Z", `+
|
|
||||||
`"expires": "2026-10-07T00:00:00Z", "cause": "abuseipdb"}, `+
|
|
||||||
`{"netblock": "203.0.113.11/32", "start": "2026-10-05T00:00:00Z", `+
|
|
||||||
`"expires": "2026-10-05T01:00:00Z", "cause": "limit"}]}`)
|
|
||||||
|
|
||||||
params := newParams(dir)
|
|
||||||
load(t, params)
|
|
||||||
|
|
||||||
for netblock, want := range map[netip.Prefix]int{
|
|
||||||
ended: 0, lasting: 1, limited: 1,
|
|
||||||
} {
|
|
||||||
if got := len(params.Ledger.Bans(netblock)); got != want {
|
|
||||||
t.Errorf("the ledger holds %d bans on %s, want %d",
|
|
||||||
got, netblock, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBanLiftedByAnEditWhileRunning(t *testing.T) {
|
func TestBanLiftedByAnEditWhileRunning(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -2448,7 +2414,7 @@ func newParams(dir string) state.Params {
|
|||||||
AttackBanDuration: 7 * 24 * time.Hour,
|
AttackBanDuration: 7 * 24 * time.Hour,
|
||||||
MaxBans: 5000,
|
MaxBans: 5000,
|
||||||
}),
|
}),
|
||||||
Limiter: ratelimit.New(ratelimit.Params{MaxClients: 20000}),
|
Limiter: ratelimit.New(ratelimit.Limits{}, 20000),
|
||||||
GeoJS: geojs.New(geojs.Params{Now: midnight}),
|
GeoJS: geojs.New(geojs.Params{Now: midnight}),
|
||||||
Lists: reputation.New(reputation.Params{
|
Lists: reputation.New(reputation.Params{
|
||||||
BlocklistURLs: []string{
|
BlocklistURLs: []string{
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -23,11 +23,11 @@ import (
|
|||||||
// setting for either.
|
// setting for either.
|
||||||
func RunRProxy(ctx context.Context, params Params) int {
|
func RunRProxy(ctx context.Context, params Params) int {
|
||||||
hostname, _ := os.Hostname() // "" when the host has no name to give
|
hostname, _ := os.Hostname() // "" when the host has no name to give
|
||||||
processLog := processlog.New(processlog.Params{
|
processLog := requestlog.NewProcessLogger(
|
||||||
Output: params.Stdout,
|
params.Stdout,
|
||||||
InstanceName: hostname,
|
hostname,
|
||||||
Level: slog.LevelInfo,
|
slog.LevelInfo,
|
||||||
})
|
)
|
||||||
|
|
||||||
cfg, err := config.RProxyFromEnvironment(params.Viper)
|
cfg, err := config.RProxyFromEnvironment(params.Viper)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -552,9 +552,7 @@ func TestAStatusWAFDoesNotTakeIsReportedAs599(t *testing.T) {
|
|||||||
// startRProxyProcess runs `sww rproxy` in a process of its own, with the
|
// startRProxyProcess runs `sww rproxy` in a process of its own, with the
|
||||||
// settings in env, each as NAME=value. It returns where rproxy listens and a
|
// settings in env, each as NAME=value. It returns where rproxy listens and a
|
||||||
// function that sends rproxy SIGTERM and fails the test unless it exits with
|
// function that sends rproxy SIGTERM and fails the test unless it exits with
|
||||||
// status 0 within waitLimit. It waits a minute for waf's answers unless env
|
// status 0 within waitLimit.
|
||||||
// sets SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS, so that a busy host cannot make it
|
|
||||||
// drop a question.
|
|
||||||
func startRProxyProcess(t *testing.T, env ...string) (string, func()) {
|
func startRProxyProcess(t *testing.T, env ...string) (string, func()) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -570,8 +568,7 @@ func startRProxyProcess(t *testing.T, env ...string) (string, func()) {
|
|||||||
|
|
||||||
cmd.Env = append(os.Environ(),
|
cmd.Env = append(os.Environ(),
|
||||||
commandForMain+"=rproxy",
|
commandForMain+"=rproxy",
|
||||||
"SWW_RPROXY_LISTEN_ADDR="+localhost+":0",
|
"SWW_RPROXY_LISTEN_ADDR="+localhost+":0")
|
||||||
wafBackendTimeout+"=60000")
|
|
||||||
cmd.Env = append(cmd.Env, env...)
|
cmd.Env = append(cmd.Env, env...)
|
||||||
|
|
||||||
exited := startProcess(t, cmd)
|
exited := startProcess(t, cmd)
|
||||||
|
|||||||
@@ -660,9 +660,7 @@ func TestCoreRuleSetExclusionsFileThatCannotBeUsedStopsTheStartWhileTheCoreRuleS
|
|||||||
|
|
||||||
status := run(t.Context(), map[string]string{
|
status := run(t.Context(), map[string]string{
|
||||||
listenAddr: localhost + ":0", stateDir: t.TempDir(), ruleFilesDir: t.TempDir(),
|
listenAddr: localhost + ":0", stateDir: t.TempDir(), ruleFilesDir: t.TempDir(),
|
||||||
"SWW_WAF_CORE_RULE_SET_MODE": string(
|
"SWW_WAF_CORE_RULE_SET_MODE": config.CoreRuleSetModeOff,
|
||||||
config.CoreRuleSetModeOff,
|
|
||||||
),
|
|
||||||
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE": path,
|
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE": path,
|
||||||
}, out)
|
}, out)
|
||||||
if status != 1 {
|
if status != 1 {
|
||||||
|
|||||||
+43
-113
@@ -2,7 +2,6 @@ package sww
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net"
|
"net"
|
||||||
@@ -19,10 +18,10 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/state"
|
"sneak.berlin/go/smallwebwaf/internal/state"
|
||||||
@@ -43,11 +42,8 @@ const remoteLogStopTimeout = 2 * time.Second
|
|||||||
func RunWAF(ctx context.Context, params Params) int {
|
func RunWAF(ctx context.Context, params Params) int {
|
||||||
// Until the settings are read, the one message is an invalid setting's
|
// Until the settings are read, the one message is an invalid setting's
|
||||||
// error, which every SWW_WAF_LOG_LEVEL lets through.
|
// error, which every SWW_WAF_LOG_LEVEL lets through.
|
||||||
processLog := processlog.New(processlog.Params{
|
processLog := requestlog.NewProcessLogger(params.Stdout,
|
||||||
Output: params.Stdout,
|
config.InstanceName(params.Viper), slog.LevelError)
|
||||||
InstanceName: config.InstanceName(params.Viper),
|
|
||||||
Level: slog.LevelError,
|
|
||||||
})
|
|
||||||
|
|
||||||
cfg, err := config.WAFFromEnvironment(params.Viper)
|
cfg, err := config.WAFFromEnvironment(params.Viper)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -66,11 +62,11 @@ func RunWAF(ctx context.Context, params Params) int {
|
|||||||
stdout = io.MultiWriter(params.Stdout, remote)
|
stdout = io.MultiWriter(params.Stdout, remote)
|
||||||
}
|
}
|
||||||
|
|
||||||
processLog = processlog.New(processlog.Params{
|
processLog = requestlog.NewProcessLogger(
|
||||||
Output: stdout,
|
stdout,
|
||||||
InstanceName: cfg.InstanceName,
|
cfg.InstanceName,
|
||||||
Level: cfg.LogLevel,
|
cfg.LogLevel,
|
||||||
})
|
)
|
||||||
|
|
||||||
if remote != nil {
|
if remote != nil {
|
||||||
stopSending := startSending(ctx, remote, processLog)
|
stopSending := startSending(ctx, remote, processLog)
|
||||||
@@ -121,8 +117,7 @@ func RunWAF(ctx context.Context, params Params) int {
|
|||||||
"settings", cfg)
|
"settings", cfg)
|
||||||
|
|
||||||
// The last write follows the stop of every hook, and so the end of every
|
// The last write follows the stop of every hook, and so the end of every
|
||||||
// loop that changes the state or raises an alert, and of every ask of
|
// loop that changes the state or raises an alert. A hook of the state
|
||||||
// GeoJS, the DNSBL zones, AbuseIPDB and routewatch. A hook of the state
|
|
||||||
// files would stop before the loops of the parts built ahead of them.
|
// files would stop before the loops of the parts built ahead of them.
|
||||||
return run(ctx, app, failed, processLog, func() error {
|
return run(ctx, app, failed, processLog, func() error {
|
||||||
err := files.WriteAll()
|
err := files.WriteAll()
|
||||||
@@ -180,35 +175,11 @@ func newRuleFiles(
|
|||||||
return ruleFiles, nil
|
return ruleFiles, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// newCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
|
// newCoreRuleSet returns the Core Rule Set, as waf.NewCoreRuleSet does.
|
||||||
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
|
|
||||||
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
|
|
||||||
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
|
|
||||||
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
|
|
||||||
// coreruleset.New refuses is an error in every mode. While the mode is off, a
|
|
||||||
// set file is loaded with the Core Rule Set only to check it.
|
|
||||||
func newCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
|
func newCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
|
||||||
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
|
crs, err := waf.NewCoreRuleSet(cfg)
|
||||||
cfg.CoreRuleSetExclusionsFile == "" {
|
|
||||||
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
|
||||||
}
|
|
||||||
|
|
||||||
crs, err := coreruleset.New(coreruleset.Params{
|
|
||||||
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
|
|
||||||
DisabledRules: cfg.CoreRuleSetDisabledRules,
|
|
||||||
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
|
|
||||||
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
|
|
||||||
Apps: cfg.Apps,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, startError{
|
return nil, startError{"cannot use the Core Rule Set", err}
|
||||||
"cannot use the Core Rule Set",
|
|
||||||
fmt.Errorf("SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w", err),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
|
|
||||||
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return crs, nil
|
return crs, nil
|
||||||
@@ -221,7 +192,7 @@ func newLookupFile(
|
|||||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||||
alertQueue *alerts.Queue,
|
alertQueue *alerts.Queue,
|
||||||
) (*lookup.File, error) {
|
) (*lookup.File, error) {
|
||||||
if cfg.LookupSource != config.LookupSourceFile {
|
if cfg.LookupSource != "file" {
|
||||||
return nil, nil //nolint:nilnil // no lookup database, and no error
|
return nil, nil //nolint:nilnil // no lookup database, and no error
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -242,17 +213,14 @@ func newLookupFile(
|
|||||||
|
|
||||||
// newLimiter returns the table of clients, with the rate and byte limits.
|
// newLimiter returns the table of clients, with the rate and byte limits.
|
||||||
func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
|
func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
|
||||||
return ratelimit.New(ratelimit.Params{
|
return ratelimit.New(ratelimit.Limits{
|
||||||
Limits: ratelimit.Limits{
|
PerMinute: cfg.RateLimitPerMinute,
|
||||||
PerMinute: cfg.RateLimitPerMinute,
|
PerHour: cfg.RateLimitPerHour,
|
||||||
PerHour: cfg.RateLimitPerHour,
|
PerDay: cfg.RateLimitPerDay,
|
||||||
PerDay: cfg.RateLimitPerDay,
|
BytesPerMinute: cfg.BytesLimitPerMinute,
|
||||||
BytesPerMinute: cfg.BytesLimitPerMinute,
|
BytesPerHour: cfg.BytesLimitPerHour,
|
||||||
BytesPerHour: cfg.BytesLimitPerHour,
|
BytesPerDay: cfg.BytesLimitPerDay,
|
||||||
BytesPerDay: cfg.BytesLimitPerDay,
|
}, cfg.MaxTrackedClients)
|
||||||
},
|
|
||||||
MaxClients: cfg.MaxTrackedClients,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// newLedger returns the ledger of the bans, with the settings for them.
|
// newLedger returns the ledger of the bans, with the settings for them.
|
||||||
@@ -304,13 +272,11 @@ func newLists(
|
|||||||
return lists
|
return lists
|
||||||
}
|
}
|
||||||
|
|
||||||
// newDNSBL returns what asks the DNSBL zones, none asked yet. At the stop it
|
// newDNSBL returns what asks the DNSBL zones, none asked yet.
|
||||||
// ends the queries under way, as reputation.DNSBL.StopAsking says.
|
|
||||||
func newDNSBL(
|
func newDNSBL(
|
||||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
cfg *config.WAF, processLog *slog.Logger, alertQueue *alerts.Queue,
|
||||||
alertQueue *alerts.Queue,
|
|
||||||
) *reputation.DNSBL {
|
) *reputation.DNSBL {
|
||||||
dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{
|
return reputation.NewDNSBL(reputation.DNSBLParams{
|
||||||
Zones: cfg.DNSBLZones,
|
Zones: cfg.DNSBLZones,
|
||||||
Resolver: cfg.DNSBLResolver,
|
Resolver: cfg.DNSBLResolver,
|
||||||
CacheTTL: cfg.IPReputationCacheTTL,
|
CacheTTL: cfg.IPReputationCacheTTL,
|
||||||
@@ -319,66 +285,40 @@ func newDNSBL(
|
|||||||
ProcessLog: processLog,
|
ProcessLog: processLog,
|
||||||
Alerts: alertQueue,
|
Alerts: alertQueue,
|
||||||
})
|
})
|
||||||
|
|
||||||
lc.Append(fx.StopHook(dnsbl.StopAsking))
|
|
||||||
|
|
||||||
return dnsbl
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// newAbuseIPDB returns what checks clients with AbuseIPDB while
|
// newAbuseIPDB returns what checks clients with AbuseIPDB while
|
||||||
// SWW_WAF_ABUSEIPDB_KEY is set, none checked yet. Until waf stops, it
|
// SWW_WAF_ABUSEIPDB_KEY is set, none checked yet.
|
||||||
// fetches AbuseIPDB's blacklist as it is due, and bans the clients on it in
|
|
||||||
// ledger, as waf.BanAbuseIPDBBlacklist says. At the stop it ends the checks
|
|
||||||
// under way, as reputation.AbuseIPDB.StopChecking says.
|
|
||||||
func newAbuseIPDB(
|
func newAbuseIPDB(
|
||||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
cfg *config.WAF, processLog *slog.Logger, alertQueue *alerts.Queue,
|
||||||
alertQueue *alerts.Queue, ledger *bans.Ledger,
|
|
||||||
) *reputation.AbuseIPDB {
|
) *reputation.AbuseIPDB {
|
||||||
abuseIPDB := reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
|
return reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
|
||||||
URL: reputation.AbuseIPDBURL,
|
URL: reputation.AbuseIPDBURL,
|
||||||
Key: cfg.AbuseIPDBKey,
|
Key: cfg.AbuseIPDBKey,
|
||||||
MinScore: cfg.AbuseIPDBMinScore,
|
MinScore: cfg.AbuseIPDBMinScore,
|
||||||
DailyBudget: cfg.AbuseIPDBDailyBudget,
|
DailyBudget: cfg.AbuseIPDBDailyBudget,
|
||||||
CacheTTL: cfg.IPReputationCacheTTL,
|
CacheTTL: cfg.IPReputationCacheTTL,
|
||||||
Timeout: cfg.DNSBLAbuseIPDBTimeout,
|
Timeout: cfg.DNSBLAbuseIPDBTimeout,
|
||||||
BlacklistURL: reputation.AbuseIPDBBlacklistURL,
|
Now: utcNow,
|
||||||
Threshold: cfg.AbuseIPDBThreshold,
|
ProcessLog: processLog,
|
||||||
BlacklistRefresh: cfg.AbuseIPDBBlacklistRefresh,
|
Alerts: alertQueue,
|
||||||
Blacklisted: func(listed []reputation.BlacklistEntry) {
|
|
||||||
waf.BanAbuseIPDBBlacklist(
|
|
||||||
cfg, ledger, processLog, utcNow(), listed,
|
|
||||||
)
|
|
||||||
},
|
|
||||||
Now: utcNow,
|
|
||||||
ProcessLog: processLog,
|
|
||||||
Alerts: alertQueue,
|
|
||||||
})
|
})
|
||||||
|
|
||||||
lc.Append(background(abuseIPDB.RunBlacklist))
|
|
||||||
lc.Append(fx.StopHook(abuseIPDB.StopChecking))
|
|
||||||
|
|
||||||
return abuseIPDB
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// newRouteWatch returns what asks routewatch about clients, or nil while
|
// newRouteWatch returns what asks routewatch about clients, or nil while
|
||||||
// SWW_WAF_ROUTEWATCH_URL is unset. At the stop it ends the queries under
|
// SWW_WAF_ROUTEWATCH_URL is unset.
|
||||||
// way, as routewatch.Client.StopAsking says.
|
|
||||||
func newRouteWatch(
|
func newRouteWatch(
|
||||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
cfg *config.WAF, processLog *slog.Logger,
|
||||||
) *routewatch.Client {
|
) *routewatch.Client {
|
||||||
if cfg.RouteWatchURL == "" {
|
if cfg.RouteWatchURL == "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
routeWatch := routewatch.New(routewatch.Params{
|
return routewatch.New(routewatch.Params{
|
||||||
URL: cfg.RouteWatchURL,
|
URL: cfg.RouteWatchURL,
|
||||||
Now: utcNow,
|
Now: utcNow,
|
||||||
ProcessLog: processLog,
|
ProcessLog: processLog,
|
||||||
})
|
})
|
||||||
|
|
||||||
lc.Append(fx.StopHook(routeWatch.StopAsking))
|
|
||||||
|
|
||||||
return routeWatch
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// newMetrics returns the metrics, with those read from the other parts as
|
// newMetrics returns the metrics, with those read from the other parts as
|
||||||
@@ -416,18 +356,12 @@ func newMetrics(
|
|||||||
|
|
||||||
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
|
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
|
||||||
// answer goes to the limiter and the ledger, as waf.AddLookup says, and each
|
// answer goes to the limiter and the ledger, as waf.AddLookup says, and each
|
||||||
// failure is handled as waf.GeoJSFailed says. At the stop it ends the request
|
// failure is handled as waf.GeoJSFailed says.
|
||||||
// under way, as geojs.GeoJS.StopAsking says.
|
|
||||||
func newGeoJS(
|
func newGeoJS(
|
||||||
lc fx.Lifecycle,
|
cfg *config.WAF, m *metrics.Metrics, limiter *ratelimit.Limiter,
|
||||||
cfg *config.WAF,
|
ledger *bans.Ledger, alertQueue *alerts.Queue, processLog *slog.Logger,
|
||||||
m *metrics.Metrics,
|
|
||||||
limiter *ratelimit.Limiter,
|
|
||||||
ledger *bans.Ledger,
|
|
||||||
alertQueue *alerts.Queue,
|
|
||||||
processLog *slog.Logger,
|
|
||||||
) *geojs.GeoJS {
|
) *geojs.GeoJS {
|
||||||
geoJS := geojs.New(geojs.Params{
|
return geojs.New(geojs.Params{
|
||||||
URL: geojs.URL,
|
URL: geojs.URL,
|
||||||
Answered: func(answer geojs.Answer) {
|
Answered: func(answer geojs.Answer) {
|
||||||
waf.AddLookup(cfg, limiter, ledger, answer)
|
waf.AddLookup(cfg, limiter, ledger, answer)
|
||||||
@@ -439,10 +373,6 @@ func newGeoJS(
|
|||||||
},
|
},
|
||||||
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
||||||
})
|
})
|
||||||
|
|
||||||
lc.Append(fx.StopHook(geoJS.StopAsking))
|
|
||||||
|
|
||||||
return geoJS
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// newWAFServer returns the server waf runs, with its parts and the time
|
// newWAFServer returns the server waf runs, with its parts and the time
|
||||||
|
|||||||
@@ -90,8 +90,6 @@ func TestMainRunsWAF(t *testing.T) {
|
|||||||
cmd := exec.CommandContext(t.Context(), binary)
|
cmd := exec.CommandContext(t.Context(), binary)
|
||||||
cmd.Stdout = out
|
cmd.Stdout = out
|
||||||
|
|
||||||
// The Core Rule Set is off. Main keeps waf's own limit for its checks,
|
|
||||||
// and on a busy host the Core Rule Set's inspection can pass it.
|
|
||||||
cmd.Env = append(os.Environ(),
|
cmd.Env = append(os.Environ(),
|
||||||
"TZ="+zone,
|
"TZ="+zone,
|
||||||
commandForMain+"="+wafCommand,
|
commandForMain+"="+wafCommand,
|
||||||
@@ -99,8 +97,7 @@ func TestMainRunsWAF(t *testing.T) {
|
|||||||
stateDir+"="+dir,
|
stateDir+"="+dir,
|
||||||
cacheDir+"="+dir,
|
cacheDir+"="+dir,
|
||||||
ruleFilesDir+"="+t.TempDir(),
|
ruleFilesDir+"="+t.TempDir(),
|
||||||
lookupSource+"=off",
|
lookupSource+"=off")
|
||||||
"SWW_WAF_CORE_RULE_SET_MODE=off")
|
|
||||||
|
|
||||||
exited := startProcess(t, cmd)
|
exited := startProcess(t, cmd)
|
||||||
|
|
||||||
|
|||||||
@@ -54,7 +54,7 @@ func (abuseIPDB) check(
|
|||||||
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
|
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
|
||||||
)
|
)
|
||||||
|
|
||||||
if rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
|
if rq.h.config.DNSBLAbuseIPDBAction != deny {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,46 +0,0 @@
|
|||||||
package waf
|
|
||||||
|
|
||||||
// The bans for AbuseIPDB's blacklist. They read SWW_WAF_ABUSEIPDB_BAN_DAYS and
|
|
||||||
// SWW_WAF_MODE; the blacklist is fetched by the reputation package.
|
|
||||||
|
|
||||||
import (
|
|
||||||
"log/slog"
|
|
||||||
"net/netip"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
|
||||||
)
|
|
||||||
|
|
||||||
// BanAbuseIPDBBlacklist bans in ledger the netblock of each client on
|
|
||||||
// AbuseIPDB's blacklist, listed, for SWW_WAF_ABUSEIPDB_BAN_DAYS from now, as
|
|
||||||
// the ledger's BanForAbuseIPDB describes. It logs to processLog how many
|
|
||||||
// clients were listed and how many bans were made. Clients in one netblock
|
|
||||||
// make one ban, with the highest of their scores. In observe mode it makes
|
|
||||||
// no ban.
|
|
||||||
func BanAbuseIPDBBlacklist(
|
|
||||||
cfg *config.WAF, ledger *bans.Ledger, processLog *slog.Logger,
|
|
||||||
now time.Time, listed []reputation.BlacklistEntry,
|
|
||||||
) {
|
|
||||||
scores := make(map[netip.Prefix]int64, len(listed))
|
|
||||||
|
|
||||||
for _, client := range listed {
|
|
||||||
netblock := banNetblock(cfg, client.Addr)
|
|
||||||
scores[netblock] = max(scores[netblock], client.Score)
|
|
||||||
}
|
|
||||||
|
|
||||||
made := 0
|
|
||||||
|
|
||||||
if !cfg.Observe {
|
|
||||||
made = ledger.BanForAbuseIPDB(
|
|
||||||
now, now.Add(cfg.AbuseIPDBBanDuration), scores,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
processLog.Info(
|
|
||||||
"fetched AbuseIPDB's blacklist",
|
|
||||||
"clients", len(listed),
|
|
||||||
"bans_made", made,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -1,100 +0,0 @@
|
|||||||
package waf_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net/http"
|
|
||||||
"net/netip"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
||||||
)
|
|
||||||
|
|
||||||
// abuseIPDBBanDays is the setting of how long a ban for AbuseIPDB's
|
|
||||||
// blacklist lasts, in days.
|
|
||||||
const abuseIPDBBanDays = "SWW_WAF_ABUSEIPDB_BAN_DAYS"
|
|
||||||
|
|
||||||
func TestClientOnTheFetchedAbuseIPDBBlacklistIsRefused(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s, clk, server := startWithClock(t, "", map[string]string{
|
|
||||||
abuseIPDBKey: accountKey,
|
|
||||||
})
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(t.Context())
|
|
||||||
stopped := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
server.AbuseIPDB.RunBlacklist(ctx)
|
|
||||||
close(stopped)
|
|
||||||
}()
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
cancel()
|
|
||||||
<-stopped
|
|
||||||
})
|
|
||||||
|
|
||||||
waitUntil(func() bool {
|
|
||||||
active, _ := server.Ledger.Count(clk.Now())
|
|
||||||
|
|
||||||
return active == 1
|
|
||||||
})
|
|
||||||
|
|
||||||
s.get(client, http.StatusForbidden, requestlog.ActionTemporarilyBanned)
|
|
||||||
s.get(otherClient, http.StatusOK, requestlog.ActionForward)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestClientsOnTheAbuseIPDBBlacklistAreBannedForTheBanDays(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s, clk, server := startWithClock(t, "", map[string]string{
|
|
||||||
abuseIPDBBanDays: "2", banScopeV4Prefix: "24",
|
|
||||||
})
|
|
||||||
|
|
||||||
// Two clients in one netblock make one ban, with the higher score.
|
|
||||||
server.BanAbuseIPDBBlacklist([]reputation.BlacklistEntry{
|
|
||||||
{Addr: netip.MustParseAddr(client), Score: 95},
|
|
||||||
{Addr: netip.MustParseAddr(otherClient), Score: 100},
|
|
||||||
{Addr: netip.MustParseAddr("2001:db8:5::1"), Score: 90},
|
|
||||||
})
|
|
||||||
|
|
||||||
held := server.Ledger.Bans(netip.MustParsePrefix(clientsNet))
|
|
||||||
if len(held) != 1 || held[0].Cause != bans.CauseAbuseIPDB ||
|
|
||||||
*held[0].Notes.Reputation[0].Score != 100 {
|
|
||||||
t.Errorf("bans on %s %+v, want one for the blacklist, with 100",
|
|
||||||
clientsNet, held)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Every client of a banned netblock is refused, the IPv6 address's /64
|
|
||||||
// included.
|
|
||||||
for _, from := range []string{"203.0.113.200", "2001:db8:5::ffff:1"} {
|
|
||||||
s.get(from, http.StatusForbidden, requestlog.ActionTemporarilyBanned)
|
|
||||||
}
|
|
||||||
|
|
||||||
s.get("2001:db8:5:1::1", http.StatusOK, requestlog.ActionForward)
|
|
||||||
|
|
||||||
clk.advance(2 * 24 * time.Hour)
|
|
||||||
s.get(client, http.StatusOK, requestlog.ActionForward)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAbuseIPDBBlacklistMakesNoBanInObserveMode(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s, clk, server := startWithClock(
|
|
||||||
t,
|
|
||||||
"",
|
|
||||||
map[string]string{mode: "observe"},
|
|
||||||
)
|
|
||||||
|
|
||||||
server.BanAbuseIPDBBlacklist([]reputation.BlacklistEntry{
|
|
||||||
{Addr: netip.MustParseAddr(client), Score: 100},
|
|
||||||
})
|
|
||||||
|
|
||||||
if active, _ := server.Ledger.Count(clk.Now()); active != 0 {
|
|
||||||
t.Errorf("%d bans active, want none", active)
|
|
||||||
}
|
|
||||||
|
|
||||||
s.get(client, http.StatusOK, requestlog.ActionForward)
|
|
||||||
}
|
|
||||||
@@ -86,14 +86,13 @@ func (rq *request) countedBytes() int64 {
|
|||||||
response, request := rq.out.bytes, rq.requestBytes
|
response, request := rq.out.bytes, rq.requestBytes
|
||||||
|
|
||||||
switch rq.h.config.BytesCounted {
|
switch rq.h.config.BytesCounted {
|
||||||
case config.BytesCountedResponse:
|
case "response":
|
||||||
return response
|
return response
|
||||||
case config.BytesCountedRequest:
|
case "request":
|
||||||
return request
|
return request
|
||||||
case config.BytesCountedBoth:
|
default: // both
|
||||||
|
return response + request
|
||||||
}
|
}
|
||||||
|
|
||||||
return response + request
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// banForLimit bans the client's netblock at now for a broken limit, the
|
// banForLimit bans the client's netblock at now for a broken limit, the
|
||||||
|
|||||||
@@ -63,8 +63,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
blocklisted := percentage{percent: whole}
|
blocklisted := percentage{percent: whole}
|
||||||
if rq.blocklisted &&
|
if rq.blocklisted && cfg.BlocklistAction == "limit" {
|
||||||
cfg.BlocklistAction == config.ReputationActionLimit {
|
|
||||||
blocklisted = percentage{
|
blocklisted = percentage{
|
||||||
cfg.BlocklistLimitPercent,
|
cfg.BlocklistLimitPercent,
|
||||||
"SWW_WAF_BLOCKLIST_ACTION",
|
"SWW_WAF_BLOCKLIST_ACTION",
|
||||||
@@ -73,7 +72,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
|
|
||||||
dnsblAbuseIPDBListed := percentage{percent: whole}
|
dnsblAbuseIPDBListed := percentage{percent: whole}
|
||||||
if (rq.dnsblListed || rq.abuseIPDBHit) &&
|
if (rq.dnsblListed || rq.abuseIPDBHit) &&
|
||||||
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit {
|
cfg.DNSBLAbuseIPDBAction == "limit" {
|
||||||
dnsblAbuseIPDBListed = percentage{
|
dnsblAbuseIPDBListed = percentage{
|
||||||
cfg.DNSBLAbuseIPDBLimitPercent,
|
cfg.DNSBLAbuseIPDBLimitPercent,
|
||||||
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
|
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -42,6 +41,5 @@ func (blocklists) check(
|
|||||||
// client, given listedBy, the blocklists that list it: it does when one lists
|
// client, given listedBy, the blocklists that list it: it does when one lists
|
||||||
// it and SWW_WAF_BLOCKLIST_ACTION is deny.
|
// it and SWW_WAF_BLOCKLIST_ACTION is deny.
|
||||||
func (rq *request) refusedByBlocklists(listedBy []string) bool {
|
func (rq *request) refusedByBlocklists(listedBy []string) bool {
|
||||||
return len(listedBy) > 0 && rq.h.config.BlocklistAction ==
|
return len(listedBy) > 0 && rq.h.config.BlocklistAction == deny
|
||||||
config.ReputationActionDeny
|
|
||||||
}
|
}
|
||||||
@@ -47,7 +47,7 @@ func (coreRuleSet) check(
|
|||||||
rq.line.CoreRuleSetScore = &result.Score
|
rq.line.CoreRuleSetScore = &result.Score
|
||||||
|
|
||||||
for _, id := range result.RuleIDs {
|
for _, id := range result.RuleIDs {
|
||||||
rq.h.metrics.CoreRuleSetMatched(string(cfg.CoreRuleSetMode), id)
|
rq.h.metrics.CoreRuleSetMatched(cfg.CoreRuleSetMode, id)
|
||||||
}
|
}
|
||||||
|
|
||||||
threshold := cfg.CoreRuleSetScoreThreshold
|
threshold := cfg.CoreRuleSetScoreThreshold
|
||||||
@@ -99,7 +99,7 @@ func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
|
|||||||
|
|
||||||
switch {
|
switch {
|
||||||
case rq.h.config.CoreRuleSetMode == config.CoreRuleSetModeDetect:
|
case rq.h.config.CoreRuleSetMode == config.CoreRuleSetModeDetect:
|
||||||
detail["mode"] = string(config.CoreRuleSetModeDetect)
|
detail["mode"] = config.CoreRuleSetModeDetect
|
||||||
case rq.h.config.Observe:
|
case rq.h.config.Observe:
|
||||||
detail["mode"] = "observe"
|
detail["mode"] = "observe"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,8 +32,7 @@ func (dnsbl) check(ctx context.Context, rq *request, _ time.Time) *verdict {
|
|||||||
rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone))
|
rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone))
|
||||||
}
|
}
|
||||||
|
|
||||||
if !rq.dnsblListed ||
|
if !rq.dnsblListed || rq.h.config.DNSBLAbuseIPDBAction != deny {
|
||||||
rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,13 +1,4 @@
|
|||||||
package waf
|
package waf
|
||||||
|
|
||||||
import "sneak.berlin/go/smallwebwaf/internal/reputation"
|
|
||||||
|
|
||||||
// BesideLookupTimeLimit is besideLookupTimeLimit.
|
// BesideLookupTimeLimit is besideLookupTimeLimit.
|
||||||
const BesideLookupTimeLimit = besideLookupTimeLimit
|
const BesideLookupTimeLimit = besideLookupTimeLimit
|
||||||
|
|
||||||
// BanAbuseIPDBBlacklist does on s what a fetch of AbuseIPDB's blacklist with
|
|
||||||
// the clients on listed does.
|
|
||||||
func (s *Server) BanAbuseIPDBBlacklist(listed []reputation.BlacklistEntry) {
|
|
||||||
h := s.Handler.(*handler) //nolint:forcetypeassert // New makes it one
|
|
||||||
BanAbuseIPDBBlacklist(h.config, h.ledger, h.processLog, h.now(), listed)
|
|
||||||
}
|
|
||||||
@@ -49,8 +49,7 @@ func (rq *request) startLookUp(
|
|||||||
) <-chan lookupResult {
|
) <-chan lookupResult {
|
||||||
result := make(chan lookupResult, 1)
|
result := make(chan lookupResult, 1)
|
||||||
|
|
||||||
if rq.h.config.LookupSource == config.LookupSourceOff ||
|
if rq.h.config.LookupSource == "off" || !canBePlaced(rq.client) {
|
||||||
!canBePlaced(rq.client) {
|
|
||||||
result <- lookupResult{}
|
result <- lookupResult{}
|
||||||
|
|
||||||
return result
|
return result
|
||||||
@@ -58,7 +57,7 @@ func (rq *request) startLookUp(
|
|||||||
|
|
||||||
client := clientGroup(rq.h.config, rq.client)
|
client := clientGroup(rq.h.config, rq.client)
|
||||||
|
|
||||||
if rq.h.config.LookupSource == config.LookupSourceFile {
|
if rq.h.config.LookupSource == "file" {
|
||||||
result <- lookupResult{
|
result <- lookupResult{
|
||||||
lookedUp: true, answer: rq.h.lookupFile.LookUp(client),
|
lookedUp: true, answer: rq.h.lookupFile.LookUp(client),
|
||||||
}
|
}
|
||||||
|
|||||||
+38
-4
@@ -1,11 +1,11 @@
|
|||||||
// Package waf is the server the waf command runs. It answers
|
// Package waf is the server the waf command runs. It answers
|
||||||
// rproxy's questions, takes rproxy's reports and answers its own endpoints, and
|
// rproxy's questions, takes rproxy's reports and answers its own endpoints, and
|
||||||
// writes a log line for each. It also bans the clients on AbuseIPDB's
|
// writes a log line for each.
|
||||||
// blacklist each time the reputation package fetches it.
|
|
||||||
package waf
|
package waf
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"cmp"
|
"cmp"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -82,8 +82,8 @@ type Params struct {
|
|||||||
// LookupFile is the lookup database they are looked up in while
|
// LookupFile is the lookup database they are looked up in while
|
||||||
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
|
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
|
||||||
LookupFile *lookup.File
|
LookupFile *lookup.File
|
||||||
// CoreRuleSet is what inspects requests while
|
// CoreRuleSet is what NewCoreRuleSet returns: what inspects requests
|
||||||
// SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
|
// while SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
|
||||||
CoreRuleSet CoreRuleSetInspector
|
CoreRuleSet CoreRuleSetInspector
|
||||||
// Now tells the time by which requests are counted for the rate
|
// Now tells the time by which requests are counted for the rate
|
||||||
// limits, bans are made and run out, and GeoJS's answers are kept,
|
// limits, bans are made and run out, and GeoJS's answers are kept,
|
||||||
@@ -173,6 +173,40 @@ func newHandler(params Params) *handler {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
|
||||||
|
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
|
||||||
|
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
|
||||||
|
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
|
||||||
|
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
|
||||||
|
// coreruleset.New refuses is an error in every mode. While the mode is off, a
|
||||||
|
// set file is loaded with the Core Rule Set only to check it.
|
||||||
|
func NewCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
|
||||||
|
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
|
||||||
|
cfg.CoreRuleSetExclusionsFile == "" {
|
||||||
|
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
||||||
|
}
|
||||||
|
|
||||||
|
coreRuleSet, err := coreruleset.New(coreruleset.Params{
|
||||||
|
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
|
||||||
|
DisabledRules: cfg.CoreRuleSetDisabledRules,
|
||||||
|
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
|
||||||
|
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
|
||||||
|
Apps: cfg.Apps,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w",
|
||||||
|
err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
|
||||||
|
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
||||||
|
}
|
||||||
|
|
||||||
|
return coreRuleSet, nil
|
||||||
|
}
|
||||||
|
|
||||||
// handler is waf's handler. It holds what every request shares; what
|
// handler is waf's handler. It holds what every request shares; what
|
||||||
// belongs to one request is in a request.
|
// belongs to one request is in a request.
|
||||||
type handler struct {
|
type handler struct {
|
||||||
|
|||||||
+43
-78
@@ -20,11 +20,9 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
@@ -399,11 +397,11 @@ func startRProxy(
|
|||||||
server := rproxy.New(rproxy.Params{
|
server := rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: io.Discard,
|
RequestLog: io.Discard,
|
||||||
ProcessLog: processlog.New(processlog.Params{
|
ProcessLog: requestlog.NewProcessLogger(
|
||||||
Output: io.Discard,
|
io.Discard,
|
||||||
InstanceName: "rproxy",
|
"rproxy",
|
||||||
Level: slog.LevelInfo,
|
slog.LevelInfo,
|
||||||
}),
|
),
|
||||||
})
|
})
|
||||||
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(
|
listener, err := (&net.ListenConfig{}).Listen(
|
||||||
@@ -433,8 +431,7 @@ func startRProxy(
|
|||||||
// With no geojsURL, there is no stand-in for GeoJS to look clients up at, and
|
// With no geojsURL, there is no stand-in for GeoJS to look clients up at, and
|
||||||
// SWW_WAF_LOOKUP_SOURCE is off unless env sets it. While it is file, the lookup
|
// SWW_WAF_LOOKUP_SOURCE is off unless env sets it. While it is file, the lookup
|
||||||
// database SWW_WAF_LOOKUP_DB_PATH names is read. Clients are checked with AbuseIPDB
|
// database SWW_WAF_LOOKUP_DB_PATH names is read. Clients are checked with AbuseIPDB
|
||||||
// at abuseIPDBURL while env sets SWW_WAF_ABUSEIPDB_KEY, and its blacklist is
|
// at abuseIPDBURL while env sets SWW_WAF_ABUSEIPDB_KEY.
|
||||||
// fetched from abuseIPDBBlacklistURL.
|
|
||||||
// SWW_WAF_CORE_RULE_SET_MODE is off unless env sets it, so that only the tests of the
|
// SWW_WAF_CORE_RULE_SET_MODE is off unless env sets it, so that only the tests of the
|
||||||
// Core Rule Set have their requests inspected by it.
|
// Core Rule Set have their requests inspected by it.
|
||||||
func newWAF(
|
func newWAF(
|
||||||
@@ -472,11 +469,11 @@ func newWAFWith(
|
|||||||
}
|
}
|
||||||
|
|
||||||
out := &output{}
|
out := &output{}
|
||||||
processLog := processlog.New(processlog.Params{
|
processLog := requestlog.NewProcessLogger(
|
||||||
Output: out,
|
out,
|
||||||
InstanceName: cfg.InstanceName,
|
cfg.InstanceName,
|
||||||
Level: cfg.LogLevel,
|
cfg.LogLevel,
|
||||||
})
|
)
|
||||||
|
|
||||||
ruleFiles, err := rules.Load(rules.Params{
|
ruleFiles, err := rules.Load(rules.Params{
|
||||||
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
|
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
|
||||||
@@ -506,20 +503,24 @@ func newWAFWith(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
coreRuleSet, err := waf.NewCoreRuleSet(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Core Rule Set: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
params := waf.Params{
|
params := waf.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: out,
|
RequestLog: out,
|
||||||
ProcessLog: processLog,
|
ProcessLog: processLog,
|
||||||
LookupFile: lookupFile,
|
LookupFile: lookupFile,
|
||||||
Now: now,
|
CoreRuleSet: coreRuleSet,
|
||||||
Rules: ruleFiles,
|
Now: now,
|
||||||
Alerts: alertQueue,
|
Rules: ruleFiles,
|
||||||
|
Alerts: alertQueue,
|
||||||
// The tests of the limit set it back, and hold the Core Rule Set
|
// The tests of the limit set it back, and hold the Core Rule Set
|
||||||
// with a stand-in.
|
// with a stand-in.
|
||||||
BesideLookupTimeLimit: time.Hour,
|
BesideLookupTimeLimit: time.Hour,
|
||||||
}
|
}
|
||||||
|
|
||||||
addCoreRuleSet(t, ¶ms)
|
|
||||||
addCounters(¶ms)
|
addCounters(¶ms)
|
||||||
addOutsideSources(¶ms, geojsURL, abuseIPDBCheckURL)
|
addOutsideSources(¶ms, geojsURL, abuseIPDBCheckURL)
|
||||||
|
|
||||||
@@ -528,30 +529,6 @@ func newWAFWith(
|
|||||||
return server, out, alertQueue
|
return server, out, alertQueue
|
||||||
}
|
}
|
||||||
|
|
||||||
// addCoreRuleSet adds to params the Core Rule Set, with the settings for it,
|
|
||||||
// while SWW_WAF_CORE_RULE_SET_MODE is not off.
|
|
||||||
func addCoreRuleSet(t *testing.T, params *waf.Params) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
cfg := params.Config
|
|
||||||
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
coreRuleSet, err := coreruleset.New(coreruleset.Params{
|
|
||||||
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
|
|
||||||
DisabledRules: cfg.CoreRuleSetDisabledRules,
|
|
||||||
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
|
|
||||||
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
|
|
||||||
Apps: cfg.Apps,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Core Rule Set: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
params.CoreRuleSet = coreRuleSet
|
|
||||||
}
|
|
||||||
|
|
||||||
// addCounters adds to params the metrics, the table of clients, the ledger
|
// addCounters adds to params the metrics, the table of clients, the ledger
|
||||||
// and the anomaly counters, as sww builds them from params' settings. The
|
// and the anomaly counters, as sww builds them from params' settings. The
|
||||||
// metrics read the ledger, the table of clients and the rules.
|
// metrics read the ledger, the table of clients and the rules.
|
||||||
@@ -562,17 +539,14 @@ func addCounters(params *waf.Params) {
|
|||||||
cfg.MetricsBusiestCountriesAndASNs,
|
cfg.MetricsBusiestCountriesAndASNs,
|
||||||
cfg.InstanceName,
|
cfg.InstanceName,
|
||||||
)
|
)
|
||||||
params.Limiter = ratelimit.New(ratelimit.Params{
|
params.Limiter = ratelimit.New(ratelimit.Limits{
|
||||||
Limits: ratelimit.Limits{
|
PerMinute: cfg.RateLimitPerMinute,
|
||||||
PerMinute: cfg.RateLimitPerMinute,
|
PerHour: cfg.RateLimitPerHour,
|
||||||
PerHour: cfg.RateLimitPerHour,
|
PerDay: cfg.RateLimitPerDay,
|
||||||
PerDay: cfg.RateLimitPerDay,
|
BytesPerMinute: cfg.BytesLimitPerMinute,
|
||||||
BytesPerMinute: cfg.BytesLimitPerMinute,
|
BytesPerHour: cfg.BytesLimitPerHour,
|
||||||
BytesPerHour: cfg.BytesLimitPerHour,
|
BytesPerDay: cfg.BytesLimitPerDay,
|
||||||
BytesPerDay: cfg.BytesLimitPerDay,
|
}, cfg.MaxTrackedClients)
|
||||||
},
|
|
||||||
MaxClients: cfg.MaxTrackedClients,
|
|
||||||
})
|
|
||||||
params.Ledger = bans.New(bans.Params{
|
params.Ledger = bans.New(bans.Params{
|
||||||
LimitBanDuration: cfg.LimitBanDuration,
|
LimitBanDuration: cfg.LimitBanDuration,
|
||||||
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
|
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
|
||||||
@@ -601,9 +575,8 @@ func addCounters(params *waf.Params) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// addOutsideSources adds to params the lists fetched from URLs, the DNSBL
|
// addOutsideSources adds to params the lists fetched from URLs, the DNSBL
|
||||||
// zones' verdicts, AbuseIPDB's scores from abuseIPDBCheckURL and its
|
// zones' verdicts, AbuseIPDB's scores from abuseIPDBCheckURL, routewatch's
|
||||||
// blacklist from abuseIPDBBlacklistURL, routewatch's answers while its
|
// answers while its setting is set, and GeoJS at geojsURL, as sww builds
|
||||||
// setting is set, and GeoJS at geojsURL, as sww builds
|
|
||||||
// them from params' settings and the parts addCounters added. The metrics
|
// them from params' settings and the parts addCounters added. The metrics
|
||||||
// read the lists, the zones' verdicts, and AbuseIPDB's while its key is set.
|
// read the lists, the zones' verdicts, and AbuseIPDB's while its key is set.
|
||||||
func addOutsideSources(
|
func addOutsideSources(
|
||||||
@@ -632,23 +605,15 @@ func addOutsideSources(
|
|||||||
Alerts: alertQueue,
|
Alerts: alertQueue,
|
||||||
})
|
})
|
||||||
params.AbuseIPDB = reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
|
params.AbuseIPDB = reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
|
||||||
URL: abuseIPDBCheckURL,
|
URL: abuseIPDBCheckURL,
|
||||||
Key: cfg.AbuseIPDBKey,
|
Key: cfg.AbuseIPDBKey,
|
||||||
MinScore: cfg.AbuseIPDBMinScore,
|
MinScore: cfg.AbuseIPDBMinScore,
|
||||||
DailyBudget: cfg.AbuseIPDBDailyBudget,
|
DailyBudget: cfg.AbuseIPDBDailyBudget,
|
||||||
CacheTTL: cfg.IPReputationCacheTTL,
|
CacheTTL: cfg.IPReputationCacheTTL,
|
||||||
Timeout: cfg.DNSBLAbuseIPDBTimeout,
|
Timeout: cfg.DNSBLAbuseIPDBTimeout,
|
||||||
BlacklistURL: abuseIPDBBlacklistURL,
|
Now: params.Now,
|
||||||
Threshold: cfg.AbuseIPDBThreshold,
|
ProcessLog: processLog,
|
||||||
BlacklistRefresh: cfg.AbuseIPDBBlacklistRefresh,
|
Alerts: alertQueue,
|
||||||
Blacklisted: func(listed []reputation.BlacklistEntry) {
|
|
||||||
waf.BanAbuseIPDBBlacklist(
|
|
||||||
cfg, ledger, processLog, params.Now(), listed,
|
|
||||||
)
|
|
||||||
},
|
|
||||||
Now: params.Now,
|
|
||||||
ProcessLog: processLog,
|
|
||||||
Alerts: alertQueue,
|
|
||||||
})
|
})
|
||||||
|
|
||||||
if cfg.RouteWatchURL != "" {
|
if cfg.RouteWatchURL != "" {
|
||||||
|
|||||||
@@ -7,6 +7,10 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// deny is the SWW_WAF_BLOCKLIST_ACTION and the SWW_WAF_DNSBL_ABUSEIPDB_ACTION that
|
||||||
|
// refuses the requests of a client a source lists.
|
||||||
|
const deny = "deny"
|
||||||
|
|
||||||
// noteListed notes each of sources, the URLs of the blocklists or the
|
// noteListed notes each of sources, the URLs of the blocklists or the
|
||||||
// DNSBL zones, their keys masked, that list the client, as noteHit does,
|
// DNSBL zones, their keys masked, that list the client, as noteHit does,
|
||||||
// with reason.
|
// with reason.
|
||||||
|
|||||||
@@ -822,39 +822,22 @@ const abuseipdb = reputation.AbuseIPDBSource
|
|||||||
// through which AbuseIPDB is asked.
|
// through which AbuseIPDB is asked.
|
||||||
const abuseIPDBURL = "abuseipdb://stand-in/api/v2/check"
|
const abuseIPDBURL = "abuseipdb://stand-in/api/v2/check"
|
||||||
|
|
||||||
// abuseIPDBBlacklistURL is where newWAF has AbuseIPDB's blacklist fetched
|
|
||||||
// from: at abuseIPDBStandIn too.
|
|
||||||
const abuseIPDBBlacklistURL = "abuseipdb://stand-in/api/v2/blacklist"
|
|
||||||
|
|
||||||
// abuseIPDBStandIn is a stand-in for AbuseIPDB that gives every client the
|
// abuseIPDBStandIn is a stand-in for AbuseIPDB that gives every client the
|
||||||
// score 100, at once and without the network. Its blacklist lists client
|
// score 100, at once and without the network.
|
||||||
// alone, with the score 100, to a fetch with accountKey that asks for the
|
|
||||||
// clients scored 90 or more, the default threshold; it refuses any other
|
|
||||||
// fetch of it with 400.
|
|
||||||
type abuseIPDBStandIn struct{}
|
type abuseIPDBStandIn struct{}
|
||||||
|
|
||||||
// RoundTrip answers req with the score 100, or with the blacklist.
|
// RoundTrip answers req with the score 100.
|
||||||
func (abuseIPDBStandIn) RoundTrip(
|
func (abuseIPDBStandIn) RoundTrip(
|
||||||
req *http.Request,
|
req *http.Request,
|
||||||
) (*http.Response, error) {
|
) (*http.Response, error) {
|
||||||
status, body := http.StatusOK, `{"data":{"abuseConfidenceScore":100}}`
|
|
||||||
|
|
||||||
if req.URL.Path == "/api/v2/blacklist" {
|
|
||||||
body = `{"data":[{"ipAddress":"` + client +
|
|
||||||
`","abuseConfidenceScore":100}]}`
|
|
||||||
|
|
||||||
if req.Header.Get("Key") != accountKey ||
|
|
||||||
req.URL.RawQuery != "confidenceMinimum=90" {
|
|
||||||
status, body = http.StatusBadRequest, `{"errors":[]}`
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return &http.Response{
|
return &http.Response{
|
||||||
StatusCode: status,
|
StatusCode: http.StatusOK,
|
||||||
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
|
Status: "200 OK",
|
||||||
Header: http.Header{},
|
Header: http.Header{},
|
||||||
Body: io.NopCloser(strings.NewReader(body)),
|
Body: io.NopCloser(
|
||||||
Request: req,
|
strings.NewReader(`{"data":{"abuseConfidenceScore":100}}`),
|
||||||
|
),
|
||||||
|
Request: req,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -421,7 +421,7 @@ func (rq *request) answerAtTheEnd() (geojs.Answer, bool) {
|
|||||||
return geojs.Answer{}, false
|
return geojs.Answer{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
if rq.h.config.LookupSource == config.LookupSourceFile {
|
if rq.h.config.LookupSource == "file" {
|
||||||
return rq.lookupAnswer, true
|
return rq.lookupAnswer, true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user