Compare commits
1
Commits
e68848d1eb
...
cc50b04f2a
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cc50b04f2a |
No files matched your search
@@ -257,14 +257,14 @@ Every setting the code reads, with its default:
|
|||||||
| `SWW_LOOKUP_DB_PATH` | empty | The IPinfo Lite database file, in its `.mmdb` form, read while the source is `file`. |
|
| `SWW_LOOKUP_DB_PATH` | empty | The IPinfo Lite database file, in its `.mmdb` form, read while the source is `file`. |
|
||||||
| `SWW_LOOKUP_TIMEOUT` | `800ms` | How long a request waits for its client's first GeoJS answer when a setting needs it, and how long a request to GeoJS may take. |
|
| `SWW_LOOKUP_TIMEOUT` | `800ms` | How long a request waits for its client's first GeoJS answer when a setting needs it, and how long a request to GeoJS may take. |
|
||||||
| `SWW_ADD_LOOKUP_HEADERS` | `false` | `true` passes the app the client's AS number and country in `X-Client-ASN` and `X-Client-Country`. |
|
| `SWW_ADD_LOOKUP_HEADERS` | `false` | `true` passes the app the client's AS number and country in `X-Client-ASN` and `X-Client-Country`. |
|
||||||
| `SWW_DENIED_COUNTRIES` | empty | Countries whose clients are refused, such as `cn,ru`. |
|
|
||||||
| `SWW_EXCLUSIVELY_ALLOWED_COUNTRIES` | empty | When set, the only countries whose clients get through; a client the lookup cannot place is refused too. |
|
| `SWW_EXCLUSIVELY_ALLOWED_COUNTRIES` | empty | When set, the only countries whose clients get through; a client the lookup cannot place is refused too. |
|
||||||
| `SWW_ASN_LIMIT_PERCENT` | empty | AS numbers whose clients get a percentage of every request and byte limit, such as `AS14061:50,AS16276:25`. |
|
| `SWW_TAG_LISTS` | empty | Address lists that each add a tag to their clients' requests, as `name=URL` pairs, such as `torexit=https://example.com/tor.txt`. |
|
||||||
| `SWW_COUNTRY_LIMIT_PERCENT` | empty | The same by country, such as `cn:25`. |
|
| `SWW_TAG_DENY` | empty | Tags whose requests are refused, such as `torexit,country=CN`. |
|
||||||
| `SWW_ASN_BYTES_PERCENT` | empty | A percentage of the byte limits alone for the AS numbers listed, in place of their `SWW_ASN_LIMIT_PERCENT`. |
|
| `SWW_TAG_LIMIT_PERCENT` | empty | Tags whose clients get a percentage, from 1 to 99, of every request and byte limit, such as `vpn:50,asn=14061:25`; the lowest applies. |
|
||||||
| `SWW_COUNTRY_BYTES_PERCENT` | empty | A percentage of the byte limits alone for the countries listed, in place of their `SWW_COUNTRY_LIMIT_PERCENT`. |
|
| `SWW_ASN_BYTES_PERCENT` | empty | A percentage of the byte limits alone for the AS numbers listed, in place of their `asn=` tag's percentage. |
|
||||||
|
| `SWW_COUNTRY_BYTES_PERCENT` | empty | A percentage of the byte limits alone for the countries listed, in place of their `country=` tag's percentage. |
|
||||||
| `SWW_UNKNOWN_LIMIT_PERCENT` | `100` | The percentage of every limit a client without a country gets. |
|
| `SWW_UNKNOWN_LIMIT_PERCENT` | `100` | The percentage of every limit a client without a country gets. |
|
||||||
| `SWW_ASN_LIMIT_PERCENT_URL` | unset | The URL of a text file of `AS<number>:<percent>` lines, fetched like a blocklist; where it and `SWW_ASN_LIMIT_PERCENT` both give a percentage, the lower applies. |
|
| `SWW_ASN_LIMIT_PERCENT_URL` | unset | The URL of a text file of `AS<number>:<percent>` lines, fetched like a blocklist; each line counts as an `asn=` tag in `SWW_TAG_LIMIT_PERCENT` (the lowest wins). |
|
||||||
| `SWW_RULES_DIR` | `/etc/sww/rules.d` | The directory of the rule files, read at start and again when a file there changes. |
|
| `SWW_RULES_DIR` | `/etc/sww/rules.d` | The directory of the rule files, read at start and again when a file there changes. |
|
||||||
| `SWW_RULES_ENABLED` | `true` | `false` reads no rule file. |
|
| `SWW_RULES_ENABLED` | `true` | `false` reads no rule file. |
|
||||||
| `SWW_TRAP_PATHS` | empty | Paths only scanners ask for, such as `/wp-login.php,/xmlrpc.php`; a request for one bans the client. |
|
| `SWW_TRAP_PATHS` | empty | Paths only scanners ask for, such as `/wp-login.php,/xmlrpc.php`; a request for one bans the client. |
|
||||||
@@ -414,8 +414,7 @@ services:
|
|||||||
SWW_RATE_LIMIT_PER_HOUR: "600"
|
SWW_RATE_LIMIT_PER_HOUR: "600"
|
||||||
SWW_RATE_LIMIT_PER_DAY: "3000"
|
SWW_RATE_LIMIT_PER_DAY: "3000"
|
||||||
# Hosting networks and one country get a share of those limits.
|
# Hosting networks and one country get a share of those limits.
|
||||||
SWW_ASN_LIMIT_PERCENT: AS14061:50,AS16276:25
|
SWW_TAG_LIMIT_PERCENT: asn=14061:50,asn=16276:25,country=CN:25
|
||||||
SWW_COUNTRY_LIMIT_PERCENT: cn:25
|
|
||||||
# Paths only scanners ask for; one request bans the client.
|
# Paths only scanners ask for; one request bans the client.
|
||||||
SWW_TRAP_PATHS: /wp-login.php,/xmlrpc.php
|
SWW_TRAP_PATHS: /wp-login.php,/xmlrpc.php
|
||||||
# Answer a refused client with 429 rather than 403.
|
# Answer a refused client with 429 rather than 403.
|
||||||
@@ -507,31 +506,34 @@ of the request reaches the app.
|
|||||||
|
|
||||||
`discriminator` takes the checks:
|
`discriminator` takes the checks:
|
||||||
|
|
||||||
3. The static lists, `SWW_ALLOW_NETS` and `SWW_DENY_NETS`.
|
3. The tag lists.
|
||||||
4. The bans.
|
4. The static lists, `SWW_ALLOW_NETS` and `SWW_DENY_NETS`.
|
||||||
5. The lookup of the client's AS number and country.
|
5. The bans.
|
||||||
6. The country lists.
|
6. The lookup of the client's AS number and country.
|
||||||
7. The reputation sources.
|
7. The country list.
|
||||||
8. The client's limit percentage, from its AS number, its country or a
|
8. The reputation sources.
|
||||||
reputation hit.
|
9. The tag refusals of `SWW_TAG_DENY`.
|
||||||
9. The rate limits.
|
10. The client's limit percentage, from its tags, its AS number, its country or
|
||||||
10. The trap paths.
|
a reputation hit.
|
||||||
11. The rule files.
|
11. The rate limits.
|
||||||
12. The header limits, and then the Core Rule Set.
|
12. The trap paths.
|
||||||
13. A path under `/_sww/` is answered `404`, so that the app never sees that
|
13. The rule files.
|
||||||
|
14. The header limits, and then the Core Rule Set.
|
||||||
|
15. The tag refusals again, so that `exploitsignature` can refuse too.
|
||||||
|
16. A path under `/_sww/` is answered `404`, so that the app never sees that
|
||||||
prefix.
|
prefix.
|
||||||
|
|
||||||
`rproxy` takes the rest, with `discriminator` counting at the end:
|
`rproxy` takes the rest, with `discriminator` counting at the end:
|
||||||
|
|
||||||
14. The request is passed to the app, streaming.
|
17. The request is passed to the app, streaming.
|
||||||
15. After the answer, `rproxy` sends `discriminator` a report with the status
|
18. After the answer, `rproxy` sends `discriminator` a report with the status
|
||||||
and the bytes each way. For an allowed request, `discriminator` counts the
|
and the bytes each way. For an allowed request, `discriminator` counts the
|
||||||
bytes toward the byte limits and adds the request to the client's history.
|
bytes toward the byte limits and adds the request to the client's history.
|
||||||
|
|
||||||
A refused visitor is answered with `SWW_BAN_RESPONSE`. A `block` rule or a Core
|
A refused visitor is answered with `SWW_BAN_RESPONSE`. A `block` rule or a Core
|
||||||
Rule Set match answers `403` and bans nobody, since the Core Rule Set has false
|
Rule Set match answers `403` and bans nobody, since the Core Rule Set has false
|
||||||
positives; such refusals count toward `SWW_ERROR_BURST_THRESHOLD` instead. A
|
positives; such refusals count toward `SWW_ERROR_BURST_THRESHOLD` instead. A
|
||||||
request over the header limits of step 12 is answered `431`, and bans nobody
|
request over the header limits of step 14 is answered `431`, and bans nobody
|
||||||
either. `rproxy` itself answers `405` to a method it does not pass on, and the
|
either. `rproxy` itself answers `405` to a method it does not pass on, and the
|
||||||
status for a size or time limit, such as `413` for a body too large.
|
status for a size or time limit, such as `413` for a body too large.
|
||||||
|
|
||||||
@@ -550,6 +552,21 @@ status for a size or time limit, such as `413` for a body too large.
|
|||||||
bans. An admin adds, keeps or lifts a ban by editing `bans.json` or through
|
bans. An admin adds, keeps or lifts a ban by editing `bans.json` or through
|
||||||
the ban endpoints (see [`docs/admin-endpoints.md`](docs/admin-endpoints.md)).
|
the ban endpoints (see [`docs/admin-endpoints.md`](docs/admin-endpoints.md)).
|
||||||
|
|
||||||
|
### Request tags
|
||||||
|
|
||||||
|
Each request gets tags from what `discriminator` knows of its client, written in
|
||||||
|
the `tags` field of its log line:
|
||||||
|
|
||||||
|
- `asn=<number>` and `country=<code>`, from the lookup;
|
||||||
|
- one tag per list of `SWW_TAG_LISTS` that holds the client, such as `torexit`
|
||||||
|
or `vpn=mullvad`;
|
||||||
|
- `crowdsec`, `dnsbl=<zone>` and `abuseipdb`, when that reputation source lists
|
||||||
|
the client;
|
||||||
|
- `exploitsignature`, when a rule file or the Core Rule Set matched the request.
|
||||||
|
|
||||||
|
`SWW_TAG_DENY` refuses requests by tag, and `SWW_TAG_LIMIT_PERCENT` lowers a
|
||||||
|
client's limits by tag. See "Request tags" in [`SPEC.md`](SPEC.md).
|
||||||
|
|
||||||
### Attack detection
|
### Attack detection
|
||||||
|
|
||||||
- The Core Rule Set reads the method, the URL with its query string, and the
|
- The Core Rule Set reads the method, the URL with its query string, and the
|
||||||
|
|||||||
@@ -119,9 +119,9 @@ to them.
|
|||||||
|
|
||||||
## What happens to a request
|
## What happens to a request
|
||||||
|
|
||||||
`rproxy` runs steps 1, 2 and 14, and sends the report of step 15.
|
`rproxy` runs steps 1, 2 and 17, and sends the report of step 18.
|
||||||
`discriminator` runs steps 3 to 13, and does the counting of step 15. The first
|
`discriminator` runs steps 3 to 16, and does the counting of step 18. The first
|
||||||
step that produces a final answer ends the checks. Step 15 runs for each request
|
step that produces a final answer ends the checks. Step 18 runs for each request
|
||||||
`discriminator` answers with `allow`, `deny` or `close`. `rproxy`'s size and
|
`discriminator` answers with `allow`, `deny` or `close`. `rproxy`'s size and
|
||||||
time limits apply to the whole exchange. Each step names the section that holds
|
time limits apply to the whole exchange. Each step names the section that holds
|
||||||
its rules. A step that names a heuristic in parentheses runs that heuristic. The
|
its rules. A step that names a heuristic in parentheses runs that heuristic. The
|
||||||
@@ -143,46 +143,54 @@ request log's `heuristic` field gives that name. `refusal_limit` and
|
|||||||
- An IPv6 client is its group: the netblock of `SWW_IPV6_GROUP_PREFIX`
|
- An IPv6 client is its group: the netblock of `SWW_IPV6_GROUP_PREFIX`
|
||||||
around its address, a /64 by default, since one abuser usually controls a
|
around its address, a /64 by default, since one abuser usually controls a
|
||||||
whole /64. A client is therefore one IPv4 address or one IPv6 group.
|
whole /64. A client is therefore one IPv4 address or one IPv6 group.
|
||||||
3. The static lists. A client in `SWW_ALLOW_NETS` skips steps 4 to 12, the byte
|
3. The tag lists (`tag_lists`; see "Request tags"). Each list that holds the
|
||||||
|
client adds its tag. A client in `SWW_ALLOW_NETS` gets no tag.
|
||||||
|
4. The static lists. A client in `SWW_ALLOW_NETS` skips steps 5 to 15, the byte
|
||||||
limits and the error burst. A client in `SWW_DENY_NETS` is refused
|
limits and the error burst. A client in `SWW_DENY_NETS` is refused
|
||||||
(`deny_nets`); a client in both lists is let through.
|
(`deny_nets`); a client in both lists is let through.
|
||||||
4. Bans. A client under a ban is refused (`active_ban`; see "Bans").
|
5. Bans. A client under a ban is refused (`active_ban`; see "Bans").
|
||||||
5. The lookup, unless `SWW_LOOKUP_SOURCE` is `off` (see "Lookup of AS number and
|
6. The lookup, unless `SWW_LOOKUP_SOURCE` is `off` (see "Lookup of AS number and
|
||||||
country"). The request goes on at once. It waits for the first GeoJS answer,
|
country"). The request goes on at once. It waits for the first GeoJS answer,
|
||||||
up to `SWW_LOOKUP_TIMEOUT`, only when one of these needs it:
|
up to `SWW_LOOKUP_TIMEOUT`, only when one of these needs it:
|
||||||
- a country list;
|
- the country list;
|
||||||
- a biased threshold;
|
- a biased threshold;
|
||||||
|
- `SWW_TAG_DENY` or `SWW_TAG_LIMIT_PERCENT` naming an `asn=` or `country=`
|
||||||
|
tag;
|
||||||
- `SWW_ADD_LOOKUP_HEADERS`.
|
- `SWW_ADD_LOOKUP_HEADERS`.
|
||||||
|
|
||||||
A request that ends before the answer comes is logged without it.
|
A request that ends before the answer comes is logged without it.
|
||||||
|
|
||||||
6. The country lists (`countries`; see "Country lists").
|
7. The country list (`countries`; see "Country list").
|
||||||
7. The reputation sources (see "Reputation"), in this order:
|
8. The reputation sources (see "Reputation"), in this order:
|
||||||
- A client on a fetched blocklist gets `SWW_BLOCKLIST_ACTION`
|
- A client on a fetched blocklist gets `SWW_BLOCKLIST_ACTION`
|
||||||
(`blocklists`).
|
(`blocklists`).
|
||||||
- A client on the CrowdSec decision list is banned (`crowdsec`).
|
- A client on the CrowdSec decision list is banned (`crowdsec`).
|
||||||
- A client listed by a DNS blocklist gets `SWW_REPUTATION_ACTION` (`dnsbl`).
|
- A client listed by a DNS blocklist gets `SWW_REPUTATION_ACTION` (`dnsbl`).
|
||||||
- A client scored at least `SWW_ABUSEIPDB_MIN_SCORE` by AbuseIPDB gets
|
- A client scored at least `SWW_ABUSEIPDB_MIN_SCORE` by AbuseIPDB gets
|
||||||
`SWW_REPUTATION_ACTION` (`abuseipdb`).
|
`SWW_REPUTATION_ACTION` (`abuseipdb`).
|
||||||
8. The client's limit percentage (see "Biased thresholds").
|
9. The tag refusals (`tag_deny`; see "Request tags"). A request carrying a tag
|
||||||
9. Rate limiting (`rate_limits`; see "Request rate limits, per client"). A
|
in `SWW_TAG_DENY` is refused.
|
||||||
|
10. The client's limit percentage (see "Biased thresholds").
|
||||||
|
11. Rate limiting (`rate_limits`; see "Request rate limits, per client"). A
|
||||||
request over a limit times the percentage is refused and bans the client.
|
request over a limit times the percentage is refused and bans the client.
|
||||||
10. The trap paths (`trap_paths`; see "Attack detection"). A request for one is
|
12. The trap paths (`trap_paths`; see "Attack detection"). A request for one is
|
||||||
refused and bans the client.
|
refused and bans the client.
|
||||||
11. The rule files (`rule_files`; see "Rule files"). Matching stops at the first
|
13. The rule files (`rule_files`; see "Rule files"). Matching stops at the first
|
||||||
rule that refuses.
|
rule that refuses.
|
||||||
12. The header limits (`header_limits`; see "Answering in time"), and then the
|
14. The header limits (`header_limits`; see "Answering in time"), and then the
|
||||||
Core Rule Set (`core_rule_set`; see "Attack detection"). A Core Rule Set
|
Core Rule Set (`core_rule_set`; see "Attack detection"). A Core Rule Set
|
||||||
match is refused with `403` while `SWW_WAF_MODE` is `block`, and only logged
|
match is refused with `403` while `SWW_WAF_MODE` is `block`, and only logged
|
||||||
and alerted while it is `detect`.
|
and alerted while it is `detect`.
|
||||||
13. A path under `/_sww/` is answered `404`, so that the app never sees that
|
15. The tag refusals again (`tag_deny`), so that `exploitsignature` can refuse
|
||||||
|
too.
|
||||||
|
16. A path under `/_sww/` is answered `404`, so that the app never sees that
|
||||||
prefix.
|
prefix.
|
||||||
14. `rproxy` passes the request to `SWRP_UPSTREAM_URL`, streaming (see "Passing
|
17. `rproxy` passes the request to `SWRP_UPSTREAM_URL`, streaming (see "Passing
|
||||||
a request to the app"). With `SWW_ADD_LOOKUP_HEADERS`, `X-Client-ASN` and
|
a request to the app"). With `SWW_ADD_LOOKUP_HEADERS`, `X-Client-ASN` and
|
||||||
`X-Client-Country` are added for the app's own logs; a header whose value is
|
`X-Client-Country` are added for the app's own logs; a header whose value is
|
||||||
unknown is left out. Whatever that setting says, any such header the client
|
unknown is left out. Whatever that setting says, any such header the client
|
||||||
sent is removed.
|
sent is removed.
|
||||||
15. After the visitor's answer has ended, `rproxy` sends the report (see "The
|
18. After the visitor's answer has ended, `rproxy` sends the report (see "The
|
||||||
report"). `discriminator` then counts:
|
report"). `discriminator` then counts:
|
||||||
- For a request it allowed, the response bytes, the request bytes or both
|
- For a request it allowed, the response bytes, the request bytes or both
|
||||||
are added to the byte counters, as `SWW_BYTES_COUNT` says. For a
|
are added to the byte counters, as `SWW_BYTES_COUNT` says. For a
|
||||||
@@ -285,7 +293,8 @@ no offence:
|
|||||||
|
|
||||||
- `SWW_DENY_NETS`;
|
- `SWW_DENY_NETS`;
|
||||||
- a blocklist or reputation hit whose action is `deny`;
|
- a blocklist or reputation hit whose action is `deny`;
|
||||||
- the country lists.
|
- the country list;
|
||||||
|
- `SWW_TAG_DENY`.
|
||||||
|
|
||||||
Every ban carries notes with what an admin needs to decide whether to lift it,
|
Every ban carries notes with what an admin needs to decide whether to lift it,
|
||||||
drawn from what `discriminator` already knows. No lookup is made for the notes
|
drawn from what `discriminator` already knows. No lookup is made for the notes
|
||||||
@@ -392,7 +401,7 @@ Conventions:
|
|||||||
|
|
||||||
### Static lists
|
### Static lists
|
||||||
|
|
||||||
- `SWW_ALLOW_NETS` skips the checks step 3 names, for monitoring or the owner's
|
- `SWW_ALLOW_NETS` skips the checks step 4 names, for monitoring or the owner's
|
||||||
own networks. `SWW_RATE_LIMIT_EXEMPT_NETS` bypasses the request and byte
|
own networks. `SWW_RATE_LIMIT_EXEMPT_NETS` bypasses the request and byte
|
||||||
limits only; the error burst, attack detection and bans still apply.
|
limits only; the error burst, attack detection and bans still apply.
|
||||||
- Many people can share one address, as in an office, on a mobile carrier or
|
- Many people can share one address, as in an office, on a mobile carrier or
|
||||||
@@ -442,7 +451,7 @@ the same way.
|
|||||||
By default there is no minute limit, so a client can download in a burst at full
|
By default there is no minute limit, so a client can download in a burst at full
|
||||||
speed. The hour limit defaults to 50 GiB and the day limit to 100 GiB. They
|
speed. The hour limit defaults to 50 GiB and the day limit to 100 GiB. They
|
||||||
allow several downloads of a few GiB each. A response's bytes are counted when
|
allow several downloads of a few GiB each. A response's bytes are counted when
|
||||||
it ends (see step 15).
|
it ends (see step 18).
|
||||||
|
|
||||||
### Lookup of AS number and country
|
### Lookup of AS number and country
|
||||||
|
|
||||||
@@ -471,12 +480,12 @@ service that keeps its visitors' addresses on its own host.
|
|||||||
dropped from that table. The answer reaches the client's history and ban notes
|
dropped from that table. The answer reaches the client's history and ban notes
|
||||||
when it comes.
|
when it comes.
|
||||||
- `SWW_LOOKUP_TIMEOUT` is how long a visitor's request waits for its client's
|
- `SWW_LOOKUP_TIMEOUT` is how long a visitor's request waits for its client's
|
||||||
first answer, when a setting needs it (see step 5). It also bounds each
|
first answer, when a setting needs it (see step 6). It also bounds each
|
||||||
request to GeoJS. A client whose answer has not come in time counts as unknown
|
request to GeoJS. A client whose answer has not come in time counts as unknown
|
||||||
until it comes, and its later requests do not wait. Its address is asked about
|
until it comes, and its later requests do not wait. Its address is asked about
|
||||||
again in the background.
|
again in the background.
|
||||||
- A client the lookup cannot place has an unknown country and AS number, which
|
- A client the lookup cannot place has an unknown country and AS number, which
|
||||||
the country lists and the biased thresholds treat as described there. Nothing
|
the country list and the biased thresholds treat as described there. Nothing
|
||||||
else treats it differently.
|
else treats it differently.
|
||||||
- GeoJS publishes no rate limit, but its terms forbid "an excessive amount of
|
- GeoJS publishes no rate limit, but its terms forbid "an excessive amount of
|
||||||
API requests", judged by GeoJS alone, and let it block a caller. While GeoJS
|
API requests", judged by GeoJS alone, and let it block a caller. While GeoJS
|
||||||
@@ -496,12 +505,12 @@ service that keeps its visitors' addresses on its own host.
|
|||||||
`file_error` alert and ignored. The database already loaded stays in use. A
|
`file_error` alert and ignored. The database already loaded stays in use. A
|
||||||
client whose address the file does not hold counts as unknown.
|
client whose address the file does not hold counts as unknown.
|
||||||
|
|
||||||
### Country lists
|
### Country list
|
||||||
|
|
||||||
Both are empty by default: the defaults judge a client by what it does, not by
|
`SWW_EXCLUSIVELY_ALLOWED_COUNTRIES` is empty by default: the defaults judge a
|
||||||
where it comes from.
|
client by what it does, not by where it comes from. A `country=` tag in
|
||||||
|
`SWW_TAG_DENY` refuses one country (see "Request tags").
|
||||||
|
|
||||||
- `SWW_DENIED_COUNTRIES` refuses every request from a listed country.
|
|
||||||
- `SWW_EXCLUSIVELY_ALLOWED_COUNTRIES` lets through only clients placed in a
|
- `SWW_EXCLUSIVELY_ALLOWED_COUNTRIES` lets through only clients placed in a
|
||||||
listed country. Every request from any other country is refused. So is every
|
listed country. Every request from any other country is refused. So is every
|
||||||
request from a client the lookup cannot place, such as internal monitoring or
|
request from a client the lookup cannot place, such as internal monitoring or
|
||||||
@@ -510,45 +519,84 @@ where it comes from.
|
|||||||
client in whenever GeoJS stops answering. The cost: while GeoJS fails, this
|
client in whenever GeoJS stops answering. The cost: while GeoJS fails, this
|
||||||
list refuses every new client. No other failure of GeoJS, a reputation source
|
list refuses every new client. No other failure of GeoJS, a reputation source
|
||||||
or an alert destination comes this close to taking the service down.
|
or an alert destination comes this close to taking the service down.
|
||||||
- Both may be set. `SWW_DENIED_COUNTRIES` then adds nothing, since the exclusive
|
- A code in this list whose `country=` tag is in `SWW_TAG_DENY` stops the start
|
||||||
list already refuses every other country. A code on both lists stops the start
|
|
||||||
with a message naming it.
|
with a message naming it.
|
||||||
- A refused request is answered with `SWW_BAN_RESPONSE`, as soon as the client's
|
- A refused request is answered with `SWW_BAN_RESPONSE`, as soon as the client's
|
||||||
address has been looked up. It is logged with the action `country_denied`, and
|
address has been looked up. It is logged with the action `country_denied`, and
|
||||||
counted in the metrics.
|
counted in the metrics.
|
||||||
|
|
||||||
|
### Request tags
|
||||||
|
|
||||||
|
Each request gets tags from what `discriminator` knows of its client and what
|
||||||
|
the checks found. The operator refuses or slows clients by tag. Tags appear in
|
||||||
|
the log line as `tags`, sorted, with empty lists left out (see
|
||||||
|
`docs/request-log.md`).
|
||||||
|
|
||||||
|
Each tag and where it comes from:
|
||||||
|
|
||||||
|
- `asn=<number>` and `country=<code>`, such as `asn=64496` and `country=DE`,
|
||||||
|
from the lookup. A client the lookup cannot place has neither. A request
|
||||||
|
refused by `deny_nets` or `active_ban` has neither, because those checks come
|
||||||
|
before the lookup.
|
||||||
|
- One tag per list of `SWW_TAG_LISTS` that holds the client (`tag_lists`), the
|
||||||
|
tag being the list's name, such as `torexit` or `vpn=mullvad`. These lists are
|
||||||
|
fetched and kept like a blocklist. `discriminator` ships none of its own.
|
||||||
|
- `crowdsec`, `dnsbl=<zone>` and `abuseipdb`, when that reputation source lists
|
||||||
|
the client, whatever the source's action. A zone with a key is named with the
|
||||||
|
key masked, as the `reputation` field gives it.
|
||||||
|
- `exploitsignature`, when a rule file or the Core Rule Set matched the request.
|
||||||
|
For the Core Rule Set a match is a score at or over
|
||||||
|
`SWW_WAF_ANOMALY_THRESHOLD`.
|
||||||
|
|
||||||
|
A client in `SWW_ALLOW_NETS` skips the checks and gets no tag. Every tag is in
|
||||||
|
lower case, except a country code, which is in capitals. `SWW_TAG_DENY` and
|
||||||
|
`SWW_TAG_LIMIT_PERCENT` accept tags in either case.
|
||||||
|
|
||||||
|
Two settings act on tags:
|
||||||
|
|
||||||
|
- `SWW_TAG_DENY` lists tags whose requests are refused with the action `denied`
|
||||||
|
(`tag_deny`). The check runs twice: just before the rate limits, and again
|
||||||
|
after the Core Rule Set, so that `exploitsignature` can refuse too.
|
||||||
|
- `SWW_TAG_LIMIT_PERCENT` lists tags with a percentage from 1 to 99. Their
|
||||||
|
clients get that percentage of every request and byte limit. The lowest
|
||||||
|
percentage among a client's tags applies. `exploitsignature` cannot be listed,
|
||||||
|
since a request gets it only after the rate limits have counted it.
|
||||||
|
|
||||||
|
The tag settings are empty by default, for the same reason as the country list.
|
||||||
|
The lookup waits for GeoJS when either names an `asn=` or `country=` tag. A
|
||||||
|
setting that names one stops the start while `SWW_LOOKUP_SOURCE` is `off`.
|
||||||
|
|
||||||
### Biased thresholds
|
### Biased thresholds
|
||||||
|
|
||||||
The biased thresholds lower the limits of clients in listed AS numbers and
|
The biased thresholds lower the limits of clients by AS number, country and tag.
|
||||||
countries. The lists are empty by default, for the same reason as the country
|
The lists are empty by default, for the same reason as the country list. The
|
||||||
lists. The request log and the metrics show which AS numbers and countries a
|
request log and the metrics show which AS numbers and countries a service's
|
||||||
service's abuse comes from.
|
abuse comes from.
|
||||||
|
|
||||||
- Each list gives AS numbers or countries a percentage, such as `AS14061:50`.
|
- `SWW_TAG_LIMIT_PERCENT` gives a tag a percentage, such as `asn=14061:50` (see
|
||||||
Their clients get that percentage of every request and byte limit
|
"Request tags"). A lowered limit is rounded down.
|
||||||
(`SWW_ASN_LIMIT_PERCENT`, `SWW_COUNTRY_LIMIT_PERCENT`). A lowered limit is
|
- `SWW_UNKNOWN_LIMIT_PERCENT` at `0` is a zero allowance: the client's first
|
||||||
rounded down. `0` is a zero allowance: the client's first request breaks a
|
request breaks a limit and bans it.
|
||||||
limit and bans it.
|
|
||||||
- `SWW_ASN_BYTES_PERCENT` and `SWW_COUNTRY_BYTES_PERCENT` set the byte limits'
|
- `SWW_ASN_BYTES_PERCENT` and `SWW_COUNTRY_BYTES_PERCENT` set the byte limits'
|
||||||
percentage alone, when it should differ from the request limits'. For an AS
|
percentage alone, when it should differ from the request limits'. For an AS
|
||||||
number they list, they replace its percentage from `SWW_ASN_LIMIT_PERCENT` and
|
number they list, they replace its `asn=` tag's percentage and its entry in
|
||||||
`SWW_ASN_LIMIT_PERCENT_URL`; for a country, the one from
|
`SWW_ASN_LIMIT_PERCENT_URL`; for a country, its `country=` tag's percentage.
|
||||||
`SWW_COUNTRY_LIMIT_PERCENT`.
|
|
||||||
- `SWW_ASN_LIMIT_PERCENT_URL` names a text file of `AS:percent` lines, so one
|
- `SWW_ASN_LIMIT_PERCENT_URL` names a text file of `AS:percent` lines, so one
|
||||||
abuse-source list can be shared by every `discriminator` in the fleet. It is
|
abuse-source list can be shared by every `discriminator` in the fleet. It is
|
||||||
fetched and kept like a blocklist (see "Reputation"). Where it and
|
fetched and kept like a blocklist (see "Reputation"). Each entry counts like
|
||||||
`SWW_ASN_LIMIT_PERCENT` both give a percentage, the lower applies.
|
an `asn=` tag in `SWW_TAG_LIMIT_PERCENT`. Where both give a percentage, the
|
||||||
|
lower applies.
|
||||||
- A client's limit percentage is the lowest of those that apply to it: its AS
|
- A client's limit percentage is the lowest of those that apply to it: its AS
|
||||||
number's, its country's, or a reputation hit's `limit:<percent>`. A client
|
number's, its country's, its tags', or a reputation hit's `limit:<percent>`. A
|
||||||
without a country gets `SWW_UNKNOWN_LIMIT_PERCENT` in place of a country's.
|
client without a country gets `SWW_UNKNOWN_LIMIT_PERCENT` in place of a
|
||||||
The percentage is 100 if none applies. Clients given a lower percentage are
|
country's. The percentage is 100 if none applies. Clients given a lower
|
||||||
banned after fewer requests, first for a while and then, for repeated abuse,
|
percentage are banned after fewer requests, first for a while and then, for
|
||||||
permanently. The log line of each counted request, and the notes of each ban,
|
repeated abuse, permanently. The log line of each counted request, and the
|
||||||
say which setting lowered the limit.
|
notes of each ban, say which setting lowered the limit.
|
||||||
- No budget is shared by a whole country or AS number. One abuser could use it
|
- No budget is shared by a whole country or AS number. One abuser could use it
|
||||||
up and lock out everyone else there: a denial of service nobody chose, and the
|
up and lock out everyone else there: a denial of service nobody chose, and the
|
||||||
one this tool exists to prevent. Refusing a whole country is left to the
|
one this tool exists to prevent. Refusing a whole country is left to the
|
||||||
operator's choice, through the country lists.
|
operator's choice, through `SWW_TAG_DENY` or the country list.
|
||||||
|
|
||||||
### Attack detection
|
### Attack detection
|
||||||
|
|
||||||
@@ -903,7 +951,7 @@ left unset sends no alert. An alert needs a destination only the operator can
|
|||||||
supply, and what counts as unusual depends on each service's normal traffic,
|
supply, and what counts as unusual depends on each service's normal traffic,
|
||||||
which the metrics show. They reveal a widely distributed scraper, thousands of
|
which the metrics show. They reveal a widely distributed scraper, thousands of
|
||||||
addresses at low rates each, which the per-client limits do not see. A low
|
addresses at low rates each, which the per-client limits do not see. A low
|
||||||
`SWW_ASN_LIMIT_PERCENT` for its AS number is the response.
|
percentage for its `asn=` tag in `SWW_TAG_LIMIT_PERCENT` is the response.
|
||||||
|
|
||||||
- Requests and bytes are counted per minute and per hour, for each of:
|
- Requests and bytes are counted per minute and per hour, for each of:
|
||||||
- a client;
|
- a client;
|
||||||
@@ -1181,7 +1229,7 @@ the metrics.
|
|||||||
|
|
||||||
### The answer
|
### The answer
|
||||||
|
|
||||||
`discriminator` runs steps 3 to 13 of "What happens to a request", in that
|
`discriminator` runs steps 3 to 16 of "What happens to a request", in that
|
||||||
order. Every answer but the last below carries `X-Sww-Disposition`.
|
order. Every answer but the last below carries `X-Sww-Disposition`.
|
||||||
|
|
||||||
- `X-Sww-Disposition: allow`, status `200`, no body. `rproxy` passes the request
|
- `X-Sww-Disposition: allow`, status `200`, no body. `rproxy` passes the request
|
||||||
@@ -1220,13 +1268,13 @@ seconds, and `SWRP_DEFAULT_DISPOSITION` then decides. So at the defaults,
|
|||||||
and whatever its outside sources do.
|
and whatever its outside sources do.
|
||||||
|
|
||||||
GeoJS is the only outside source a question can wait on. A question waits only
|
GeoJS is the only outside source a question can wait on. A question waits only
|
||||||
when step 5 says so, and for at most `SWW_LOOKUP_TIMEOUT`, by default 800
|
when step 6 says so, and for at most `SWW_LOOKUP_TIMEOUT`, by default 800
|
||||||
milliseconds. A client without an answer by then counts as unknown. No question
|
milliseconds. A client without an answer by then counts as unknown. No question
|
||||||
waits on these, which run in the background:
|
waits on these, which run in the background:
|
||||||
|
|
||||||
- the DNS blocklists and AbuseIPDB;
|
- the DNS blocklists and AbuseIPDB;
|
||||||
- fetching the blocklists, the CrowdSec decision list and the file
|
- fetching the blocklists, the tag lists, the CrowdSec decision list and the
|
||||||
`SWW_ASN_LIMIT_PERCENT_URL` names.
|
file `SWW_ASN_LIMIT_PERCENT_URL` names.
|
||||||
|
|
||||||
In `enforce` mode, a question does not wait for GeoJS when it is a request:
|
In `enforce` mode, a question does not wait for GeoJS when it is a request:
|
||||||
|
|
||||||
|
|||||||
+5
-2
@@ -96,8 +96,8 @@ Every one of these lines has:
|
|||||||
|
|
||||||
- `action`: what was done:
|
- `action`: what was done:
|
||||||
- `forward`: answered `allow`, so that `rproxy` passes it to the app;
|
- `forward`: answered `allow`, so that `rproxy` passes it to the app;
|
||||||
- `denied`: refused for `SWW_DENY_NETS`, or for a reputation source with the
|
- `denied`: refused for `SWW_DENY_NETS`, for a reputation source with the
|
||||||
action `deny`;
|
action `deny`, or for a tag in `SWW_TAG_DENY`;
|
||||||
- `banned`: refused under a ban, or for a `ban` rule, a trap path or a
|
- `banned`: refused under a ban, or for a `ban` rule, a trap path or a
|
||||||
CrowdSec decision, each of which bans the client;
|
CrowdSec decision, each of which bans the client;
|
||||||
- `country_denied`, `rate_limited`, `rule_blocked` and `waf_blocked`:
|
- `country_denied`, `rate_limited`, `rule_blocked` and `waf_blocked`:
|
||||||
@@ -115,6 +115,7 @@ Every one of these lines has:
|
|||||||
- `deny_nets`: `netblock`, the entry of `SWW_DENY_NETS` the client is in;
|
- `deny_nets`: `netblock`, the entry of `SWW_DENY_NETS` the client is in;
|
||||||
- `active_ban`: `netblock`, the ban's netblock, and `ban_expires`;
|
- `active_ban`: `netblock`, the ban's netblock, and `ban_expires`;
|
||||||
- `countries`: `country`, and `setting`, the country list that refused it;
|
- `countries`: `country`, and `setting`, the country list that refused it;
|
||||||
|
- `tag_deny`: `tag`, the tag of `SWW_TAG_DENY` the request carries;
|
||||||
- `blocklists`: `lists`, the blocklist URLs that list the client;
|
- `blocklists`: `lists`, the blocklist URLs that list the client;
|
||||||
- `crowdsec`: `scenario`, the decision's scenario, and `expires`, when the
|
- `crowdsec`: `scenario`, the decision's scenario, and `expires`, when the
|
||||||
decision ends;
|
decision ends;
|
||||||
@@ -168,6 +169,8 @@ Every one of these lines has:
|
|||||||
- the CrowdSec decision list's URL;
|
- the CrowdSec decision list's URL;
|
||||||
- DNS blocklist zones;
|
- DNS blocklist zones;
|
||||||
- `abuseipdb`.
|
- `abuseipdb`.
|
||||||
|
- `tags`: the request's tags, sorted, left out when it has none. Each tag and
|
||||||
|
where it comes from is under "Request tags" in [`SPEC.md`](../SPEC.md).
|
||||||
- `offence`: `limit`, for a request that broke a limit.
|
- `offence`: `limit`, for a request that broke a limit.
|
||||||
- `ban_expires`: when the client's ban ends, for a request that made one or was
|
- `ban_expires`: when the client's ban ends, for a request that made one or was
|
||||||
refused under one: a time, or `permanent`.
|
refused under one: a time, or `permanent`.
|
||||||
|
|||||||
+49
-16
@@ -90,7 +90,7 @@ between the two (see "The image's command" below).
|
|||||||
## Static lists
|
## Static lists
|
||||||
|
|
||||||
- `SWW_ALLOW_NETS` (default empty): netblocks you trust, such as your monitoring
|
- `SWW_ALLOW_NETS` (default empty): netblocks you trust, such as your monitoring
|
||||||
(see step 3 in [`SPEC.md`](../SPEC.md)).
|
(see step 4 in [`SPEC.md`](../SPEC.md)).
|
||||||
- `SWW_RATE_LIMIT_EXEMPT_NETS` (default empty): netblocks whose clients are
|
- `SWW_RATE_LIMIT_EXEMPT_NETS` (default empty): netblocks whose clients are
|
||||||
exempt from rate limiting and the byte limits.
|
exempt from rate limiting and the byte limits.
|
||||||
- `SWW_DENY_NETS` (default empty): netblocks whose clients are refused (see step
|
- `SWW_DENY_NETS` (default empty): netblocks whose clients are refused (see step
|
||||||
@@ -130,9 +130,11 @@ between the two (see "The image's command" below).
|
|||||||
|
|
||||||
- `SWW_LOOKUP_SOURCE` (default `geojs`): `geojs`, `file` or `off`. `off` stops
|
- `SWW_LOOKUP_SOURCE` (default `geojs`): `geojs`, `file` or `off`. `off` stops
|
||||||
the start while a setting that needs a lookup is set:
|
the start while a setting that needs a lookup is set:
|
||||||
- the two country lists;
|
- `SWW_EXCLUSIVELY_ALLOWED_COUNTRIES`;
|
||||||
- `SWW_ADD_LOOKUP_HEADERS`;
|
- `SWW_ADD_LOOKUP_HEADERS`;
|
||||||
- the four biased threshold lists and `SWW_ASN_LIMIT_PERCENT_URL`;
|
- `SWW_TAG_DENY` or `SWW_TAG_LIMIT_PERCENT` naming an `asn=` or `country=`
|
||||||
|
tag;
|
||||||
|
- the two byte percentage lists and `SWW_ASN_LIMIT_PERCENT_URL`;
|
||||||
- `SWW_UNKNOWN_LIMIT_PERCENT` below `100`;
|
- `SWW_UNKNOWN_LIMIT_PERCENT` below `100`;
|
||||||
- an anomaly threshold per AS number.
|
- an anomaly threshold per AS number.
|
||||||
- `SWW_LOOKUP_DB_PATH` (default empty): the IPinfo Lite database file, in its
|
- `SWW_LOOKUP_DB_PATH` (default empty): the IPinfo Lite database file, in its
|
||||||
@@ -144,28 +146,59 @@ between the two (see "The image's command" below).
|
|||||||
the client's AS number in `X-Client-ASN` and its country in
|
the client's AS number in `X-Client-ASN` and its country in
|
||||||
`X-Client-Country`.
|
`X-Client-Country`.
|
||||||
|
|
||||||
## Country lists
|
## Country list
|
||||||
|
|
||||||
- `SWW_DENIED_COUNTRIES` (default empty): countries whose clients are refused,
|
|
||||||
such as `cn,ru,kp`.
|
|
||||||
- `SWW_EXCLUSIVELY_ALLOWED_COUNTRIES` (default empty): when set, the only
|
- `SWW_EXCLUSIVELY_ALLOWED_COUNTRIES` (default empty): when set, the only
|
||||||
countries whose clients get through, such as `us,de`. A country on both lists
|
countries whose clients get through, such as `us,de`. A country whose
|
||||||
stops the start.
|
`country=` tag is in `SWW_TAG_DENY` stops the start.
|
||||||
|
|
||||||
|
## Request tags
|
||||||
|
|
||||||
|
Every request gets tags from what `discriminator` knows of its client. See
|
||||||
|
"Request tags" in [`SPEC.md`](../SPEC.md).
|
||||||
|
|
||||||
|
- `SWW_TAG_LISTS` (default empty): address lists that each add a tag, as
|
||||||
|
`name=URL` pairs, such as `torexit=https://example.com/tor.txt`. A name is the
|
||||||
|
tag, and may hold `=`, such as `vpn=mullvad`. Each list is an `http` or
|
||||||
|
`https` URL fetched like a blocklist. A name or URL listed twice is refused.
|
||||||
|
So is a URL that is also in `SWW_BLOCKLIST_URLS`, `SWW_ASN_LIMIT_PERCENT_URL`,
|
||||||
|
or the CrowdSec decision list.
|
||||||
|
- `SWW_TAG_DENY` (default empty): tags whose requests are refused, such as
|
||||||
|
`torexit,country=CN`. A tag listed twice, an `asn=` tag without a number, or a
|
||||||
|
`country=` tag without a country, is refused.
|
||||||
|
- `SWW_TAG_LIMIT_PERCENT` (default empty): tags that lower a client's limits, as
|
||||||
|
tag and percentage pairs, such as `vpn:50,asn=14061:25`. A percentage is from
|
||||||
|
`1` to `99`, of every rate limit and byte limit. The lowest percentage among a
|
||||||
|
client's tags applies. A tag listed twice is refused. So is
|
||||||
|
`exploitsignature`: a request gets that tag after the rate limits have counted
|
||||||
|
it.
|
||||||
|
|
||||||
|
`SWW_TAG_DENY` and `SWW_TAG_LIMIT_PERCENT` accept tags in either case. A tag
|
||||||
|
that no request can get stops the start. A request can get only these:
|
||||||
|
|
||||||
|
- a name in `SWW_TAG_LISTS`;
|
||||||
|
- an `asn=` or `country=` tag;
|
||||||
|
- a `dnsbl=` tag of a zone in `SWW_DNSBL_ZONES`, in lower case, with a key
|
||||||
|
masked as the `reputation` field shows it;
|
||||||
|
- `abuseipdb`, while `SWW_ABUSEIPDB_KEY` is set;
|
||||||
|
- `exploitsignature`, in `SWW_TAG_DENY` only.
|
||||||
|
|
||||||
|
`crowdsec` stops the start in both settings, because a CrowdSec decision refuses
|
||||||
|
the request first.
|
||||||
|
|
||||||
## Biased thresholds
|
## Biased thresholds
|
||||||
|
|
||||||
- `SWW_ASN_LIMIT_PERCENT` (default empty): a list of AS numbers, each with `:`
|
- `SWW_ASN_BYTES_PERCENT` and `SWW_COUNTRY_BYTES_PERCENT` (default empty): a
|
||||||
and a percentage, such as `AS14061:50,AS16276:50,AS45102:25`. An AS number
|
percentage of the byte limits alone, for the AS numbers or countries listed,
|
||||||
listed twice is refused.
|
such as `AS14061:50` or `cn:25`. It takes the place of the percentage
|
||||||
- `SWW_COUNTRY_LIMIT_PERCENT` (default empty): the same by country, such as
|
`SWW_TAG_LIMIT_PERCENT` gives their `asn=` or `country=` tag. An AS number or
|
||||||
`cn:25,ru:50`.
|
country listed twice is refused.
|
||||||
- `SWW_ASN_BYTES_PERCENT` and `SWW_COUNTRY_BYTES_PERCENT` (default empty): the
|
|
||||||
same, for the byte limits alone.
|
|
||||||
- `SWW_UNKNOWN_LIMIT_PERCENT` (default `100`): the percentage for a client the
|
- `SWW_UNKNOWN_LIMIT_PERCENT` (default `100`): the percentage for a client the
|
||||||
lookup cannot place.
|
lookup cannot place.
|
||||||
- `SWW_ASN_LIMIT_PERCENT_URL` (default unset): the `http` or `https` URL of a
|
- `SWW_ASN_LIMIT_PERCENT_URL` (default unset): the `http` or `https` URL of a
|
||||||
text file with one AS number and percentage on each line, such as
|
text file with one AS number and percentage on each line, such as
|
||||||
`AS64496:50`. A URL that is also in `SWW_BLOCKLIST_URLS` is refused.
|
`AS64496:50`. Each entry counts like an `asn=` tag in `SWW_TAG_LIMIT_PERCENT`.
|
||||||
|
A URL that is also in `SWW_BLOCKLIST_URLS` is refused.
|
||||||
|
|
||||||
## Attack detection
|
## Attack detection
|
||||||
|
|
||||||
|
|||||||
+303
-28
@@ -93,29 +93,30 @@ type Config struct {
|
|||||||
LookupDBPath string
|
LookupDBPath string
|
||||||
LookupTimeout time.Duration
|
LookupTimeout time.Duration
|
||||||
AddLookupHeaders bool
|
AddLookupHeaders bool
|
||||||
// DeniedCountries are the countries whose clients are refused
|
// ExclusivelyAllowedCountries, when not empty, are the only countries
|
||||||
// (SWW_DENIED_COUNTRIES). ExclusivelyAllowedCountries, when not
|
// whose clients are let through (SWW_EXCLUSIVELY_ALLOWED_COUNTRIES),
|
||||||
// empty, are the only countries whose clients are let through
|
// two-letter codes in capitals, as GeoJS gives them.
|
||||||
// (SWW_EXCLUSIVELY_ALLOWED_COUNTRIES). Both hold two-letter codes in
|
|
||||||
// capitals, as GeoJS gives them.
|
|
||||||
DeniedCountries []string
|
|
||||||
ExclusivelyAllowedCountries []string
|
ExclusivelyAllowedCountries []string
|
||||||
// The biased thresholds. ASNLimitPercent and CountryLimitPercent give
|
// TagLists are the address lists that each add a tag to the requests of
|
||||||
// the clients of the AS numbers and the countries they list that
|
// the clients they hold (SWW_TAG_LISTS). TagDeny are the tags whose
|
||||||
// percentage of every rate limit and byte limit
|
// requests are refused (SWW_TAG_DENY). TagLimitPercent gives the clients
|
||||||
// (SWW_ASN_LIMIT_PERCENT and SWW_COUNTRY_LIMIT_PERCENT).
|
// with a tag that percentage of every rate limit and byte limit, from 1
|
||||||
// ASNBytesPercent and CountryBytesPercent give those they list a
|
// to 99 (SWW_TAG_LIMIT_PERCENT). All hold tags as requests carry them: in
|
||||||
// percentage of the byte limits in place of that one
|
// lower case, a country code in capitals.
|
||||||
|
TagLists []TagList
|
||||||
|
TagDeny []string
|
||||||
|
TagLimitPercent map[string]int64
|
||||||
|
// The biased thresholds. ASNBytesPercent and CountryBytesPercent give
|
||||||
|
// those they list a percentage of the byte limits in place of the one
|
||||||
|
// TagLimitPercent gives their asn= or country= tag
|
||||||
// (SWW_ASN_BYTES_PERCENT and SWW_COUNTRY_BYTES_PERCENT). Each holds
|
// (SWW_ASN_BYTES_PERCENT and SWW_COUNTRY_BYTES_PERCENT). Each holds
|
||||||
// percentages from 0 to 100, by AS number, written as AS64496, or by
|
// percentages from 0 to 100, by AS number, written as AS64496, or by
|
||||||
// country, a two-letter code in capitals, as the lookup gives them.
|
// country, a two-letter code in capitals, as the lookup gives them.
|
||||||
// UnknownLimitPercent is the percentage of every limit a client without
|
// UnknownLimitPercent is the percentage of every limit a client without
|
||||||
// a country gets (SWW_UNKNOWN_LIMIT_PERCENT). ASNLimitPercentURL is
|
// a country gets (SWW_UNKNOWN_LIMIT_PERCENT). ASNLimitPercentURL is
|
||||||
// where a file of AS:percent lines is fetched from, whose percentages
|
// where a file of AS:percent lines is fetched from, whose percentages
|
||||||
// count as those of ASNLimitPercent do (SWW_ASN_LIMIT_PERCENT_URL), ""
|
// count like asn= tags in TagLimitPercent (SWW_ASN_LIMIT_PERCENT_URL),
|
||||||
// while it is unset.
|
// "" while it is unset.
|
||||||
ASNLimitPercent map[string]int64
|
|
||||||
CountryLimitPercent map[string]int64
|
|
||||||
ASNBytesPercent map[string]int64
|
ASNBytesPercent map[string]int64
|
||||||
CountryBytesPercent map[string]int64
|
CountryBytesPercent map[string]int64
|
||||||
UnknownLimitPercent int64
|
UnknownLimitPercent int64
|
||||||
@@ -164,8 +165,8 @@ type Config struct {
|
|||||||
// BanResponse is the status a refused client is answered with, 403
|
// BanResponse is the status a refused client is answered with, 403
|
||||||
// or 429, or 0 to close the connection without an answer
|
// or 429, or 0 to close the connection without an answer
|
||||||
// (SWW_BAN_RESPONSE). It answers a banned client, a request that
|
// (SWW_BAN_RESPONSE). It answers a banned client, a request that
|
||||||
// breaks a rate limit or matches a ban rule, SWW_DENY_NETS and the
|
// breaks a rate limit or matches a ban rule, SWW_DENY_NETS,
|
||||||
// country lists.
|
// SWW_EXCLUSIVELY_ALLOWED_COUNTRIES and SWW_TAG_DENY.
|
||||||
BanResponse int
|
BanResponse int
|
||||||
// LimitBanDuration is the ban for a first broken rate limit
|
// LimitBanDuration is the ban for a first broken rate limit
|
||||||
// (SWW_LIMIT_BAN_DURATION). A limit broken again within
|
// (SWW_LIMIT_BAN_DURATION). A limit broken again within
|
||||||
@@ -367,7 +368,7 @@ var (
|
|||||||
errHeaderTakenOut = errors.New(
|
errHeaderTakenOut = errors.New(
|
||||||
"is taken out of every request by Go's HTTP server, so it can never " +
|
"is taken out of every request by Go's HTTP server, so it can never " +
|
||||||
"be logged")
|
"be logged")
|
||||||
errOnBothLists = errors.New("is in SWW_DENIED_COUNTRIES too")
|
errOnBothLists = errors.New("is in SWW_TAG_DENY too")
|
||||||
errNotLookupSource = errors.New("is not geojs, file or off")
|
errNotLookupSource = errors.New("is not geojs, file or off")
|
||||||
errNeedsLookups = errors.New("it needs each client looked up")
|
errNeedsLookups = errors.New("it needs each client looked up")
|
||||||
errNeedsDBPath = errors.New("it names the file to look clients up in")
|
errNeedsDBPath = errors.New("it names the file to look clients up in")
|
||||||
@@ -465,6 +466,24 @@ var (
|
|||||||
errLAPIKeyUnused = errors.New("it is sent only to the engine at that URL")
|
errLAPIKeyUnused = errors.New("it is sent only to the engine at that URL")
|
||||||
errAnotherList = errors.New(
|
errAnotherList = errors.New(
|
||||||
"is in SWW_BLOCKLIST_URLS or is SWW_ASN_LIMIT_PERCENT_URL too")
|
"is in SWW_BLOCKLIST_URLS or is SWW_ASN_LIMIT_PERCENT_URL too")
|
||||||
|
errFetchedOtherwise = errors.New("is in SWW_BLOCKLIST_URLS, is " +
|
||||||
|
"SWW_ASN_LIMIT_PERCENT_URL or is the CrowdSec decision list too")
|
||||||
|
errNotTagList = errors.New(
|
||||||
|
"is not a name, = and an http or https URL, such as " +
|
||||||
|
"torexit=https://example.com/tor.txt")
|
||||||
|
errNotTagPercentItem = errors.New(
|
||||||
|
"is not a tag, : and a percentage, such as vpn:50 or asn=14061:25")
|
||||||
|
errNotTagPercent = errors.New("is not a percentage from 1 to 99")
|
||||||
|
errNotASNTag = errors.New("is not asn= and a number, such as asn=14061")
|
||||||
|
errExploitTag = errors.New(
|
||||||
|
"cannot lower limits: the rate limits count a request before its " +
|
||||||
|
"rules are matched")
|
||||||
|
errCrowdSecTag = errors.New(
|
||||||
|
"cannot be used: a CrowdSec decision refuses the request first")
|
||||||
|
errNoRequestGetsTag = errors.New(
|
||||||
|
"is a tag no request can get: it is not a name in SWW_TAG_LISTS, an " +
|
||||||
|
"asn= or country= tag, a dnsbl= tag of a zone in SWW_DNSBL_ZONES, " +
|
||||||
|
"or abuseipdb with SWW_ABUSEIPDB_KEY set")
|
||||||
)
|
)
|
||||||
|
|
||||||
// FromEnvironment reads the settings with lookupEnv, normally
|
// FromEnvironment reads the settings with lookupEnv, normally
|
||||||
@@ -499,11 +518,11 @@ func FromEnvironment(
|
|||||||
LookupDBPath: env.value("SWW_LOOKUP_DB_PATH", ""),
|
LookupDBPath: env.value("SWW_LOOKUP_DB_PATH", ""),
|
||||||
LookupTimeout: env.durationNotOff("SWW_LOOKUP_TIMEOUT", "800ms"),
|
LookupTimeout: env.durationNotOff("SWW_LOOKUP_TIMEOUT", "800ms"),
|
||||||
AddLookupHeaders: env.boolean("SWW_ADD_LOOKUP_HEADERS", "false"),
|
AddLookupHeaders: env.boolean("SWW_ADD_LOOKUP_HEADERS", "false"),
|
||||||
DeniedCountries: env.countries("SWW_DENIED_COUNTRIES", ""),
|
|
||||||
ExclusivelyAllowedCountries: env.countries(
|
ExclusivelyAllowedCountries: env.countries(
|
||||||
"SWW_EXCLUSIVELY_ALLOWED_COUNTRIES", ""),
|
"SWW_EXCLUSIVELY_ALLOWED_COUNTRIES", ""),
|
||||||
ASNLimitPercent: env.percents("SWW_ASN_LIMIT_PERCENT", ParseASN),
|
TagLists: env.tagLists("SWW_TAG_LISTS"),
|
||||||
CountryLimitPercent: env.percents("SWW_COUNTRY_LIMIT_PERCENT", parseCountry),
|
TagDeny: env.tags("SWW_TAG_DENY"),
|
||||||
|
TagLimitPercent: env.tagPercents("SWW_TAG_LIMIT_PERCENT"),
|
||||||
ASNBytesPercent: env.percents("SWW_ASN_BYTES_PERCENT", ParseASN),
|
ASNBytesPercent: env.percents("SWW_ASN_BYTES_PERCENT", ParseASN),
|
||||||
CountryBytesPercent: env.percents("SWW_COUNTRY_BYTES_PERCENT", parseCountry),
|
CountryBytesPercent: env.percents("SWW_COUNTRY_BYTES_PERCENT", parseCountry),
|
||||||
UnknownLimitPercent: env.percent("SWW_UNKNOWN_LIMIT_PERCENT", "100"),
|
UnknownLimitPercent: env.percent("SWW_UNKNOWN_LIMIT_PERCENT", "100"),
|
||||||
@@ -583,6 +602,8 @@ func FromEnvironment(
|
|||||||
env.checkCountriesAndLookups(cfg)
|
env.checkCountriesAndLookups(cfg)
|
||||||
env.checkASNLimitPercentURL(cfg)
|
env.checkASNLimitPercentURL(cfg)
|
||||||
env.checkCrowdSec(cfg)
|
env.checkCrowdSec(cfg)
|
||||||
|
env.checkTagLists(cfg)
|
||||||
|
env.checkTags(cfg)
|
||||||
|
|
||||||
if env.err != nil {
|
if env.err != nil {
|
||||||
return nil, env.err
|
return nil, env.err
|
||||||
@@ -860,6 +881,31 @@ func (e *environment) percents(
|
|||||||
return percents
|
return percents
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// tagLists reads SWW_TAG_LISTS, which is empty by default.
|
||||||
|
func (e *environment) tagLists(name string) []TagList {
|
||||||
|
lists, err := parseTagLists(e.value(name, ""))
|
||||||
|
e.check(name, err)
|
||||||
|
|
||||||
|
return lists
|
||||||
|
}
|
||||||
|
|
||||||
|
// tags reads a setting that is a list of tags. It is empty by default.
|
||||||
|
func (e *environment) tags(name string) []string {
|
||||||
|
tags, err := parseTags(e.value(name, ""))
|
||||||
|
e.check(name, err)
|
||||||
|
|
||||||
|
return tags
|
||||||
|
}
|
||||||
|
|
||||||
|
// tagPercents reads a setting that is a list of tags, each with a
|
||||||
|
// percentage. It is empty by default.
|
||||||
|
func (e *environment) tagPercents(name string) map[string]int64 {
|
||||||
|
percents, err := parseTagPercents(e.value(name, ""))
|
||||||
|
e.check(name, err)
|
||||||
|
|
||||||
|
return percents
|
||||||
|
}
|
||||||
|
|
||||||
// percent reads a setting that is a percentage, from 0 to 100.
|
// percent reads a setting that is a percentage, from 0 to 100.
|
||||||
func (e *environment) percent(name, defaultValue string) int64 {
|
func (e *environment) percent(name, defaultValue string) int64 {
|
||||||
percent, err := ParsePercent(e.value(name, defaultValue))
|
percent, err := ParsePercent(e.value(name, defaultValue))
|
||||||
@@ -997,15 +1043,17 @@ func (e *environment) checkLookupDBPath(cfg *Config) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkCountriesAndLookups refuses a country on both country lists, and,
|
// checkCountriesAndLookups refuses a country in
|
||||||
|
// SWW_EXCLUSIVELY_ALLOWED_COUNTRIES whose tag is in SWW_TAG_DENY, and,
|
||||||
// while SWW_LOOKUP_SOURCE is off, each setting that needs clients looked
|
// while SWW_LOOKUP_SOURCE is off, each setting that needs clients looked
|
||||||
// up: the country lists, SWW_ADD_LOOKUP_HEADERS, the biased thresholds,
|
// up: the country list, SWW_ADD_LOOKUP_HEADERS, the tag settings that name
|
||||||
|
// an asn= or country= tag, the biased thresholds,
|
||||||
// SWW_ASN_LIMIT_PERCENT_URL among them, of which
|
// SWW_ASN_LIMIT_PERCENT_URL among them, of which
|
||||||
// SWW_UNKNOWN_LIMIT_PERCENT needs them only below 100, where it lowers a
|
// SWW_UNKNOWN_LIMIT_PERCENT needs them only below 100, where it lowers a
|
||||||
// limit, and the anomaly thresholds per AS number.
|
// limit, and the anomaly thresholds per AS number.
|
||||||
func (e *environment) checkCountriesAndLookups(cfg *Config) {
|
func (e *environment) checkCountriesAndLookups(cfg *Config) {
|
||||||
for _, country := range cfg.ExclusivelyAllowedCountries {
|
for _, country := range cfg.ExclusivelyAllowedCountries {
|
||||||
if slices.Contains(cfg.DeniedCountries, country) {
|
if slices.Contains(cfg.TagDeny, CountryTagPrefix+country) {
|
||||||
e.check("SWW_EXCLUSIVELY_ALLOWED_COUNTRIES",
|
e.check("SWW_EXCLUSIVELY_ALLOWED_COUNTRIES",
|
||||||
fmt.Errorf("%q %w", country, errOnBothLists))
|
fmt.Errorf("%q %w", country, errOnBothLists))
|
||||||
}
|
}
|
||||||
@@ -1019,11 +1067,10 @@ func (e *environment) checkCountriesAndLookups(cfg *Config) {
|
|||||||
name string
|
name string
|
||||||
set bool
|
set bool
|
||||||
}{
|
}{
|
||||||
{"SWW_DENIED_COUNTRIES", len(cfg.DeniedCountries) > 0},
|
|
||||||
{"SWW_EXCLUSIVELY_ALLOWED_COUNTRIES", len(cfg.ExclusivelyAllowedCountries) > 0},
|
{"SWW_EXCLUSIVELY_ALLOWED_COUNTRIES", len(cfg.ExclusivelyAllowedCountries) > 0},
|
||||||
{"SWW_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders},
|
{"SWW_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders},
|
||||||
{"SWW_ASN_LIMIT_PERCENT", len(cfg.ASNLimitPercent) > 0},
|
{"SWW_TAG_DENY", slices.ContainsFunc(cfg.TagDeny, isLookupTag)},
|
||||||
{"SWW_COUNTRY_LIMIT_PERCENT", len(cfg.CountryLimitPercent) > 0},
|
{"SWW_TAG_LIMIT_PERCENT", tagLimitNeedsLookups(cfg)},
|
||||||
{"SWW_ASN_BYTES_PERCENT", len(cfg.ASNBytesPercent) > 0},
|
{"SWW_ASN_BYTES_PERCENT", len(cfg.ASNBytesPercent) > 0},
|
||||||
{"SWW_COUNTRY_BYTES_PERCENT", len(cfg.CountryBytesPercent) > 0},
|
{"SWW_COUNTRY_BYTES_PERCENT", len(cfg.CountryBytesPercent) > 0},
|
||||||
{"SWW_UNKNOWN_LIMIT_PERCENT", cfg.UnknownLimitPercent < 100},
|
{"SWW_UNKNOWN_LIMIT_PERCENT", cfg.UnknownLimitPercent < 100},
|
||||||
@@ -1069,6 +1116,85 @@ func (e *environment) checkCrowdSec(cfg *Config) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// checkTagLists refuses a list of SWW_TAG_LISTS whose URL is fetched as
|
||||||
|
// another list too: the file at a URL is read as one kind of list.
|
||||||
|
func (e *environment) checkTagLists(cfg *Config) {
|
||||||
|
for _, list := range cfg.TagLists {
|
||||||
|
if slices.Contains(cfg.BlocklistURLs, list.URL) ||
|
||||||
|
list.URL == cfg.ASNLimitPercentURL || list.URL == cfg.CrowdSecDecisionsURL {
|
||||||
|
e.check("SWW_TAG_LISTS", fmt.Errorf("%q %w", list.URL, errFetchedOtherwise))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkTags refuses a tag of SWW_TAG_DENY or SWW_TAG_LIMIT_PERCENT that would
|
||||||
|
// never apply: crowdsec, whose request is refused before tags are acted on, and
|
||||||
|
// any tag these settings cannot give a request.
|
||||||
|
func (e *environment) checkTags(cfg *Config) {
|
||||||
|
for _, tag := range cfg.TagDeny {
|
||||||
|
e.check("SWW_TAG_DENY", checkTag(cfg, tag))
|
||||||
|
}
|
||||||
|
|
||||||
|
for tag := range cfg.TagLimitPercent {
|
||||||
|
e.check("SWW_TAG_LIMIT_PERCENT", checkTag(cfg, tag))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkTag returns an error when tag would never apply, as checkTags says.
|
||||||
|
func checkTag(cfg *Config, tag string) error {
|
||||||
|
if tag == CrowdSecTag {
|
||||||
|
return fmt.Errorf("%q %w", tag, errCrowdSecTag)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !requestCanGetTag(cfg, tag) {
|
||||||
|
return fmt.Errorf("%q %w", MaskZoneKey(tag), errNoRequestGetsTag)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// requestCanGetTag reports whether anything cfg sets up can give a request tag.
|
||||||
|
func requestCanGetTag(cfg *Config, tag string) bool {
|
||||||
|
isListName := func(list TagList) bool { return list.Name == tag }
|
||||||
|
if isLookupTag(tag) || tag == ExploitSignatureTag ||
|
||||||
|
slices.ContainsFunc(cfg.TagLists, isListName) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
if tag == AbuseIPDBTag {
|
||||||
|
return cfg.AbuseIPDBKey != ""
|
||||||
|
}
|
||||||
|
|
||||||
|
zone, isZoneTag := strings.CutPrefix(tag, DNSBLTagPrefix)
|
||||||
|
|
||||||
|
return isZoneTag && slices.ContainsFunc(cfg.DNSBLZones, func(each string) bool {
|
||||||
|
return strings.ToLower(MaskZoneKey(each)) == zone
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// TagsNeedLookup reports whether SWW_TAG_DENY or SWW_TAG_LIMIT_PERCENT
|
||||||
|
// names an asn= or country= tag, which only the lookup can give a request.
|
||||||
|
func (c *Config) TagsNeedLookup() bool {
|
||||||
|
return slices.ContainsFunc(c.TagDeny, isLookupTag) || tagLimitNeedsLookups(c)
|
||||||
|
}
|
||||||
|
|
||||||
|
// tagLimitNeedsLookups reports whether SWW_TAG_LIMIT_PERCENT names an asn=
|
||||||
|
// or country= tag.
|
||||||
|
func tagLimitNeedsLookups(cfg *Config) bool {
|
||||||
|
for tag := range cfg.TagLimitPercent {
|
||||||
|
if isLookupTag(tag) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// isLookupTag reports whether tag is an asn= or country= tag.
|
||||||
|
func isLookupTag(tag string) bool {
|
||||||
|
return strings.HasPrefix(tag, ASNTagPrefix) || strings.HasPrefix(tag, CountryTagPrefix)
|
||||||
|
}
|
||||||
|
|
||||||
// headerNames reads a setting that is a list of header names, and
|
// headerNames reads a setting that is a list of header names, and
|
||||||
// returns them in lower case.
|
// returns them in lower case.
|
||||||
func (e *environment) headerNames(name, defaultValue string) []string {
|
func (e *environment) headerNames(name, defaultValue string) []string {
|
||||||
@@ -1785,6 +1911,155 @@ func ParseASN(value string) (string, error) {
|
|||||||
return "AS" + strconv.FormatUint(number, 10), nil
|
return "AS" + strconv.FormatUint(number, 10), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The tags a request can get that the settings can name, as discriminator
|
||||||
|
// writes them. The prefixes are followed by a value.
|
||||||
|
const (
|
||||||
|
ASNTagPrefix = "asn="
|
||||||
|
CountryTagPrefix = "country="
|
||||||
|
DNSBLTagPrefix = "dnsbl="
|
||||||
|
AbuseIPDBTag = "abuseipdb"
|
||||||
|
CrowdSecTag = "crowdsec"
|
||||||
|
// ExploitSignatureTag is the tag of a request a rule file or the Core
|
||||||
|
// Rule Set matched.
|
||||||
|
ExploitSignatureTag = "exploitsignature"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TagList is an address list of SWW_TAG_LISTS: the tag it adds to the
|
||||||
|
// requests of the clients it holds, and the URL it is fetched from.
|
||||||
|
type TagList struct {
|
||||||
|
Name string
|
||||||
|
URL string
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseTagLists reads a comma-separated list of name=URL pairs, such as
|
||||||
|
// torexit=https://example.com/tor.txt, and returns each list with its name
|
||||||
|
// in lower case. A name can hold a =, as vpn=mullvad does, and ends at the
|
||||||
|
// = before http. A name or a URL listed twice is an error.
|
||||||
|
func parseTagLists(value string) ([]TagList, error) {
|
||||||
|
items, err := parseList(value)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
lists := make([]TagList, 0, len(items))
|
||||||
|
|
||||||
|
for _, item := range items {
|
||||||
|
end := strings.Index(item, "=http")
|
||||||
|
if end <= 0 || !isListURL(item[end+1:]) {
|
||||||
|
return nil, fmt.Errorf("%q %w", item, errNotTagList)
|
||||||
|
}
|
||||||
|
|
||||||
|
list := TagList{Name: strings.ToLower(item[:end]), URL: item[end+1:]}
|
||||||
|
|
||||||
|
if slices.ContainsFunc(lists, func(earlier TagList) bool {
|
||||||
|
return earlier.Name == list.Name || earlier.URL == list.URL
|
||||||
|
}) {
|
||||||
|
return nil, fmt.Errorf("%q %w", item, errListedTwice)
|
||||||
|
}
|
||||||
|
|
||||||
|
lists = append(lists, list)
|
||||||
|
}
|
||||||
|
|
||||||
|
return lists, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseTags reads a comma-separated list of tags, which parseTag reads. A
|
||||||
|
// tag listed twice is an error.
|
||||||
|
func parseTags(value string) ([]string, error) {
|
||||||
|
items, err := parseList(value)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
tags := make([]string, 0, len(items))
|
||||||
|
|
||||||
|
for _, item := range items {
|
||||||
|
tag, err := parseTag(item)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if slices.Contains(tags, tag) {
|
||||||
|
return nil, fmt.Errorf("%q %w", item, errListedTwice)
|
||||||
|
}
|
||||||
|
|
||||||
|
tags = append(tags, tag)
|
||||||
|
}
|
||||||
|
|
||||||
|
return tags, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseTag reads a tag in either case, with spaces around it, and returns it
|
||||||
|
// as a request carries it: in lower case, but for a country code, which is in
|
||||||
|
// capitals, and an asn= tag's number without leading zeros. An asn= or
|
||||||
|
// country= tag must name a number or a country, so that a mistyped one stops
|
||||||
|
// the start instead of never matching.
|
||||||
|
func parseTag(value string) (string, error) {
|
||||||
|
tag := strings.ToLower(strings.TrimSpace(value))
|
||||||
|
|
||||||
|
if digits, isASN := strings.CutPrefix(tag, ASNTagPrefix); isASN {
|
||||||
|
number, err := strconv.ParseUint(digits, 10, 32)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("%q %w", value, errNotASNTag)
|
||||||
|
}
|
||||||
|
|
||||||
|
return ASNTagPrefix + strconv.FormatUint(number, 10), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if code, isCountry := strings.CutPrefix(tag, CountryTagPrefix); isCountry {
|
||||||
|
country, err := parseCountry(code)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return CountryTagPrefix + country, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return tag, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseTagPercents reads a comma-separated list of tags, each with : and a
|
||||||
|
// percentage from 1 to 99, such as vpn:50 or asn=14061:25. A tag listed
|
||||||
|
// twice is an error, and so is ExploitSignatureTag, which a request gets
|
||||||
|
// only after the rate limits have counted it.
|
||||||
|
func parseTagPercents(value string) (map[string]int64, error) {
|
||||||
|
items, err := parseList(value)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
percents := make(map[string]int64, len(items))
|
||||||
|
|
||||||
|
for _, item := range items {
|
||||||
|
colon := strings.LastIndex(item, ":")
|
||||||
|
if colon <= 0 {
|
||||||
|
return nil, fmt.Errorf("%q %w", item, errNotTagPercentItem)
|
||||||
|
}
|
||||||
|
|
||||||
|
tag, err := parseTag(item[:colon])
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
percent, err := strconv.ParseInt(item[colon+1:], 10, 64)
|
||||||
|
if err != nil || percent < 1 || percent > 99 {
|
||||||
|
return nil, fmt.Errorf("%q %w", item[colon+1:], errNotTagPercent)
|
||||||
|
}
|
||||||
|
|
||||||
|
if tag == ExploitSignatureTag {
|
||||||
|
return nil, fmt.Errorf("%q %w", tag, errExploitTag)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, listed := percents[tag]; listed {
|
||||||
|
return nil, fmt.Errorf("%q %w", item[:colon], errListedTwice)
|
||||||
|
}
|
||||||
|
|
||||||
|
percents[tag] = percent
|
||||||
|
}
|
||||||
|
|
||||||
|
return percents, nil
|
||||||
|
}
|
||||||
|
|
||||||
// parsePercents reads a comma-separated list of items, each an AS number
|
// parsePercents reads a comma-separated list of items, each an AS number
|
||||||
// or a country, which parseCode reads, then : and a percentage, such as
|
// or a country, which parseCode reads, then : and a percentage, such as
|
||||||
// AS64496:50 or cn:25, and returns each one's percentage. An empty value
|
// AS64496:50 or cn:25, and returns each one's percentage. An empty value
|
||||||
|
|||||||
+185
-49
@@ -42,10 +42,10 @@ const (
|
|||||||
lookupDBPath = "SWW_LOOKUP_DB_PATH"
|
lookupDBPath = "SWW_LOOKUP_DB_PATH"
|
||||||
lookupTimeout = "SWW_LOOKUP_TIMEOUT"
|
lookupTimeout = "SWW_LOOKUP_TIMEOUT"
|
||||||
addLookupHeaders = "SWW_ADD_LOOKUP_HEADERS"
|
addLookupHeaders = "SWW_ADD_LOOKUP_HEADERS"
|
||||||
deniedCountries = "SWW_DENIED_COUNTRIES"
|
|
||||||
allowedCountries = "SWW_EXCLUSIVELY_ALLOWED_COUNTRIES"
|
allowedCountries = "SWW_EXCLUSIVELY_ALLOWED_COUNTRIES"
|
||||||
asnLimitPercent = "SWW_ASN_LIMIT_PERCENT"
|
tagLists = "SWW_TAG_LISTS"
|
||||||
countryLimitPercent = "SWW_COUNTRY_LIMIT_PERCENT"
|
tagDeny = "SWW_TAG_DENY"
|
||||||
|
tagLimitPercent = "SWW_TAG_LIMIT_PERCENT"
|
||||||
asnBytesPercent = "SWW_ASN_BYTES_PERCENT"
|
asnBytesPercent = "SWW_ASN_BYTES_PERCENT"
|
||||||
countryBytesPercent = "SWW_COUNTRY_BYTES_PERCENT"
|
countryBytesPercent = "SWW_COUNTRY_BYTES_PERCENT"
|
||||||
unknownLimitPercent = "SWW_UNKNOWN_LIMIT_PERCENT"
|
unknownLimitPercent = "SWW_UNKNOWN_LIMIT_PERCENT"
|
||||||
@@ -274,9 +274,13 @@ func TestDefaults(t *testing.T) {
|
|||||||
wantNetblocks(t, cfg.AllowNets)
|
wantNetblocks(t, cfg.AllowNets)
|
||||||
wantNetblocks(t, cfg.RateLimitExemptNets)
|
wantNetblocks(t, cfg.RateLimitExemptNets)
|
||||||
wantNetblocks(t, cfg.DenyNets)
|
wantNetblocks(t, cfg.DenyNets)
|
||||||
wantCountries(t, deniedCountries, cfg.DeniedCountries)
|
|
||||||
wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries)
|
wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries)
|
||||||
|
|
||||||
|
if len(cfg.TagLists) != 0 || len(cfg.TagDeny) != 0 || len(cfg.TagLimitPercent) != 0 {
|
||||||
|
t.Errorf("tag settings %v, %v and %v by default, want all empty",
|
||||||
|
cfg.TagLists, cfg.TagDeny, cfg.TagLimitPercent)
|
||||||
|
}
|
||||||
|
|
||||||
hostname, err := os.Hostname()
|
hostname, err := os.Hostname()
|
||||||
if err != nil || hostname == "" || cfg.InstanceName != hostname {
|
if err != nil || hostname == "" || cfg.InstanceName != hostname {
|
||||||
t.Errorf("%s is %q, want the host's name %q (%v)", instanceName,
|
t.Errorf("%s is %q, want the host's name %q (%v)", instanceName,
|
||||||
@@ -327,7 +331,6 @@ func TestValuesAsSet(t *testing.T) {
|
|||||||
rateLimitPerMinute: "60",
|
rateLimitPerMinute: "60",
|
||||||
rateLimitPerHour: "600",
|
rateLimitPerHour: "600",
|
||||||
rateLimitPerDay: "6000",
|
rateLimitPerDay: "6000",
|
||||||
deniedCountries: "cn, RU,kp,Xk",
|
|
||||||
allowedCountries: "de",
|
allowedCountries: "de",
|
||||||
banResponse: "429",
|
banResponse: "429",
|
||||||
limitBanDuration: "15m",
|
limitBanDuration: "15m",
|
||||||
@@ -370,7 +373,6 @@ func TestValuesAsSet(t *testing.T) {
|
|||||||
wantNetblocks(t, cfg.AllowNets, "192.0.2.7/32")
|
wantNetblocks(t, cfg.AllowNets, "192.0.2.7/32")
|
||||||
wantNetblocks(t, cfg.RateLimitExemptNets, "2001:db8::/48", "10.9.8.7/32")
|
wantNetblocks(t, cfg.RateLimitExemptNets, "2001:db8::/48", "10.9.8.7/32")
|
||||||
wantNetblocks(t, cfg.DenyNets, "198.51.100.0/24")
|
wantNetblocks(t, cfg.DenyNets, "198.51.100.0/24")
|
||||||
wantCountries(t, deniedCountries, cfg.DeniedCountries, "CN", "RU", "KP", "XK")
|
|
||||||
wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries, "DE")
|
wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries, "DE")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1101,7 +1103,7 @@ func TestCodeOnBothCountryListsStopsTheStart(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
_, err := config.FromEnvironment(environment{
|
_, err := config.FromEnvironment(environment{
|
||||||
deniedCountries: "cn,ru",
|
tagDeny: "country=cn,country=ru",
|
||||||
allowedCountries: "de,RU",
|
allowedCountries: "de,RU",
|
||||||
}.lookupEnv)
|
}.lookupEnv)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -1176,12 +1178,11 @@ func TestSettingNeedingLookupsStopsTheStartWhileTheyAreOff(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
for name, value := range map[string]string{
|
for name, value := range map[string]string{
|
||||||
deniedCountries: "kp",
|
|
||||||
allowedCountries: "de",
|
allowedCountries: "de",
|
||||||
addLookupHeaders: enabled,
|
addLookupHeaders: enabled,
|
||||||
|
tagDeny: "country=KP",
|
||||||
|
tagLimitPercent: "asn=64496:50",
|
||||||
asnLimitPercentURL: asnURL,
|
asnLimitPercentURL: asnURL,
|
||||||
asnLimitPercent: "AS64496:50",
|
|
||||||
countryLimitPercent: "cn:25",
|
|
||||||
asnBytesPercent: "AS64496:50",
|
asnBytesPercent: "AS64496:50",
|
||||||
countryBytesPercent: "cn:25",
|
countryBytesPercent: "cn:25",
|
||||||
unknownLimitPercent: "99",
|
unknownLimitPercent: "99",
|
||||||
@@ -1204,12 +1205,13 @@ func TestSettingNeedingLookupsStopsTheStartWhileTheyAreOff(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Set empty, the lists need nothing looked up, and nor does
|
// Set empty, the lists need nothing looked up, and nor does
|
||||||
// SWW_UNKNOWN_LIMIT_PERCENT at 100, which lowers no limit, an anomaly
|
// SWW_UNKNOWN_LIMIT_PERCENT at 100, which lowers no limit, a tag the
|
||||||
// threshold per AS number that is off, any other anomaly threshold, or
|
// lookup does not give, an anomaly threshold per AS number that is off,
|
||||||
// a blocklist.
|
// any other anomaly threshold, or a blocklist.
|
||||||
env := environment{
|
env := environment{
|
||||||
lookupSource: off, deniedCountries: "", allowedCountries: "",
|
lookupSource: off, allowedCountries: "", asnBytesPercent: "",
|
||||||
asnLimitPercent: "", countryLimitPercent: "", asnBytesPercent: "",
|
tagLists: "torexit=https://example.com/tor.txt,vpn=https://example.com/vpn.txt",
|
||||||
|
tagDeny: torExitTag, tagLimitPercent: "vpn:50",
|
||||||
countryBytesPercent: "", unknownLimitPercent: "100", asnLimitPercentURL: "",
|
countryBytesPercent: "", unknownLimitPercent: "100", asnLimitPercentURL: "",
|
||||||
blocklistURLs: dropURL,
|
blocklistURLs: dropURL,
|
||||||
}
|
}
|
||||||
@@ -1355,34 +1357,152 @@ func TestInvalidAnomalySettingStopsTheStartSayingWhatIsWrong(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// torExitTag and zenTag are tags these tests name.
|
||||||
|
const (
|
||||||
|
torExitTag = "torexit"
|
||||||
|
zenTag = "dnsbl=zen.example"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestTagSettingsAsSet(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const torURL = "https://example.com/tor.txt"
|
||||||
|
|
||||||
|
// Names and tags in either case, an AS number with leading zeros, and a
|
||||||
|
// name holding a =.
|
||||||
|
cfg := fromEnvironment(t, environment{
|
||||||
|
tagLists: "TorExit=" + torURL +
|
||||||
|
", vpn=mullvad=https://example.com/mullvad.txt",
|
||||||
|
dnsblZones: "Zen.example",
|
||||||
|
tagDeny: "TorExit, Country=kp, asn=014061",
|
||||||
|
tagLimitPercent: "vpn=mullvad :50, asn=014061:25, COUNTRY=cn:1, dnsbl=zen.example:99",
|
||||||
|
})
|
||||||
|
|
||||||
|
wantLists := []config.TagList{
|
||||||
|
{Name: torExitTag, URL: torURL},
|
||||||
|
{Name: "vpn=mullvad", URL: "https://example.com/mullvad.txt"},
|
||||||
|
}
|
||||||
|
if !slices.Equal(cfg.TagLists, wantLists) {
|
||||||
|
t.Errorf("%s gave %+v, want %+v", tagLists, cfg.TagLists, wantLists)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantDeny := []string{torExitTag, "country=KP", "asn=14061"}
|
||||||
|
if !slices.Equal(cfg.TagDeny, wantDeny) {
|
||||||
|
t.Errorf("%s gave %v, want %v", tagDeny, cfg.TagDeny, wantDeny)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantPercents := map[string]int64{
|
||||||
|
"vpn=mullvad": 50, "asn=14061": 25, "country=CN": 1, zenTag: 99,
|
||||||
|
}
|
||||||
|
if !maps.Equal(cfg.TagLimitPercent, wantPercents) {
|
||||||
|
t.Errorf("%s gave %v, want %v", tagLimitPercent, cfg.TagLimitPercent, wantPercents)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTagThatNoRequestCanGetIsToldApartFromOneItCan(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const (
|
||||||
|
list = "torexit=https://example.com/tor.txt"
|
||||||
|
key = "abc123"
|
||||||
|
zone = key + ".xbl.dq.spamhaus.net"
|
||||||
|
masked = "********.xbl.dq.spamhaus.net"
|
||||||
|
)
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
env environment
|
||||||
|
want string // "" is accepted
|
||||||
|
}{
|
||||||
|
{"a list's name", environment{tagLists: list, tagDeny: torExitTag}, ""},
|
||||||
|
{
|
||||||
|
"a mistyped list's name", environment{tagLists: list, tagDeny: "torexti"},
|
||||||
|
`"torexti"`,
|
||||||
|
},
|
||||||
|
{"a masked zone", environment{dnsblZones: zone, tagDeny: "dnsbl=" + masked}, ""},
|
||||||
|
{
|
||||||
|
"a zone with its key", environment{dnsblZones: zone, tagDeny: "dnsbl=" + zone},
|
||||||
|
masked,
|
||||||
|
},
|
||||||
|
{"another zone", environment{dnsblZones: zone, tagDeny: zenTag}, zenTag},
|
||||||
|
{
|
||||||
|
"abuseipdb with a key",
|
||||||
|
environment{abuseIPDBKey: "k", tagLimitPercent: "abuseipdb:50"}, "",
|
||||||
|
},
|
||||||
|
{"crowdsec, though a list has its name", environment{
|
||||||
|
tagLists: "crowdsec=https://example.com/c.txt", tagDeny: "crowdsec",
|
||||||
|
}, `"crowdsec" cannot be used`},
|
||||||
|
{"exploitsignature", environment{tagDeny: "exploitsignature"}, ""},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
_, err := config.FromEnvironment(tc.env.lookupEnv)
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case tc.want == "" && err != nil:
|
||||||
|
t.Errorf("error %v, want none", err)
|
||||||
|
case tc.want != "" && (err == nil || !strings.Contains(err.Error(), tc.want)):
|
||||||
|
t.Errorf("error %v, want one holding %s", err, tc.want)
|
||||||
|
case err != nil && strings.Contains(err.Error(), key):
|
||||||
|
t.Errorf("error %v shows the zone's key", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTagListURLFetchedAsAnotherListStopsTheStart(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const lapi = "http://172.17.0.1:8080"
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, listURL string
|
||||||
|
env environment
|
||||||
|
}{
|
||||||
|
{"a blocklist", dropURL, environment{blocklistURLs: dropURL}},
|
||||||
|
{"the file of AS:percent lines", asnURL, environment{asnLimitPercentURL: asnURL}},
|
||||||
|
{
|
||||||
|
"the CrowdSec decision list", lapi + "/v1/decisions",
|
||||||
|
environment{"SWW_CROWDSEC_LAPI_URL": lapi, "SWW_CROWDSEC_LAPI_KEY": "key"},
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tc.env[tagLists] = "dup=" + tc.listURL
|
||||||
|
|
||||||
|
want := tagLists + `: "` + tc.listURL + `" is in SWW_BLOCKLIST_URLS, is ` +
|
||||||
|
"SWW_ASN_LIMIT_PERCENT_URL or is the CrowdSec decision list too"
|
||||||
|
|
||||||
|
_, err := config.FromEnvironment(tc.env.lookupEnv)
|
||||||
|
if err == nil || err.Error() != want {
|
||||||
|
t.Errorf("error %v, want %s", err, want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestBiasedThresholdsAsSet(t *testing.T) {
|
func TestBiasedThresholdsAsSet(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
cfg := fromEnvironment(t, environment{})
|
cfg := fromEnvironment(t, environment{})
|
||||||
if len(cfg.ASNLimitPercent) != 0 || len(cfg.CountryLimitPercent) != 0 ||
|
if len(cfg.ASNBytesPercent) != 0 || len(cfg.CountryBytesPercent) != 0 ||
|
||||||
len(cfg.ASNBytesPercent) != 0 || len(cfg.CountryBytesPercent) != 0 ||
|
|
||||||
cfg.UnknownLimitPercent != 100 {
|
cfg.UnknownLimitPercent != 100 {
|
||||||
t.Errorf("biased thresholds %v, %v, %v, %v and %d by default, "+
|
t.Errorf("biased thresholds %v, %v and %d by default, "+
|
||||||
"want four empty lists and 100", cfg.ASNLimitPercent, cfg.CountryLimitPercent,
|
"want two empty lists and 100",
|
||||||
cfg.ASNBytesPercent, cfg.CountryBytesPercent, cfg.UnknownLimitPercent)
|
cfg.ASNBytesPercent, cfg.CountryBytesPercent, cfg.UnknownLimitPercent)
|
||||||
}
|
}
|
||||||
|
|
||||||
// AS numbers and countries in either case, an AS number with leading
|
// AS numbers and countries in either case, an AS number with leading
|
||||||
// zeros, 0 and 100.
|
// zeros, 0 and 100.
|
||||||
cfg = fromEnvironment(t, environment{
|
cfg = fromEnvironment(t, environment{
|
||||||
asnLimitPercent: "AS14061:50, as16276:0,AS045102:100",
|
|
||||||
countryLimitPercent: "cn:25,RU:50",
|
|
||||||
asnBytesPercent: "as16276:75",
|
asnBytesPercent: "as16276:75",
|
||||||
countryBytesPercent: "ru:10",
|
countryBytesPercent: "ru:10",
|
||||||
unknownLimitPercent: "0",
|
unknownLimitPercent: "0",
|
||||||
})
|
})
|
||||||
|
|
||||||
for name, tc := range map[string]struct{ got, want map[string]int64 }{
|
for name, tc := range map[string]struct{ got, want map[string]int64 }{
|
||||||
asnLimitPercent: {
|
|
||||||
cfg.ASNLimitPercent,
|
|
||||||
map[string]int64{"AS14061": 50, "AS16276": 0, "AS45102": 100},
|
|
||||||
},
|
|
||||||
countryLimitPercent: {cfg.CountryLimitPercent, map[string]int64{"CN": 25, "RU": 50}},
|
|
||||||
asnBytesPercent: {cfg.ASNBytesPercent, map[string]int64{"AS16276": 75}},
|
asnBytesPercent: {cfg.ASNBytesPercent, map[string]int64{"AS16276": 75}},
|
||||||
countryBytesPercent: {cfg.CountryBytesPercent, map[string]int64{"RU": 10}},
|
countryBytesPercent: {cfg.CountryBytesPercent, map[string]int64{"RU": 10}},
|
||||||
} {
|
} {
|
||||||
@@ -1406,16 +1526,16 @@ func TestInvalidBiasedThresholdStopsTheStartSayingWhatIsWrong(t *testing.T) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
for _, tc := range []struct{ name, value, want string }{
|
for _, tc := range []struct{ name, value, want string }{
|
||||||
{asnLimitPercent, "14061:50", `"14061"` + notASN},
|
{asnBytesPercent, "14061:50", `"14061"` + notASN},
|
||||||
{asnLimitPercent, "AS4294967296:50", `"AS4294967296"` + notASN},
|
{asnBytesPercent, "AS4294967296:50", `"AS4294967296"` + notASN},
|
||||||
{asnLimitPercent, "AS14061", `"AS14061"` + notItem},
|
{asnBytesPercent, "AS14061", `"AS14061"` + notItem},
|
||||||
{asnLimitPercent, "AS14061:101", `"101"` + notPercent},
|
{asnBytesPercent, "AS14061:101", `"101"` + notPercent},
|
||||||
{asnLimitPercent, "AS14061:50,as14061:25", `"as14061" is listed twice`},
|
{asnBytesPercent, "AS14061:50,as14061:25", `"as14061" is listed twice`},
|
||||||
{
|
{
|
||||||
countryLimitPercent, "nk:25",
|
countryBytesPercent, "nk:25",
|
||||||
`"nk" is not a two-letter country code such as de or kp`,
|
`"nk" is not a two-letter country code such as de or kp`,
|
||||||
},
|
},
|
||||||
{countryLimitPercent, "cn:25,CN:50", `"CN" is listed twice`},
|
{countryBytesPercent, "cn:25,CN:50", `"CN" is listed twice`},
|
||||||
{asnBytesPercent, "AS14061:-1", `"-1"` + notPercent},
|
{asnBytesPercent, "AS14061:-1", `"-1"` + notPercent},
|
||||||
{countryBytesPercent, "cn:50%", `"50%"` + notPercent},
|
{countryBytesPercent, "cn:50%", `"50%"` + notPercent},
|
||||||
{unknownLimitPercent, "101", `"101"` + notPercent},
|
{unknownLimitPercent, "101", `"101"` + notPercent},
|
||||||
@@ -2003,23 +2123,39 @@ func TestInvalidValueStopsTheStart(t *testing.T) {
|
|||||||
{lookupSource, "ipinfo"}, {lookupSource, "GeoJS"}, {lookupSource, ""},
|
{lookupSource, "ipinfo"}, {lookupSource, "GeoJS"}, {lookupSource, ""},
|
||||||
{lookupTimeout, off}, {lookupTimeout, "0s"}, {lookupTimeout, "1"},
|
{lookupTimeout, off}, {lookupTimeout, "0s"}, {lookupTimeout, "1"},
|
||||||
{addLookupHeaders, "yes"},
|
{addLookupHeaders, "yes"},
|
||||||
{deniedCountries, "nk"},
|
{allowedCountries, "nk"},
|
||||||
{deniedCountries, "kp,,ir"},
|
{allowedCountries, "kp,,ir"},
|
||||||
{deniedCountries, "prk"},
|
{allowedCountries, "prk"},
|
||||||
{deniedCountries, "408"},
|
{allowedCountries, "408"},
|
||||||
{deniedCountries, "k"},
|
{allowedCountries, "k"},
|
||||||
{deniedCountries, "eu"},
|
{allowedCountries, "eu"},
|
||||||
{deniedCountries, "un"},
|
{allowedCountries, "un"},
|
||||||
{deniedCountries, "su"},
|
{allowedCountries, "su"},
|
||||||
{allowedCountries, "ac"},
|
{allowedCountries, "ac"},
|
||||||
{allowedCountries, "uk"},
|
{allowedCountries, "uk"},
|
||||||
{allowedCountries, "zz"},
|
{allowedCountries, "zz"},
|
||||||
{allowedCountries, "de,germany"},
|
{allowedCountries, "de,germany"},
|
||||||
{asnLimitPercent, "AS14061:50,,AS16276:50"}, {asnLimitPercent, "ASX:50"},
|
{asnBytesPercent, "AS14061:50,,AS16276:50"}, {asnBytesPercent, "ASX:50"},
|
||||||
{asnLimitPercent, "AS14061:"}, {asnLimitPercent, "AS14061 :50"},
|
{asnBytesPercent, "AS14061:"}, {asnBytesPercent, "AS14061 :50"},
|
||||||
{asnLimitPercent, "AS14061:1.5"}, {asnLimitPercent, "AS-1:50"},
|
{asnBytesPercent, "AS14061:1.5"}, {asnBytesPercent, "AS-1:50"},
|
||||||
{countryLimitPercent, "cn"}, {countryLimitPercent, "cn:"},
|
{countryBytesPercent, "cn"}, {countryBytesPercent, "cn:"},
|
||||||
{countryLimitPercent, "cn:25:50"}, {countryLimitPercent, "china:25"},
|
{countryBytesPercent, "cn:25:50"}, {countryBytesPercent, "china:25"},
|
||||||
|
{tagDeny, "a,,b"}, {tagDeny, "country=nk"}, {tagDeny, "asn=abc"},
|
||||||
|
{tagDeny, "asn="}, {tagDeny, "torexit,TOREXIT"},
|
||||||
|
{tagDeny, "torexti"}, {tagDeny, zenTag},
|
||||||
|
{tagDeny, "abuseipdb"}, {tagDeny, "crowdsec"},
|
||||||
|
{tagLimitPercent, "torexti:50"}, {tagLimitPercent, "dnsbl=zen.example:50"},
|
||||||
|
{tagLimitPercent, "abuseipdb:50"}, {tagLimitPercent, "crowdsec:50"},
|
||||||
|
{tagLimitPercent, "vpn :50"},
|
||||||
|
{tagLimitPercent, "vpn"}, {tagLimitPercent, ":50"}, {tagLimitPercent, "vpn:"},
|
||||||
|
{tagLimitPercent, "vpn:0"}, {tagLimitPercent, "vpn:100"},
|
||||||
|
{tagLimitPercent, "vpn:1.5"}, {tagLimitPercent, "vpn:50,VPN:25"},
|
||||||
|
{tagLimitPercent, "exploitsignature:50"}, {tagLimitPercent, "asn=abc:50"},
|
||||||
|
{tagLists, torExitTag}, {tagLists, "=https://example.com/a.txt"},
|
||||||
|
{tagLists, "a,b=https://example.com/a.txt"},
|
||||||
|
{tagLists, "a=ftp://example.com/a.txt"},
|
||||||
|
{tagLists, "a=https://example.com/a.txt,A=https://example.com/b.txt"},
|
||||||
|
{tagLists, "a=https://example.com/a.txt,b=https://example.com/a.txt"},
|
||||||
{asnBytesPercent, "AS14061:101"}, {countryBytesPercent, "su:50"},
|
{asnBytesPercent, "AS14061:101"}, {countryBytesPercent, "su:50"},
|
||||||
{unknownLimitPercent, ""}, {unknownLimitPercent, "-1"},
|
{unknownLimitPercent, ""}, {unknownLimitPercent, "-1"},
|
||||||
{unknownLimitPercent, "50%"},
|
{unknownLimitPercent, "50%"},
|
||||||
@@ -2286,10 +2422,10 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
|
|||||||
lookupDBPath: "",
|
lookupDBPath: "",
|
||||||
lookupTimeout: "800ms",
|
lookupTimeout: "800ms",
|
||||||
addLookupHeaders: "false",
|
addLookupHeaders: "false",
|
||||||
deniedCountries: "",
|
|
||||||
allowedCountries: "",
|
allowedCountries: "",
|
||||||
asnLimitPercent: "",
|
tagLists: "",
|
||||||
countryLimitPercent: "",
|
tagDeny: "",
|
||||||
|
tagLimitPercent: "",
|
||||||
asnBytesPercent: "",
|
asnBytesPercent: "",
|
||||||
countryBytesPercent: "",
|
countryBytesPercent: "",
|
||||||
unknownLimitPercent: "100",
|
unknownLimitPercent: "100",
|
||||||
|
|||||||
@@ -8,18 +8,20 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// abuseIPDB notes AbuseIPDB, as noteHit does, with the score, when its score
|
// abuseIPDB adds the tag abuseipdb and notes AbuseIPDB, as noteHit does, with
|
||||||
// of the client is a hit, and refuses the request when SWW_REPUTATION_ACTION
|
// the score, when its score of the client is a hit, and refuses the request
|
||||||
// is deny, as dnsbl does for a zone. While SWW_ABUSEIPDB_KEY is unset it does
|
// when SWW_REPUTATION_ACTION is deny, as dnsbl does for a zone. While
|
||||||
// nothing. A client without a score is checked in the background, by the
|
// SWW_ABUSEIPDB_KEY is unset it does nothing. A client without a score is
|
||||||
// request's address, if its history counts an offence, and the request does
|
// checked in the background, by the request's address, if its history counts
|
||||||
// not wait for the answer. The score is then used for each address of the
|
// an offence, and the request does not wait for the answer. The score is then
|
||||||
// client. Its figures are the score and SWW_ABUSEIPDB_MIN_SCORE.
|
// used for each address of the client. Its figures are the score and
|
||||||
|
// SWW_ABUSEIPDB_MIN_SCORE.
|
||||||
type abuseIPDB struct{}
|
type abuseIPDB struct{}
|
||||||
|
|
||||||
func (abuseIPDB) name() string { return "abuseipdb" }
|
func (abuseIPDB) name() string { return "abuseipdb" }
|
||||||
@@ -39,6 +41,7 @@ func (abuseIPDB) check(ctx context.Context, rq *request, _ time.Time) *verdict {
|
|||||||
}
|
}
|
||||||
|
|
||||||
rq.abuseIPDBHit = true
|
rq.abuseIPDBHit = true
|
||||||
|
rq.addTag(config.AbuseIPDBTag)
|
||||||
rq.noteHit(bans.ReputationHit{Source: reputation.AbuseIPDBSource, Score: &score},
|
rq.noteHit(bans.ReputationHit{Source: reputation.AbuseIPDBSource, Score: &score},
|
||||||
"scored by AbuseIPDB at or over SWW_ABUSEIPDB_MIN_SCORE")
|
"scored by AbuseIPDB at or over SWW_ABUSEIPDB_MIN_SCORE")
|
||||||
|
|
||||||
|
|||||||
@@ -335,7 +335,7 @@ func newDiscriminatorInBubble(
|
|||||||
) (*discriminator.Server, *output) {
|
) (*discriminator.Server, *output) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
settings := map[string]string{deniedCountries: "KP"}
|
settings := map[string]string{tagDeny: "country=KP"}
|
||||||
maps.Copy(settings, env)
|
maps.Copy(settings, env)
|
||||||
|
|
||||||
server, out, _ := newDiscriminator(t, unansweredGeoJSURL, time.Now, settings)
|
server, out, _ := newDiscriminator(t, unansweredGeoJSURL, time.Now, settings)
|
||||||
@@ -542,7 +542,7 @@ func TestDiscriminatorAnswersWithinTheTimeLimitWhileEveryOutsideSourceHangs(
|
|||||||
|
|
||||||
return discriminator.New(params)
|
return discriminator.New(params)
|
||||||
}, held, time.Now, map[string]string{
|
}, held, time.Now, map[string]string{
|
||||||
deniedCountries: "KP", wafMode: block,
|
tagDeny: "country=KP", wafMode: block,
|
||||||
dnsblZones: dnsblZone, dnsblResolver: resolver.LocalAddr().String(),
|
dnsblZones: dnsblZone, dnsblResolver: resolver.LocalAddr().String(),
|
||||||
abuseIPDBKey: accountKey,
|
abuseIPDBKey: accountKey,
|
||||||
blocklistURLs: held + "/drop.txt", asnLimitPercentURL: held + "/asn.txt",
|
blocklistURLs: held + "/drop.txt", asnLimitPercentURL: held + "/asn.txt",
|
||||||
|
|||||||
@@ -211,7 +211,7 @@ func TestBannedClientIsRefusedBeforeItsCountryIsLookedUp(t *testing.T) {
|
|||||||
lookupTimeout: "1h",
|
lookupTimeout: "1h",
|
||||||
rateLimitPerMinute: "1",
|
rateLimitPerMinute: "1",
|
||||||
banScopeV4Prefix: "24",
|
banScopeV4Prefix: "24",
|
||||||
deniedCountries: "kp",
|
tagDeny: denyKP,
|
||||||
})
|
})
|
||||||
|
|
||||||
// fromDE's ban covers otherClient, which is refused unasked about.
|
// fromDE's ban covers otherClient, which is refused unasked about.
|
||||||
@@ -266,7 +266,7 @@ func TestBanResponseAnswersEveryRefusalButTheSizeLimits(t *testing.T) {
|
|||||||
lookupTimeout: "1h",
|
lookupTimeout: "1h",
|
||||||
rateLimitPerMinute: "1",
|
rateLimitPerMinute: "1",
|
||||||
denyNets: denied,
|
denyNets: denied,
|
||||||
deniedCountries: "kp",
|
tagDeny: denyKP,
|
||||||
}
|
}
|
||||||
|
|
||||||
if tc.setting != "" {
|
if tc.setting != "" {
|
||||||
@@ -276,7 +276,7 @@ func TestBanResponseAnswersEveryRefusalButTheSizeLimits(t *testing.T) {
|
|||||||
s, _, _ := startWithClock(t, geojsURL, env)
|
s, _, _ := startWithClock(t, geojsURL, env)
|
||||||
|
|
||||||
s.get(denied, tc.status, requestlog.ActionDenied)
|
s.get(denied, tc.status, requestlog.ActionDenied)
|
||||||
s.get(fromKP, tc.status, requestlog.ActionCountryDenied)
|
s.get(fromKP, tc.status, requestlog.ActionDenied)
|
||||||
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
||||||
s.get(fromDE, tc.status, requestlog.ActionRateLimited)
|
s.get(fromDE, tc.status, requestlog.ActionRateLimited)
|
||||||
s.get(fromDE, tc.status, requestlog.ActionBanned)
|
s.get(fromDE, tc.status, requestlog.ActionBanned)
|
||||||
@@ -291,7 +291,7 @@ func TestBanNotes(t *testing.T) {
|
|||||||
s, clk, server := startWithClock(t, geojsURL, map[string]string{
|
s, clk, server := startWithClock(t, geojsURL, map[string]string{
|
||||||
lookupTimeout: "1h",
|
lookupTimeout: "1h",
|
||||||
rateLimitPerMinute: "1",
|
rateLimitPerMinute: "1",
|
||||||
deniedCountries: "kp",
|
tagDeny: denyKP,
|
||||||
})
|
})
|
||||||
start := clk.Now()
|
start := clk.Now()
|
||||||
|
|
||||||
|
|||||||
@@ -17,30 +17,31 @@ type percentage struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// biasedThresholdsSet reports whether a biased threshold can lower a
|
// biasedThresholdsSet reports whether a biased threshold can lower a
|
||||||
// client's limits: one of its lists is not empty,
|
// client's limits by its lookup: SWW_ASN_BYTES_PERCENT or
|
||||||
// SWW_UNKNOWN_LIMIT_PERCENT is below 100, or SWW_ASN_LIMIT_PERCENT_URL
|
// SWW_COUNTRY_BYTES_PERCENT is not empty, SWW_UNKNOWN_LIMIT_PERCENT is
|
||||||
// is set. The client's lookup is then needed before its request goes on.
|
// below 100, or SWW_ASN_LIMIT_PERCENT_URL is set. The client's lookup is
|
||||||
|
// then needed before its request goes on.
|
||||||
func biasedThresholdsSet(cfg *config.Config) bool {
|
func biasedThresholdsSet(cfg *config.Config) bool {
|
||||||
return len(cfg.ASNLimitPercent) > 0 || len(cfg.CountryLimitPercent) > 0 ||
|
return len(cfg.ASNBytesPercent) > 0 || len(cfg.CountryBytesPercent) > 0 ||
|
||||||
len(cfg.ASNBytesPercent) > 0 || len(cfg.CountryBytesPercent) > 0 ||
|
|
||||||
cfg.UnknownLimitPercent < whole || cfg.ASNLimitPercentURL != ""
|
cfg.UnknownLimitPercent < whole || cfg.ASNLimitPercentURL != ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// limitPercentages returns the client's limit percentages, for the rate
|
// limitPercentages returns the client's limit percentages, for the rate
|
||||||
// limits and for the byte limits, by its AS number and country as looked
|
// limits and for the byte limits, by its tags, its AS number and country
|
||||||
// up, each "" when unknown, and the blocklists, DNSBL zones and AbuseIPDB
|
// as looked up, each "" when unknown, and the blocklists, DNSBL zones and
|
||||||
// that list it. Each is the lowest of those the settings give it, the
|
// AbuseIPDB that list it. Each is the lowest of those the settings give
|
||||||
// first of them in the order below when several are lowest: the
|
// it, the first of them in the order below when several are lowest: the
|
||||||
// percentage SWW_ASN_LIMIT_PERCENT gives its AS number, the one the file
|
// percentage SWW_TAG_LIMIT_PERCENT gives its asn= tag, the one the file
|
||||||
// SWW_ASN_LIMIT_PERCENT_URL names gives it, the one
|
// SWW_ASN_LIMIT_PERCENT_URL names gives its AS number, the one
|
||||||
// SWW_COUNTRY_LIMIT_PERCENT gives its country, for a client without a
|
// SWW_TAG_LIMIT_PERCENT gives its country= tag, the lowest it gives its
|
||||||
// country, unless its request skipped the wait for GeoJS,
|
// other tags, for a client without a country, unless its request skipped
|
||||||
// SWW_UNKNOWN_LIMIT_PERCENT, for a client a blocklist lists, the
|
// the wait for GeoJS, SWW_UNKNOWN_LIMIT_PERCENT, for a client a blocklist
|
||||||
// percentage of SWW_BLOCKLIST_ACTION while it is limit, and for a client
|
// lists, the percentage of SWW_BLOCKLIST_ACTION while it is limit, and
|
||||||
// a DNSBL zone's verdict lists, or whose AbuseIPDB score is a hit, the
|
// for a client a DNSBL zone's verdict lists, or whose AbuseIPDB score is
|
||||||
// percentage of SWW_REPUTATION_ACTION while it is limit. For the byte
|
// a hit, the percentage of SWW_REPUTATION_ACTION while it is limit. For
|
||||||
// limits, SWW_ASN_BYTES_PERCENT and SWW_COUNTRY_BYTES_PERCENT take the
|
// the byte limits, SWW_ASN_BYTES_PERCENT and SWW_COUNTRY_BYTES_PERCENT
|
||||||
// place of the first three for an AS number or a country they list.
|
// take the place of the first three for an AS number or a country they
|
||||||
|
// list.
|
||||||
func (rq *request) limitPercentages() (percentage, percentage) {
|
func (rq *request) limitPercentages() (percentage, percentage) {
|
||||||
cfg := rq.h.config
|
cfg := rq.h.config
|
||||||
asn, country := rq.line.ASN, rq.line.Country
|
asn, country := rq.line.ASN, rq.line.Country
|
||||||
@@ -65,10 +66,18 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
reputationListed = percentage{cfg.ReputationLimitPercent, "SWW_REPUTATION_ACTION"}
|
reputationListed = percentage{cfg.ReputationLimitPercent, "SWW_REPUTATION_ACTION"}
|
||||||
}
|
}
|
||||||
|
|
||||||
asnRequests := lowest(given(cfg.ASNLimitPercent, asn, "SWW_ASN_LIMIT_PERCENT"),
|
const tagSetting = "SWW_TAG_LIMIT_PERCENT"
|
||||||
fetched)
|
|
||||||
countryRequests := given(cfg.CountryLimitPercent, country,
|
asnRequests := lowest(given(cfg.TagLimitPercent, asnTag(asn), tagSetting), fetched)
|
||||||
"SWW_COUNTRY_LIMIT_PERCENT")
|
countryRequests := given(cfg.TagLimitPercent, countryTag(country), tagSetting)
|
||||||
|
|
||||||
|
otherTags := percentage{percent: whole}
|
||||||
|
|
||||||
|
for _, tag := range rq.line.Tags {
|
||||||
|
if tag != asnTag(asn) && tag != countryTag(country) {
|
||||||
|
otherTags = lowest(otherTags, given(cfg.TagLimitPercent, tag, tagSetting))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
asnBytes, countryBytes := asnRequests, countryRequests
|
asnBytes, countryBytes := asnRequests, countryRequests
|
||||||
if _, listed := cfg.ASNBytesPercent[asn]; listed {
|
if _, listed := cfg.ASNBytesPercent[asn]; listed {
|
||||||
@@ -79,12 +88,13 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
countryBytes = given(cfg.CountryBytesPercent, country, "SWW_COUNTRY_BYTES_PERCENT")
|
countryBytes = given(cfg.CountryBytesPercent, country, "SWW_COUNTRY_BYTES_PERCENT")
|
||||||
}
|
}
|
||||||
|
|
||||||
return lowest(asnRequests, countryRequests, unknown, blocklisted, reputationListed),
|
return lowest(asnRequests, countryRequests, otherTags, unknown, blocklisted,
|
||||||
lowest(asnBytes, countryBytes, unknown, blocklisted, reputationListed)
|
reputationListed),
|
||||||
|
lowest(asnBytes, countryBytes, otherTags, unknown, blocklisted, reputationListed)
|
||||||
}
|
}
|
||||||
|
|
||||||
// given returns the percentage percents, the setting named setting, gives
|
// given returns the percentage percents, the setting named setting, gives
|
||||||
// code, an AS number or a country, or whole when it does not list code.
|
// code, an AS number, a country or a tag, or whole when it does not list code.
|
||||||
func given(percents map[string]int64, code, setting string) percentage {
|
func given(percents map[string]int64, code, setting string) percentage {
|
||||||
percent, listed := percents[code]
|
percent, listed := percents[code]
|
||||||
if !listed {
|
if !listed {
|
||||||
|
|||||||
@@ -22,8 +22,7 @@ import (
|
|||||||
|
|
||||||
// The biased thresholds.
|
// The biased thresholds.
|
||||||
const (
|
const (
|
||||||
asnLimitPercent = "SWW_ASN_LIMIT_PERCENT"
|
tagLimitPercent = "SWW_TAG_LIMIT_PERCENT"
|
||||||
countryLimitPercent = "SWW_COUNTRY_LIMIT_PERCENT"
|
|
||||||
asnBytesPercent = "SWW_ASN_BYTES_PERCENT"
|
asnBytesPercent = "SWW_ASN_BYTES_PERCENT"
|
||||||
countryBytesPercent = "SWW_COUNTRY_BYTES_PERCENT"
|
countryBytesPercent = "SWW_COUNTRY_BYTES_PERCENT"
|
||||||
unknownLimitPercent = "SWW_UNKNOWN_LIMIT_PERCENT"
|
unknownLimitPercent = "SWW_UNKNOWN_LIMIT_PERCENT"
|
||||||
@@ -35,6 +34,11 @@ const (
|
|||||||
asnDEHalf = asnDE + ":50"
|
asnDEHalf = asnDE + ":50"
|
||||||
asnDEQuarter = asnDE + ":25"
|
asnDEQuarter = asnDE + ":25"
|
||||||
countryDEHalf = "de:50"
|
countryDEHalf = "de:50"
|
||||||
|
// The same, as SWW_TAG_LIMIT_PERCENT names fromDE's tags.
|
||||||
|
asnTagDEHalf = "asn=64496:50"
|
||||||
|
asnTagDEQuarter = "asn=64496:25"
|
||||||
|
countryTagDEHalf = "country=DE:50"
|
||||||
|
countryTagDEQuarter = "country=DE:25"
|
||||||
// noCountry is in an AS of its own, AS64500, and in no country.
|
// noCountry is in an AS of its own, AS64500, and in no country.
|
||||||
noCountry = "192.0.2.80"
|
noCountry = "192.0.2.80"
|
||||||
// fourAMinute is the rate limit these tests set: half of it is 2
|
// fourAMinute is the rate limit these tests set: half of it is 2
|
||||||
@@ -54,8 +58,8 @@ func TestEachBiasedThresholdLowersTheRateLimits(t *testing.T) {
|
|||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
setting, value, from string
|
setting, value, from string
|
||||||
}{
|
}{
|
||||||
{asnLimitPercent, asnDEHalf, fromDE},
|
{tagLimitPercent, asnTagDEHalf, fromDE},
|
||||||
{countryLimitPercent, countryDEHalf, fromDE},
|
{tagLimitPercent, countryTagDEHalf, fromDE},
|
||||||
{unknownLimitPercent, "50", unplaced},
|
{unknownLimitPercent, "50", unplaced},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.setting, func(t *testing.T) {
|
t.Run(tc.setting, func(t *testing.T) {
|
||||||
@@ -96,8 +100,8 @@ func TestEachBiasedThresholdLowersTheByteLimits(t *testing.T) {
|
|||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
setting, value, from string
|
setting, value, from string
|
||||||
}{
|
}{
|
||||||
{asnLimitPercent, asnDEHalf, fromDE},
|
{tagLimitPercent, asnTagDEHalf, fromDE},
|
||||||
{countryLimitPercent, "DE:50", fromDE},
|
{tagLimitPercent, "Country=de:50", fromDE},
|
||||||
{unknownLimitPercent, "50", unplaced},
|
{unknownLimitPercent, "50", unplaced},
|
||||||
{asnBytesPercent, "as64496:50", fromDE},
|
{asnBytesPercent, "as64496:50", fromDE},
|
||||||
{countryBytesPercent, countryDEHalf, fromDE},
|
{countryBytesPercent, countryDEHalf, fromDE},
|
||||||
@@ -151,13 +155,13 @@ func TestBytesPercentSettingsTakeThePlaceOfTheOthersForByteLimits(t *testing.T)
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"raising the byte limits back",
|
"raising the byte limits back",
|
||||||
map[string]string{asnLimitPercent: asnDEHalf, asnBytesPercent: asnDE + ":100"},
|
map[string]string{tagLimitPercent: asnTagDEHalf, asnBytesPercent: asnDE + ":100"},
|
||||||
"50 from " + asnLimitPercent, none, "",
|
"50 from " + tagLimitPercent, none, "",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"raising the byte limits back, by country",
|
"raising the byte limits back, by country",
|
||||||
map[string]string{countryLimitPercent: countryDEHalf, countryBytesPercent: "de:100"},
|
map[string]string{tagLimitPercent: countryTagDEHalf, countryBytesPercent: "de:100"},
|
||||||
"50 from " + countryLimitPercent, none, "",
|
"50 from " + tagLimitPercent, none, "",
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
@@ -184,19 +188,19 @@ func TestBytesPercentSettingsTakeThePlaceOfTheOthersForByteLimits(t *testing.T)
|
|||||||
func TestZeroPercentIsAZeroAllowance(t *testing.T) {
|
func TestZeroPercentIsAZeroAllowance(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
s, _, _ := startWithLookups(t, map[string]string{asnLimitPercent: asnDE + ":0"})
|
s, _, _ := startWithLookups(t, map[string]string{unknownLimitPercent: "0"})
|
||||||
|
|
||||||
// The first request breaks the limit, and bans the client; the log line
|
// The first request breaks the limit, and bans the client; the log line
|
||||||
// gives the 0.
|
// gives the 0.
|
||||||
line := s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited)
|
line := s.get(unplaced, http.StatusForbidden, requestlog.ActionRateLimited)
|
||||||
if line.fields["limit_percent"] != float64(0) ||
|
if line.fields["limit_percent"] != float64(0) ||
|
||||||
line.fields["limit_percent_setting"] != asnLimitPercent {
|
line.fields["limit_percent_setting"] != unknownLimitPercent {
|
||||||
t.Errorf("log line has limit_percent %v from %v, want 0 from %s",
|
t.Errorf("log line has limit_percent %v from %v, want 0 from %s",
|
||||||
line.fields["limit_percent"], line.fields["limit_percent_setting"],
|
line.fields["limit_percent"], line.fields["limit_percent_setting"],
|
||||||
asnLimitPercent)
|
unknownLimitPercent)
|
||||||
}
|
}
|
||||||
|
|
||||||
s.get(fromDE, http.StatusForbidden, requestlog.ActionBanned)
|
s.get(unplaced, http.StatusForbidden, requestlog.ActionBanned)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLowestPercentageApplies(t *testing.T) {
|
func TestLowestPercentageApplies(t *testing.T) {
|
||||||
@@ -211,29 +215,29 @@ func TestLowestPercentageApplies(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
"the country's",
|
"the country's",
|
||||||
map[string]string{asnLimitPercent: asnDEHalf, countryLimitPercent: "de:25"},
|
map[string]string{tagLimitPercent: asnTagDEHalf + "," + countryTagDEQuarter},
|
||||||
fromDE, "25 from " + countryLimitPercent,
|
fromDE, "25 from " + tagLimitPercent,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"the AS number's",
|
"the AS number's",
|
||||||
map[string]string{asnLimitPercent: asnDEQuarter, countryLimitPercent: countryDEHalf},
|
map[string]string{tagLimitPercent: asnTagDEQuarter + "," + countryTagDEHalf},
|
||||||
fromDE, "25 from " + asnLimitPercent,
|
fromDE, "25 from " + tagLimitPercent,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"the AS number's, the first of two alike",
|
"the AS number's, the first of two alike",
|
||||||
map[string]string{asnLimitPercent: asnDEQuarter, countryLimitPercent: "de:25"},
|
map[string]string{tagLimitPercent: asnTagDEQuarter + "," + countryTagDEQuarter},
|
||||||
fromDE, "25 from " + asnLimitPercent,
|
fromDE, "25 from " + tagLimitPercent,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"that for a client without a country",
|
"that for a client without a country",
|
||||||
map[string]string{asnLimitPercent: "AS64500:50", unknownLimitPercent: "25"},
|
map[string]string{tagLimitPercent: "asn=64500:50", unknownLimitPercent: "25"},
|
||||||
noCountry, "25 from " + unknownLimitPercent,
|
noCountry, "25 from " + unknownLimitPercent,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
// SWW_UNKNOWN_LIMIT_PERCENT is left at its default, 100.
|
// SWW_UNKNOWN_LIMIT_PERCENT is left at its default, 100.
|
||||||
"the AS number's, for a client without a country",
|
"the AS number's, for a client without a country",
|
||||||
map[string]string{asnLimitPercent: "AS64500:25"},
|
map[string]string{tagLimitPercent: "asn=64500:25"},
|
||||||
noCountry, "25 from " + asnLimitPercent,
|
noCountry, "25 from " + tagLimitPercent,
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
@@ -304,8 +308,12 @@ func TestRequestWaitsForItsLookupWhileABiasedThresholdIsSet(t *testing.T) {
|
|||||||
setting, value string
|
setting, value string
|
||||||
waits bool
|
waits bool
|
||||||
}{
|
}{
|
||||||
{asnLimitPercent, asnDEHalf, true},
|
{tagLimitPercent, asnTagDEHalf, true},
|
||||||
{countryLimitPercent, countryDEHalf, true},
|
{tagLimitPercent, countryTagDEHalf, true},
|
||||||
|
{tagDeny, denyKP, true},
|
||||||
|
// A tag the lookup does not give needs no answer.
|
||||||
|
{tagLimitPercent, "vpn:50", false},
|
||||||
|
{tagDeny, "vpn", false},
|
||||||
{asnBytesPercent, asnDEHalf, true},
|
{asnBytesPercent, asnDEHalf, true},
|
||||||
{countryBytesPercent, countryDEHalf, true},
|
{countryBytesPercent, countryDEHalf, true},
|
||||||
{unknownLimitPercent, "99", true},
|
{unknownLimitPercent, "99", true},
|
||||||
@@ -320,7 +328,9 @@ func TestRequestWaitsForItsLookupWhileABiasedThresholdIsSet(t *testing.T) {
|
|||||||
// says, with a GeoJS that never answers.
|
// says, with a GeoJS that never answers.
|
||||||
synctest.Test(t, func(t *testing.T) {
|
synctest.Test(t, func(t *testing.T) {
|
||||||
server, out, _ := newDiscriminator(t, unansweredGeoJSURL, time.Now,
|
server, out, _ := newDiscriminator(t, unansweredGeoJSURL, time.Now,
|
||||||
map[string]string{lookupTimeout: timeout.String(), tc.setting: tc.value})
|
map[string]string{
|
||||||
|
lookupTimeout: timeout.String(), tagLists: "vpn=" + vpnList, tc.setting: tc.value,
|
||||||
|
})
|
||||||
began := time.Now()
|
began := time.Now()
|
||||||
|
|
||||||
serveFromDE(t, server, http.MethodPost, strings.NewReader("ab"))
|
serveFromDE(t, server, http.MethodPost, strings.NewReader("ab"))
|
||||||
@@ -363,15 +373,15 @@ func TestBanForALoweredLimitGivesThePercentageInItsNotesAndItsAlert(t *testing.T
|
|||||||
{
|
{
|
||||||
// A quarter of 12 requests a minute is 3: the fourth breaks it.
|
// A quarter of 12 requests a minute is 3: the fourth breaks it.
|
||||||
"a rate limit",
|
"a rate limit",
|
||||||
map[string]string{rateLimitPerMinute: "12", asnLimitPercent: asnDEQuarter},
|
map[string]string{rateLimitPerMinute: "12", tagLimitPercent: asnTagDEQuarter},
|
||||||
3, http.StatusForbidden, requestlog.ActionRateLimited,
|
3, http.StatusForbidden, requestlog.ActionRateLimited,
|
||||||
"requests per minute over the limit of 3", "25 from " + asnLimitPercent,
|
"requests per minute over the limit of 3", "25 from " + tagLimitPercent,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
// The byte limits' percentage, not the rate limits'.
|
// The byte limits' percentage, not the rate limits'.
|
||||||
"a byte limit",
|
"a byte limit",
|
||||||
map[string]string{
|
map[string]string{
|
||||||
bytesLimitPerMinute: twoUploads, asnLimitPercent: asnDEQuarter,
|
bytesLimitPerMinute: twoUploads, tagLimitPercent: asnTagDEQuarter,
|
||||||
asnBytesPercent: asnDEHalf,
|
asnBytesPercent: asnDEHalf,
|
||||||
},
|
},
|
||||||
0, http.StatusOK, requestlog.ActionForward,
|
0, http.StatusOK, requestlog.ActionForward,
|
||||||
@@ -487,7 +497,7 @@ func wantPercent(t *testing.T, what string, percent *int64, setting, want string
|
|||||||
}
|
}
|
||||||
|
|
||||||
// percentText gives a limit percentage and the setting that gave it as
|
// percentText gives a limit percentage and the setting that gave it as
|
||||||
// text, such as "50 from SWW_ASN_LIMIT_PERCENT", or none when both are
|
// text, such as "50 from SWW_TAG_LIMIT_PERCENT", or none when both are
|
||||||
// left out.
|
// left out.
|
||||||
func percentText(percent *int64, setting string) string {
|
func percentText(percent *int64, setting string) string {
|
||||||
switch {
|
switch {
|
||||||
|
|||||||
@@ -17,9 +17,10 @@ import (
|
|||||||
// SWW_WAF_MODE is off or SWW_WAF_EXEMPT_PATHS exempts its path, as
|
// SWW_WAF_MODE is off or SWW_WAF_EXEMPT_PATHS exempts its path, as
|
||||||
// pathExempt decides, and notes the rules it matched and its score in the
|
// pathExempt decides, and notes the rules it matched and its score in the
|
||||||
// log line, and the rules in the metrics. A score at or over
|
// log line, and the rules in the metrics. A score at or over
|
||||||
// SWW_WAF_ANOMALY_THRESHOLD is a match: it raises the waf_block alert,
|
// SWW_WAF_ANOMALY_THRESHOLD is a match: it adds the tag exploitsignature,
|
||||||
// and in block mode refuses the request with 403, which is an offence its
|
// raises the waf_block alert, and in block mode refuses the request with 403,
|
||||||
// client's history counts. Its figures are the score and the threshold.
|
// which is an offence its client's history counts. Its figures are the score
|
||||||
|
// and the threshold.
|
||||||
type coreRuleSet struct{}
|
type coreRuleSet struct{}
|
||||||
|
|
||||||
func (coreRuleSet) name() string { return "core_rule_set" }
|
func (coreRuleSet) name() string { return "core_rule_set" }
|
||||||
@@ -50,6 +51,7 @@ func (coreRuleSet) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
rq.addTag(config.ExploitSignatureTag)
|
||||||
rq.alertWAFBlock(result)
|
rq.alertWAFBlock(result)
|
||||||
|
|
||||||
if cfg.WAFMode == config.WAFModeDetect {
|
if cfg.WAFMode == config.WAFModeDetect {
|
||||||
|
|||||||
@@ -61,6 +61,41 @@ func wantWAF(t *testing.T, line logLine, score *int, ruleIDs ...int) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCoreRuleSetMatchAddsTheExploitSignatureTag(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
mode string
|
||||||
|
status int
|
||||||
|
action string
|
||||||
|
}{
|
||||||
|
{block, block, http.StatusForbidden, requestlog.ActionWAFBlocked},
|
||||||
|
{detect, detect, http.StatusOK, requestlog.ActionForward},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s, _, _ := startWithClock(t, "", map[string]string{wafMode: tc.mode})
|
||||||
|
|
||||||
|
wantTags(t, s.request(client, sqlInjection, tc.status, tc.action),
|
||||||
|
"exploitsignature")
|
||||||
|
wantTags(t, s.get(client, http.StatusOK, requestlog.ActionForward))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDetectedAttackIsRefusedByTagDenyLast(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s, _, _ := startWithClock(t, "", map[string]string{
|
||||||
|
wafMode: detect, tagDeny: "exploitsignature",
|
||||||
|
})
|
||||||
|
|
||||||
|
line := s.request(client, sqlInjection, http.StatusForbidden, requestlog.ActionDenied)
|
||||||
|
wantHeuristic(t, line, "tag_deny", `{"tag": "exploitsignature"}`)
|
||||||
|
}
|
||||||
|
|
||||||
func TestCoreRuleSetRefusesAttacksInBlockModeAndOnlyLogsThemInDetectMode(t *testing.T) {
|
func TestCoreRuleSetRefusesAttacksInBlockModeAndOnlyLogsThemInDetectMode(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
package discriminator
|
package discriminator
|
||||||
|
|
||||||
// The heuristic countries. It reads SWW_DENIED_COUNTRIES and
|
// The heuristic countries. It reads SWW_EXCLUSIVELY_ALLOWED_COUNTRIES.
|
||||||
// SWW_EXCLUSIVELY_ALLOWED_COUNTRIES.
|
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -11,32 +10,26 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// countries refuses a request by the client's country as it was looked up. A
|
// countries refuses a request when SWW_EXCLUSIVELY_ALLOWED_COUNTRIES is set and
|
||||||
// client without a country, or whose country cannot be found, is refused only
|
// does not hold the client's country as it was looked up. A client without a
|
||||||
// by SWW_EXCLUSIVELY_ALLOWED_COUNTRIES. Its figures are the country and the
|
// country, or whose country cannot be found, is refused. Its figures are the
|
||||||
// setting that refused it.
|
// country and the setting that refused it.
|
||||||
type countries struct{}
|
type countries struct{}
|
||||||
|
|
||||||
func (countries) name() string { return "countries" }
|
func (countries) name() string { return "countries" }
|
||||||
|
|
||||||
func (countries) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
func (countries) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
||||||
denied := rq.h.config.DeniedCountries
|
|
||||||
allowed := rq.h.config.ExclusivelyAllowedCountries
|
allowed := rq.h.config.ExclusivelyAllowedCountries
|
||||||
country := rq.line.Country
|
country := rq.line.Country
|
||||||
|
|
||||||
var setting string
|
if len(allowed) == 0 || slices.Contains(allowed, country) {
|
||||||
|
|
||||||
switch {
|
|
||||||
case slices.Contains(denied, country):
|
|
||||||
setting = "SWW_DENIED_COUNTRIES"
|
|
||||||
case len(allowed) > 0 && !slices.Contains(allowed, country):
|
|
||||||
setting = "SWW_EXCLUSIVELY_ALLOWED_COUNTRIES"
|
|
||||||
default:
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return &verdict{
|
return &verdict{
|
||||||
action: requestlog.ActionCountryDenied,
|
action: requestlog.ActionCountryDenied,
|
||||||
figures: map[string]any{"country": country, "setting": setting},
|
figures: map[string]any{
|
||||||
|
"country": country, "setting": "SWW_EXCLUSIVELY_ALLOWED_COUNTRIES",
|
||||||
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -33,13 +33,13 @@ func TestCountryLists(t *testing.T) {
|
|||||||
env map[string]string
|
env map[string]string
|
||||||
refused []string
|
refused []string
|
||||||
}{
|
}{
|
||||||
{"denied", map[string]string{deniedCountries: "kp"}, []string{fromKP}},
|
|
||||||
{
|
{
|
||||||
"exclusively allowed", map[string]string{allowedCountries: "DE"},
|
"exclusively allowed", map[string]string{allowedCountries: "DE"},
|
||||||
[]string{fromKP, unplaced},
|
[]string{fromKP, unplaced},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"both", map[string]string{deniedCountries: "kp", allowedCountries: "de,fr"},
|
"with a country in SWW_TAG_DENY",
|
||||||
|
map[string]string{tagDeny: denyKP, allowedCountries: "de,fr"},
|
||||||
[]string{fromKP, unplaced},
|
[]string{fromKP, unplaced},
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
@@ -95,12 +95,12 @@ func TestCountryRefusalNamesItsHeuristicCountryAndSetting(t *testing.T) {
|
|||||||
|
|
||||||
geojsURL, _ := startGeoJS(t)
|
geojsURL, _ := startGeoJS(t)
|
||||||
s, _, _ := startWithClock(t, geojsURL, map[string]string{
|
s, _, _ := startWithClock(t, geojsURL, map[string]string{
|
||||||
lookupTimeout: "1h", deniedCountries: "kp", allowedCountries: "de",
|
lookupTimeout: "1h", allowedCountries: "de",
|
||||||
})
|
})
|
||||||
|
|
||||||
line := s.get(fromKP, http.StatusForbidden, requestlog.ActionCountryDenied)
|
line := s.get(fromKP, http.StatusForbidden, requestlog.ActionCountryDenied)
|
||||||
wantHeuristic(t, line, "countries",
|
wantHeuristic(t, line, "countries",
|
||||||
`{"country": "KP", "setting": "SWW_DENIED_COUNTRIES"}`)
|
`{"country": "KP", "setting": "SWW_EXCLUSIVELY_ALLOWED_COUNTRIES"}`)
|
||||||
|
|
||||||
line = s.get(unplaced, http.StatusForbidden, requestlog.ActionCountryDenied)
|
line = s.get(unplaced, http.StatusForbidden, requestlog.ActionCountryDenied)
|
||||||
wantHeuristic(t, line, "countries",
|
wantHeuristic(t, line, "countries",
|
||||||
@@ -119,7 +119,7 @@ func TestCountryRefusalKeepsARequestFromTheApp(t *testing.T) {
|
|||||||
addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, map[string]string{
|
addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, map[string]string{
|
||||||
trustedProxies: trustLocalhost,
|
trustedProxies: trustLocalhost,
|
||||||
lookupTimeout: "1h",
|
lookupTimeout: "1h",
|
||||||
deniedCountries: "kp",
|
allowedCountries: "de",
|
||||||
})
|
})
|
||||||
|
|
||||||
req := newRequest(t, http.MethodPost, addr, "/", strings.NewReader("a body"))
|
req := newRequest(t, http.MethodPost, addr, "/", strings.NewReader("a body"))
|
||||||
@@ -202,7 +202,7 @@ func TestPrivateAddressIsNeverLookedUp(t *testing.T) {
|
|||||||
env map[string]string
|
env map[string]string
|
||||||
}{
|
}{
|
||||||
{"no setting needs the lookup", nil},
|
{"no setting needs the lookup", nil},
|
||||||
{"a country list is set", map[string]string{deniedCountries: "kp"}},
|
{"a tag setting needs it", map[string]string{tagDeny: denyKP}},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|||||||
@@ -8,14 +8,15 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// crowdSec refuses a request when a decision of the CrowdSec decision list on
|
// crowdSec refuses a request when a decision of the CrowdSec decision list on
|
||||||
// the client is in force at now. It then notes the list, as noteHit does, and
|
// the client is in force at now. It then adds the tag crowdsec, notes the
|
||||||
// bans the client until that decision ends. Its figures are the decision's
|
// list, as noteHit does, and bans the client until that decision ends. Its
|
||||||
// scenario and when it ends.
|
// figures are the decision's scenario and when it ends.
|
||||||
type crowdSec struct{}
|
type crowdSec struct{}
|
||||||
|
|
||||||
func (crowdSec) name() string { return "crowdsec" }
|
func (crowdSec) name() string { return "crowdsec" }
|
||||||
@@ -26,6 +27,7 @@ func (crowdSec) check(_ context.Context, rq *request, now time.Time) *verdict {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
rq.addTag(config.CrowdSecTag)
|
||||||
rq.noteHit(bans.ReputationHit{Source: rq.h.config.CrowdSecDecisionsURL},
|
rq.noteHit(bans.ReputationHit{Source: rq.h.config.CrowdSecDecisionsURL},
|
||||||
"listed by the CrowdSec decision list")
|
"listed by the CrowdSec decision list")
|
||||||
rq.banForCrowdSec(now, decision)
|
rq.banForCrowdSec(now, decision)
|
||||||
|
|||||||
@@ -108,6 +108,7 @@ func TestCrowdSecBanNamesItsHeuristicScenarioAndEnd(t *testing.T) {
|
|||||||
line := s.get(client, http.StatusForbidden, requestlog.ActionBanned)
|
line := s.get(client, http.StatusForbidden, requestlog.ActionBanned)
|
||||||
wantHeuristic(t, line, "crowdsec", `{"scenario": "crowdsecurity/ssh-bf", `+
|
wantHeuristic(t, line, "crowdsec", `{"scenario": "crowdsecurity/ssh-bf", `+
|
||||||
`"expires": "`+requestlog.FormatTime(clk.Now().Add(4*time.Hour))+`"}`)
|
`"expires": "`+requestlog.FormatTime(clk.Now().Add(4*time.Hour))+`"}`)
|
||||||
|
wantTags(t, line, "crowdsec")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEndedCrowdSecDecisionNoLongerBansThoughTheCopyStillHoldsIt(t *testing.T) {
|
func TestEndedCrowdSecDecisionNoLongerBansThoughTheCopyStillHoldsIt(t *testing.T) {
|
||||||
|
|||||||
@@ -414,7 +414,7 @@ func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
|
|||||||
|
|
||||||
geojsURL, asked, release := startHeldGeoJS(t)
|
geojsURL, asked, release := startHeldGeoJS(t)
|
||||||
s := startDiscriminator(t, geojsURL, map[string]string{
|
s := startDiscriminator(t, geojsURL, map[string]string{
|
||||||
lookupTimeout: "1h", deniedCountries: "kp", bytesLimitPerMinute: byteLimit,
|
lookupTimeout: "1h", tagDeny: denyKP, bytesLimitPerMinute: byteLimit,
|
||||||
})
|
})
|
||||||
|
|
||||||
first := s.question(siteA, client, "/")
|
first := s.question(siteA, client, "/")
|
||||||
@@ -765,7 +765,7 @@ func TestAllowCarriesTheLookupHeadersWhileTheSettingIsSet(t *testing.T) {
|
|||||||
|
|
||||||
geojsURL, _ := startGeoJS(t)
|
geojsURL, _ := startGeoJS(t)
|
||||||
s := startDiscriminator(t, geojsURL, map[string]string{
|
s := startDiscriminator(t, geojsURL, map[string]string{
|
||||||
addLookupHeaders: tc.setting, lookupTimeout: "1h", deniedCountries: "kp",
|
addLookupHeaders: tc.setting, lookupTimeout: "1h", tagDeny: denyKP,
|
||||||
})
|
})
|
||||||
|
|
||||||
got := s.ask(fromDE, "/")
|
got := s.ask(fromDE, "/")
|
||||||
|
|||||||
@@ -5,13 +5,16 @@ package discriminator
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// dnsbl notes the DNSBL zones whose verdict lists the client, as noteListed
|
// dnsbl notes the DNSBL zones whose verdict lists the client, as noteListed
|
||||||
// does, and refuses the request when SWW_REPUTATION_ACTION is deny. Being
|
// does, adds the tag dnsbl=<zone> for each, the zone's key masked, and refuses
|
||||||
|
// the request when SWW_REPUTATION_ACTION is deny. Being
|
||||||
// limit, it lowers the client's limits instead (see limitPercentages), and
|
// limit, it lowers the client's limits instead (see limitPercentages), and
|
||||||
// being log, it does nothing more. A zone without a verdict on the client is
|
// being log, it does nothing more. A zone without a verdict on the client is
|
||||||
// asked about it in the background, and the request does not wait for the
|
// asked about it in the background, and the request does not wait for the
|
||||||
@@ -25,6 +28,10 @@ func (dnsbl) check(ctx context.Context, rq *request, _ time.Time) *verdict {
|
|||||||
rq.dnsblListed = len(listedBy) > 0
|
rq.dnsblListed = len(listedBy) > 0
|
||||||
rq.noteListed(listedBy, "listed by a DNSBL zone")
|
rq.noteListed(listedBy, "listed by a DNSBL zone")
|
||||||
|
|
||||||
|
for _, zone := range listedBy {
|
||||||
|
rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone))
|
||||||
|
}
|
||||||
|
|
||||||
if !rq.dnsblListed || rq.h.config.ReputationAction != deny {
|
if !rq.dnsblListed || rq.h.config.ReputationAction != deny {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -43,12 +43,15 @@ const (
|
|||||||
// country are looked up, so that a client they refuse is not looked up.
|
// country are looked up, so that a client they refuse is not looked up.
|
||||||
func beforeLookup() []heuristic {
|
func beforeLookup() []heuristic {
|
||||||
return []heuristic{
|
return []heuristic{
|
||||||
|
tagLists{},
|
||||||
denyNets{},
|
denyNets{},
|
||||||
activeBan{},
|
activeBan{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// afterLookup are the heuristics that run once the lookup has been made.
|
// afterLookup are the heuristics that run once the lookup has been made.
|
||||||
|
// tag_deny runs twice: before the rate limits, on the tags known by then, and
|
||||||
|
// last, on the tag rule_files and core_rule_set can add.
|
||||||
func afterLookup() []heuristic {
|
func afterLookup() []heuristic {
|
||||||
return []heuristic{
|
return []heuristic{
|
||||||
countries{},
|
countries{},
|
||||||
@@ -56,11 +59,13 @@ func afterLookup() []heuristic {
|
|||||||
crowdSec{},
|
crowdSec{},
|
||||||
dnsbl{},
|
dnsbl{},
|
||||||
abuseIPDB{},
|
abuseIPDB{},
|
||||||
|
tagDeny{},
|
||||||
rateLimits{},
|
rateLimits{},
|
||||||
trapPaths{},
|
trapPaths{},
|
||||||
ruleFiles{},
|
ruleFiles{},
|
||||||
headerLimits{},
|
headerLimits{},
|
||||||
coreRuleSet{},
|
coreRuleSet{},
|
||||||
|
tagDeny{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ func TestHistoryKeepsEachRequestOfTheClient(t *testing.T) {
|
|||||||
s, clk, server := startWithClock(t, geojsURL, map[string]string{
|
s, clk, server := startWithClock(t, geojsURL, map[string]string{
|
||||||
lookupTimeout: "1h",
|
lookupTimeout: "1h",
|
||||||
rateLimitPerMinute: "2",
|
rateLimitPerMinute: "2",
|
||||||
deniedCountries: "kp",
|
tagDeny: denyKP,
|
||||||
})
|
})
|
||||||
start := clk.Now()
|
start := clk.Now()
|
||||||
|
|
||||||
|
|||||||
@@ -4,8 +4,10 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -19,7 +21,8 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// lookUp looks up the client's AS number and country, in the lookup
|
// lookUp looks up the client's AS number and country, in the lookup
|
||||||
// database or through GeoJS, and notes them for the log line, unless
|
// database or through GeoJS, and notes them for the log line and as the
|
||||||
|
// request's asn= and country= tags, unless
|
||||||
// SWW_LOOKUP_SOURCE is off or the client is on a private, loopback or
|
// SWW_LOOKUP_SOURCE is off or the client is on a private, loopback or
|
||||||
// link-local address, which no lookup can place. The lookup database
|
// link-local address, which no lookup can place. The lookup database
|
||||||
// answers at once. With GeoJS, while a setting needs the answer, such as a
|
// answers at once. With GeoJS, while a setting needs the answer, such as a
|
||||||
@@ -47,6 +50,34 @@ func (rq *request) lookUp(ctx context.Context, now time.Time) {
|
|||||||
rq.line.ASN = rq.lookupAnswer.ASN
|
rq.line.ASN = rq.lookupAnswer.ASN
|
||||||
rq.line.ASName = rq.lookupAnswer.ASName
|
rq.line.ASName = rq.lookupAnswer.ASName
|
||||||
rq.line.Country = rq.lookupAnswer.Country
|
rq.line.Country = rq.lookupAnswer.Country
|
||||||
|
|
||||||
|
if tag := asnTag(rq.line.ASN); tag != "" {
|
||||||
|
rq.addTag(tag)
|
||||||
|
}
|
||||||
|
|
||||||
|
if tag := countryTag(rq.line.Country); tag != "" {
|
||||||
|
rq.addTag(tag)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// asnTag returns the tag of asn, an AS number as the lookup gives it, such as
|
||||||
|
// asn=64496 for AS64496, or "" for no AS number.
|
||||||
|
func asnTag(asn string) string {
|
||||||
|
if asn == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
return config.ASNTagPrefix + strings.TrimPrefix(asn, "AS")
|
||||||
|
}
|
||||||
|
|
||||||
|
// countryTag returns the tag of country, a code as the lookup gives it, such
|
||||||
|
// as country=DE for DE, or "" for no country.
|
||||||
|
func countryTag(country string) string {
|
||||||
|
if country == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
return config.CountryTagPrefix + country
|
||||||
}
|
}
|
||||||
|
|
||||||
// addLookup adds answer, an answer about a client from the lookup
|
// addLookup adds answer, an answer about a client from the lookup
|
||||||
|
|||||||
@@ -96,7 +96,7 @@ func TestRequestThatWaitedForItsLookupIsCountedWhenItGoesOn(t *testing.T) {
|
|||||||
map[string]string{
|
map[string]string{
|
||||||
trustedProxies: trustLocalhost,
|
trustedProxies: trustLocalhost,
|
||||||
lookupTimeout: "1h",
|
lookupTimeout: "1h",
|
||||||
deniedCountries: "kp",
|
tagDeny: denyKP,
|
||||||
})
|
})
|
||||||
|
|
||||||
requestFromDE := func() *http.Request {
|
requestFromDE := func() *http.Request {
|
||||||
|
|||||||
@@ -198,7 +198,7 @@ func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
|
|||||||
trustedProxies: trustLocalhost,
|
trustedProxies: trustLocalhost,
|
||||||
metricsToken: token,
|
metricsToken: token,
|
||||||
metricsTopN: "2",
|
metricsTopN: "2",
|
||||||
deniedCountries: "kp",
|
allowedCountries: "de,fr",
|
||||||
}
|
}
|
||||||
geojsURL, _ := startGeoJS(t)
|
geojsURL, _ := startGeoJS(t)
|
||||||
addr, out, server := startProxyWithClock(t, app.URL, geojsURL, time.Now, env)
|
addr, out, server := startProxyWithClock(t, app.URL, geojsURL, time.Now, env)
|
||||||
@@ -324,11 +324,11 @@ func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) {
|
|||||||
map[string]string{
|
map[string]string{
|
||||||
trustedProxies: trustLocalhost,
|
trustedProxies: trustLocalhost,
|
||||||
metricsToken: token,
|
metricsToken: token,
|
||||||
deniedCountries: "kp",
|
tagDeny: denyKP,
|
||||||
})
|
})
|
||||||
|
|
||||||
// GeoJS fails, so the client counts as coming from an unknown country,
|
// GeoJS fails, so the client has no country= tag, and SWW_TAG_DENY does
|
||||||
// which SWW_DENIED_COUNTRIES does not refuse.
|
// not refuse it.
|
||||||
req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
|
req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
|
||||||
req.Header.Set(forwardedFor, fromDE)
|
req.Header.Set(forwardedFor, fromDE)
|
||||||
wantStatus(t, do(t, req), http.StatusOK)
|
wantStatus(t, do(t, req), http.StatusOK)
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ func TestObserveModeForwardsWhatEnforceModeRefuses(t *testing.T) {
|
|||||||
lookupTimeout: "1h",
|
lookupTimeout: "1h",
|
||||||
rateLimitPerMinute: "1",
|
rateLimitPerMinute: "1",
|
||||||
denyNets: denied,
|
denyNets: denied,
|
||||||
deniedCountries: "kp",
|
tagDeny: denyKP,
|
||||||
}
|
}
|
||||||
|
|
||||||
if tc.setting != "" {
|
if tc.setting != "" {
|
||||||
@@ -71,7 +71,7 @@ func TestObserveModeForwardsWhatEnforceModeRefuses(t *testing.T) {
|
|||||||
for _, sent := range []struct{ from, refusal string }{
|
for _, sent := range []struct{ from, refusal string }{
|
||||||
{denied, requestlog.ActionDenied},
|
{denied, requestlog.ActionDenied},
|
||||||
{banned, requestlog.ActionBanned},
|
{banned, requestlog.ActionBanned},
|
||||||
{fromKP, requestlog.ActionCountryDenied},
|
{fromKP, requestlog.ActionDenied},
|
||||||
{fromDE, requestlog.ActionRateLimited},
|
{fromDE, requestlog.ActionRateLimited},
|
||||||
} {
|
} {
|
||||||
if !tc.observe {
|
if !tc.observe {
|
||||||
|
|||||||
@@ -155,11 +155,12 @@ func newHandler(params Params) *handler {
|
|||||||
h.geojs = lookup.New(lookup.Params{
|
h.geojs = lookup.New(lookup.Params{
|
||||||
URL: params.GeoJSURL,
|
URL: params.GeoJSURL,
|
||||||
Timeout: params.Config.LookupTimeout,
|
Timeout: params.Config.LookupTimeout,
|
||||||
// The country lists, the headers and the biased thresholds act on
|
// The country list, the headers, the tag settings that name an asn=
|
||||||
// the answer before the request goes on.
|
// or country= tag and the biased thresholds act on the answer before
|
||||||
Wait: len(params.Config.DeniedCountries) > 0 ||
|
// the request goes on.
|
||||||
len(params.Config.ExclusivelyAllowedCountries) > 0 ||
|
Wait: len(params.Config.ExclusivelyAllowedCountries) > 0 ||
|
||||||
params.Config.AddLookupHeaders || biasedThresholdsSet(params.Config),
|
params.Config.AddLookupHeaders || params.Config.TagsNeedLookup() ||
|
||||||
|
biasedThresholdsSet(params.Config),
|
||||||
Answered: h.addLookup,
|
Answered: h.addLookup,
|
||||||
Now: params.Now,
|
Now: params.Now,
|
||||||
ProcessLog: params.ProcessLog,
|
ProcessLog: params.ProcessLog,
|
||||||
@@ -183,6 +184,7 @@ func newReputation(
|
|||||||
lists := reputation.New(reputation.Params{
|
lists := reputation.New(reputation.Params{
|
||||||
BlocklistURLs: cfg.BlocklistURLs, Refresh: cfg.BlocklistRefresh,
|
BlocklistURLs: cfg.BlocklistURLs, Refresh: cfg.BlocklistRefresh,
|
||||||
ASNLimitPercentURL: cfg.ASNLimitPercentURL,
|
ASNLimitPercentURL: cfg.ASNLimitPercentURL,
|
||||||
|
TagLists: cfg.TagLists,
|
||||||
CrowdSecDecisionsURL: cfg.CrowdSecDecisionsURL, CrowdSecKey: cfg.CrowdSecKey,
|
CrowdSecDecisionsURL: cfg.CrowdSecDecisionsURL, CrowdSecKey: cfg.CrowdSecKey,
|
||||||
Now: params.Now, ProcessLog: params.ProcessLog, Alerts: params.Alerts,
|
Now: params.Now, ProcessLog: params.ProcessLog, Alerts: params.Alerts,
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -69,7 +69,9 @@ const (
|
|||||||
lookupDBPath = "SWW_LOOKUP_DB_PATH"
|
lookupDBPath = "SWW_LOOKUP_DB_PATH"
|
||||||
lookupTimeout = "SWW_LOOKUP_TIMEOUT"
|
lookupTimeout = "SWW_LOOKUP_TIMEOUT"
|
||||||
addLookupHeaders = "SWW_ADD_LOOKUP_HEADERS"
|
addLookupHeaders = "SWW_ADD_LOOKUP_HEADERS"
|
||||||
deniedCountries = "SWW_DENIED_COUNTRIES"
|
tagDeny = "SWW_TAG_DENY"
|
||||||
|
// denyKP is the value of SWW_TAG_DENY that refuses fromKP's country.
|
||||||
|
denyKP = "country=KP"
|
||||||
allowedCountries = "SWW_EXCLUSIVELY_ALLOWED_COUNTRIES"
|
allowedCountries = "SWW_EXCLUSIVELY_ALLOWED_COUNTRIES"
|
||||||
banResponse = "SWW_BAN_RESPONSE"
|
banResponse = "SWW_BAN_RESPONSE"
|
||||||
limitBanDuration = "SWW_LIMIT_BAN_DURATION"
|
limitBanDuration = "SWW_LIMIT_BAN_DURATION"
|
||||||
|
|||||||
@@ -136,7 +136,7 @@ func TestRateLimitExemptPathsAreNeitherCountedNorRefused(t *testing.T) {
|
|||||||
rateLimitPerMinute: "1",
|
rateLimitPerMinute: "1",
|
||||||
rateLimitExemptPaths: "/assets/,/favicon.ico",
|
rateLimitExemptPaths: "/assets/,/favicon.ico",
|
||||||
denyNets: denied,
|
denyNets: denied,
|
||||||
deniedCountries: "kp",
|
tagDeny: denyKP,
|
||||||
})
|
})
|
||||||
|
|
||||||
// The answers are kept before the requests, so that none waits for
|
// The answers are kept before the requests, so that none waits for
|
||||||
@@ -162,12 +162,12 @@ func TestRateLimitExemptPathsAreNeitherCountedNorRefused(t *testing.T) {
|
|||||||
// /assets/, and breaks the limit.
|
// /assets/, and breaks the limit.
|
||||||
s.request(client, "/assets", http.StatusForbidden, requestlog.ActionRateLimited)
|
s.request(client, "/assets", http.StatusForbidden, requestlog.ActionRateLimited)
|
||||||
|
|
||||||
// A ban, SWW_DENY_NETS and the country lists still refuse a path
|
// A ban, SWW_DENY_NETS and SWW_TAG_DENY still refuse a path under a
|
||||||
// under a prefix.
|
// prefix.
|
||||||
s.request(client, "/assets/app.js", http.StatusForbidden, requestlog.ActionBanned)
|
s.request(client, "/assets/app.js", http.StatusForbidden, requestlog.ActionBanned)
|
||||||
s.request(denied, "/assets/app.js", http.StatusForbidden, requestlog.ActionDenied)
|
s.request(denied, "/assets/app.js", http.StatusForbidden, requestlog.ActionDenied)
|
||||||
s.request(fromKP, "/assets/app.js",
|
s.request(fromKP, "/assets/app.js",
|
||||||
http.StatusForbidden, requestlog.ActionCountryDenied)
|
http.StatusForbidden, requestlog.ActionDenied)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRateLimitCountsPathsThatAreNotExempt(t *testing.T) {
|
func TestRateLimitCountsPathsThatAreNotExempt(t *testing.T) {
|
||||||
|
|||||||
@@ -191,7 +191,7 @@ func TestErrorBurstIsNotLoweredForAClientWithLowerLimits(t *testing.T) {
|
|||||||
lookupTimeout: "1h",
|
lookupTimeout: "1h",
|
||||||
errorBurstThreshold: "2",
|
errorBurstThreshold: "2",
|
||||||
rulesDir: writeRules(t, testRules),
|
rulesDir: writeRules(t, testRules),
|
||||||
countryLimitPercent: countryDEHalf,
|
tagLimitPercent: countryTagDEHalf,
|
||||||
})
|
})
|
||||||
|
|
||||||
// Half of the threshold would be one, which the second refusal is over.
|
// Half of the threshold would be one, which the second refusal is over.
|
||||||
|
|||||||
@@ -172,7 +172,7 @@ func TestBlocklistsComeAfterTheCountryListsAndSkipAllowNets(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
s, server, queue := startWithLookups(t, map[string]string{
|
s, server, queue := startWithLookups(t, map[string]string{
|
||||||
blocklistURLs: dropURL, deniedCountries: "kp", allowNets: fromDE,
|
blocklistURLs: dropURL, allowedCountries: "de", allowNets: fromDE,
|
||||||
})
|
})
|
||||||
loadLists(t, server, map[string][]string{dropURL: {fromDE, fromKP}})
|
loadLists(t, server, map[string][]string{dropURL: {fromDE, fromKP}})
|
||||||
|
|
||||||
@@ -211,18 +211,18 @@ func TestBlocklistLimitTakesPartInTheLowestPercentageOfEveryLimit(t *testing.T)
|
|||||||
// percentText gives them, and limitHit its limit_hit.
|
// percentText gives them, and limitHit its limit_hit.
|
||||||
want, limitHit string
|
want, limitHit string
|
||||||
}{
|
}{
|
||||||
{limitHalf, asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes},
|
{limitHalf, asnTagDEQuarter, "25 from " + tagLimitPercent, minuteBytes},
|
||||||
{limitQuarter, asnDEHalf, "25 from " + blocklistAction, minuteBytes},
|
{limitQuarter, asnTagDEHalf, "25 from " + blocklistAction, minuteBytes},
|
||||||
// The AS number's, the first of two alike.
|
// The AS number's, the first of two alike.
|
||||||
{limitQuarter, asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes},
|
{limitQuarter, asnTagDEQuarter, "25 from " + tagLimitPercent, minuteBytes},
|
||||||
{actionLog, asnDE + ":100", none, ""},
|
{actionLog, "asn=64511:25", none, ""},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.action+" "+tc.asnPercent, func(t *testing.T) {
|
t.Run(tc.action+" "+tc.asnPercent, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
s, server, _ := startWithLookups(t, map[string]string{
|
s, server, _ := startWithLookups(t, map[string]string{
|
||||||
bytesLimitPerMinute: twoUploads, blocklistURLs: dropURL,
|
bytesLimitPerMinute: twoUploads, blocklistURLs: dropURL,
|
||||||
blocklistAction: tc.action, asnLimitPercent: tc.asnPercent,
|
blocklistAction: tc.action, tagLimitPercent: tc.asnPercent,
|
||||||
})
|
})
|
||||||
loadLists(t, server, map[string][]string{dropURL: {fromDE}})
|
loadLists(t, server, map[string][]string{dropURL: {fromDE}})
|
||||||
|
|
||||||
@@ -246,7 +246,7 @@ func TestASNLimitPercentFileCountsAsTheSettingDoesTheLowerWinning(t *testing.T)
|
|||||||
|
|
||||||
const (
|
const (
|
||||||
fromURL = "25 from " + asnLimitPercentURL
|
fromURL = "25 from " + asnLimitPercentURL
|
||||||
fromSetting = "25 from " + asnLimitPercent
|
fromSetting = "25 from " + tagLimitPercent
|
||||||
)
|
)
|
||||||
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
@@ -260,16 +260,16 @@ func TestASNLimitPercentFileCountsAsTheSettingDoesTheLowerWinning(t *testing.T)
|
|||||||
{"the file's alone", nil, asnDEQuarter, fromURL, fromURL},
|
{"the file's alone", nil, asnDEQuarter, fromURL, fromURL},
|
||||||
{
|
{
|
||||||
"the file's, lower than the setting's",
|
"the file's, lower than the setting's",
|
||||||
map[string]string{asnLimitPercent: asnDEHalf}, asnDEQuarter, fromURL, fromURL,
|
map[string]string{tagLimitPercent: asnTagDEHalf}, asnDEQuarter, fromURL, fromURL,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"the setting's, lower than the file's",
|
"the setting's, lower than the file's",
|
||||||
map[string]string{asnLimitPercent: asnDEQuarter}, asnDEHalf,
|
map[string]string{tagLimitPercent: asnTagDEQuarter}, asnDEHalf,
|
||||||
fromSetting, fromSetting,
|
fromSetting, fromSetting,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"the setting's, the first of two alike",
|
"the setting's, the first of two alike",
|
||||||
map[string]string{asnLimitPercent: asnDEQuarter}, asnDEQuarter,
|
map[string]string{tagLimitPercent: asnTagDEQuarter}, asnDEQuarter,
|
||||||
fromSetting, fromSetting,
|
fromSetting, fromSetting,
|
||||||
},
|
},
|
||||||
{"none, for an AS number the file does not list", nil, asnKP + ":25", none, none},
|
{"none, for an AS number the file does not list", nil, asnKP + ":25", none, none},
|
||||||
@@ -482,6 +482,46 @@ func TestDNSBLZonesComeAfterTheBlocklistsAndSkipAllowNets(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestReputationSourcesTagTheRequestWhateverTheirAction(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
env map[string]string
|
||||||
|
load func(*discriminator.Server)
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
"dnsbl",
|
||||||
|
map[string]string{
|
||||||
|
dnsblZones: dnsblZone, dnsblResolver: noResolver, reputationAction: actionLog,
|
||||||
|
},
|
||||||
|
func(server *discriminator.Server) {
|
||||||
|
loadVerdicts(server, map[string][]string{fromDE: {dnsblZone}})
|
||||||
|
},
|
||||||
|
[]string{"asn=64496", "country=DE", "dnsbl=" + dnsblZone},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
abuseipdb,
|
||||||
|
map[string]string{abuseIPDBKey: accountKey, reputationAction: actionLog},
|
||||||
|
func(server *discriminator.Server) {
|
||||||
|
loadScores(server, map[string]int64{fromDE: 80})
|
||||||
|
},
|
||||||
|
[]string{abuseipdb, "asn=64496", "country=DE"},
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s, server, _ := startWithLookups(t, tc.env)
|
||||||
|
tc.load(server)
|
||||||
|
|
||||||
|
wantTags(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward),
|
||||||
|
tc.want...)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestObserveModeForwardsAClientADNSBLZoneDeniesAndAlertsIt(t *testing.T) {
|
func TestObserveModeForwardsAClientADNSBLZoneDeniesAndAlertsIt(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -138,6 +138,15 @@ func (h *handler) startRequest(
|
|||||||
return rq
|
return rq
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// addTag adds tag to the request's tags, which stay sorted and hold each tag
|
||||||
|
// once.
|
||||||
|
func (rq *request) addTag(tag string) {
|
||||||
|
at, found := slices.BinarySearch(rq.line.Tags, tag)
|
||||||
|
if !found {
|
||||||
|
rq.line.Tags = slices.Insert(rq.line.Tags, at, tag)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// requestHeaders returns the headers of r that names lists, by name in
|
// requestHeaders returns the headers of r that names lists, by name in
|
||||||
// lower case, each with its values joined by ", ". Authorization, Cookie
|
// lower case, each with its values joined by ", ". Authorization, Cookie
|
||||||
// and Set-Cookie are never among them, whatever names says.
|
// and Set-Cookie are never among them, whatever names says.
|
||||||
|
|||||||
@@ -8,12 +8,14 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ruleFiles checks the request against the rules of the rule files at now,
|
// ruleFiles checks the request against the rules of the rule files at now,
|
||||||
// notes the ids of those it matches in the log line, and refuses the request
|
// notes the ids of those it matches in the log line, adds the tag
|
||||||
|
// exploitsignature when it matches any, and refuses the request
|
||||||
// when the last rule it matches refuses it: with ActionRuleBlocked for a block
|
// when the last rule it matches refuses it: with ActionRuleBlocked for a block
|
||||||
// rule, and ActionBanned for a ban rule. A ban rule bans the client's netblock
|
// rule, and ActionBanned for a ban rule. A ban rule bans the client's netblock
|
||||||
// for a clear sign of attack, or in observe mode raises the alert for the ban
|
// for a clear sign of attack, or in observe mode raises the alert for the ban
|
||||||
@@ -35,6 +37,8 @@ func (ruleFiles) check(_ context.Context, rq *request, now time.Time) *verdict {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
rq.addTag(config.ExploitSignatureTag)
|
||||||
|
|
||||||
// Only the last rule matched can refuse the request.
|
// Only the last rule matched can refuse the request.
|
||||||
last := matched[len(matched)-1]
|
last := matched[len(matched)-1]
|
||||||
figures := map[string]any{"rule_id": last.ID, "rule_action": last.Action}
|
figures := map[string]any{"rule_id": last.ID, "rule_action": last.Action}
|
||||||
|
|||||||
@@ -93,6 +93,20 @@ func TestEachRuleAction(t *testing.T) {
|
|||||||
s.get(client, http.StatusTooManyRequests, requestlog.ActionBanned)
|
s.get(client, http.StatusTooManyRequests, requestlog.ActionBanned)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestEachRuleActionAddsTheExploitSignatureTag(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s, _, _ := startWithClock(t, "", map[string]string{
|
||||||
|
rulesDir: writeRules(t, "noted path log ^/noted$\n"+testRules),
|
||||||
|
})
|
||||||
|
|
||||||
|
wantTags(t, s.request(client, "/noted", http.StatusOK, requestlog.ActionForward),
|
||||||
|
"exploitsignature")
|
||||||
|
wantTags(t, s.request(client, "/blocked", http.StatusForbidden,
|
||||||
|
requestlog.ActionRuleBlocked), "exploitsignature")
|
||||||
|
wantTags(t, s.get(client, http.StatusOK, requestlog.ActionForward))
|
||||||
|
}
|
||||||
|
|
||||||
func TestRuleRefusalNamesItsHeuristicAndTheRule(t *testing.T) {
|
func TestRuleRefusalNamesItsHeuristicAndTheRule(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -32,13 +32,12 @@ func TestAllowNetsSkipEveryCheck(t *testing.T) {
|
|||||||
trustedProxies: trustLocalhost,
|
trustedProxies: trustLocalhost,
|
||||||
allowNets: "198.51.100.0/24",
|
allowNets: "198.51.100.0/24",
|
||||||
denyNets: fromKP,
|
denyNets: fromKP,
|
||||||
deniedCountries: "kp",
|
tagDeny: denyKP,
|
||||||
rateLimitPerMinute: "1",
|
rateLimitPerMinute: "1",
|
||||||
})
|
})
|
||||||
|
|
||||||
// Neither SWW_DENY_NETS, the country lists nor the limit of one
|
// Neither SWW_DENY_NETS, SWW_TAG_DENY nor the limit of one request a
|
||||||
// request a minute refuses the client, and its country is not looked
|
// minute refuses the client, and its country is not looked up.
|
||||||
// up.
|
|
||||||
wantAnswers(t, addr, out, []sentRequest{
|
wantAnswers(t, addr, out, []sentRequest{
|
||||||
{fromKP, http.StatusOK, requestlog.ActionForward},
|
{fromKP, http.StatusOK, requestlog.ActionForward},
|
||||||
{fromKP, http.StatusOK, requestlog.ActionForward},
|
{fromKP, http.StatusOK, requestlog.ActionForward},
|
||||||
@@ -85,7 +84,7 @@ func TestDenyNetsRefuseBeforeTheLookup(t *testing.T) {
|
|||||||
addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, map[string]string{
|
addr, out := startProxyWithGeoJS(t, app.URL, geojsURL, map[string]string{
|
||||||
trustedProxies: trustLocalhost,
|
trustedProxies: trustLocalhost,
|
||||||
denyNets: "203.0.113.0/24",
|
denyNets: "203.0.113.0/24",
|
||||||
deniedCountries: "kp",
|
tagDeny: denyKP,
|
||||||
})
|
})
|
||||||
|
|
||||||
req := newRequest(t, http.MethodPost, addr, "/", strings.NewReader("a body"))
|
req := newRequest(t, http.MethodPost, addr, "/", strings.NewReader("a body"))
|
||||||
@@ -142,19 +141,19 @@ func TestRateLimitExemptNetsAreNeitherCountedNorRefused(t *testing.T) {
|
|||||||
trustedProxies: trustLocalhost,
|
trustedProxies: trustLocalhost,
|
||||||
lookupTimeout: "1h",
|
lookupTimeout: "1h",
|
||||||
rateLimitExemptNets: listedAddr + "," + fromKP,
|
rateLimitExemptNets: listedAddr + "," + fromKP,
|
||||||
deniedCountries: "kp",
|
tagDeny: denyKP,
|
||||||
rateLimitPerMinute: "1",
|
rateLimitPerMinute: "1",
|
||||||
})
|
})
|
||||||
|
|
||||||
// listedAddr's requests are neither refused nor counted, so the first
|
// listedAddr's requests are neither refused nor counted, so the first
|
||||||
// request from unlistedAddr is within the limit of one a minute. The
|
// request from unlistedAddr is within the limit of one a minute.
|
||||||
// country lists still refuse an exempt client.
|
// SWW_TAG_DENY still refuses an exempt client.
|
||||||
wantAnswers(t, addr, out, []sentRequest{
|
wantAnswers(t, addr, out, []sentRequest{
|
||||||
{listedAddr, http.StatusOK, requestlog.ActionForward},
|
{listedAddr, http.StatusOK, requestlog.ActionForward},
|
||||||
{listedAddr, http.StatusOK, requestlog.ActionForward},
|
{listedAddr, http.StatusOK, requestlog.ActionForward},
|
||||||
{unlistedAddr, http.StatusOK, requestlog.ActionForward},
|
{unlistedAddr, http.StatusOK, requestlog.ActionForward},
|
||||||
{unlistedAddr, http.StatusForbidden, requestlog.ActionRateLimited},
|
{unlistedAddr, http.StatusForbidden, requestlog.ActionRateLimited},
|
||||||
{fromKP, http.StatusForbidden, requestlog.ActionCountryDenied},
|
{fromKP, http.StatusForbidden, requestlog.ActionDenied},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
package discriminator
|
||||||
|
|
||||||
|
// The heuristic tag_deny. It reads SWW_TAG_DENY.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"slices"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
|
)
|
||||||
|
|
||||||
|
// tagDeny refuses a request that carries a tag in SWW_TAG_DENY. Its figure is
|
||||||
|
// the first such tag, in sorted order.
|
||||||
|
type tagDeny struct{}
|
||||||
|
|
||||||
|
func (tagDeny) name() string { return "tag_deny" }
|
||||||
|
|
||||||
|
func (tagDeny) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
||||||
|
for _, tag := range rq.line.Tags {
|
||||||
|
if slices.Contains(rq.h.config.TagDeny, tag) {
|
||||||
|
return &verdict{
|
||||||
|
action: requestlog.ActionDenied,
|
||||||
|
figures: map[string]any{"tag": tag},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,105 @@
|
|||||||
|
package discriminator_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestTagDenyRefusesALookupTagAndNamesTheTag(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// The setting takes a country code in either case; the request's tag is
|
||||||
|
// in capitals.
|
||||||
|
s, _, _ := startWithLookups(t, map[string]string{tagDeny: "Country=kp"})
|
||||||
|
|
||||||
|
line := s.get(fromKP, http.StatusForbidden, requestlog.ActionDenied)
|
||||||
|
wantHeuristic(t, line, "tag_deny", `{"tag": "country=KP"}`)
|
||||||
|
wantTags(t, line, "asn=64511", "country=KP")
|
||||||
|
|
||||||
|
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTagDenyRefusesAnASNTag(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s, _, _ := startWithLookups(t, map[string]string{tagDeny: "asn=64511"})
|
||||||
|
|
||||||
|
wantHeuristic(t, s.get(fromKP, http.StatusForbidden, requestlog.ActionDenied),
|
||||||
|
"tag_deny", `{"tag": "asn=64511"}`)
|
||||||
|
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTagDenyRefusesAListTagBeforeTheRateLimitsCount(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s, server, _ := startWithLookups(t, map[string]string{
|
||||||
|
tagLists: "torexit=" + torList, tagDeny: "torexit", rateLimitPerMinute: "1",
|
||||||
|
})
|
||||||
|
loadLists(t, server, map[string][]string{torList: {fromDE}})
|
||||||
|
|
||||||
|
// Each is refused for its tag: none is counted to break the limit of one.
|
||||||
|
for range 3 {
|
||||||
|
wantHeuristic(t, s.get(fromDE, http.StatusForbidden, requestlog.ActionDenied),
|
||||||
|
"tag_deny", `{"tag": "torexit"}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTagDenyRefusesAnExploitSignatureOfARuleThatOnlyLogs(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s, _, _ := startWithClock(t, "", map[string]string{
|
||||||
|
rulesDir: writeRules(t, "noted path log ^/\n"), tagDeny: "exploitsignature",
|
||||||
|
})
|
||||||
|
|
||||||
|
line := s.get(client, http.StatusForbidden, requestlog.ActionDenied)
|
||||||
|
wantHeuristic(t, line, "tag_deny", `{"tag": "exploitsignature"}`)
|
||||||
|
wantTags(t, line, "exploitsignature")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTagLimitPercentLowersALimitTheLowestWinning(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s, server, _ := startWithLookups(t, map[string]string{
|
||||||
|
rateLimitPerMinute: fourAMinute,
|
||||||
|
tagLists: "vpn=" + vpnList + ",tor=" + torList,
|
||||||
|
tagLimitPercent: "vpn:50,tor:25",
|
||||||
|
})
|
||||||
|
loadLists(t, server, map[string][]string{vpnList: {fromDE}, torList: {fromDE}})
|
||||||
|
|
||||||
|
// A quarter of 4 requests a minute: the second breaks the limit.
|
||||||
|
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
||||||
|
|
||||||
|
line := s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited)
|
||||||
|
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
|
||||||
|
"25 from "+tagLimitPercent)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTagLimitPercentLowersAByteLimitByAListTag(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s, server, _ := startWithLookups(t, map[string]string{
|
||||||
|
bytesLimitPerMinute: twoUploads,
|
||||||
|
tagLists: "vpn=" + vpnList,
|
||||||
|
tagLimitPercent: "vpn:50",
|
||||||
|
})
|
||||||
|
loadLists(t, server, map[string][]string{vpnList: {fromDE}})
|
||||||
|
|
||||||
|
// The upload's 100 bytes are over half of 199, 99.
|
||||||
|
line := s.uploadFrom(fromDE)
|
||||||
|
if line.LimitHit != minuteBytes {
|
||||||
|
t.Errorf("log line has limit_hit %q, want %s", line.LimitHit, minuteBytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting,
|
||||||
|
"50 from "+tagLimitPercent)
|
||||||
|
|
||||||
|
// fromKP, which the list does not hold, has the whole limit.
|
||||||
|
line = s.uploadFrom(fromKP)
|
||||||
|
if line.LimitHit != "" {
|
||||||
|
t.Errorf("log line for %s has limit_hit %q, want none", fromKP, line.LimitHit)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting, none)
|
||||||
|
}
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
package discriminator
|
||||||
|
|
||||||
|
// The heuristic tag_lists. It reads the address lists of SWW_TAG_LISTS through
|
||||||
|
// the reputation lists.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// tagLists adds to the request one tag for each list of SWW_TAG_LISTS that
|
||||||
|
// holds the client, the tag being the list's name. It refuses nothing, and runs
|
||||||
|
// first so that every refusal's log line carries these tags.
|
||||||
|
type tagLists struct{}
|
||||||
|
|
||||||
|
func (tagLists) name() string { return "tag_lists" }
|
||||||
|
|
||||||
|
func (tagLists) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
||||||
|
for _, name := range rq.h.lists.TagsOf(rq.client) {
|
||||||
|
rq.addTag(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
package discriminator_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
|
)
|
||||||
|
|
||||||
|
// tagLists is SWW_TAG_LISTS, and vpnList and torList lists it names.
|
||||||
|
const (
|
||||||
|
tagLists = "SWW_TAG_LISTS"
|
||||||
|
vpnList = "https://lists.example/vpn.txt"
|
||||||
|
torList = "https://lists.example/torexit.txt"
|
||||||
|
)
|
||||||
|
|
||||||
|
// wantTags checks the log line's tags.
|
||||||
|
func wantTags(t *testing.T, line logLine, want ...string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
got, _ := line.fields["tags"].([]any)
|
||||||
|
|
||||||
|
tags := make([]string, 0, len(got))
|
||||||
|
for _, tag := range got {
|
||||||
|
text, _ := tag.(string)
|
||||||
|
tags = append(tags, text)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !slices.Equal(tags, want) {
|
||||||
|
t.Errorf("log line has tags %v, want %v", tags, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLookupAndTagListsGiveTheRequestItsTagsSorted(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s, server, _ := startWithLookups(t, map[string]string{
|
||||||
|
tagLists: "vpn=mullvad=" + vpnList + ",TorExit=" + torList,
|
||||||
|
})
|
||||||
|
loadLists(t, server, map[string][]string{
|
||||||
|
vpnList: {fromDE}, torList: {fromDE + "/32", fromKP},
|
||||||
|
})
|
||||||
|
|
||||||
|
wantTags(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward),
|
||||||
|
"asn=64496", "country=DE", "torexit", "vpn=mullvad")
|
||||||
|
wantTags(t, s.get(fromKP, http.StatusOK, requestlog.ActionForward),
|
||||||
|
"asn=64511", "country=KP", "torexit")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRequestWithoutATagLeavesTheFieldOut(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// With no lookup, a client is not placed and no setting gives it a tag.
|
||||||
|
s, _, _ := startWithClock(t, "", map[string]string{lookupSource: "off"})
|
||||||
|
|
||||||
|
line := s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
||||||
|
if _, present := line.fields["tags"]; present {
|
||||||
|
t.Errorf("log line has tags %v, want none", line.fields["tags"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTagListsTagEvenARequestTheCountryListRefuses(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s, server, _ := startWithLookups(t, map[string]string{
|
||||||
|
tagLists: "vpn=" + vpnList, allowedCountries: "de",
|
||||||
|
})
|
||||||
|
loadLists(t, server, map[string][]string{vpnList: {fromKP}})
|
||||||
|
|
||||||
|
wantTags(t, s.get(fromKP, http.StatusForbidden, requestlog.ActionCountryDenied),
|
||||||
|
"asn=64511", "country=KP", "vpn")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTagListsTagARequestRefusedBeforeTheLookup(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s, server, _ := startWithLookups(t, map[string]string{
|
||||||
|
tagLists: "vpn=" + vpnList, denyNets: fromKP, rateLimitPerMinute: "1",
|
||||||
|
})
|
||||||
|
loadLists(t, server, map[string][]string{vpnList: {fromDE, fromKP}})
|
||||||
|
|
||||||
|
line := s.get(fromKP, http.StatusForbidden, requestlog.ActionDenied)
|
||||||
|
if line.Heuristic != "deny_nets" {
|
||||||
|
t.Errorf("log line has heuristic %q, want deny_nets", line.Heuristic)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantTags(t, line, "vpn")
|
||||||
|
|
||||||
|
// The second request breaks the limit of one a minute, which bans the client.
|
||||||
|
s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
||||||
|
s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited)
|
||||||
|
|
||||||
|
line = s.get(fromDE, http.StatusForbidden, requestlog.ActionBanned)
|
||||||
|
if line.Heuristic != "active_ban" {
|
||||||
|
t.Errorf("log line has heuristic %q, want active_ban", line.Heuristic)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantTags(t, line, "vpn")
|
||||||
|
}
|
||||||
@@ -25,7 +25,7 @@ type busiest struct {
|
|||||||
requests *prometheus.CounterVec
|
requests *prometheus.CounterVec
|
||||||
requestBytes *prometheus.CounterVec
|
requestBytes *prometheus.CounterVec
|
||||||
responseBytes *prometheus.CounterVec
|
responseBytes *prometheus.CounterVec
|
||||||
// refused are, by country, the requests the country lists refused; nil
|
// refused are, by country, the requests the country list refused; nil
|
||||||
// by AS number.
|
// by AS number.
|
||||||
refused *prometheus.CounterVec
|
refused *prometheus.CounterVec
|
||||||
|
|
||||||
@@ -42,7 +42,7 @@ type busiest struct {
|
|||||||
func newCountries(topN int) *busiest {
|
func newCountries(topN int) *busiest {
|
||||||
countries := newBusiest(topN, "country", "the client's country")
|
countries := newBusiest(topN, "country", "the client's country")
|
||||||
countries.refused = counterVec("sww_country_list_refusals_total",
|
countries.refused = counterVec("sww_country_list_refusals_total",
|
||||||
"Requests the country lists refused, by the client's country.",
|
"Requests SWW_EXCLUSIVELY_ALLOWED_COUNTRIES refused, by the client's country.",
|
||||||
[]string{"country"})
|
[]string{"country"})
|
||||||
|
|
||||||
return countries
|
return countries
|
||||||
@@ -88,7 +88,7 @@ func (b *busiest) Collect(ch chan<- prometheus.Metric) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// vecs returns the metrics: by AS number, those of requests and bytes; by
|
// vecs returns the metrics: by AS number, those of requests and bytes; by
|
||||||
// country, the refusals by the country lists as well.
|
// country, the refusals by the country list as well.
|
||||||
func (b *busiest) vecs() []*prometheus.CounterVec {
|
func (b *busiest) vecs() []*prometheus.CounterVec {
|
||||||
vecs := []*prometheus.CounterVec{b.requests, b.requestBytes, b.responseBytes}
|
vecs := []*prometheus.CounterVec{b.requests, b.requestBytes, b.responseBytes}
|
||||||
if b.refused != nil {
|
if b.refused != nil {
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
// Package reputation fetches the lists the settings name by URL: the
|
// Package reputation fetches the lists the settings name by URL: the
|
||||||
// blocklists of SWW_BLOCKLIST_URLS, the file of AS:percent lines
|
// blocklists of SWW_BLOCKLIST_URLS, the file of AS:percent lines
|
||||||
// SWW_ASN_LIMIT_PERCENT_URL names, and the decision list of the CrowdSec
|
// SWW_ASN_LIMIT_PERCENT_URL names, the address lists of SWW_TAG_LISTS, and
|
||||||
// engine SWW_CROWDSEC_LAPI_URL names. It keeps the last good copy of
|
// the decision list of the CrowdSec engine SWW_CROWDSEC_LAPI_URL names. It
|
||||||
|
// keeps the last good copy of
|
||||||
// each, whole, comment lines included, which is used while a fetch fails,
|
// each, whole, comment lines included, which is used while a fetch fails,
|
||||||
// and when each was last tried. It also asks the DNSBL zones of
|
// and when each was last tried. It also asks the DNSBL zones of
|
||||||
// SWW_DNSBL_ZONES about clients, and keeps their verdicts, and checks
|
// SWW_DNSBL_ZONES about clients, and keeps their verdicts, and checks
|
||||||
@@ -71,6 +72,8 @@ type Params struct {
|
|||||||
// (SWW_ASN_LIMIT_PERCENT_URL), "" while it is unset.
|
// (SWW_ASN_LIMIT_PERCENT_URL), "" while it is unset.
|
||||||
BlocklistURLs []string
|
BlocklistURLs []string
|
||||||
ASNLimitPercentURL string
|
ASNLimitPercentURL string
|
||||||
|
// TagLists are the address lists that add a tag (SWW_TAG_LISTS).
|
||||||
|
TagLists []config.TagList
|
||||||
// CrowdSecDecisionsURL is the CrowdSec decision list, "" while
|
// CrowdSecDecisionsURL is the CrowdSec decision list, "" while
|
||||||
// SWW_CROWDSEC_LAPI_URL is unset, fetched with CrowdSecKey
|
// SWW_CROWDSEC_LAPI_URL is unset, fetched with CrowdSecKey
|
||||||
// (SWW_CROWDSEC_LAPI_KEY).
|
// (SWW_CROWDSEC_LAPI_KEY).
|
||||||
@@ -112,11 +115,11 @@ type list struct {
|
|||||||
failures int
|
failures int
|
||||||
}
|
}
|
||||||
|
|
||||||
// entries are what the lines of a copy say: for a blocklist, the netblocks
|
// entries are what the lines of a copy say: for a blocklist or a tag list, the
|
||||||
// it names, with the lengths among them, for the file of AS:percent lines,
|
// netblocks it names, with the lengths among them, for the file of AS:percent
|
||||||
// the percentage it gives each AS number, and for the CrowdSec decision
|
// lines, the percentage it gives each AS number, and for the CrowdSec decision
|
||||||
// list, the decision on each netblock that ends last, with the lengths
|
// list, the decision on each netblock that ends last, with the lengths among
|
||||||
// among them.
|
// them.
|
||||||
type entries struct {
|
type entries struct {
|
||||||
netblocks map[netip.Prefix]bool
|
netblocks map[netip.Prefix]bool
|
||||||
lengths []int
|
lengths []int
|
||||||
@@ -152,14 +155,18 @@ func New(params Params) *Lists {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// URLs returns the URL of every list: the blocklists' in the order
|
// URLs returns the URL of every list: the blocklists' in the order
|
||||||
// SWW_BLOCKLIST_URLS names them, then SWW_ASN_LIMIT_PERCENT_URL, then
|
// SWW_BLOCKLIST_URLS names them, then SWW_ASN_LIMIT_PERCENT_URL, then the
|
||||||
// the CrowdSec decision list's.
|
// tag lists', then the CrowdSec decision list's.
|
||||||
func (l *Lists) URLs() []string {
|
func (l *Lists) URLs() []string {
|
||||||
urls := slices.Clone(l.params.BlocklistURLs)
|
urls := slices.Clone(l.params.BlocklistURLs)
|
||||||
if l.params.ASNLimitPercentURL != "" {
|
if l.params.ASNLimitPercentURL != "" {
|
||||||
urls = append(urls, l.params.ASNLimitPercentURL)
|
urls = append(urls, l.params.ASNLimitPercentURL)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for _, list := range l.params.TagLists {
|
||||||
|
urls = append(urls, list.URL)
|
||||||
|
}
|
||||||
|
|
||||||
if l.params.CrowdSecDecisionsURL != "" {
|
if l.params.CrowdSecDecisionsURL != "" {
|
||||||
urls = append(urls, l.params.CrowdSecDecisionsURL)
|
urls = append(urls, l.params.CrowdSecDecisionsURL)
|
||||||
}
|
}
|
||||||
@@ -184,6 +191,22 @@ func (l *Lists) ListedBy(addr netip.Addr) []string {
|
|||||||
return listedBy
|
return listedBy
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TagsOf returns the names of the tag lists whose copy holds addr.
|
||||||
|
func (l *Lists) TagsOf(addr netip.Addr) []string {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
|
||||||
|
var names []string
|
||||||
|
|
||||||
|
for _, list := range l.params.TagLists {
|
||||||
|
if l.lists[list.URL].entries.contain(addr) {
|
||||||
|
names = append(names, list.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
|
||||||
// ASNLimitPercent returns the percentage the copy of the file of
|
// ASNLimitPercent returns the percentage the copy of the file of
|
||||||
// AS:percent lines gives asn, and whether it lists asn.
|
// AS:percent lines gives asn, and whether it lists asn.
|
||||||
func (l *Lists) ASNLimitPercent(asn string) (int64, bool) {
|
func (l *Lists) ASNLimitPercent(asn string) (int64, bool) {
|
||||||
@@ -481,10 +504,10 @@ func (l *Lists) get(ctx context.Context, listURL string) ([]string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// parse reads the lines of the list at listURL, fetched at fetched: those
|
// parse reads the lines of the list at listURL, fetched at fetched: those
|
||||||
// of a blocklist, of the file of AS:percent lines, or of the CrowdSec
|
// of a blocklist or a tag list, of the file of AS:percent lines, or of the
|
||||||
// decision list. In the first two, anything after a ; or a # on a line is
|
// CrowdSec decision list. In all but the decision list, anything after a ; or
|
||||||
// left out, and so is a line left blank. Any other line that does not read
|
// a # on a line is left out, and so is a line left blank. Any other line that
|
||||||
// is an error naming it by its number.
|
// does not read is an error naming it by its number.
|
||||||
func (l *Lists) parse(
|
func (l *Lists) parse(
|
||||||
listURL string, lines []string, fetched time.Time,
|
listURL string, lines []string, fetched time.Time,
|
||||||
) (entries, error) {
|
) (entries, error) {
|
||||||
@@ -498,8 +521,8 @@ func (l *Lists) parse(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseNetblocks reads a blocklist's lines, each an address or a netblock
|
// parseNetblocks reads the lines of a blocklist or a tag list, each an address
|
||||||
// as the settings take them.
|
// or a netblock as the settings take them.
|
||||||
func parseNetblocks(lines []string) (entries, error) {
|
func parseNetblocks(lines []string) (entries, error) {
|
||||||
found := entries{netblocks: map[netip.Prefix]bool{}}
|
found := entries{netblocks: map[netip.Prefix]bool{}}
|
||||||
|
|
||||||
@@ -551,7 +574,7 @@ func parseNetblock(text string) (netip.Prefix, bool) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// parsePercents reads the lines of the file of AS:percent lines, each an
|
// parsePercents reads the lines of the file of AS:percent lines, each an
|
||||||
// AS number, : and a percentage, as SWW_ASN_LIMIT_PERCENT takes them. An
|
// AS number, : and a percentage, such as AS64496:50. An
|
||||||
// AS number listed more than once gets the lowest of its percentages.
|
// AS number listed more than once gets the lowest of its percentages.
|
||||||
func parsePercents(lines []string) (entries, error) {
|
func parsePercents(lines []string) (entries, error) {
|
||||||
found := entries{percents: map[string]int64{}}
|
found := entries{percents: map[string]int64{}}
|
||||||
|
|||||||
@@ -44,10 +44,11 @@ const (
|
|||||||
// ActionDenied is a request refused because its client is in
|
// ActionDenied is a request refused because its client is in
|
||||||
// SWW_DENY_NETS, in a blocklist while SWW_BLOCKLIST_ACTION is deny,
|
// SWW_DENY_NETS, in a blocklist while SWW_BLOCKLIST_ACTION is deny,
|
||||||
// or listed by a DNSBL zone, or scored a hit by AbuseIPDB, while
|
// or listed by a DNSBL zone, or scored a hit by AbuseIPDB, while
|
||||||
// SWW_REPUTATION_ACTION is deny.
|
// SWW_REPUTATION_ACTION is deny, or that carries a tag in SWW_TAG_DENY.
|
||||||
ActionDenied = "denied"
|
ActionDenied = "denied"
|
||||||
|
|
||||||
// ActionCountryDenied is a request refused for its client's country.
|
// ActionCountryDenied is a request refused because its client's country
|
||||||
|
// is not in SWW_EXCLUSIVELY_ALLOWED_COUNTRIES.
|
||||||
ActionCountryDenied = "country_denied"
|
ActionCountryDenied = "country_denied"
|
||||||
// ActionAdmin is a request discriminator answered at one of its own
|
// ActionAdmin is a request discriminator answered at one of its own
|
||||||
// endpoints, or a question about a path under /_sww/.
|
// endpoints, or a question about a path under /_sww/.
|
||||||
@@ -179,6 +180,8 @@ type Line struct {
|
|||||||
// whose verdict lists it, their keys masked, then abuseipdb when its
|
// whose verdict lists it, their keys masked, then abuseipdb when its
|
||||||
// score is a hit.
|
// score is a hit.
|
||||||
Reputation []string `json:"reputation,omitempty"`
|
Reputation []string `json:"reputation,omitempty"`
|
||||||
|
// Tags are the request's tags, sorted.
|
||||||
|
Tags []string `json:"tags,omitempty"`
|
||||||
// Offence is the offence the request was held as, OffenceLimit.
|
// Offence is the offence the request was held as, OffenceLimit.
|
||||||
Offence string `json:"offence,omitempty"`
|
Offence string `json:"offence,omitempty"`
|
||||||
// BanExpires is when the ban the request made, or was refused under,
|
// BanExpires is when the ban the request made, or was refused under,
|
||||||
|
|||||||
@@ -618,7 +618,7 @@ func TestLookupDatabaseReplacedWhileRunningTakesEffect(t *testing.T) {
|
|||||||
instanceName: instance,
|
instanceName: instance,
|
||||||
lookupSource: "file",
|
lookupSource: "file",
|
||||||
lookupDBPath: path,
|
lookupDBPath: path,
|
||||||
"SWW_DENIED_COUNTRIES": "kp",
|
"SWW_TAG_DENY": "country=KP",
|
||||||
// The requests sent until a replacement takes effect, and those
|
// The requests sent until a replacement takes effect, and those
|
||||||
// for the metrics, must not break a rate limit, whose ban would
|
// for the metrics, must not break a rate limit, whose ban would
|
||||||
// refuse them too.
|
// refuse them too.
|
||||||
@@ -1280,7 +1280,7 @@ func wantStartingLine(t *testing.T, line map[string]any, dir string) {
|
|||||||
"SWW_RATE_LIMIT_PER_HOUR": "1500",
|
"SWW_RATE_LIMIT_PER_HOUR": "1500",
|
||||||
rateLimitPerDay: "6000",
|
rateLimitPerDay: "6000",
|
||||||
"SWW_RATE_LIMIT_EXEMPT_PATHS": "",
|
"SWW_RATE_LIMIT_EXEMPT_PATHS": "",
|
||||||
"SWW_DENIED_COUNTRIES": "",
|
"SWW_TAG_DENY": "",
|
||||||
"SWW_EXCLUSIVELY_ALLOWED_COUNTRIES": "",
|
"SWW_EXCLUSIVELY_ALLOWED_COUNTRIES": "",
|
||||||
"SWW_BAN_RESPONSE": "403",
|
"SWW_BAN_RESPONSE": "403",
|
||||||
"SWW_LIMIT_BAN_DURATION": "1h",
|
"SWW_LIMIT_BAN_DURATION": "1h",
|
||||||
|
|||||||
Reference in new issue
Block a user