Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
828f1c7bc5 | ||
|
|
8e3879a0c9 |
No files matched your search
@@ -906,11 +906,9 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
|
|||||||
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
||||||
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
||||||
`internal/alerts`: one check or source each.
|
`internal/alerts`: one check or source each.
|
||||||
- `internal/state`: the state and cache files.
|
- `internal/state`, `internal/requestlog`, `internal/remotelog` and
|
||||||
- `internal/requestlog`: the request log.
|
`internal/metrics`: the state and cache files, the log lines, the syslog
|
||||||
- `internal/processlog`: the process's own messages.
|
sender and the metrics.
|
||||||
- `internal/remotelog`: the syslog sender.
|
|
||||||
- `internal/metrics`: the metrics.
|
|
||||||
|
|
||||||
## Rationale
|
## Rationale
|
||||||
|
|
||||||
|
|||||||
@@ -104,10 +104,8 @@ never as a time.
|
|||||||
its own.
|
its own.
|
||||||
- `internal/state`: reads the state and cache files at start, takes in edits
|
- `internal/state`: reads the state and cache files at start, takes in edits
|
||||||
while running, and writes the files when due and at the stop.
|
while running, and writes the files when due and at the stop.
|
||||||
- `internal/requestlog`: the request log, as JSON lines on stdout.
|
- `internal/requestlog` and `internal/remotelog`: the lines on stdout, and
|
||||||
- `internal/processlog`: the process lines on stdout, which hold the
|
sending them to the syslog server.
|
||||||
process's own messages.
|
|
||||||
- `internal/remotelog`: sends the lines on stdout to the syslog server.
|
|
||||||
- `internal/metrics`: the metrics, in the Prometheus text format.
|
- `internal/metrics`: the metrics, in the Prometheus text format.
|
||||||
- `Dockerfile`: the lint and test phases, then the image.
|
- `Dockerfile`: the lint and test phases, then the image.
|
||||||
`deploy/example-app` is an app built on the image, with `rproxy` in its
|
`deploy/example-app` is an app built on the image, with `rproxy` in its
|
||||||
@@ -819,8 +817,8 @@ The supported apps are:
|
|||||||
- `vaultwarden`;
|
- `vaultwarden`;
|
||||||
- `wordpress`.
|
- `wordpress`.
|
||||||
|
|
||||||
Each app's fixes are in one file, such as `internal/coreruleset/gitea.conf`, and
|
Each app's fixes are in one file named for the app, such as
|
||||||
one entry of `apps` in `internal/coreruleset/coreruleset.go`. gitea's fixes are:
|
`internal/coreruleset/apps/gitea.conf`. gitea's fixes are:
|
||||||
|
|
||||||
1. Ten parameters are not checked against the Core Rule Set's lists of system
|
1. Ten parameters are not checked against the Core Rule Set's lists of system
|
||||||
files (930120), shell paths (932160) and command names (932260), on any path:
|
files (930120), shell paths (932160) and command names (932260), on any path:
|
||||||
@@ -853,8 +851,8 @@ one entry of `apps` in `internal/coreruleset/coreruleset.go`. gitea's fixes are:
|
|||||||
3. On gitea's paths, some fields skip the rules that refused ordinary gitea
|
3. On gitea's paths, some fields skip the rules that refused ordinary gitea
|
||||||
requests there. They are the fields where gitea sends a search, a file's name
|
requests there. They are the fields where gitea sends a search, a file's name
|
||||||
or text, a commit message, a branch name or a repository's name.
|
or text, a commit message, a branch name or a repository's name.
|
||||||
`internal/coreruleset/gitea.conf` lists each rule and the request it refused.
|
`internal/coreruleset/apps/gitea.conf` lists each rule and the request it
|
||||||
The fields are:
|
refused. The fields are:
|
||||||
- `q` on gitea's searches of code, of issues and pull requests, and of
|
- `q` on gitea's searches of code, of issues and pull requests, and of
|
||||||
commits. A code search holds code, such as `<script>`,
|
commits. A code search holds code, such as `<script>`,
|
||||||
`SELECT * FROM users WHERE` or `../`. The searches of issues and pull
|
`SELECT * FROM users WHERE` or `../`. The searches of issues and pull
|
||||||
@@ -1039,7 +1037,7 @@ one entry of `apps` in `internal/coreruleset/coreruleset.go`. gitea's fixes are:
|
|||||||
- The `Content-Range` header of an artifact upload from
|
- The `Content-Range` header of an artifact upload from
|
||||||
`actions/upload-artifact@v3` (920450).
|
`actions/upload-artifact@v3` (920450).
|
||||||
- The passwords in a body, for each rule that random passwords set off. The
|
- The passwords in a body, for each rule that random passwords set off. The
|
||||||
numbers are in `internal/coreruleset/gitea.conf`.
|
numbers are in `internal/coreruleset/apps/gitea.conf`.
|
||||||
- `password` on these paths:
|
- `password` on these paths:
|
||||||
- `/user/login`, `/user/sign_up`, `/user/activate`;
|
- `/user/login`, `/user/sign_up`, `/user/activate`;
|
||||||
- `/user/link_account_signin` and `/user/link_account_signup`;
|
- `/user/link_account_signin` and `/user/link_account_signup`;
|
||||||
@@ -1071,8 +1069,8 @@ one entry of `apps` in `internal/coreruleset/coreruleset.go`. gitea's fixes are:
|
|||||||
such as `owner/python3-app` would have that push refused.
|
such as `owner/python3-app` would have that push refused.
|
||||||
|
|
||||||
WordPress's fixes: on WordPress's paths, some fields skip the rules that refused
|
WordPress's fixes: on WordPress's paths, some fields skip the rules that refused
|
||||||
ordinary WordPress requests there. `internal/coreruleset/wordpress.conf` lists
|
ordinary WordPress requests there. `internal/coreruleset/apps/wordpress.conf`
|
||||||
each rule and the request it refused. The REST API has four addresses:
|
lists each rule and the request it refused. The REST API has four addresses:
|
||||||
|
|
||||||
- `/wp-json/...`
|
- `/wp-json/...`
|
||||||
- `/index.php/wp-json/...`
|
- `/index.php/wp-json/...`
|
||||||
@@ -1164,9 +1162,9 @@ Elsewhere WordPress's fixes do not apply. The paths are those of WordPress at
|
|||||||
the site root.
|
the site root.
|
||||||
|
|
||||||
Drupal's fixes: on Drupal's paths, some fields skip the rules that refused
|
Drupal's fixes: on Drupal's paths, some fields skip the rules that refused
|
||||||
ordinary Drupal requests there. `internal/coreruleset/drupal.conf` lists each
|
ordinary Drupal requests there. `internal/coreruleset/apps/drupal.conf` lists
|
||||||
rule and the request it refused. These query fields skip the lists of system
|
each rule and the request it refused. These query fields skip the lists of
|
||||||
files and command names (930120, 932120, 932250, 932260):
|
system files and command names (930120, 932120, 932250, 932260):
|
||||||
|
|
||||||
- On every page under `/admin`: `title`, `name`, `user`, `filename`, `filemime`,
|
- On every page under `/admin`: `title`, `name`, `user`, `filename`, `filemime`,
|
||||||
`info`, `subject`, `author_name`, `search`, `q` and `destination`.
|
`info`, `subject`, `author_name`, `search`, `q` and `destination`.
|
||||||
@@ -1256,10 +1254,10 @@ first path segment of two or three lowercase letters, optionally followed by a
|
|||||||
hyphen and more letters, so `/api` counts too.
|
hyphen and more letters, so `/api` counts too.
|
||||||
|
|
||||||
MediaWiki's fixes: on MediaWiki's paths, some fields skip the rules that refused
|
MediaWiki's fixes: on MediaWiki's paths, some fields skip the rules that refused
|
||||||
ordinary MediaWiki requests there. `internal/coreruleset/mediawiki.conf` lists
|
ordinary MediaWiki requests there. `internal/coreruleset/apps/mediawiki.conf`
|
||||||
each rule and the request it refused. These fields skip the lists of system
|
lists each rule and the request it refused. These fields skip the lists of
|
||||||
files and command names (930120, 932120, 932250, 932260), in the query or the
|
system files and command names (930120, 932120, 932250, 932260), in the query or
|
||||||
body:
|
the body:
|
||||||
|
|
||||||
- On MediaWiki's pages, which are `/index.php` and the paths under `/index.php/`
|
- On MediaWiki's pages, which are `/index.php` and the paths under `/index.php/`
|
||||||
and `/wiki/`:
|
and `/wiki/`:
|
||||||
@@ -1365,9 +1363,9 @@ Elsewhere these fields keep every rule. The paths are those of MediaWiki at the
|
|||||||
site root or under `/w/`.
|
site root or under `/w/`.
|
||||||
|
|
||||||
Mastodon's fixes: on Mastodon's paths, some fields skip the rules that refused
|
Mastodon's fixes: on Mastodon's paths, some fields skip the rules that refused
|
||||||
ordinary Mastodon requests there. `internal/coreruleset/mastodon.conf` lists
|
ordinary Mastodon requests there. `internal/coreruleset/apps/mastodon.conf`
|
||||||
each rule and the request it refused. These fields skip the lists of system
|
lists each rule and the request it refused. These fields skip the lists of
|
||||||
files and command names (930120, 932120, 932250, 932260), in the query:
|
system files and command names (930120, 932120, 932250, 932260), in the query:
|
||||||
|
|
||||||
- On the API's search, accounts and admin accounts, under `/api/v1/` and
|
- On the API's search, accounts and admin accounts, under `/api/v1/` and
|
||||||
`/api/v2/`, the searches and names `q`, `acct`, `username`, `display_name`,
|
`/api/v2/`, the searches and names `q`, `acct`, `username`, `display_name`,
|
||||||
@@ -1528,7 +1526,7 @@ every rule.
|
|||||||
|
|
||||||
Vaultwarden's fixes: on Vaultwarden's paths, some fields skip the rules that
|
Vaultwarden's fixes: on Vaultwarden's paths, some fields skip the rules that
|
||||||
refused ordinary Vaultwarden requests there.
|
refused ordinary Vaultwarden requests there.
|
||||||
`internal/coreruleset/vaultwarden.conf` lists each rule and the request it
|
`internal/coreruleset/apps/vaultwarden.conf` lists each rule and the request it
|
||||||
refused. The apps encrypt vault items, folders, collections and sends before
|
refused. The apps encrypt vault items, folders, collections and sends before
|
||||||
sending them, so those fields skip no rules. The fields below hold plain text.
|
sending them, so those fields skip no rules. The fields below hold plain text.
|
||||||
|
|
||||||
@@ -1579,8 +1577,8 @@ has a parameter of the same name.
|
|||||||
|
|
||||||
`token` in the body of `/admin`, the admin token typed on the admin page, skips
|
`token` in the body of `/admin`, the admin token typed on the admin page, skips
|
||||||
each rule that random passwords set off. The numbers are in
|
each rule that random passwords set off. The numbers are in
|
||||||
`internal/coreruleset/vaultwarden.conf`. It applies only when the query has no
|
`internal/coreruleset/apps/vaultwarden.conf`. It applies only when the query has
|
||||||
`token` parameter. `' OR '1'='1`, `../../etc/passwd`, `; cat /etc/passwd` and
|
no `token` parameter. `' OR '1'='1`, `../../etc/passwd`, `; cat /etc/passwd` and
|
||||||
`<img src=x onerror=x()>` sent as the token reach Vaultwarden;
|
`<img src=x onerror=x()>` sent as the token reach Vaultwarden;
|
||||||
`<script>alert(1)</script>` and `javascript:alert(1)` are still refused.
|
`<script>alert(1)</script>` and `javascript:alert(1)` are still refused.
|
||||||
|
|
||||||
@@ -1602,7 +1600,7 @@ Elsewhere these fields keep every rule.
|
|||||||
|
|
||||||
Nextcloud's fixes: on Nextcloud's paths, some fields skip the rules that refused
|
Nextcloud's fixes: on Nextcloud's paths, some fields skip the rules that refused
|
||||||
ordinary Nextcloud requests there, and its WebDAV methods are allowed.
|
ordinary Nextcloud requests there, and its WebDAV methods are allowed.
|
||||||
`internal/coreruleset/nextcloud.conf` lists each rule and the request it
|
`internal/coreruleset/apps/nextcloud.conf` lists each rule and the request it
|
||||||
refused.
|
refused.
|
||||||
|
|
||||||
The sync client and the Files app use WebDAV on these paths:
|
The sync client and the Files app use WebDAV on these paths:
|
||||||
@@ -1612,7 +1610,8 @@ The sync client and the Files app use WebDAV on these paths:
|
|||||||
- `/remote.php/webdav/`;
|
- `/remote.php/webdav/`;
|
||||||
- `/public.php/webdav/`.
|
- `/public.php/webdav/`.
|
||||||
|
|
||||||
These methods are allowed there, besides those of the first change:
|
These methods are allowed there, besides `PUT`, `PATCH` and `DELETE`, which
|
||||||
|
every app gets:
|
||||||
|
|
||||||
- `PROPFIND` and `PROPPATCH`;
|
- `PROPFIND` and `PROPPATCH`;
|
||||||
- `MKCOL`;
|
- `MKCOL`;
|
||||||
@@ -1756,7 +1755,7 @@ gets gitea's fixes too. Forgejo's own fixes hold only what Forgejo sends and
|
|||||||
gitea does not.
|
gitea does not.
|
||||||
|
|
||||||
On Forgejo's paths, some fields skip the rules that refused ordinary Forgejo
|
On Forgejo's paths, some fields skip the rules that refused ordinary Forgejo
|
||||||
requests there. `internal/coreruleset/forgejo.conf` lists each rule and the
|
requests there. `internal/coreruleset/apps/forgejo.conf` lists each rule and the
|
||||||
request it refused.
|
request it refused.
|
||||||
|
|
||||||
`q` skips rules on some of Forgejo's searches. The searches are:
|
`q` skips rules on some of Forgejo's searches. The searches are:
|
||||||
@@ -1802,10 +1801,10 @@ The Core Rule Set reads the request without the `flash` cookie, in which Forgejo
|
|||||||
puts what gitea puts in `gitea_flash` (gitea's second fix).
|
puts what gitea puts in `gitea_flash` (gitea's second fix).
|
||||||
|
|
||||||
Ghost's fixes: on Ghost's paths, some fields skip the rules that refused
|
Ghost's fixes: on Ghost's paths, some fields skip the rules that refused
|
||||||
ordinary Ghost requests there. `internal/coreruleset/ghost.conf` lists each rule
|
ordinary Ghost requests there. `internal/coreruleset/apps/ghost.conf` lists each
|
||||||
and the request it refused. Ghost's admin and integrations call the Admin API
|
rule and the request it refused. Ghost's admin and integrations call the Admin
|
||||||
under `/ghost/api/admin/`. Themes and Ghost's scripts call the Content API under
|
API under `/ghost/api/admin/`. Themes and Ghost's scripts call the Content API
|
||||||
`/ghost/api/content/`, and members call `/members/api/`. The admin calls
|
under `/ghost/api/content/`, and members call `/members/api/`. The admin calls
|
||||||
ActivityPub, a service of its own, under `/.ghost/activitypub/`.
|
ActivityPub, a service of its own, under `/.ghost/activitypub/`.
|
||||||
|
|
||||||
Eight rules refused words that name commands and files, such as `python3`,
|
Eight rules refused words that name commands and files, such as `python3`,
|
||||||
@@ -1943,9 +1942,9 @@ That name holds `.profile`, a file on the list of system files, so the name
|
|||||||
skips 930120 when a profile is saved.
|
skips 930120 when a profile is saved.
|
||||||
|
|
||||||
The passwords skip each rule that random passwords set off. The numbers are in
|
The passwords skip each rule that random passwords set off. The numbers are in
|
||||||
`internal/coreruleset/ghost.conf`. They are `password` when signing in, and the
|
`internal/coreruleset/apps/ghost.conf`. They are `password` when signing in, and
|
||||||
passwords sent when creating the first staff user, accepting an invitation, and
|
the passwords sent when creating the first staff user, accepting an invitation,
|
||||||
resetting or changing a password. `' OR '1'='1`, `../../etc/passwd`,
|
and resetting or changing a password. `' OR '1'='1`, `../../etc/passwd`,
|
||||||
`; cat /etc/passwd` and `<img src=x onerror=x()>` sent as a password reach
|
`; cat /etc/passwd` and `<img src=x onerror=x()>` sent as a password reach
|
||||||
Ghost; `<script>alert(1)</script>` and `javascript:alert(1)` are still refused.
|
Ghost; `<script>alert(1)</script>` and `javascript:alert(1)` are still refused.
|
||||||
|
|
||||||
|
|||||||
+32
-93
@@ -75,7 +75,7 @@ type WAF struct {
|
|||||||
BytesLimitPerMinute int64
|
BytesLimitPerMinute int64
|
||||||
BytesLimitPerHour int64
|
BytesLimitPerHour int64
|
||||||
BytesLimitPerDay int64
|
BytesLimitPerDay int64
|
||||||
BytesCounted BytesCounted
|
BytesCounted string
|
||||||
// LookupSource is where each client's AS number and country are looked
|
// LookupSource is where each client's AS number and country are looked
|
||||||
// up (SWW_WAF_LOOKUP_SOURCE): geojs, file, or off for nowhere.
|
// up (SWW_WAF_LOOKUP_SOURCE): geojs, file, or off for nowhere.
|
||||||
// LookupDBPath is the lookup database, the IPinfo Lite file looked up in
|
// LookupDBPath is the lookup database, the IPinfo Lite file looked up in
|
||||||
@@ -86,7 +86,7 @@ type WAF struct {
|
|||||||
// AddLookupHeaders is true when the app is passed the client's AS number
|
// AddLookupHeaders is true when the app is passed the client's AS number
|
||||||
// and country in headers (SWW_WAF_ADD_LOOKUP_HEADERS), by default unless
|
// and country in headers (SWW_WAF_ADD_LOOKUP_HEADERS), by default unless
|
||||||
// LookupSource is off.
|
// LookupSource is off.
|
||||||
LookupSource LookupSource
|
LookupSource string
|
||||||
LookupDBPath string
|
LookupDBPath string
|
||||||
ASNAndCountryLookupTimeout time.Duration
|
ASNAndCountryLookupTimeout time.Duration
|
||||||
AddLookupHeaders bool
|
AddLookupHeaders bool
|
||||||
@@ -131,7 +131,7 @@ type WAF struct {
|
|||||||
// BlocklistLimitPercent is the percentage of every limit it gets.
|
// BlocklistLimitPercent is the percentage of every limit it gets.
|
||||||
BlocklistURLs []string
|
BlocklistURLs []string
|
||||||
BlocklistRefresh time.Duration
|
BlocklistRefresh time.Duration
|
||||||
BlocklistAction ReputationAction
|
BlocklistAction string
|
||||||
BlocklistLimitPercent int64
|
BlocklistLimitPercent int64
|
||||||
// DNSBLZones are the DNSBL zones clients are asked about
|
// DNSBLZones are the DNSBL zones clients are asked about
|
||||||
// (SWW_WAF_DNSBL_ZONES), through DNSBLResolver (SWW_WAF_DNSBL_RESOLVER), or
|
// (SWW_WAF_DNSBL_ZONES), through DNSBLResolver (SWW_WAF_DNSBL_RESOLVER), or
|
||||||
@@ -154,7 +154,7 @@ type WAF struct {
|
|||||||
AbuseIPDBKey string
|
AbuseIPDBKey string
|
||||||
AbuseIPDBMinScore int64
|
AbuseIPDBMinScore int64
|
||||||
AbuseIPDBDailyBudget int
|
AbuseIPDBDailyBudget int
|
||||||
DNSBLAbuseIPDBAction ReputationAction
|
DNSBLAbuseIPDBAction string
|
||||||
DNSBLAbuseIPDBLimitPercent int64
|
DNSBLAbuseIPDBLimitPercent int64
|
||||||
IPReputationCacheTTL time.Duration
|
IPReputationCacheTTL time.Duration
|
||||||
DNSBLAbuseIPDBTimeout time.Duration
|
DNSBLAbuseIPDBTimeout time.Duration
|
||||||
@@ -237,7 +237,7 @@ type WAF struct {
|
|||||||
// reads none.
|
// reads none.
|
||||||
// CoreRuleSetExclusionsFile is the path of the file of its exclusions
|
// CoreRuleSetExclusionsFile is the path of the file of its exclusions
|
||||||
// (SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE), "" while there is none.
|
// (SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE), "" while there is none.
|
||||||
CoreRuleSetMode CoreRuleSetMode
|
CoreRuleSetMode string
|
||||||
CoreRuleSetParanoiaLevel int
|
CoreRuleSetParanoiaLevel int
|
||||||
CoreRuleSetScoreThreshold int
|
CoreRuleSetScoreThreshold int
|
||||||
CoreRuleSetDisabledRules []int
|
CoreRuleSetDisabledRules []int
|
||||||
@@ -316,66 +316,20 @@ type WAF struct {
|
|||||||
settings []slog.Attr
|
settings []slog.Attr
|
||||||
}
|
}
|
||||||
|
|
||||||
// BytesCounted is which body bytes count toward the byte limits and the
|
|
||||||
// byte anomaly thresholds, a value of SWW_WAF_BYTES_COUNTED.
|
|
||||||
type BytesCounted string
|
|
||||||
|
|
||||||
// The values of SWW_WAF_BYTES_COUNTED.
|
|
||||||
const (
|
|
||||||
// BytesCountedResponse counts the response's body bytes.
|
|
||||||
BytesCountedResponse BytesCounted = "response"
|
|
||||||
// BytesCountedRequest counts the request's body bytes.
|
|
||||||
BytesCountedRequest BytesCounted = "request"
|
|
||||||
// BytesCountedBoth counts the body bytes of both.
|
|
||||||
BytesCountedBoth BytesCounted = "both"
|
|
||||||
)
|
|
||||||
|
|
||||||
// LookupSource is where each client's AS number and country are looked up,
|
|
||||||
// a value of SWW_WAF_LOOKUP_SOURCE.
|
|
||||||
type LookupSource string
|
|
||||||
|
|
||||||
// The values of SWW_WAF_LOOKUP_SOURCE.
|
|
||||||
const (
|
|
||||||
// LookupSourceGeoJS asks GeoJS.
|
|
||||||
LookupSourceGeoJS LookupSource = "geojs"
|
|
||||||
// LookupSourceFile looks clients up in the lookup database, the file
|
|
||||||
// SWW_WAF_LOOKUP_DB_PATH names.
|
|
||||||
LookupSourceFile LookupSource = "file"
|
|
||||||
// LookupSourceOff looks no client up.
|
|
||||||
LookupSourceOff LookupSource = "off"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ReputationAction is what is done with a client a reputation source
|
|
||||||
// lists, a value of SWW_WAF_BLOCKLIST_ACTION and of
|
|
||||||
// SWW_WAF_DNSBL_ABUSEIPDB_ACTION.
|
|
||||||
type ReputationAction string
|
|
||||||
|
|
||||||
// The values of SWW_WAF_BLOCKLIST_ACTION and SWW_WAF_DNSBL_ABUSEIPDB_ACTION.
|
|
||||||
const (
|
|
||||||
// ReputationActionDeny refuses the client's requests.
|
|
||||||
ReputationActionDeny ReputationAction = "deny"
|
|
||||||
// ReputationActionLimit gives the client a percentage of every limit,
|
|
||||||
// written limit:<percent> in the setting.
|
|
||||||
ReputationActionLimit ReputationAction = "limit"
|
|
||||||
// ReputationActionLog only notes the hit in the log line and raises an
|
|
||||||
// alert.
|
|
||||||
ReputationActionLog ReputationAction = "log"
|
|
||||||
)
|
|
||||||
|
|
||||||
// CoreRuleSetMode is what the Core Rule Set does, a value of
|
|
||||||
// SWW_WAF_CORE_RULE_SET_MODE.
|
|
||||||
type CoreRuleSetMode string
|
|
||||||
|
|
||||||
// The values of SWW_WAF_CORE_RULE_SET_MODE.
|
// The values of SWW_WAF_CORE_RULE_SET_MODE.
|
||||||
const (
|
const (
|
||||||
// CoreRuleSetModeOff runs no request through the Core Rule Set.
|
// CoreRuleSetModeOff runs no request through the Core Rule Set.
|
||||||
CoreRuleSetModeOff CoreRuleSetMode = "off"
|
CoreRuleSetModeOff = off
|
||||||
// CoreRuleSetModeDetect logs and alerts a match, and refuses nothing.
|
// CoreRuleSetModeDetect logs and alerts a match, and refuses nothing.
|
||||||
CoreRuleSetModeDetect CoreRuleSetMode = "detect"
|
CoreRuleSetModeDetect = "detect"
|
||||||
// CoreRuleSetModeBlock refuses a match with 403.
|
// CoreRuleSetModeBlock refuses a match with 403.
|
||||||
CoreRuleSetModeBlock CoreRuleSetMode = "block"
|
CoreRuleSetModeBlock = "block"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// fileSource is the SWW_WAF_LOOKUP_SOURCE that looks clients up in the
|
||||||
|
// lookup database, the file SWW_WAF_LOOKUP_DB_PATH names.
|
||||||
|
const fileSource = "file"
|
||||||
|
|
||||||
const (
|
const (
|
||||||
day = 24 * time.Hour
|
day = 24 * time.Hour
|
||||||
ipv4Bits = 32
|
ipv4Bits = 32
|
||||||
@@ -604,13 +558,10 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
|||||||
),
|
),
|
||||||
BytesLimitPerHour: env.size("SWW_WAF_BYTES_LIMIT_PER_HOUR", "50G"),
|
BytesLimitPerHour: env.size("SWW_WAF_BYTES_LIMIT_PER_HOUR", "50G"),
|
||||||
BytesLimitPerDay: env.size("SWW_WAF_BYTES_LIMIT_PER_DAY", "100G"),
|
BytesLimitPerDay: env.size("SWW_WAF_BYTES_LIMIT_PER_DAY", "100G"),
|
||||||
BytesCounted: env.bytesCounted(
|
BytesCounted: env.bytesCounted("SWW_WAF_BYTES_COUNTED", "both"),
|
||||||
"SWW_WAF_BYTES_COUNTED",
|
|
||||||
BytesCountedBoth,
|
|
||||||
),
|
|
||||||
LookupSource: env.lookupSource(
|
LookupSource: env.lookupSource(
|
||||||
"SWW_WAF_LOOKUP_SOURCE",
|
"SWW_WAF_LOOKUP_SOURCE",
|
||||||
LookupSourceGeoJS,
|
"geojs",
|
||||||
),
|
),
|
||||||
LookupDBPath: env.value("SWW_WAF_LOOKUP_DB_PATH", ""),
|
LookupDBPath: env.value("SWW_WAF_LOOKUP_DB_PATH", ""),
|
||||||
ASNAndCountryLookupTimeout: env.milliseconds(
|
ASNAndCountryLookupTimeout: env.milliseconds(
|
||||||
@@ -807,9 +758,9 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
|||||||
cfg.InstanceName, cfg.LogRemoteURL != nil)
|
cfg.InstanceName, cfg.LogRemoteURL != nil)
|
||||||
// The app gets the headers by default, unless no client is looked up.
|
// The app gets the headers by default, unless no client is looked up.
|
||||||
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
|
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
|
||||||
strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
|
strconv.FormatBool(cfg.LookupSource != off))
|
||||||
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
|
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
|
||||||
"SWW_WAF_BLOCKLIST_ACTION", string(ReputationActionDeny))
|
"SWW_WAF_BLOCKLIST_ACTION", "deny")
|
||||||
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
|
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
|
||||||
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "limit:25")
|
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "limit:25")
|
||||||
|
|
||||||
@@ -907,12 +858,9 @@ func (e *environment) count(name, defaultValue string) int64 {
|
|||||||
|
|
||||||
// bytesCounted reads the setting that is which body bytes count toward the
|
// bytesCounted reads the setting that is which body bytes count toward the
|
||||||
// byte limits and the byte anomaly thresholds: response, request or both.
|
// byte limits and the byte anomaly thresholds: response, request or both.
|
||||||
func (e *environment) bytesCounted(
|
func (e *environment) bytesCounted(name, defaultValue string) string {
|
||||||
name string, defaultValue BytesCounted,
|
value := e.value(name, defaultValue)
|
||||||
) BytesCounted {
|
if value != "response" && value != "request" && value != "both" {
|
||||||
value := BytesCounted(e.value(name, string(defaultValue)))
|
|
||||||
if value != BytesCountedResponse && value != BytesCountedRequest &&
|
|
||||||
value != BytesCountedBoth {
|
|
||||||
e.check(name, fmt.Errorf("%q %w", value, errNotBytesCounted))
|
e.check(name, fmt.Errorf("%q %w", value, errNotBytesCounted))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -957,10 +905,8 @@ func (e *environment) trapPaths(name string) []string {
|
|||||||
|
|
||||||
// coreRuleSetMode reads the setting that is what the Core Rule Set does: off,
|
// coreRuleSetMode reads the setting that is what the Core Rule Set does: off,
|
||||||
// detect or block.
|
// detect or block.
|
||||||
func (e *environment) coreRuleSetMode(
|
func (e *environment) coreRuleSetMode(name, defaultValue string) string {
|
||||||
name string, defaultValue CoreRuleSetMode,
|
mode := e.value(name, defaultValue)
|
||||||
) CoreRuleSetMode {
|
|
||||||
mode := CoreRuleSetMode(e.value(name, string(defaultValue)))
|
|
||||||
if mode != CoreRuleSetModeOff && mode != CoreRuleSetModeDetect &&
|
if mode != CoreRuleSetModeOff && mode != CoreRuleSetModeDetect &&
|
||||||
mode != CoreRuleSetModeBlock {
|
mode != CoreRuleSetModeBlock {
|
||||||
e.check(name, fmt.Errorf("%q %w", mode, errNotCoreRuleSetMode))
|
e.check(name, fmt.Errorf("%q %w", mode, errNotCoreRuleSetMode))
|
||||||
@@ -1110,24 +1056,20 @@ func (e *environment) refresh(name, defaultValue string) time.Duration {
|
|||||||
// action reads a setting that is what is done with a client a blocklist
|
// action reads a setting that is what is done with a client a blocklist
|
||||||
// or a DNSBL zone lists: deny, log, or limit:<percent>, which it returns
|
// or a DNSBL zone lists: deny, log, or limit:<percent>, which it returns
|
||||||
// as limit and the percentage.
|
// as limit and the percentage.
|
||||||
func (e *environment) action(
|
func (e *environment) action(name, defaultValue string) (string, int64) {
|
||||||
name, defaultValue string,
|
value := e.value(name, defaultValue)
|
||||||
) (ReputationAction, int64) {
|
if value == "deny" || value == "log" {
|
||||||
value := ReputationAction(e.value(name, defaultValue))
|
|
||||||
if value == ReputationActionDeny || value == ReputationActionLog {
|
|
||||||
return value, 0
|
return value, 0
|
||||||
}
|
}
|
||||||
|
|
||||||
percentText, isLimit := strings.CutPrefix(
|
percentText, isLimit := strings.CutPrefix(value, "limit:")
|
||||||
string(value), string(ReputationActionLimit)+":",
|
|
||||||
)
|
|
||||||
|
|
||||||
percent, err := ParsePercent(percentText)
|
percent, err := ParsePercent(percentText)
|
||||||
if !isLimit || err != nil {
|
if !isLimit || err != nil {
|
||||||
e.check(name, fmt.Errorf("%q %w", value, errNotAction))
|
e.check(name, fmt.Errorf("%q %w", value, errNotAction))
|
||||||
}
|
}
|
||||||
|
|
||||||
return ReputationActionLimit, percent
|
return "limit", percent
|
||||||
}
|
}
|
||||||
|
|
||||||
// zones reads the setting that is the list of DNSBL zones. It is empty by
|
// zones reads the setting that is the list of DNSBL zones. It is empty by
|
||||||
@@ -1178,12 +1120,9 @@ func (e *environment) routeWatchURL(name string) string {
|
|||||||
|
|
||||||
// lookupSource reads the setting that is where clients are looked up:
|
// lookupSource reads the setting that is where clients are looked up:
|
||||||
// geojs, file, or off.
|
// geojs, file, or off.
|
||||||
func (e *environment) lookupSource(
|
func (e *environment) lookupSource(name, defaultValue string) string {
|
||||||
name string, defaultValue LookupSource,
|
source := e.value(name, defaultValue)
|
||||||
) LookupSource {
|
if source != "geojs" && source != fileSource && source != off {
|
||||||
source := LookupSource(e.value(name, string(defaultValue)))
|
|
||||||
if source != LookupSourceGeoJS && source != LookupSourceFile &&
|
|
||||||
source != LookupSourceOff {
|
|
||||||
e.check(name, fmt.Errorf("%q %w", source, errNotLookupSource))
|
e.check(name, fmt.Errorf("%q %w", source, errNotLookupSource))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1195,12 +1134,12 @@ func (e *environment) lookupSource(
|
|||||||
// one source at a time.
|
// one source at a time.
|
||||||
func (e *environment) checkLookupDBPath(cfg *WAF) {
|
func (e *environment) checkLookupDBPath(cfg *WAF) {
|
||||||
switch {
|
switch {
|
||||||
case cfg.LookupSource == LookupSourceFile && cfg.LookupDBPath == "":
|
case cfg.LookupSource == fileSource && cfg.LookupDBPath == "":
|
||||||
e.check("SWW_WAF_LOOKUP_SOURCE", fmt.Errorf(
|
e.check("SWW_WAF_LOOKUP_SOURCE", fmt.Errorf(
|
||||||
"is file while SWW_WAF_LOOKUP_DB_PATH is unset; %w",
|
"is file while SWW_WAF_LOOKUP_DB_PATH is unset; %w",
|
||||||
errNeedsDBPath,
|
errNeedsDBPath,
|
||||||
))
|
))
|
||||||
case cfg.LookupSource != LookupSourceFile && cfg.LookupDBPath != "":
|
case cfg.LookupSource != fileSource && cfg.LookupDBPath != "":
|
||||||
e.check("SWW_WAF_LOOKUP_DB_PATH", fmt.Errorf(
|
e.check("SWW_WAF_LOOKUP_DB_PATH", fmt.Errorf(
|
||||||
"is set while SWW_WAF_LOOKUP_SOURCE is %s; %w",
|
"is set while SWW_WAF_LOOKUP_SOURCE is %s; %w",
|
||||||
cfg.LookupSource,
|
cfg.LookupSource,
|
||||||
@@ -1225,7 +1164,7 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.LookupSource != LookupSourceOff {
|
if cfg.LookupSource != off {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -614,7 +614,7 @@ func TestCoreRuleSetSettings(t *testing.T) {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
environment{
|
environment{
|
||||||
coreRuleSetMode: string(config.CoreRuleSetModeDetect),
|
coreRuleSetMode: config.CoreRuleSetModeDetect,
|
||||||
coreRuleSetParanoiaLevel: "4",
|
coreRuleSetParanoiaLevel: "4",
|
||||||
coreRuleSetScoreThreshold: "10",
|
coreRuleSetScoreThreshold: "10",
|
||||||
coreRuleSetDisabledRules: "942100, 920350",
|
coreRuleSetDisabledRules: "942100, 920350",
|
||||||
@@ -2033,9 +2033,8 @@ func TestReputationSettingsAsSet(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
value string
|
value, action string
|
||||||
action config.ReputationAction
|
percent int64
|
||||||
percent int64
|
|
||||||
}{
|
}{
|
||||||
{actionDeny, actionDeny, 0},
|
{actionDeny, actionDeny, 0},
|
||||||
{actionLog, actionLog, 0},
|
{actionLog, actionLog, 0},
|
||||||
@@ -2168,7 +2167,7 @@ func TestDNSBLSettingsAsSet(t *testing.T) {
|
|||||||
// want are the zones, resolver, action and percent config.WAF gives.
|
// want are the zones, resolver, action and percent config.WAF gives.
|
||||||
want []string
|
want []string
|
||||||
wantResolver string
|
wantResolver string
|
||||||
wantAction config.ReputationAction
|
wantAction string
|
||||||
wantPercent int64
|
wantPercent int64
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
@@ -3050,7 +3049,7 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
|
|||||||
logLevel: "info",
|
logLevel: "info",
|
||||||
ruleFilesDir: "/etc/sww/rules.d",
|
ruleFilesDir: "/etc/sww/rules.d",
|
||||||
ruleFilesEnabled: "true",
|
ruleFilesEnabled: "true",
|
||||||
coreRuleSetMode: string(config.CoreRuleSetModeBlock),
|
coreRuleSetMode: config.CoreRuleSetModeBlock,
|
||||||
coreRuleSetParanoiaLevel: "1",
|
coreRuleSetParanoiaLevel: "1",
|
||||||
coreRuleSetScoreThreshold: "5",
|
coreRuleSetScoreThreshold: "5",
|
||||||
coreRuleSetDisabledRules: defaultCoreRuleSetDisabledRules,
|
coreRuleSetDisabledRules: defaultCoreRuleSetDisabledRules,
|
||||||
|
|||||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -37,9 +37,10 @@
|
|||||||
SecRule &TX:app_nextcloud "@eq 0" \
|
SecRule &TX:app_nextcloud "@eq 0" \
|
||||||
"id:900090,phase:1,pass,nolog,skipAfter:END_NEXTCLOUD"
|
"id:900090,phase:1,pass,nolog,skipAfter:END_NEXTCLOUD"
|
||||||
|
|
||||||
# The sync client and the Files app use these WebDAV methods besides those
|
# The sync client and the Files app use these WebDAV methods besides PUT,
|
||||||
# the first change allows, on /remote.php/dav/ and /public.php/dav/ and on the
|
# PATCH and DELETE, which sww allows for every app, on /remote.php/dav/ and
|
||||||
# older /remote.php/webdav/ and /public.php/webdav/. 911100 refused each:
|
# /public.php/dav/ and on the older /remote.php/webdav/ and /public.php/webdav/.
|
||||||
|
# 911100 refused each:
|
||||||
# PROPFIND, PROPPATCH: listing a folder, marking a file a favourite
|
# PROPFIND, PROPPATCH: listing a folder, marking a file a favourite
|
||||||
# MKCOL: making a folder, and starting a chunked upload
|
# MKCOL: making a folder, and starting a chunked upload
|
||||||
# COPY, MOVE: copying, moving and renaming, and ending a chunked upload
|
# COPY, MOVE: copying, moving and renaming, and ending a chunked upload
|
||||||
File renamed without changes.
File renamed without changes.
@@ -12,11 +12,10 @@
|
|||||||
package coreruleset
|
package coreruleset
|
||||||
|
|
||||||
import (
|
import (
|
||||||
_ "embed"
|
"embed"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"maps"
|
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
@@ -35,7 +34,7 @@ import (
|
|||||||
// directives are the Core Rule Set as sww runs it, with the
|
// directives are the Core Rule Set as sww runs it, with the
|
||||||
// paranoia level for %d, bodyDirectives for the first %s while
|
// paranoia level for %d, bodyDirectives for the first %s while
|
||||||
// SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE is set, the exclusions file for
|
// SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE is set, the exclusions file for
|
||||||
// the second, and the directives of every app in apps for the third.
|
// the second, and the fixes of every app in appFixes for the third.
|
||||||
// Each rule sww adds has an id from 900000 to 900999, the ids the
|
// Each rule sww adds has an id from 900000 to 900999, the ids the
|
||||||
// Core Rule Set keeps for the rules that set it up, which
|
// Core Rule Set keeps for the rules that set it up, which
|
||||||
// SWW_WAF_CORE_RULE_SET_DISABLED_RULES and the exclusions file refuse, so that no
|
// SWW_WAF_CORE_RULE_SET_DISABLED_RULES and the exclusions file refuse, so that no
|
||||||
@@ -54,13 +53,12 @@ Include @crs-setup.conf.example
|
|||||||
SecAction "id:900000,phase:1,pass,nolog,\
|
SecAction "id:900000,phase:1,pass,nolog,\
|
||||||
setvar:tx.blocking_paranoia_level=%d"
|
setvar:tx.blocking_paranoia_level=%d"
|
||||||
|
|
||||||
# The first change: PUT, PATCH and DELETE are allowed besides GET, HEAD,
|
# PUT, PATCH and DELETE are allowed besides GET, HEAD, POST and OPTIONS.
|
||||||
# POST and OPTIONS.
|
|
||||||
SecAction "id:900200,phase:1,pass,nolog,\
|
SecAction "id:900200,phase:1,pass,nolog,\
|
||||||
setvar:'tx.allowed_methods=GET HEAD POST OPTIONS PUT PATCH DELETE'"
|
setvar:'tx.allowed_methods=GET HEAD POST OPTIONS PUT PATCH DELETE'"
|
||||||
|
|
||||||
# The second: Expect and Content-Encoding are taken off the Core Rule Set's
|
# Expect and Content-Encoding are taken off the Core Rule Set's list of the
|
||||||
# list of the headers it refuses. Content-Encoding goes back on it for a
|
# headers it refuses. Content-Encoding goes back on it for a
|
||||||
# body the Core Rule Set reads (900260 in bodyDirectives).
|
# body the Core Rule Set reads (900260 in bodyDirectives).
|
||||||
SecAction "id:900250,phase:1,pass,nolog,\
|
SecAction "id:900250,phase:1,pass,nolog,\
|
||||||
setvar:'tx.restricted_headers_basic=/proxy/ /lock-token/ /content-range/ \
|
setvar:'tx.restricted_headers_basic=/proxy/ /lock-token/ /content-range/ \
|
||||||
@@ -86,18 +84,17 @@ SecRule ARGUMENTS_LIMIT_REACHED "@eq 1" "id:900300,phase:2,pass,\
|
|||||||
# The apps' fixes come before the rules for the same reason.
|
# The apps' fixes come before the rules for the same reason.
|
||||||
%s
|
%s
|
||||||
|
|
||||||
# The fifth: only the rules for requests are loaded, and no response is
|
# Only the rules for requests are loaded, and no response is inspected.
|
||||||
# inspected.
|
|
||||||
Include @owasp_crs/REQUEST-*.conf
|
Include @owasp_crs/REQUEST-*.conf
|
||||||
|
|
||||||
# The third: redirect_uri is not checked for a URL naming an IP address or
|
# redirect_uri is not checked for a URL naming an IP address or
|
||||||
# localhost. Coraza matches a parameter name here without regard to case.
|
# localhost. Coraza matches a parameter name here without regard to case.
|
||||||
# ARGS holds the fields of a form data or multipart body Coraza reads as
|
# ARGS holds the fields of a form data or multipart body Coraza reads as
|
||||||
# well as the query parameters, so a field of this name is left out too.
|
# well as the query parameters, so a field of this name is left out too.
|
||||||
SecRuleUpdateTargetById 931100 "!ARGS:redirect_uri"
|
SecRuleUpdateTargetById 931100 "!ARGS:redirect_uri"
|
||||||
SecRuleUpdateTargetById 934110 "!ARGS:redirect_uri"
|
SecRuleUpdateTargetById 934110 "!ARGS:redirect_uri"
|
||||||
|
|
||||||
# The fourth: Referer is not checked for a Unix command without arguments
|
# Referer is not checked for a Unix command without arguments
|
||||||
# or for Java starting a process.
|
# or for Java starting a process.
|
||||||
SecRuleUpdateTargetById 932340 "!REQUEST_HEADERS:Referer"
|
SecRuleUpdateTargetById 932340 "!REQUEST_HEADERS:Referer"
|
||||||
SecRuleUpdateTargetById 944110 "!REQUEST_HEADERS:Referer"
|
SecRuleUpdateTargetById 944110 "!REQUEST_HEADERS:Referer"
|
||||||
@@ -122,9 +119,9 @@ SecRule REQUEST_HEADERS:Content-Type \
|
|||||||
"@rx ^(?:application|text)/(?:[a-z0-9.-]+[+])?json" \
|
"@rx ^(?:application|text)/(?:[a-z0-9.-]+[+])?json" \
|
||||||
"id:900420,phase:1,pass,nolog,t:none,t:lowercase,ctl:requestBodyProcessor=JSON"
|
"id:900420,phase:1,pass,nolog,t:none,t:lowercase,ctl:requestBodyProcessor=JSON"
|
||||||
|
|
||||||
# The rest of the second change: Content-Encoding is refused again on a
|
# Content-Encoding goes back on the list of the headers the Core Rule Set
|
||||||
# body of a kind the Core Rule Set reads, since a compressed body cannot be
|
# refuses (900250 in directives) for a body of a kind it reads, since a
|
||||||
# inspected.
|
# compressed body cannot be inspected.
|
||||||
SecRule REQBODY_PROCESSOR "@rx ^(?:URLENCODED|MULTIPART|JSON|XML)$" \
|
SecRule REQBODY_PROCESSOR "@rx ^(?:URLENCODED|MULTIPART|JSON|XML)$" \
|
||||||
"id:900260,phase:1,pass,nolog,\
|
"id:900260,phase:1,pass,nolog,\
|
||||||
setvar:'tx.restricted_headers_basic=%%{tx.restricted_headers_basic} \
|
setvar:'tx.restricted_headers_basic=%%{tx.restricted_headers_basic} \
|
||||||
@@ -145,15 +142,19 @@ SecRule MULTIPART_STRICT_ERROR "!@eq 0" "id:900450,phase:2,pass,\
|
|||||||
severity:'CRITICAL',setvar:'tx.inbound_anomaly_score_pl1=+5'"
|
severity:'CRITICAL',setvar:'tx.inbound_anomaly_score_pl1=+5'"
|
||||||
`
|
`
|
||||||
|
|
||||||
// app is an app whose ordinary requests the Core Rule Set would refuse,
|
// appFixes holds the fixes of each app whose ordinary requests the Core
|
||||||
// with the fixes that let them pass.
|
// Rule Set would refuse, in a file named for the app by the name
|
||||||
|
// SWW_WAF_APPS gives it, such as apps/gitea.conf. The fixes are Coraza
|
||||||
|
// exclusions, loaded before the Core Rule Set's rules. Their first rule
|
||||||
|
// skips to the marker they end with, unless Inspect sets the transaction
|
||||||
|
// variable named for the app, such as app_gitea. The skip reaches only
|
||||||
|
// phase 1, so a rule of a later phase must check that variable itself.
|
||||||
|
//
|
||||||
|
//go:embed apps/*.conf
|
||||||
|
var appFixes embed.FS
|
||||||
|
|
||||||
|
// app is what an app's file in appFixes cannot say about it.
|
||||||
type app struct {
|
type app struct {
|
||||||
// directives are Coraza exclusions, loaded before the Core Rule Set's
|
|
||||||
// rules. Their first rule skips to the marker they end with, unless
|
|
||||||
// Inspect sets the transaction variable named for the app, such as
|
|
||||||
// app_gitea. The skip reaches only phase 1, so a rule of a later phase
|
|
||||||
// must check that variable itself.
|
|
||||||
directives string
|
|
||||||
// cookies are the cookies the Core Rule Set reads a request without.
|
// cookies are the cookies the Core Rule Set reads a request without.
|
||||||
cookies []string
|
cookies []string
|
||||||
// forkOf names the app this one is a fork of, "" for none. A hostname
|
// forkOf names the app this one is a fork of, "" for none. A hostname
|
||||||
@@ -161,60 +162,34 @@ type app struct {
|
|||||||
forkOf string
|
forkOf string
|
||||||
}
|
}
|
||||||
|
|
||||||
//go:embed gitea.conf
|
// apps are the apps that need more than their file in appFixes, by the
|
||||||
var giteaDirectives string
|
// name SWW_WAF_APPS gives each. flash is Forgejo's name for gitea_flash.
|
||||||
|
|
||||||
//go:embed wordpress.conf
|
|
||||||
var wordpressDirectives string
|
|
||||||
|
|
||||||
//go:embed drupal.conf
|
|
||||||
var drupalDirectives string
|
|
||||||
|
|
||||||
//go:embed mediawiki.conf
|
|
||||||
var mediawikiDirectives string
|
|
||||||
|
|
||||||
//go:embed mastodon.conf
|
|
||||||
var mastodonDirectives string
|
|
||||||
|
|
||||||
//go:embed vaultwarden.conf
|
|
||||||
var vaultwardenDirectives string
|
|
||||||
|
|
||||||
//go:embed nextcloud.conf
|
|
||||||
var nextcloudDirectives string
|
|
||||||
|
|
||||||
//go:embed forgejo.conf
|
|
||||||
var forgejoDirectives string
|
|
||||||
|
|
||||||
//go:embed ghost.conf
|
|
||||||
var ghostDirectives string
|
|
||||||
|
|
||||||
// apps are the apps whose fixes sww has, by the name SWW_WAF_APPS gives each.
|
|
||||||
// flash is Forgejo's name for gitea_flash.
|
|
||||||
//
|
//
|
||||||
//nolint:gochecknoglobals // a constant cannot be a map
|
//nolint:gochecknoglobals // a constant cannot be a map
|
||||||
var apps = map[string]app{
|
var apps = map[string]app{
|
||||||
"drupal": {directives: drupalDirectives},
|
|
||||||
"forgejo": {
|
"forgejo": {
|
||||||
directives: forgejoDirectives,
|
cookies: []string{"flash"},
|
||||||
cookies: []string{"flash"},
|
forkOf: "gitea",
|
||||||
forkOf: "gitea",
|
|
||||||
},
|
},
|
||||||
"ghost": {directives: ghostDirectives},
|
|
||||||
"gitea": {
|
"gitea": {
|
||||||
directives: giteaDirectives,
|
cookies: []string{"gitea_flash", "redirect_to"},
|
||||||
cookies: []string{"gitea_flash", "redirect_to"},
|
|
||||||
},
|
},
|
||||||
"mastodon": {directives: mastodonDirectives},
|
|
||||||
"mediawiki": {directives: mediawikiDirectives},
|
|
||||||
"nextcloud": {directives: nextcloudDirectives},
|
|
||||||
"vaultwarden": {directives: vaultwardenDirectives},
|
|
||||||
"wordpress": {directives: wordpressDirectives},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// AppNames returns the names of the apps whose fixes sww has, in
|
// AppNames returns the names of the apps whose fixes sww has, in
|
||||||
// alphabetical order.
|
// alphabetical order.
|
||||||
func AppNames() []string {
|
func AppNames() []string {
|
||||||
return slices.Sorted(maps.Keys(apps))
|
// An embedded directory always reads.
|
||||||
|
files, _ := appFixes.ReadDir("apps")
|
||||||
|
|
||||||
|
names := make([]string, len(files))
|
||||||
|
for i, file := range files {
|
||||||
|
names[i] = strings.TrimSuffix(file.Name(), ".conf")
|
||||||
|
}
|
||||||
|
|
||||||
|
slices.Sort(names)
|
||||||
|
|
||||||
|
return names
|
||||||
}
|
}
|
||||||
|
|
||||||
// Params are what New needs.
|
// Params are what New needs.
|
||||||
@@ -373,8 +348,11 @@ func New(params Params) (*CoreRuleSet, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var fixes strings.Builder
|
var fixes strings.Builder
|
||||||
|
|
||||||
for _, name := range AppNames() {
|
for _, name := range AppNames() {
|
||||||
fixes.WriteString(apps[name].directives + "\n")
|
// An embedded file always reads.
|
||||||
|
file, _ := appFixes.ReadFile("apps/" + name + ".conf")
|
||||||
|
fixes.WriteString(string(file) + "\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
text := fmt.Sprintf(directives, params.ParanoiaLevel, body, exclusions,
|
text := fmt.Sprintf(directives, params.ParanoiaLevel, body, exclusions,
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// giteaSearches are searches of the pages gitea and Forgejo share whose q
|
// giteaSearches are searches of the pages gitea and Forgejo share whose q
|
||||||
// skips the lists of system files and command names, in the eighth change.
|
// skips the lists of system files and command names, in gitea's fixes.
|
||||||
// gitea serves its admin pages under /-/admin/ and Forgejo under /admin/.
|
// gitea serves its admin pages under /-/admin/ and Forgejo under /admin/.
|
||||||
//
|
//
|
||||||
//nolint:gochecknoglobals // a constant cannot be a list
|
//nolint:gochecknoglobals // a constant cannot be a list
|
||||||
@@ -222,8 +222,8 @@ func TestGiteaSearchExclusionsAreNarrow(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Under /admin/, where Forgejo serves its admin pages, only the hard
|
// Under /admin/, where Forgejo serves its admin pages, only the hard
|
||||||
// values are left out: the tenth change lets them through in q on every
|
// values are left out: Drupal's fixes let them through in q on every path
|
||||||
// path there, for Drupal.
|
// there.
|
||||||
for _, target := range []string{
|
for _, target := range []string{
|
||||||
"/admin/packages/delete?q=",
|
"/admin/packages/delete?q=",
|
||||||
"/admin/packages?other=",
|
"/admin/packages?other=",
|
||||||
|
|||||||
+37
-60
@@ -84,7 +84,12 @@ type Params struct {
|
|||||||
// one request to GeoJS is under way at a time, and it asks about every
|
// one request to GeoJS is under way at a time, and it asks about every
|
||||||
// client waiting, up to maxPerRequest. It is safe for concurrent use.
|
// client waiting, up to maxPerRequest. It is safe for concurrent use.
|
||||||
type GeoJS struct {
|
type GeoJS struct {
|
||||||
params Params
|
url string
|
||||||
|
answered func(Answer)
|
||||||
|
now func() time.Time
|
||||||
|
asked func()
|
||||||
|
failed func(err error, askingAgainIn time.Duration)
|
||||||
|
wentWithoutAnswer func()
|
||||||
// httpClient follows no redirect, so that visitors' addresses go to
|
// httpClient follows no redirect, so that visitors' addresses go to
|
||||||
// GeoJS alone: a redirect is a failure.
|
// GeoJS alone: a redirect is a failure.
|
||||||
httpClient *http.Client
|
httpClient *http.Client
|
||||||
@@ -94,13 +99,8 @@ type GeoJS struct {
|
|||||||
// waiting are the clients without an answer: those to ask GeoJS about,
|
// waiting are the clients without an answer: those to ask GeoJS about,
|
||||||
// and those it is being asked about.
|
// and those it is being asked about.
|
||||||
waiting map[netip.Prefix]*wait
|
waiting map[netip.Prefix]*wait
|
||||||
// asking is true while a request to GeoJS is under way, and endAsking
|
// asking is true while a request to GeoJS is under way.
|
||||||
// ends it. stopped is true once StopAsking has been called, and
|
asking bool
|
||||||
// askGoroutines waits for the goroutine that makes the requests.
|
|
||||||
asking bool
|
|
||||||
endAsking context.CancelFunc
|
|
||||||
stopped bool
|
|
||||||
askGoroutines sync.WaitGroup
|
|
||||||
// retryDelay is how long GeoJS is left alone after its last failure,
|
// retryDelay is how long GeoJS is left alone after its last failure,
|
||||||
// zero after an answer; retryAt is when it may be asked again.
|
// zero after an answer; retryAt is when it may be asked again.
|
||||||
retryDelay time.Duration
|
retryDelay time.Duration
|
||||||
@@ -140,7 +140,12 @@ func New(params Params) *GeoJS {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &GeoJS{
|
return &GeoJS{
|
||||||
params: params,
|
url: params.URL,
|
||||||
|
answered: params.Answered,
|
||||||
|
now: params.Now,
|
||||||
|
asked: params.Asked,
|
||||||
|
failed: params.Failed,
|
||||||
|
wentWithoutAnswer: params.WentWithoutAnswer,
|
||||||
httpClient: &http.Client{
|
httpClient: &http.Client{
|
||||||
CheckRedirect: func(*http.Request, []*http.Request) error {
|
CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||||
return http.ErrUseLastResponse
|
return http.ErrUseLastResponse
|
||||||
@@ -183,7 +188,7 @@ func (g *GeoJS) LookUp(
|
|||||||
|
|
||||||
answer, found := g.kept(client)
|
answer, found := g.kept(client)
|
||||||
if !found {
|
if !found {
|
||||||
g.params.WentWithoutAnswer()
|
g.wentWithoutAnswer()
|
||||||
}
|
}
|
||||||
|
|
||||||
w, waiting := g.waiting[client]
|
w, waiting := g.waiting[client]
|
||||||
@@ -205,23 +210,6 @@ func (g *GeoJS) LookUpWithoutWaiting(
|
|||||||
return answer
|
return answer
|
||||||
}
|
}
|
||||||
|
|
||||||
// StopAsking ends the request to GeoJS under way, and returns once the
|
|
||||||
// goroutine that made it has. A request it ends is no failure: Failed is not
|
|
||||||
// called, and GeoJS is not left alone after it. After StopAsking, GeoJS is
|
|
||||||
// asked nothing.
|
|
||||||
func (g *GeoJS) StopAsking() {
|
|
||||||
g.mu.Lock()
|
|
||||||
|
|
||||||
g.stopped = true
|
|
||||||
if g.asking {
|
|
||||||
g.endAsking()
|
|
||||||
}
|
|
||||||
|
|
||||||
g.mu.Unlock()
|
|
||||||
|
|
||||||
g.askGoroutines.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Kept returns client's answer, if one is kept, without asking GeoJS.
|
// Kept returns client's answer, if one is kept, without asking GeoJS.
|
||||||
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
|
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
|
||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
@@ -263,7 +251,7 @@ func (g *GeoJS) Load(answers []Answer) {
|
|||||||
|
|
||||||
g.answers.Purge()
|
g.answers.Purge()
|
||||||
|
|
||||||
now := g.params.Now()
|
now := g.now()
|
||||||
|
|
||||||
for _, answer := range answers {
|
for _, answer := range answers {
|
||||||
if now.Sub(answer.Answered) < keepFor {
|
if now.Sub(answer.Answered) < keepFor {
|
||||||
@@ -300,19 +288,18 @@ func (g *GeoJS) answerOrWait(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if w == nil {
|
if w == nil {
|
||||||
g.params.WentWithoutAnswer()
|
g.wentWithoutAnswer()
|
||||||
|
|
||||||
return Answer{}, nil // too many clients wait already
|
return Answer{}, nil // too many clients wait already
|
||||||
}
|
}
|
||||||
|
|
||||||
if !g.asking {
|
if !g.asking {
|
||||||
// GeoJS is left alone after a failure, or asked nothing after
|
// GeoJS is left alone after a failure, so no answer can come.
|
||||||
// StopAsking, so no answer can come.
|
|
||||||
w.late = true
|
w.late = true
|
||||||
}
|
}
|
||||||
|
|
||||||
if w.late {
|
if w.late {
|
||||||
g.params.WentWithoutAnswer()
|
g.wentWithoutAnswer()
|
||||||
|
|
||||||
return Answer{}, nil
|
return Answer{}, nil
|
||||||
}
|
}
|
||||||
@@ -323,7 +310,7 @@ func (g *GeoJS) answerOrWait(
|
|||||||
// kept returns client's answer, if GeoJS gave it less than keepFor ago,
|
// kept returns client's answer, if GeoJS gave it less than keepFor ago,
|
||||||
// and notes that it was used.
|
// and notes that it was used.
|
||||||
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
||||||
now := g.params.Now()
|
now := g.now()
|
||||||
|
|
||||||
kept, found := g.answers.Get(client)
|
kept, found := g.answers.Get(client)
|
||||||
if !found || now.Sub(kept.Answered) >= keepFor {
|
if !found || now.Sub(kept.Answered) >= keepFor {
|
||||||
@@ -336,28 +323,22 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ask starts asking GeoJS about the waiting clients, unless a request to
|
// ask starts asking GeoJS about the waiting clients, unless a request to
|
||||||
// it is under way, it is left alone after a failure, or StopAsking has
|
// it is under way or it is left alone after a failure. The requests to
|
||||||
// been called. The requests to GeoJS are for every client waiting, so they
|
// GeoJS are for every client waiting, so they go on when the caller whose
|
||||||
// go on when the caller whose ctx is given stops waiting, until StopAsking.
|
// ctx is given stops waiting.
|
||||||
func (g *GeoJS) ask(ctx context.Context) {
|
func (g *GeoJS) ask(ctx context.Context) {
|
||||||
if g.asking || g.stopped || g.params.Now().Before(g.retryAt) {
|
if g.asking || g.now().Before(g.retryAt) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
askCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
g.asking = true
|
||||||
g.asking, g.endAsking = true, end
|
|
||||||
|
|
||||||
g.askGoroutines.Go(func() {
|
go g.askAboutWaiting(context.WithoutCancel(ctx))
|
||||||
defer end()
|
|
||||||
|
|
||||||
g.askAboutWaiting(askCtx)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// askAboutWaiting asks GeoJS about the waiting clients, one request at a
|
// askAboutWaiting asks GeoJS about the waiting clients, one request at a
|
||||||
// time, until none is left, GeoJS fails, or StopAsking ends ctx. Each
|
// time, until none is left or GeoJS fails. Each answer kept is given to
|
||||||
// answer kept is given to Answered, outside the lock, since Answered takes
|
// Answered, outside the lock, since Answered takes locks of its own.
|
||||||
// locks of its own.
|
|
||||||
func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
||||||
for {
|
for {
|
||||||
clients := g.nextClients()
|
clients := g.nextClients()
|
||||||
@@ -366,15 +347,11 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
given, err := g.request(ctx, clients)
|
given, err := g.request(ctx, clients)
|
||||||
if err != nil && ctx.Err() != nil {
|
|
||||||
continue // ended by StopAsking, which is no failure
|
|
||||||
}
|
|
||||||
|
|
||||||
kept, answered := g.keep(clients, given, err)
|
kept, answered := g.keep(clients, given, err)
|
||||||
|
|
||||||
if g.params.Answered != nil {
|
if g.answered != nil {
|
||||||
for _, answer := range kept {
|
for _, answer := range kept {
|
||||||
g.params.Answered(answer)
|
g.answered(answer)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -385,13 +362,13 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// nextClients returns up to maxPerRequest of the waiting clients. When
|
// nextClients returns up to maxPerRequest of the waiting clients. When
|
||||||
// none is waiting, or StopAsking has been called, it returns none and
|
// none is waiting, it returns none and notes that no request to GeoJS is
|
||||||
// notes that no request to GeoJS is under way.
|
// under way.
|
||||||
func (g *GeoJS) nextClients() []netip.Prefix {
|
func (g *GeoJS) nextClients() []netip.Prefix {
|
||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
defer g.mu.Unlock()
|
defer g.mu.Unlock()
|
||||||
|
|
||||||
if len(g.waiting) == 0 || g.stopped {
|
if len(g.waiting) == 0 {
|
||||||
g.asking = false
|
g.asking = false
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -423,7 +400,7 @@ func (g *GeoJS) keep(
|
|||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
defer g.mu.Unlock()
|
defer g.mu.Unlock()
|
||||||
|
|
||||||
now := g.params.Now()
|
now := g.now()
|
||||||
kept := make([]Answer, 0, len(clients))
|
kept := make([]Answer, 0, len(clients))
|
||||||
leftOut := 0
|
leftOut := 0
|
||||||
|
|
||||||
@@ -467,7 +444,7 @@ func (g *GeoJS) keep(
|
|||||||
w.late = true
|
w.late = true
|
||||||
}
|
}
|
||||||
|
|
||||||
g.params.Failed(err, g.retryDelay)
|
g.failed(err, g.retryDelay)
|
||||||
|
|
||||||
return kept, false
|
return kept, false
|
||||||
}
|
}
|
||||||
@@ -497,7 +474,7 @@ func (g *GeoJS) request(
|
|||||||
req, err := http.NewRequestWithContext(
|
req, err := http.NewRequestWithContext(
|
||||||
ctx,
|
ctx,
|
||||||
http.MethodGet,
|
http.MethodGet,
|
||||||
g.params.URL,
|
g.url,
|
||||||
http.NoBody,
|
http.NoBody,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -506,7 +483,7 @@ func (g *GeoJS) request(
|
|||||||
|
|
||||||
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
|
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
|
||||||
|
|
||||||
g.params.Asked()
|
g.asked()
|
||||||
|
|
||||||
res, err := g.httpClient.Do(req)
|
res, err := g.httpClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -318,44 +318,6 @@ func TestCallerWhoseContextEndsEarlyDoesNotStopTheRequest(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestStopAskingEndsTheRequestUnderWayAsNoFailureAndAsksNoMore(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
service, _, g, reports := startWithReports()
|
|
||||||
clients := newClients()
|
|
||||||
|
|
||||||
service.set(hanging)
|
|
||||||
g.LookUpWithoutWaiting(t.Context(), clients())
|
|
||||||
waitForRequests(t, service, 1)
|
|
||||||
|
|
||||||
// The request is ended, not waited out, and is no failure.
|
|
||||||
began := time.Now()
|
|
||||||
|
|
||||||
g.StopAsking()
|
|
||||||
|
|
||||||
if waited := time.Since(began); waited != 0 {
|
|
||||||
t.Errorf("StopAsking returned after %s, want at once", waited)
|
|
||||||
}
|
|
||||||
|
|
||||||
reports.wantFailures(t)
|
|
||||||
|
|
||||||
// A new client is then not asked about, and its lookup does not
|
|
||||||
// wait.
|
|
||||||
service.set(answering)
|
|
||||||
wantCountry(t, g, clients(), "")
|
|
||||||
|
|
||||||
if waited := time.Since(began); waited != 0 {
|
|
||||||
t.Errorf("the lookup returned after %s, want at once", waited)
|
|
||||||
}
|
|
||||||
|
|
||||||
waitForRequests(t, service, 1)
|
|
||||||
reports.wantFailures(t)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRequestOver10SecondsIsAFailure(t *testing.T) {
|
func TestRequestOver10SecondsIsAFailure(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -1,44 +0,0 @@
|
|||||||
// Package processlog builds the logger for the process's own messages,
|
|
||||||
// which it writes as JSON lines marked "type":"process". The request log
|
|
||||||
// is package requestlog's.
|
|
||||||
package processlog
|
|
||||||
|
|
||||||
import (
|
|
||||||
"io"
|
|
||||||
"log/slog"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Params are what New needs.
|
|
||||||
type Params struct {
|
|
||||||
// Output is where the lines are written.
|
|
||||||
Output io.Writer
|
|
||||||
// InstanceName, SWW_WAF_INSTANCE_NAME, is each line's instance.
|
|
||||||
InstanceName string
|
|
||||||
// Level, SWW_WAF_LOG_LEVEL, is the least severe level written.
|
|
||||||
Level slog.Level
|
|
||||||
}
|
|
||||||
|
|
||||||
// New returns the logger for the process's own messages: JSON lines on
|
|
||||||
// params.Output, marked "type":"process", with the time in the same form
|
|
||||||
// as a request line's. It writes only the messages at params.Level or more
|
|
||||||
// severe; the request lines requestlog.Write writes are never held back.
|
|
||||||
func New(params Params) *slog.Logger {
|
|
||||||
handler := slog.NewJSONHandler(params.Output, &slog.HandlerOptions{
|
|
||||||
Level: params.Level,
|
|
||||||
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
|
|
||||||
if attr.Key == slog.TimeKey && len(groups) == 0 {
|
|
||||||
return slog.String(
|
|
||||||
slog.TimeKey,
|
|
||||||
requestlog.FormatTime(attr.Value.Time()),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return attr
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
return slog.New(handler).
|
|
||||||
With("type", "process", "instance", params.InstanceName)
|
|
||||||
}
|
|
||||||
@@ -1,105 +0,0 @@
|
|||||||
package processlog_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"encoding/json"
|
|
||||||
"log/slog"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
|
||||||
)
|
|
||||||
|
|
||||||
// instanceName is the instance the tests' loggers are built with.
|
|
||||||
const instanceName = "host1/blog"
|
|
||||||
|
|
||||||
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
processlog.New(processlog.Params{
|
|
||||||
Output: &out,
|
|
||||||
InstanceName: instanceName,
|
|
||||||
Level: slog.LevelInfo,
|
|
||||||
}).
|
|
||||||
Info("starting",
|
|
||||||
"version", "v1")
|
|
||||||
|
|
||||||
var fields map[string]any
|
|
||||||
|
|
||||||
err := json.Unmarshal(out.Bytes(), &fields)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("decode %q: %v", out.String(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if fields["type"] != "process" ||
|
|
||||||
fields["instance"] != "host1/blog" ||
|
|
||||||
fields["msg"] != "starting" ||
|
|
||||||
fields["level"] != "INFO" ||
|
|
||||||
fields["version"] != "v1" {
|
|
||||||
t.Errorf("process line %v", fields)
|
|
||||||
}
|
|
||||||
|
|
||||||
timeText, _ := fields["time"].(string)
|
|
||||||
|
|
||||||
logged, err := time.Parse(time.RFC3339, timeText)
|
|
||||||
if err != nil || !strings.HasSuffix(timeText, "Z") ||
|
|
||||||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
|
|
||||||
time.Since(logged) > time.Minute {
|
|
||||||
t.Errorf(
|
|
||||||
"process line time %q, want now in UTC with milliseconds",
|
|
||||||
timeText,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
levels := []slog.Level{
|
|
||||||
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, level := range levels {
|
|
||||||
t.Run(level.String(), func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
processLog := processlog.New(processlog.Params{
|
|
||||||
Output: &out,
|
|
||||||
InstanceName: instanceName,
|
|
||||||
Level: level,
|
|
||||||
})
|
|
||||||
for _, at := range levels {
|
|
||||||
processLog.Log(t.Context(), at, "message")
|
|
||||||
}
|
|
||||||
|
|
||||||
var got, want []string
|
|
||||||
|
|
||||||
for line := range strings.Lines(out.String()) {
|
|
||||||
var fields struct {
|
|
||||||
Level string `json:"level"`
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(line), &fields)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("decode %q: %v", line, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
got = append(got, fields.Level)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, written := range levels[i:] {
|
|
||||||
want = append(want, written.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(got, want) {
|
|
||||||
t.Errorf("lines at %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
func TestHistoryKeepsEveryRequest(t *testing.T) {
|
func TestHistoryKeepsEveryRequest(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -57,7 +57,7 @@ func TestHistoryKeepsEveryRequest(t *testing.T) {
|
|||||||
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
other := netip.MustParsePrefix("198.51.100.7/32")
|
other := netip.MustParsePrefix("198.51.100.7/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -100,10 +100,7 @@ func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
|||||||
func TestResetKeepsTheHistory(t *testing.T) {
|
func TestResetKeepsTheHistory(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerMinute: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -126,7 +123,7 @@ func TestResetKeepsTheHistory(t *testing.T) {
|
|||||||
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
|
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
|
|
||||||
for client, requests := range map[string]int{
|
for client, requests := range map[string]int{
|
||||||
"198.51.100.9/32": 2,
|
"198.51.100.9/32": 2,
|
||||||
|
|||||||
@@ -156,28 +156,16 @@ type Request struct {
|
|||||||
BlockedByCoreRuleSet bool
|
BlockedByCoreRuleSet bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// Params are what New needs.
|
// New returns a Limiter for limits, with no client counted yet, whose
|
||||||
type Params struct {
|
// table holds at most maxClients clients (SWW_WAF_MAX_TRACKED_CLIENTS). Past
|
||||||
// Limits are what each client's requests and bytes are counted against.
|
// it, the least recently seen client is dropped, with its history, and
|
||||||
Limits Limits
|
// starts afresh if it comes back.
|
||||||
// MaxClients is the most clients the table holds,
|
func New(limits Limits, maxClients int) *Limiter {
|
||||||
// SWW_WAF_MAX_TRACKED_CLIENTS. Past it, the least recently seen client
|
clients, err := simplelru.NewLRU[netip.Prefix, *Client](maxClients, nil)
|
||||||
// is dropped, with its history, and starts afresh if it comes back.
|
|
||||||
MaxClients int
|
|
||||||
}
|
|
||||||
|
|
||||||
// New returns a Limiter with no client counted yet.
|
|
||||||
func New(params Params) *Limiter {
|
|
||||||
clients, err := simplelru.NewLRU[netip.Prefix, *Client](
|
|
||||||
params.MaxClients,
|
|
||||||
nil,
|
|
||||||
)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err) // NewLRU fails only for a size below one
|
panic(err) // NewLRU fails only for a size below one
|
||||||
}
|
}
|
||||||
|
|
||||||
limits := params.Limits
|
|
||||||
|
|
||||||
return &Limiter{
|
return &Limiter{
|
||||||
windows: [3]window{
|
windows: [3]window{
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -33,10 +33,10 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
|
|||||||
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
now := midnight()
|
now := midnight()
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
|
ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
|
|
||||||
if limiter.OverARateLimit(client, now) {
|
if limiter.OverARateLimit(client, now) {
|
||||||
t.Error("a client never seen is over a rate limit")
|
t.Error("a client never seen is over a rate limit")
|
||||||
@@ -72,7 +72,7 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
|
|||||||
wantCount(t, limiter, client, now, minute)
|
wantCount(t, limiter, client, now, minute)
|
||||||
|
|
||||||
// A limit that is off is never gone over.
|
// A limit that is off is never gone over.
|
||||||
off := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
off := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
off.Count(client, now, whole)
|
off.Count(client, now, whole)
|
||||||
|
|
||||||
if off.OverARateLimit(client, now) {
|
if off.OverARateLimit(client, now) {
|
||||||
@@ -95,10 +95,7 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
|
|||||||
t.Run(tc.window, func(t *testing.T) {
|
t.Run(tc.window, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(tc.limits, tableSize)
|
||||||
Limits: tc.limits,
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
quarter := tc.length / 4
|
quarter := tc.length / 4
|
||||||
@@ -129,10 +126,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
|
|||||||
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerMinute: limit, PerHour: limit},
|
ratelimit.Limits{PerMinute: limit, PerHour: limit},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -162,13 +159,10 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
|||||||
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
|
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{
|
ratelimit.Limits{PerMinute: 5, BytesPerDay: math.MaxInt64},
|
||||||
PerMinute: 5,
|
tableSize,
|
||||||
BytesPerDay: math.MaxInt64,
|
)
|
||||||
},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -209,10 +203,10 @@ func TestZeroPercentIsAZeroAllowanceAndALimitOffStaysOff(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// Only the hour has limits: the minute's and the day's are off.
|
// Only the hour has limits: the minute's and the day's are off.
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
|
ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -259,10 +253,7 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
|
|||||||
t.Run(tc.window, func(t *testing.T) {
|
t.Run(tc.window, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(tc.limits, tableSize)
|
||||||
Limits: tc.limits,
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
// 600 bytes are within the limit, 600 more over it.
|
// 600 bytes are within the limit, 600 more over it.
|
||||||
@@ -296,10 +287,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
|
|||||||
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
|
ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
other := netip.MustParsePrefix("203.0.113.10/32")
|
other := netip.MustParsePrefix("203.0.113.10/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -323,7 +314,7 @@ func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
|||||||
func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -349,10 +340,7 @@ func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
|||||||
func TestResetSetsTheBytesBackToZero(t *testing.T) {
|
func TestResetSetsTheBytesBackToZero(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{BytesPerDay: 1000}, tableSize)
|
||||||
Limits: ratelimit.Limits{BytesPerDay: 1000},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -368,7 +356,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -423,7 +411,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
|
|||||||
func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -450,10 +438,10 @@ func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
|||||||
func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit},
|
ratelimit.Limits{PerMinute: limit, PerDay: limit},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -475,10 +463,7 @@ func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
|||||||
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerHour: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -497,10 +482,10 @@ func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
|||||||
func TestRefusedRequestsCount(t *testing.T) {
|
func TestRefusedRequestsCount(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(
|
||||||
Limits: ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
|
ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
|
||||||
MaxClients: tableSize,
|
tableSize,
|
||||||
})
|
)
|
||||||
refused := netip.MustParsePrefix("203.0.113.9/32")
|
refused := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
within := netip.MustParsePrefix("203.0.113.10/32")
|
within := netip.MustParsePrefix("203.0.113.10/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -533,10 +518,7 @@ func TestRefusedRequestsCount(t *testing.T) {
|
|||||||
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerMinute: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -552,10 +534,7 @@ func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
|||||||
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
|
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerHour: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -583,10 +562,7 @@ func TestKeepsAtMostMaxClientsDroppingTheLeastRecentlySeen(t *testing.T) {
|
|||||||
|
|
||||||
const maxClients = 3
|
const maxClients = 3
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{
|
limiter := ratelimit.New(ratelimit.Limits{PerMinute: 1}, maxClients)
|
||||||
Limits: ratelimit.Limits{PerMinute: 1},
|
|
||||||
MaxClients: maxClients,
|
|
||||||
})
|
|
||||||
now := midnight()
|
now := midnight()
|
||||||
|
|
||||||
clients := make([]netip.Prefix, maxClients+1)
|
clients := make([]netip.Prefix, maxClients+1)
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ func TestSnapshotListsTheClientsByAddress(t *testing.T) {
|
|||||||
"2001:db8::/64",
|
"2001:db8::/64",
|
||||||
}
|
}
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
for _, i := range []int{2, 3, 0, 1} {
|
for _, i := range []int{2, 3, 0, 1} {
|
||||||
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
|
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
|
||||||
}
|
}
|
||||||
@@ -52,10 +52,7 @@ func TestLoadedCountsCarryOn(t *testing.T) {
|
|||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
before := ratelimit.New(ratelimit.Params{
|
before := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerHour: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
for range limit {
|
for range limit {
|
||||||
wantCount(t, before, client, start, "")
|
wantCount(t, before, client, start, "")
|
||||||
}
|
}
|
||||||
@@ -63,10 +60,7 @@ func TestLoadedCountsCarryOn(t *testing.T) {
|
|||||||
// Loaded into a new limiter, as across a restart, the client has no
|
// Loaded into a new limiter, as across a restart, the client has no
|
||||||
// fresh allowance.
|
// fresh allowance.
|
||||||
later := start.Add(time.Minute)
|
later := start.Add(time.Minute)
|
||||||
after := ratelimit.New(ratelimit.Params{
|
after := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
||||||
Limits: ratelimit.Limits{PerHour: limit},
|
|
||||||
MaxClients: tableSize,
|
|
||||||
})
|
|
||||||
after.Load(before.Snapshot(), later)
|
after.Load(before.Snapshot(), later)
|
||||||
wantCount(t, after, client, later, hour)
|
wantCount(t, after, client, later, hour)
|
||||||
}
|
}
|
||||||
@@ -77,7 +71,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
|
|||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
limiter.Count(client, start, whole)
|
limiter.Count(client, start, whole)
|
||||||
limiter.CountBytes(client, start, 5, whole)
|
limiter.CountBytes(client, start, 5, whole)
|
||||||
limiter.CountRefusal(client, start, limit)
|
limiter.CountRefusal(client, start, limit)
|
||||||
@@ -86,7 +80,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
|
|||||||
loaded := func(now time.Time) ratelimit.Client {
|
loaded := func(now time.Time) ratelimit.Client {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
after := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
after := ratelimit.New(ratelimit.Limits{}, tableSize)
|
||||||
after.Load(limiter.Snapshot(), now)
|
after.Load(limiter.Snapshot(), now)
|
||||||
|
|
||||||
return after.Snapshot()[0]
|
return after.Snapshot()[0]
|
||||||
@@ -148,7 +142,7 @@ func TestLoadDropsTheLeastRecentlySeenFirst(t *testing.T) {
|
|||||||
addr = addr.Next()
|
addr = addr.Next()
|
||||||
}
|
}
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Params{MaxClients: maxClients})
|
limiter := ratelimit.New(ratelimit.Limits{}, maxClients)
|
||||||
limiter.Load(clients, midnight())
|
limiter.Load(clients, midnight())
|
||||||
|
|
||||||
got := limiter.Snapshot()
|
got := limiter.Snapshot()
|
||||||
|
|||||||
@@ -110,12 +110,8 @@ type AbuseIPDB struct {
|
|||||||
// scores are by client. Each is added as it is fetched and never moved
|
// scores are by client. Each is added as it is fetched and never moved
|
||||||
// up, so that the one fetched longest ago is the first dropped.
|
// up, so that the one fetched longest ago is the first dropped.
|
||||||
scores *simplelru.LRU[netip.Prefix, Score]
|
scores *simplelru.LRU[netip.Prefix, Score]
|
||||||
// checking are the clients whose check is under way, each with the
|
// checking are the clients whose check is under way.
|
||||||
// function that ends it. stopped is true once StopChecking has been
|
checking map[netip.Prefix]bool
|
||||||
// called, and checkGoroutines waits for the goroutine of each check.
|
|
||||||
checking map[netip.Prefix]context.CancelFunc
|
|
||||||
stopped bool
|
|
||||||
checkGoroutines sync.WaitGroup
|
|
||||||
// day is the day, in UTC, of the checks spent counts.
|
// day is the day, in UTC, of the checks spent counts.
|
||||||
day civil.Date
|
day civil.Date
|
||||||
spent int
|
spent int
|
||||||
@@ -138,7 +134,7 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
|
|||||||
params: params,
|
params: params,
|
||||||
httpClient: &http.Client{},
|
httpClient: &http.Client{},
|
||||||
scores: scores,
|
scores: scores,
|
||||||
checking: map[netip.Prefix]context.CancelFunc{},
|
checking: map[netip.Prefix]bool{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -148,11 +144,11 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
|
|||||||
// addresses its request comes from. A client without one is checked in
|
// addresses its request comes from. A client without one is checked in
|
||||||
// the background, by addr, the address its request came from, if
|
// the background, by addr, the address its request came from, if
|
||||||
// offender, if it has committed an offence, unless its check is under
|
// offender, if it has committed an offence, unless its check is under
|
||||||
// way, a check failed less than failureDelay ago, the day's checks have
|
// way, a check failed less than failureDelay ago, or the day's checks
|
||||||
// used up DailyBudget, or StopChecking has been called; Hit never waits
|
// have used up DailyBudget; Hit never waits for a check. The check that
|
||||||
// for a check. The check that uses the budget up is logged and raised as a
|
// uses the budget up is logged and raised as a source_failure alert. ctx
|
||||||
// source_failure alert. ctx is the context of the client's request, and a
|
// is the context of the client's request, and a check goes on after the
|
||||||
// check goes on after the request ends, until StopChecking.
|
// request ends.
|
||||||
func (a *AbuseIPDB) Hit(
|
func (a *AbuseIPDB) Hit(
|
||||||
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
|
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
|
||||||
) (int64, bool) {
|
) (int64, bool) {
|
||||||
@@ -171,21 +167,14 @@ func (a *AbuseIPDB) Hit(
|
|||||||
a.day, a.spent = today, 0
|
a.day, a.spent = today, 0
|
||||||
}
|
}
|
||||||
|
|
||||||
_, checking := a.checking[client]
|
check := offender && !a.checking[client] && !now.Before(a.retryAt) &&
|
||||||
|
a.spent < a.params.DailyBudget
|
||||||
check := offender && !checking && !a.stopped &&
|
|
||||||
!now.Before(a.retryAt) && a.spent < a.params.DailyBudget
|
|
||||||
if check {
|
if check {
|
||||||
checkCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
a.checking[client] = true
|
||||||
a.checking[client] = end
|
|
||||||
a.checks++
|
a.checks++
|
||||||
a.spent++
|
a.spent++
|
||||||
|
|
||||||
a.checkGoroutines.Go(func() {
|
go a.check(context.WithoutCancel(ctx), client, addr)
|
||||||
defer end()
|
|
||||||
|
|
||||||
a.check(checkCtx, client, addr)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
usedUp := check && a.spent == a.params.DailyBudget
|
usedUp := check && a.spent == a.params.DailyBudget
|
||||||
@@ -200,24 +189,6 @@ func (a *AbuseIPDB) Hit(
|
|||||||
return 0, false
|
return 0, false
|
||||||
}
|
}
|
||||||
|
|
||||||
// StopChecking ends the checks under way, and returns once their goroutines
|
|
||||||
// have. A check it ends gives no score and is no failure: it is neither
|
|
||||||
// counted, logged nor raised as an alert, and checks are not left alone
|
|
||||||
// after it. The ended check stays spent from the day's budget. After
|
|
||||||
// StopChecking, Hit checks no client.
|
|
||||||
func (a *AbuseIPDB) StopChecking() {
|
|
||||||
a.mu.Lock()
|
|
||||||
|
|
||||||
a.stopped = true
|
|
||||||
for _, end := range a.checking {
|
|
||||||
end()
|
|
||||||
}
|
|
||||||
|
|
||||||
a.mu.Unlock()
|
|
||||||
|
|
||||||
a.checkGoroutines.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Checked returns how many checks were made.
|
// Checked returns how many checks were made.
|
||||||
func (a *AbuseIPDB) Checked() int {
|
func (a *AbuseIPDB) Checked() int {
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
@@ -344,8 +315,7 @@ func (a *AbuseIPDB) RunBlacklist(ctx context.Context) {
|
|||||||
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
|
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
|
||||||
// the score as client's, and notes the check as no longer under way. A
|
// the score as client's, and notes the check as no longer under way. A
|
||||||
// check that fails gives no score: it is counted, logged and raised as a
|
// check that fails gives no score: it is counted, logged and raised as a
|
||||||
// source_failure alert, and no client is checked for failureDelay. A check
|
// source_failure alert, and no client is checked for failureDelay.
|
||||||
// ended as ctx ends, by StopChecking, is no failure.
|
|
||||||
func (a *AbuseIPDB) check(
|
func (a *AbuseIPDB) check(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
client netip.Prefix,
|
client netip.Prefix,
|
||||||
@@ -353,26 +323,24 @@ func (a *AbuseIPDB) check(
|
|||||||
) {
|
) {
|
||||||
score, err := a.ask(ctx, addr)
|
score, err := a.ask(ctx, addr)
|
||||||
now := a.params.Now()
|
now := a.params.Now()
|
||||||
isFailure := err != nil && ctx.Err() == nil
|
|
||||||
|
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
|
|
||||||
delete(a.checking, client)
|
delete(a.checking, client)
|
||||||
|
|
||||||
switch {
|
if err == nil {
|
||||||
case err == nil:
|
|
||||||
a.scores.Add(
|
a.scores.Add(
|
||||||
client,
|
client,
|
||||||
Score{Client: client, Score: score, Fetched: now},
|
Score{Client: client, Score: score, Fetched: now},
|
||||||
)
|
)
|
||||||
case isFailure:
|
} else {
|
||||||
a.failures++
|
a.failures++
|
||||||
a.retryAt = now.Add(failureDelay)
|
a.retryAt = now.Add(failureDelay)
|
||||||
}
|
}
|
||||||
|
|
||||||
a.mu.Unlock()
|
a.mu.Unlock()
|
||||||
|
|
||||||
if isFailure {
|
if err != nil {
|
||||||
a.alert("checking a client with AbuseIPDB failed", err)
|
a.alert("checking a client with AbuseIPDB failed", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -394,43 +394,6 @@ func TestCheckNotAnsweredWithinTheTimeoutFailsAndHitNeverWaits(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestStopCheckingEndsTheCheckUnderWayAsNoFailureAndChecksNoMore(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
queue := newQueue()
|
|
||||||
p := abuseIPDBParams()
|
|
||||||
p.Alerts = queue
|
|
||||||
abuseIPDB := &abuseIPDBStandIn{hanging: true}
|
|
||||||
checker := newAbuseIPDB(abuseIPDB, p)
|
|
||||||
|
|
||||||
wantScore(t, checker, suspect, true, 0, false)
|
|
||||||
synctest.Wait()
|
|
||||||
|
|
||||||
// The check is ended, not waited out, and is no failure.
|
|
||||||
began := time.Now()
|
|
||||||
|
|
||||||
checker.StopChecking()
|
|
||||||
|
|
||||||
if waited := time.Since(began); waited != 0 {
|
|
||||||
t.Errorf("StopChecking returned after %s, want at once", waited)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantFailures(t, checker, 0)
|
|
||||||
|
|
||||||
if got := waiting(queue); len(got) != 0 {
|
|
||||||
t.Errorf("alerts waiting %+v, want none", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// An offender without a score is then not checked.
|
|
||||||
wantScore(t, checker, other, true, 0, false)
|
|
||||||
synctest.Wait()
|
|
||||||
wantChecked(t, abuseIPDB, suspect)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
|
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -82,12 +82,8 @@ type DNSBL struct {
|
|||||||
// verdicts are by query. Each is added as it is fetched and never moved
|
// verdicts are by query. Each is added as it is fetched and never moved
|
||||||
// up, so that the one fetched longest ago is the first dropped.
|
// up, so that the one fetched longest ago is the first dropped.
|
||||||
verdicts *simplelru.LRU[query, Verdict]
|
verdicts *simplelru.LRU[query, Verdict]
|
||||||
// asking are the queries under way, each with the function that ends
|
// asking are the queries under way.
|
||||||
// it. stopped is true once StopAsking has been called, and
|
asking map[query]bool
|
||||||
// queryGoroutines waits for the goroutine of each query.
|
|
||||||
asking map[query]context.CancelFunc
|
|
||||||
stopped bool
|
|
||||||
queryGoroutines sync.WaitGroup
|
|
||||||
// queries and failures count, by zone, the queries made and those that
|
// queries and failures count, by zone, the queries made and those that
|
||||||
// failed, and retryAt is when a zone whose last query failed may be
|
// failed, and retryAt is when a zone whose last query failed may be
|
||||||
// asked again.
|
// asked again.
|
||||||
@@ -124,7 +120,7 @@ func NewDNSBL(params DNSBLParams) *DNSBL {
|
|||||||
params: params,
|
params: params,
|
||||||
resolver: resolver,
|
resolver: resolver,
|
||||||
verdicts: verdicts,
|
verdicts: verdicts,
|
||||||
asking: map[query]context.CancelFunc{},
|
asking: map[query]bool{},
|
||||||
queries: map[string]int{},
|
queries: map[string]int{},
|
||||||
failures: map[string]int{},
|
failures: map[string]int{},
|
||||||
retryAt: map[string]time.Time{},
|
retryAt: map[string]time.Time{},
|
||||||
@@ -142,10 +138,9 @@ func (d *DNSBL) Zones() []string {
|
|||||||
// log, the alerts and the metrics. A verdict is used until CacheTTL has
|
// log, the alerts and the metrics. A verdict is used until CacheTTL has
|
||||||
// passed since it was fetched. Each zone without one is asked about addr
|
// passed since it was fetched. Each zone without one is asked about addr
|
||||||
// in the background, unless a query about addr to it is under way, the
|
// in the background, unless a query about addr to it is under way, the
|
||||||
// zone is left alone after a failure, maxQueries are under way, or
|
// zone is left alone after a failure, or maxQueries are under way;
|
||||||
// StopAsking has been called; ListedBy never waits for a query. ctx is the
|
// ListedBy never waits for a query. ctx is the context of the client's
|
||||||
// context of the client's request, and a query goes on after the request
|
// request, and a query goes on after the request ends.
|
||||||
// ends, until StopAsking.
|
|
||||||
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
defer d.mu.Unlock()
|
defer d.mu.Unlock()
|
||||||
@@ -158,47 +153,23 @@ func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
|||||||
q := query{zone: zone, client: addr}
|
q := query{zone: zone, client: addr}
|
||||||
|
|
||||||
kept, found := d.verdicts.Peek(q)
|
kept, found := d.verdicts.Peek(q)
|
||||||
_, asking := d.asking[q]
|
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
|
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
|
||||||
if kept.Listed {
|
if kept.Listed {
|
||||||
listedBy = append(listedBy, config.MaskZoneKey(zone))
|
listedBy = append(listedBy, config.MaskZoneKey(zone))
|
||||||
}
|
}
|
||||||
case !asking && !d.stopped && !now.Before(d.retryAt[zone]) &&
|
case !d.asking[q] && !now.Before(d.retryAt[zone]) && len(d.asking) < maxQueries:
|
||||||
len(d.asking) < maxQueries:
|
d.asking[q] = true
|
||||||
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
|
||||||
d.asking[q] = end
|
|
||||||
d.queries[zone]++
|
d.queries[zone]++
|
||||||
|
|
||||||
d.queryGoroutines.Go(func() {
|
go d.ask(context.WithoutCancel(ctx), q)
|
||||||
defer end()
|
|
||||||
|
|
||||||
d.ask(queryCtx, q)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return listedBy
|
return listedBy
|
||||||
}
|
}
|
||||||
|
|
||||||
// StopAsking ends the queries under way, and returns once their
|
|
||||||
// goroutines have. A query it ends gives no verdict and is no failure: it
|
|
||||||
// is neither counted, logged nor raised as an alert, and the zone is not
|
|
||||||
// left alone after it. After StopAsking, ListedBy asks no zone.
|
|
||||||
func (d *DNSBL) StopAsking() {
|
|
||||||
d.mu.Lock()
|
|
||||||
|
|
||||||
d.stopped = true
|
|
||||||
for _, end := range d.asking {
|
|
||||||
end()
|
|
||||||
}
|
|
||||||
|
|
||||||
d.mu.Unlock()
|
|
||||||
|
|
||||||
d.queryGoroutines.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Queries returns how many queries were made to zone.
|
// Queries returns how many queries were made to zone.
|
||||||
func (d *DNSBL) Queries(zone string) int {
|
func (d *DNSBL) Queries(zone string) int {
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
@@ -267,29 +238,27 @@ func (d *DNSBL) Load(verdicts []Verdict) {
|
|||||||
// query as no longer under way. A query that fails gives no verdict: it
|
// query as no longer under way. A query that fails gives no verdict: it
|
||||||
// is counted, logged and raised as a source_failure alert, which show the
|
// is counted, logged and raised as a source_failure alert, which show the
|
||||||
// zone with its key masked, and the zone is not asked again for
|
// zone with its key masked, and the zone is not asked again for
|
||||||
// failureDelay. A query ended as ctx ends, by StopAsking, is no failure.
|
// failureDelay.
|
||||||
func (d *DNSBL) ask(ctx context.Context, q query) {
|
func (d *DNSBL) ask(ctx context.Context, q query) {
|
||||||
listed, err := d.lookUp(ctx, q)
|
listed, err := d.lookUp(ctx, q)
|
||||||
now := d.params.Now()
|
now := d.params.Now()
|
||||||
isFailure := err != nil && ctx.Err() == nil
|
|
||||||
|
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
|
|
||||||
delete(d.asking, q)
|
delete(d.asking, q)
|
||||||
|
|
||||||
switch {
|
if err == nil {
|
||||||
case err == nil:
|
|
||||||
d.verdicts.Add(q, Verdict{
|
d.verdicts.Add(q, Verdict{
|
||||||
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
|
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
|
||||||
})
|
})
|
||||||
case isFailure:
|
} else {
|
||||||
d.failures[q.zone]++
|
d.failures[q.zone]++
|
||||||
d.retryAt[q.zone] = now.Add(failureDelay)
|
d.retryAt[q.zone] = now.Add(failureDelay)
|
||||||
}
|
}
|
||||||
|
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
if isFailure {
|
if err != nil {
|
||||||
const failed = "asking a DNSBL zone failed"
|
const failed = "asking a DNSBL zone failed"
|
||||||
|
|
||||||
shown := config.MaskZoneKey(q.zone)
|
shown := config.MaskZoneKey(q.zone)
|
||||||
|
|||||||
@@ -210,45 +210,6 @@ func TestQueryNotAnsweredWithinTheTimeoutFails(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
|
||||||
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
queue := newQueue()
|
|
||||||
p := dnsblParams(zone)
|
|
||||||
p.Alerts = queue
|
|
||||||
resolver := &resolverStandIn{hanging: true}
|
|
||||||
dnsbl := newDNSBL(resolver, p)
|
|
||||||
|
|
||||||
wantZones(t, dnsbl, listed)
|
|
||||||
synctest.Wait()
|
|
||||||
|
|
||||||
// The query is ended, not waited out, and is no failure.
|
|
||||||
began := time.Now()
|
|
||||||
|
|
||||||
dnsbl.StopAsking()
|
|
||||||
|
|
||||||
if waited := time.Since(began); waited != 0 {
|
|
||||||
t.Errorf("StopAsking returned after %s, want at once", waited)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantQueries(t, dnsbl, 1, 0)
|
|
||||||
|
|
||||||
if got := waiting(queue); len(got) != 0 {
|
|
||||||
t.Errorf("alerts waiting %+v, want none", got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A client without a verdict is then asked about in no zone.
|
|
||||||
wantZones(t, dnsbl, unlisted)
|
|
||||||
synctest.Wait()
|
|
||||||
wantQueries(t, dnsbl, 1, 0)
|
|
||||||
wantAsked(t, resolver, listedName)
|
|
||||||
waitForTheResolver()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||||
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(
|
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
// Package requestlog writes the request log sww prints on stdout: JSON
|
// Package requestlog writes the lines sww prints on stdout: one
|
||||||
// lines marked "type":"request", at most two from waf and a short one
|
// JSON object per request, two from waf and a short one from rproxy,
|
||||||
// from rproxy for each request. Package processlog writes the process's
|
// marked "type":"request", and the process's own messages as JSON lines marked
|
||||||
// own messages.
|
// "type":"process".
|
||||||
package requestlog
|
package requestlog
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"log/slog"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
@@ -299,3 +300,31 @@ func Milliseconds(d time.Duration) float64 {
|
|||||||
time.Millisecond/time.Microsecond,
|
time.Millisecond/time.Microsecond,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewProcessLogger returns the logger for the process's own messages:
|
||||||
|
// JSON lines on w, marked "type":"process", with the time in the same form
|
||||||
|
// as a request line's, and instanceName, SWW_WAF_INSTANCE_NAME, as instance.
|
||||||
|
// It writes only the messages at level, SWW_WAF_LOG_LEVEL, or more severe;
|
||||||
|
// the request lines Write writes are never held back.
|
||||||
|
func NewProcessLogger(
|
||||||
|
w io.Writer,
|
||||||
|
instanceName string,
|
||||||
|
level slog.Level,
|
||||||
|
) *slog.Logger {
|
||||||
|
handler := slog.NewJSONHandler(w, &slog.HandlerOptions{
|
||||||
|
Level: level,
|
||||||
|
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
|
||||||
|
if attr.Key == slog.TimeKey && len(groups) == 0 {
|
||||||
|
return slog.String(
|
||||||
|
slog.TimeKey,
|
||||||
|
FormatTime(attr.Value.Time()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return attr
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
return slog.New(handler).
|
||||||
|
With("type", "process", "instance", instanceName)
|
||||||
|
}
|
||||||
@@ -3,6 +3,8 @@ package requestlog_test
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"log/slog"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -132,3 +134,88 @@ func TestWriteRProxyLineWritesOneJSONLineWithOnlyItsFields(t *testing.T) {
|
|||||||
t.Errorf("wrote %s, want %s", out.String(), want)
|
t.Errorf("wrote %s, want %s", out.String(), want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
|
||||||
|
requestlog.NewProcessLogger(&out, "host1/blog", slog.LevelInfo).
|
||||||
|
Info("starting",
|
||||||
|
"version", "v1")
|
||||||
|
|
||||||
|
var fields map[string]any
|
||||||
|
|
||||||
|
err := json.Unmarshal(out.Bytes(), &fields)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decode %q: %v", out.String(), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if fields["type"] != "process" ||
|
||||||
|
fields["instance"] != "host1/blog" ||
|
||||||
|
fields["msg"] != "starting" ||
|
||||||
|
fields["level"] != "INFO" ||
|
||||||
|
fields["version"] != "v1" {
|
||||||
|
t.Errorf("process line %v", fields)
|
||||||
|
}
|
||||||
|
|
||||||
|
timeText, _ := fields["time"].(string)
|
||||||
|
|
||||||
|
logged, err := time.Parse(time.RFC3339, timeText)
|
||||||
|
if err != nil || !strings.HasSuffix(timeText, "Z") ||
|
||||||
|
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
|
||||||
|
time.Since(logged) > time.Minute {
|
||||||
|
t.Errorf(
|
||||||
|
"process line time %q, want now in UTC with milliseconds",
|
||||||
|
timeText,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
levels := []slog.Level{
|
||||||
|
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, level := range levels {
|
||||||
|
t.Run(level.String(), func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
|
||||||
|
processLog := requestlog.NewProcessLogger(
|
||||||
|
&out,
|
||||||
|
"host1/blog",
|
||||||
|
level,
|
||||||
|
)
|
||||||
|
for _, at := range levels {
|
||||||
|
processLog.Log(t.Context(), at, "message")
|
||||||
|
}
|
||||||
|
|
||||||
|
var got, want []string
|
||||||
|
|
||||||
|
for line := range strings.Lines(out.String()) {
|
||||||
|
var fields struct {
|
||||||
|
Level string `json:"level"`
|
||||||
|
}
|
||||||
|
|
||||||
|
err := json.Unmarshal([]byte(line), &fields)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decode %q: %v", line, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got = append(got, fields.Level)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, written := range levels[i:] {
|
||||||
|
want = append(want, written.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
if !slices.Equal(got, want) {
|
||||||
|
t.Errorf("lines at %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -99,12 +99,8 @@ type Client struct {
|
|||||||
// answers are by client. Each is added as it comes and never moved up,
|
// answers are by client. Each is added as it comes and never moved up,
|
||||||
// so that the one that came longest ago is the first dropped.
|
// so that the one that came longest ago is the first dropped.
|
||||||
answers *simplelru.LRU[netip.Prefix, Answer]
|
answers *simplelru.LRU[netip.Prefix, Answer]
|
||||||
// asking are the clients whose query is under way, each with the
|
// asking are the clients whose query is under way.
|
||||||
// function that ends it. stopped is true once StopAsking has been
|
asking map[netip.Prefix]bool
|
||||||
// called, and queryGoroutines waits for the goroutine of each query.
|
|
||||||
asking map[netip.Prefix]context.CancelFunc
|
|
||||||
stopped bool
|
|
||||||
queryGoroutines sync.WaitGroup
|
|
||||||
// retryAt is when routewatch may be asked again after a query failed.
|
// retryAt is when routewatch may be asked again after a query failed.
|
||||||
retryAt time.Time
|
retryAt time.Time
|
||||||
}
|
}
|
||||||
@@ -120,7 +116,7 @@ func New(params Params) *Client {
|
|||||||
params: params,
|
params: params,
|
||||||
httpClient: &http.Client{},
|
httpClient: &http.Client{},
|
||||||
answers: answers,
|
answers: answers,
|
||||||
asking: map[netip.Prefix]context.CancelFunc{},
|
asking: map[netip.Prefix]bool{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -128,9 +124,8 @@ func New(params Params) *Client {
|
|||||||
// IPv6 group, or why it has no route. It never waits for a query. A client
|
// IPv6 group, or why it has no route. It never waits for a query. A client
|
||||||
// without an answer is asked about in the background, by addr, the address its
|
// without an answer is asked about in the background, by addr, the address its
|
||||||
// request came from, unless its query is under way, a query failed less than
|
// request came from, unless its query is under way, a query failed less than
|
||||||
// failureKept ago, maxQueries are under way, or StopAsking has been called. ctx
|
// failureKept ago, or maxQueries are under way. ctx is the context of the
|
||||||
// is the context of the client's request, and a query goes on after the request
|
// client's request, and a query goes on after the request ends.
|
||||||
// ends, until StopAsking.
|
|
||||||
func (c *Client) Route(
|
func (c *Client) Route(
|
||||||
ctx context.Context, client netip.Prefix, addr netip.Addr,
|
ctx context.Context, client netip.Prefix, addr netip.Addr,
|
||||||
) (Route, Missing) {
|
) (Route, Missing) {
|
||||||
@@ -152,38 +147,15 @@ func (c *Client) Route(
|
|||||||
return Route{}, RequestFailed
|
return Route{}, RequestFailed
|
||||||
}
|
}
|
||||||
|
|
||||||
_, asking := c.asking[client]
|
if !c.asking[client] && len(c.asking) < maxQueries {
|
||||||
if !asking && !c.stopped && len(c.asking) < maxQueries {
|
c.asking[client] = true
|
||||||
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
|
||||||
c.asking[client] = end
|
|
||||||
|
|
||||||
c.queryGoroutines.Go(func() {
|
go c.ask(context.WithoutCancel(ctx), client, addr)
|
||||||
defer end()
|
|
||||||
|
|
||||||
c.ask(queryCtx, client, addr)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return Route{}, NoAnswerYet
|
return Route{}, NoAnswerYet
|
||||||
}
|
}
|
||||||
|
|
||||||
// StopAsking ends the queries under way, and returns once their goroutines
|
|
||||||
// have. A query it ends gives no answer and is no failure: it is not logged,
|
|
||||||
// and routewatch is not left alone after it. After StopAsking, Route asks
|
|
||||||
// routewatch nothing.
|
|
||||||
func (c *Client) StopAsking() {
|
|
||||||
c.mu.Lock()
|
|
||||||
|
|
||||||
c.stopped = true
|
|
||||||
for _, end := range c.asking {
|
|
||||||
end()
|
|
||||||
}
|
|
||||||
|
|
||||||
c.mu.Unlock()
|
|
||||||
|
|
||||||
c.queryGoroutines.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Snapshot returns every answer still in use, sorted by client, as
|
// Snapshot returns every answer still in use, sorted by client, as
|
||||||
// routewatch.json lists them.
|
// routewatch.json lists them.
|
||||||
func (c *Client) Snapshot() []Answer {
|
func (c *Client) Snapshot() []Answer {
|
||||||
@@ -233,8 +205,7 @@ func (c *Client) Load(answers []Answer) {
|
|||||||
// ask asks routewatch about addr, keeps the answer as client's, and notes the
|
// ask asks routewatch about addr, keeps the answer as client's, and notes the
|
||||||
// query as no longer under way. A query that fails gives no answer, and
|
// query as no longer under way. A query that fails gives no answer, and
|
||||||
// routewatch is not asked again for failureKept. Only the first failure of
|
// routewatch is not asked again for failureKept. Only the first failure of
|
||||||
// those under way is logged. A query ended as ctx ends, by StopAsking, is no
|
// those under way is logged.
|
||||||
// failure.
|
|
||||||
func (c *Client) ask(
|
func (c *Client) ask(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
client netip.Prefix,
|
client netip.Prefix,
|
||||||
@@ -242,20 +213,18 @@ func (c *Client) ask(
|
|||||||
) {
|
) {
|
||||||
route, err := c.query(ctx, addr)
|
route, err := c.query(ctx, addr)
|
||||||
now := c.params.Now()
|
now := c.params.Now()
|
||||||
isFailure := err != nil && ctx.Err() == nil
|
|
||||||
|
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
|
|
||||||
delete(c.asking, client)
|
delete(c.asking, client)
|
||||||
|
|
||||||
firstFailure := isFailure && !now.Before(c.retryAt)
|
firstFailure := err != nil && !now.Before(c.retryAt)
|
||||||
|
|
||||||
switch {
|
if err == nil {
|
||||||
case err == nil:
|
|
||||||
c.answers.Add(client, Answer{
|
c.answers.Add(client, Answer{
|
||||||
Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
|
Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
|
||||||
})
|
})
|
||||||
case isFailure:
|
} else {
|
||||||
c.retryAt = now.Add(failureKept)
|
c.retryAt = now.Add(failureKept)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -361,54 +361,6 @@ func TestHangingRoutewatchNeverDelaysAndFailsTheQuery(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// In a synctest bubble, as TestHangingRoutewatchNeverDelaysAndFailsTheQuery
|
|
||||||
// says.
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
f := &fixture{t: t, log: &syncBuffer{}}
|
|
||||||
f.client = routewatch.New(routewatch.Params{
|
|
||||||
URL: silentURL, Now: time.Now,
|
|
||||||
ProcessLog: slog.New(slog.NewTextHandler(f.log, nil)),
|
|
||||||
})
|
|
||||||
|
|
||||||
f.route()
|
|
||||||
synctest.Wait()
|
|
||||||
|
|
||||||
// The query is ended, not waited out.
|
|
||||||
began := time.Now()
|
|
||||||
|
|
||||||
f.client.StopAsking()
|
|
||||||
|
|
||||||
if waited := time.Since(began); waited != 0 {
|
|
||||||
t.Errorf("StopAsking returned after %s, want at once", waited)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A query started now would fail within waitLimit. None is, and the
|
|
||||||
// one ended is no failure: nothing is logged, and routewatch is not
|
|
||||||
// left alone.
|
|
||||||
for range 2 {
|
|
||||||
if _, missing := f.route(); missing != routewatch.NoAnswerYet {
|
|
||||||
t.Errorf(
|
|
||||||
"Route gave %q, want %q",
|
|
||||||
missing,
|
|
||||||
routewatch.NoAnswerYet,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
time.Sleep(waitLimit)
|
|
||||||
synctest.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
if f.log.String() != "" {
|
|
||||||
t.Errorf("process log %q, want nothing", f.log.String())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
|
func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
@@ -51,11 +51,11 @@ func TestServerHasTheFixedLimits(t *testing.T) {
|
|||||||
server := rproxy.New(rproxy.Params{
|
server := rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: io.Discard,
|
RequestLog: io.Discard,
|
||||||
ProcessLog: processlog.New(processlog.Params{
|
ProcessLog: requestlog.NewProcessLogger(
|
||||||
Output: io.Discard,
|
io.Discard,
|
||||||
InstanceName: "test",
|
"test",
|
||||||
Level: slog.LevelInfo,
|
slog.LevelInfo,
|
||||||
}),
|
),
|
||||||
})
|
})
|
||||||
|
|
||||||
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
|
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
@@ -370,11 +370,11 @@ func startServerLogging(
|
|||||||
server := rproxy.New(rproxy.Params{
|
server := rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: requestLog,
|
RequestLog: requestLog,
|
||||||
ProcessLog: processlog.New(processlog.Params{
|
ProcessLog: requestlog.NewProcessLogger(
|
||||||
Output: processLog,
|
processLog,
|
||||||
InstanceName: "test",
|
"test",
|
||||||
Level: slog.LevelInfo,
|
slog.LevelInfo,
|
||||||
}),
|
),
|
||||||
})
|
})
|
||||||
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(
|
listener, err := (&net.ListenConfig{}).Listen(
|
||||||
|
|||||||
@@ -2448,7 +2448,7 @@ func newParams(dir string) state.Params {
|
|||||||
AttackBanDuration: 7 * 24 * time.Hour,
|
AttackBanDuration: 7 * 24 * time.Hour,
|
||||||
MaxBans: 5000,
|
MaxBans: 5000,
|
||||||
}),
|
}),
|
||||||
Limiter: ratelimit.New(ratelimit.Params{MaxClients: 20000}),
|
Limiter: ratelimit.New(ratelimit.Limits{}, 20000),
|
||||||
GeoJS: geojs.New(geojs.Params{Now: midnight}),
|
GeoJS: geojs.New(geojs.Params{Now: midnight}),
|
||||||
Lists: reputation.New(reputation.Params{
|
Lists: reputation.New(reputation.Params{
|
||||||
BlocklistURLs: []string{
|
BlocklistURLs: []string{
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -23,11 +23,11 @@ import (
|
|||||||
// setting for either.
|
// setting for either.
|
||||||
func RunRProxy(ctx context.Context, params Params) int {
|
func RunRProxy(ctx context.Context, params Params) int {
|
||||||
hostname, _ := os.Hostname() // "" when the host has no name to give
|
hostname, _ := os.Hostname() // "" when the host has no name to give
|
||||||
processLog := processlog.New(processlog.Params{
|
processLog := requestlog.NewProcessLogger(
|
||||||
Output: params.Stdout,
|
params.Stdout,
|
||||||
InstanceName: hostname,
|
hostname,
|
||||||
Level: slog.LevelInfo,
|
slog.LevelInfo,
|
||||||
})
|
)
|
||||||
|
|
||||||
cfg, err := config.RProxyFromEnvironment(params.Viper)
|
cfg, err := config.RProxyFromEnvironment(params.Viper)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -660,9 +660,7 @@ func TestCoreRuleSetExclusionsFileThatCannotBeUsedStopsTheStartWhileTheCoreRuleS
|
|||||||
|
|
||||||
status := run(t.Context(), map[string]string{
|
status := run(t.Context(), map[string]string{
|
||||||
listenAddr: localhost + ":0", stateDir: t.TempDir(), ruleFilesDir: t.TempDir(),
|
listenAddr: localhost + ":0", stateDir: t.TempDir(), ruleFilesDir: t.TempDir(),
|
||||||
"SWW_WAF_CORE_RULE_SET_MODE": string(
|
"SWW_WAF_CORE_RULE_SET_MODE": config.CoreRuleSetModeOff,
|
||||||
config.CoreRuleSetModeOff,
|
|
||||||
),
|
|
||||||
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE": path,
|
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE": path,
|
||||||
}, out)
|
}, out)
|
||||||
if status != 1 {
|
if status != 1 {
|
||||||
|
|||||||
+32
-87
@@ -2,7 +2,6 @@ package sww
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net"
|
"net"
|
||||||
@@ -19,10 +18,10 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/state"
|
"sneak.berlin/go/smallwebwaf/internal/state"
|
||||||
@@ -43,11 +42,8 @@ const remoteLogStopTimeout = 2 * time.Second
|
|||||||
func RunWAF(ctx context.Context, params Params) int {
|
func RunWAF(ctx context.Context, params Params) int {
|
||||||
// Until the settings are read, the one message is an invalid setting's
|
// Until the settings are read, the one message is an invalid setting's
|
||||||
// error, which every SWW_WAF_LOG_LEVEL lets through.
|
// error, which every SWW_WAF_LOG_LEVEL lets through.
|
||||||
processLog := processlog.New(processlog.Params{
|
processLog := requestlog.NewProcessLogger(params.Stdout,
|
||||||
Output: params.Stdout,
|
config.InstanceName(params.Viper), slog.LevelError)
|
||||||
InstanceName: config.InstanceName(params.Viper),
|
|
||||||
Level: slog.LevelError,
|
|
||||||
})
|
|
||||||
|
|
||||||
cfg, err := config.WAFFromEnvironment(params.Viper)
|
cfg, err := config.WAFFromEnvironment(params.Viper)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -66,11 +62,11 @@ func RunWAF(ctx context.Context, params Params) int {
|
|||||||
stdout = io.MultiWriter(params.Stdout, remote)
|
stdout = io.MultiWriter(params.Stdout, remote)
|
||||||
}
|
}
|
||||||
|
|
||||||
processLog = processlog.New(processlog.Params{
|
processLog = requestlog.NewProcessLogger(
|
||||||
Output: stdout,
|
stdout,
|
||||||
InstanceName: cfg.InstanceName,
|
cfg.InstanceName,
|
||||||
Level: cfg.LogLevel,
|
cfg.LogLevel,
|
||||||
})
|
)
|
||||||
|
|
||||||
if remote != nil {
|
if remote != nil {
|
||||||
stopSending := startSending(ctx, remote, processLog)
|
stopSending := startSending(ctx, remote, processLog)
|
||||||
@@ -121,8 +117,7 @@ func RunWAF(ctx context.Context, params Params) int {
|
|||||||
"settings", cfg)
|
"settings", cfg)
|
||||||
|
|
||||||
// The last write follows the stop of every hook, and so the end of every
|
// The last write follows the stop of every hook, and so the end of every
|
||||||
// loop that changes the state or raises an alert, and of every ask of
|
// loop that changes the state or raises an alert. A hook of the state
|
||||||
// GeoJS, the DNSBL zones, AbuseIPDB and routewatch. A hook of the state
|
|
||||||
// files would stop before the loops of the parts built ahead of them.
|
// files would stop before the loops of the parts built ahead of them.
|
||||||
return run(ctx, app, failed, processLog, func() error {
|
return run(ctx, app, failed, processLog, func() error {
|
||||||
err := files.WriteAll()
|
err := files.WriteAll()
|
||||||
@@ -180,35 +175,11 @@ func newRuleFiles(
|
|||||||
return ruleFiles, nil
|
return ruleFiles, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// newCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
|
// newCoreRuleSet returns the Core Rule Set, as waf.NewCoreRuleSet does.
|
||||||
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
|
|
||||||
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
|
|
||||||
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
|
|
||||||
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
|
|
||||||
// coreruleset.New refuses is an error in every mode. While the mode is off, a
|
|
||||||
// set file is loaded with the Core Rule Set only to check it.
|
|
||||||
func newCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
|
func newCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
|
||||||
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
|
crs, err := waf.NewCoreRuleSet(cfg)
|
||||||
cfg.CoreRuleSetExclusionsFile == "" {
|
|
||||||
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
|
||||||
}
|
|
||||||
|
|
||||||
crs, err := coreruleset.New(coreruleset.Params{
|
|
||||||
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
|
|
||||||
DisabledRules: cfg.CoreRuleSetDisabledRules,
|
|
||||||
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
|
|
||||||
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
|
|
||||||
Apps: cfg.Apps,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, startError{
|
return nil, startError{"cannot use the Core Rule Set", err}
|
||||||
"cannot use the Core Rule Set",
|
|
||||||
fmt.Errorf("SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w", err),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
|
|
||||||
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return crs, nil
|
return crs, nil
|
||||||
@@ -221,7 +192,7 @@ func newLookupFile(
|
|||||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||||
alertQueue *alerts.Queue,
|
alertQueue *alerts.Queue,
|
||||||
) (*lookup.File, error) {
|
) (*lookup.File, error) {
|
||||||
if cfg.LookupSource != config.LookupSourceFile {
|
if cfg.LookupSource != "file" {
|
||||||
return nil, nil //nolint:nilnil // no lookup database, and no error
|
return nil, nil //nolint:nilnil // no lookup database, and no error
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -242,17 +213,14 @@ func newLookupFile(
|
|||||||
|
|
||||||
// newLimiter returns the table of clients, with the rate and byte limits.
|
// newLimiter returns the table of clients, with the rate and byte limits.
|
||||||
func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
|
func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
|
||||||
return ratelimit.New(ratelimit.Params{
|
return ratelimit.New(ratelimit.Limits{
|
||||||
Limits: ratelimit.Limits{
|
PerMinute: cfg.RateLimitPerMinute,
|
||||||
PerMinute: cfg.RateLimitPerMinute,
|
PerHour: cfg.RateLimitPerHour,
|
||||||
PerHour: cfg.RateLimitPerHour,
|
PerDay: cfg.RateLimitPerDay,
|
||||||
PerDay: cfg.RateLimitPerDay,
|
BytesPerMinute: cfg.BytesLimitPerMinute,
|
||||||
BytesPerMinute: cfg.BytesLimitPerMinute,
|
BytesPerHour: cfg.BytesLimitPerHour,
|
||||||
BytesPerHour: cfg.BytesLimitPerHour,
|
BytesPerDay: cfg.BytesLimitPerDay,
|
||||||
BytesPerDay: cfg.BytesLimitPerDay,
|
}, cfg.MaxTrackedClients)
|
||||||
},
|
|
||||||
MaxClients: cfg.MaxTrackedClients,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// newLedger returns the ledger of the bans, with the settings for them.
|
// newLedger returns the ledger of the bans, with the settings for them.
|
||||||
@@ -304,13 +272,11 @@ func newLists(
|
|||||||
return lists
|
return lists
|
||||||
}
|
}
|
||||||
|
|
||||||
// newDNSBL returns what asks the DNSBL zones, none asked yet. At the stop it
|
// newDNSBL returns what asks the DNSBL zones, none asked yet.
|
||||||
// ends the queries under way, as reputation.DNSBL.StopAsking says.
|
|
||||||
func newDNSBL(
|
func newDNSBL(
|
||||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
cfg *config.WAF, processLog *slog.Logger, alertQueue *alerts.Queue,
|
||||||
alertQueue *alerts.Queue,
|
|
||||||
) *reputation.DNSBL {
|
) *reputation.DNSBL {
|
||||||
dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{
|
return reputation.NewDNSBL(reputation.DNSBLParams{
|
||||||
Zones: cfg.DNSBLZones,
|
Zones: cfg.DNSBLZones,
|
||||||
Resolver: cfg.DNSBLResolver,
|
Resolver: cfg.DNSBLResolver,
|
||||||
CacheTTL: cfg.IPReputationCacheTTL,
|
CacheTTL: cfg.IPReputationCacheTTL,
|
||||||
@@ -319,17 +285,12 @@ func newDNSBL(
|
|||||||
ProcessLog: processLog,
|
ProcessLog: processLog,
|
||||||
Alerts: alertQueue,
|
Alerts: alertQueue,
|
||||||
})
|
})
|
||||||
|
|
||||||
lc.Append(fx.StopHook(dnsbl.StopAsking))
|
|
||||||
|
|
||||||
return dnsbl
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// newAbuseIPDB returns what checks clients with AbuseIPDB while
|
// newAbuseIPDB returns what checks clients with AbuseIPDB while
|
||||||
// SWW_WAF_ABUSEIPDB_KEY is set, none checked yet. Until waf stops, it
|
// SWW_WAF_ABUSEIPDB_KEY is set, none checked yet. Until waf stops, it
|
||||||
// fetches AbuseIPDB's blacklist as it is due, and bans the clients on it in
|
// fetches AbuseIPDB's blacklist as it is due, and bans the clients on it in
|
||||||
// ledger, as waf.BanAbuseIPDBBlacklist says. At the stop it ends the checks
|
// ledger, as waf.BanAbuseIPDBBlacklist says.
|
||||||
// under way, as reputation.AbuseIPDB.StopChecking says.
|
|
||||||
func newAbuseIPDB(
|
func newAbuseIPDB(
|
||||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||||
alertQueue *alerts.Queue, ledger *bans.Ledger,
|
alertQueue *alerts.Queue, ledger *bans.Ledger,
|
||||||
@@ -355,30 +316,24 @@ func newAbuseIPDB(
|
|||||||
})
|
})
|
||||||
|
|
||||||
lc.Append(background(abuseIPDB.RunBlacklist))
|
lc.Append(background(abuseIPDB.RunBlacklist))
|
||||||
lc.Append(fx.StopHook(abuseIPDB.StopChecking))
|
|
||||||
|
|
||||||
return abuseIPDB
|
return abuseIPDB
|
||||||
}
|
}
|
||||||
|
|
||||||
// newRouteWatch returns what asks routewatch about clients, or nil while
|
// newRouteWatch returns what asks routewatch about clients, or nil while
|
||||||
// SWW_WAF_ROUTEWATCH_URL is unset. At the stop it ends the queries under
|
// SWW_WAF_ROUTEWATCH_URL is unset.
|
||||||
// way, as routewatch.Client.StopAsking says.
|
|
||||||
func newRouteWatch(
|
func newRouteWatch(
|
||||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
cfg *config.WAF, processLog *slog.Logger,
|
||||||
) *routewatch.Client {
|
) *routewatch.Client {
|
||||||
if cfg.RouteWatchURL == "" {
|
if cfg.RouteWatchURL == "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
routeWatch := routewatch.New(routewatch.Params{
|
return routewatch.New(routewatch.Params{
|
||||||
URL: cfg.RouteWatchURL,
|
URL: cfg.RouteWatchURL,
|
||||||
Now: utcNow,
|
Now: utcNow,
|
||||||
ProcessLog: processLog,
|
ProcessLog: processLog,
|
||||||
})
|
})
|
||||||
|
|
||||||
lc.Append(fx.StopHook(routeWatch.StopAsking))
|
|
||||||
|
|
||||||
return routeWatch
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// newMetrics returns the metrics, with those read from the other parts as
|
// newMetrics returns the metrics, with those read from the other parts as
|
||||||
@@ -416,18 +371,12 @@ func newMetrics(
|
|||||||
|
|
||||||
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
|
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
|
||||||
// answer goes to the limiter and the ledger, as waf.AddLookup says, and each
|
// answer goes to the limiter and the ledger, as waf.AddLookup says, and each
|
||||||
// failure is handled as waf.GeoJSFailed says. At the stop it ends the request
|
// failure is handled as waf.GeoJSFailed says.
|
||||||
// under way, as geojs.GeoJS.StopAsking says.
|
|
||||||
func newGeoJS(
|
func newGeoJS(
|
||||||
lc fx.Lifecycle,
|
cfg *config.WAF, m *metrics.Metrics, limiter *ratelimit.Limiter,
|
||||||
cfg *config.WAF,
|
ledger *bans.Ledger, alertQueue *alerts.Queue, processLog *slog.Logger,
|
||||||
m *metrics.Metrics,
|
|
||||||
limiter *ratelimit.Limiter,
|
|
||||||
ledger *bans.Ledger,
|
|
||||||
alertQueue *alerts.Queue,
|
|
||||||
processLog *slog.Logger,
|
|
||||||
) *geojs.GeoJS {
|
) *geojs.GeoJS {
|
||||||
geoJS := geojs.New(geojs.Params{
|
return geojs.New(geojs.Params{
|
||||||
URL: geojs.URL,
|
URL: geojs.URL,
|
||||||
Answered: func(answer geojs.Answer) {
|
Answered: func(answer geojs.Answer) {
|
||||||
waf.AddLookup(cfg, limiter, ledger, answer)
|
waf.AddLookup(cfg, limiter, ledger, answer)
|
||||||
@@ -439,10 +388,6 @@ func newGeoJS(
|
|||||||
},
|
},
|
||||||
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
||||||
})
|
})
|
||||||
|
|
||||||
lc.Append(fx.StopHook(geoJS.StopAsking))
|
|
||||||
|
|
||||||
return geoJS
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// newWAFServer returns the server waf runs, with its parts and the time
|
// newWAFServer returns the server waf runs, with its parts and the time
|
||||||
|
|||||||
@@ -54,7 +54,7 @@ func (abuseIPDB) check(
|
|||||||
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
|
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
|
||||||
)
|
)
|
||||||
|
|
||||||
if rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
|
if rq.h.config.DNSBLAbuseIPDBAction != deny {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -86,14 +86,13 @@ func (rq *request) countedBytes() int64 {
|
|||||||
response, request := rq.out.bytes, rq.requestBytes
|
response, request := rq.out.bytes, rq.requestBytes
|
||||||
|
|
||||||
switch rq.h.config.BytesCounted {
|
switch rq.h.config.BytesCounted {
|
||||||
case config.BytesCountedResponse:
|
case "response":
|
||||||
return response
|
return response
|
||||||
case config.BytesCountedRequest:
|
case "request":
|
||||||
return request
|
return request
|
||||||
case config.BytesCountedBoth:
|
default: // both
|
||||||
|
return response + request
|
||||||
}
|
}
|
||||||
|
|
||||||
return response + request
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// banForLimit bans the client's netblock at now for a broken limit, the
|
// banForLimit bans the client's netblock at now for a broken limit, the
|
||||||
|
|||||||
@@ -63,8 +63,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
blocklisted := percentage{percent: whole}
|
blocklisted := percentage{percent: whole}
|
||||||
if rq.blocklisted &&
|
if rq.blocklisted && cfg.BlocklistAction == "limit" {
|
||||||
cfg.BlocklistAction == config.ReputationActionLimit {
|
|
||||||
blocklisted = percentage{
|
blocklisted = percentage{
|
||||||
cfg.BlocklistLimitPercent,
|
cfg.BlocklistLimitPercent,
|
||||||
"SWW_WAF_BLOCKLIST_ACTION",
|
"SWW_WAF_BLOCKLIST_ACTION",
|
||||||
@@ -73,7 +72,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
|
|
||||||
dnsblAbuseIPDBListed := percentage{percent: whole}
|
dnsblAbuseIPDBListed := percentage{percent: whole}
|
||||||
if (rq.dnsblListed || rq.abuseIPDBHit) &&
|
if (rq.dnsblListed || rq.abuseIPDBHit) &&
|
||||||
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit {
|
cfg.DNSBLAbuseIPDBAction == "limit" {
|
||||||
dnsblAbuseIPDBListed = percentage{
|
dnsblAbuseIPDBListed = percentage{
|
||||||
cfg.DNSBLAbuseIPDBLimitPercent,
|
cfg.DNSBLAbuseIPDBLimitPercent,
|
||||||
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
|
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -42,6 +41,5 @@ func (blocklists) check(
|
|||||||
// client, given listedBy, the blocklists that list it: it does when one lists
|
// client, given listedBy, the blocklists that list it: it does when one lists
|
||||||
// it and SWW_WAF_BLOCKLIST_ACTION is deny.
|
// it and SWW_WAF_BLOCKLIST_ACTION is deny.
|
||||||
func (rq *request) refusedByBlocklists(listedBy []string) bool {
|
func (rq *request) refusedByBlocklists(listedBy []string) bool {
|
||||||
return len(listedBy) > 0 && rq.h.config.BlocklistAction ==
|
return len(listedBy) > 0 && rq.h.config.BlocklistAction == deny
|
||||||
config.ReputationActionDeny
|
|
||||||
}
|
}
|
||||||
@@ -47,7 +47,7 @@ func (coreRuleSet) check(
|
|||||||
rq.line.CoreRuleSetScore = &result.Score
|
rq.line.CoreRuleSetScore = &result.Score
|
||||||
|
|
||||||
for _, id := range result.RuleIDs {
|
for _, id := range result.RuleIDs {
|
||||||
rq.h.metrics.CoreRuleSetMatched(string(cfg.CoreRuleSetMode), id)
|
rq.h.metrics.CoreRuleSetMatched(cfg.CoreRuleSetMode, id)
|
||||||
}
|
}
|
||||||
|
|
||||||
threshold := cfg.CoreRuleSetScoreThreshold
|
threshold := cfg.CoreRuleSetScoreThreshold
|
||||||
@@ -99,7 +99,7 @@ func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
|
|||||||
|
|
||||||
switch {
|
switch {
|
||||||
case rq.h.config.CoreRuleSetMode == config.CoreRuleSetModeDetect:
|
case rq.h.config.CoreRuleSetMode == config.CoreRuleSetModeDetect:
|
||||||
detail["mode"] = string(config.CoreRuleSetModeDetect)
|
detail["mode"] = config.CoreRuleSetModeDetect
|
||||||
case rq.h.config.Observe:
|
case rq.h.config.Observe:
|
||||||
detail["mode"] = "observe"
|
detail["mode"] = "observe"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,8 +32,7 @@ func (dnsbl) check(ctx context.Context, rq *request, _ time.Time) *verdict {
|
|||||||
rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone))
|
rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone))
|
||||||
}
|
}
|
||||||
|
|
||||||
if !rq.dnsblListed ||
|
if !rq.dnsblListed || rq.h.config.DNSBLAbuseIPDBAction != deny {
|
||||||
rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -49,8 +49,7 @@ func (rq *request) startLookUp(
|
|||||||
) <-chan lookupResult {
|
) <-chan lookupResult {
|
||||||
result := make(chan lookupResult, 1)
|
result := make(chan lookupResult, 1)
|
||||||
|
|
||||||
if rq.h.config.LookupSource == config.LookupSourceOff ||
|
if rq.h.config.LookupSource == "off" || !canBePlaced(rq.client) {
|
||||||
!canBePlaced(rq.client) {
|
|
||||||
result <- lookupResult{}
|
result <- lookupResult{}
|
||||||
|
|
||||||
return result
|
return result
|
||||||
@@ -58,7 +57,7 @@ func (rq *request) startLookUp(
|
|||||||
|
|
||||||
client := clientGroup(rq.h.config, rq.client)
|
client := clientGroup(rq.h.config, rq.client)
|
||||||
|
|
||||||
if rq.h.config.LookupSource == config.LookupSourceFile {
|
if rq.h.config.LookupSource == "file" {
|
||||||
result <- lookupResult{
|
result <- lookupResult{
|
||||||
lookedUp: true, answer: rq.h.lookupFile.LookUp(client),
|
lookedUp: true, answer: rq.h.lookupFile.LookUp(client),
|
||||||
}
|
}
|
||||||
|
|||||||
+37
-2
@@ -6,6 +6,7 @@ package waf
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"cmp"
|
"cmp"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -82,8 +83,8 @@ type Params struct {
|
|||||||
// LookupFile is the lookup database they are looked up in while
|
// LookupFile is the lookup database they are looked up in while
|
||||||
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
|
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
|
||||||
LookupFile *lookup.File
|
LookupFile *lookup.File
|
||||||
// CoreRuleSet is what inspects requests while
|
// CoreRuleSet is what NewCoreRuleSet returns: what inspects requests
|
||||||
// SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
|
// while SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
|
||||||
CoreRuleSet CoreRuleSetInspector
|
CoreRuleSet CoreRuleSetInspector
|
||||||
// Now tells the time by which requests are counted for the rate
|
// Now tells the time by which requests are counted for the rate
|
||||||
// limits, bans are made and run out, and GeoJS's answers are kept,
|
// limits, bans are made and run out, and GeoJS's answers are kept,
|
||||||
@@ -173,6 +174,40 @@ func newHandler(params Params) *handler {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
|
||||||
|
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
|
||||||
|
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
|
||||||
|
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
|
||||||
|
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
|
||||||
|
// coreruleset.New refuses is an error in every mode. While the mode is off, a
|
||||||
|
// set file is loaded with the Core Rule Set only to check it.
|
||||||
|
func NewCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
|
||||||
|
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
|
||||||
|
cfg.CoreRuleSetExclusionsFile == "" {
|
||||||
|
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
||||||
|
}
|
||||||
|
|
||||||
|
coreRuleSet, err := coreruleset.New(coreruleset.Params{
|
||||||
|
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
|
||||||
|
DisabledRules: cfg.CoreRuleSetDisabledRules,
|
||||||
|
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
|
||||||
|
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
|
||||||
|
Apps: cfg.Apps,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w",
|
||||||
|
err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
|
||||||
|
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
||||||
|
}
|
||||||
|
|
||||||
|
return coreRuleSet, nil
|
||||||
|
}
|
||||||
|
|
||||||
// handler is waf's handler. It holds what every request shares; what
|
// handler is waf's handler. It holds what every request shares; what
|
||||||
// belongs to one request is in a request.
|
// belongs to one request is in a request.
|
||||||
type handler struct {
|
type handler struct {
|
||||||
|
|||||||
+31
-56
@@ -20,11 +20,9 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
@@ -399,11 +397,11 @@ func startRProxy(
|
|||||||
server := rproxy.New(rproxy.Params{
|
server := rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: io.Discard,
|
RequestLog: io.Discard,
|
||||||
ProcessLog: processlog.New(processlog.Params{
|
ProcessLog: requestlog.NewProcessLogger(
|
||||||
Output: io.Discard,
|
io.Discard,
|
||||||
InstanceName: "rproxy",
|
"rproxy",
|
||||||
Level: slog.LevelInfo,
|
slog.LevelInfo,
|
||||||
}),
|
),
|
||||||
})
|
})
|
||||||
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(
|
listener, err := (&net.ListenConfig{}).Listen(
|
||||||
@@ -472,11 +470,11 @@ func newWAFWith(
|
|||||||
}
|
}
|
||||||
|
|
||||||
out := &output{}
|
out := &output{}
|
||||||
processLog := processlog.New(processlog.Params{
|
processLog := requestlog.NewProcessLogger(
|
||||||
Output: out,
|
out,
|
||||||
InstanceName: cfg.InstanceName,
|
cfg.InstanceName,
|
||||||
Level: cfg.LogLevel,
|
cfg.LogLevel,
|
||||||
})
|
)
|
||||||
|
|
||||||
ruleFiles, err := rules.Load(rules.Params{
|
ruleFiles, err := rules.Load(rules.Params{
|
||||||
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
|
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
|
||||||
@@ -506,20 +504,24 @@ func newWAFWith(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
coreRuleSet, err := waf.NewCoreRuleSet(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Core Rule Set: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
params := waf.Params{
|
params := waf.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: out,
|
RequestLog: out,
|
||||||
ProcessLog: processLog,
|
ProcessLog: processLog,
|
||||||
LookupFile: lookupFile,
|
LookupFile: lookupFile,
|
||||||
Now: now,
|
CoreRuleSet: coreRuleSet,
|
||||||
Rules: ruleFiles,
|
Now: now,
|
||||||
Alerts: alertQueue,
|
Rules: ruleFiles,
|
||||||
|
Alerts: alertQueue,
|
||||||
// The tests of the limit set it back, and hold the Core Rule Set
|
// The tests of the limit set it back, and hold the Core Rule Set
|
||||||
// with a stand-in.
|
// with a stand-in.
|
||||||
BesideLookupTimeLimit: time.Hour,
|
BesideLookupTimeLimit: time.Hour,
|
||||||
}
|
}
|
||||||
|
|
||||||
addCoreRuleSet(t, ¶ms)
|
|
||||||
addCounters(¶ms)
|
addCounters(¶ms)
|
||||||
addOutsideSources(¶ms, geojsURL, abuseIPDBCheckURL)
|
addOutsideSources(¶ms, geojsURL, abuseIPDBCheckURL)
|
||||||
|
|
||||||
@@ -528,30 +530,6 @@ func newWAFWith(
|
|||||||
return server, out, alertQueue
|
return server, out, alertQueue
|
||||||
}
|
}
|
||||||
|
|
||||||
// addCoreRuleSet adds to params the Core Rule Set, with the settings for it,
|
|
||||||
// while SWW_WAF_CORE_RULE_SET_MODE is not off.
|
|
||||||
func addCoreRuleSet(t *testing.T, params *waf.Params) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
cfg := params.Config
|
|
||||||
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
coreRuleSet, err := coreruleset.New(coreruleset.Params{
|
|
||||||
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
|
|
||||||
DisabledRules: cfg.CoreRuleSetDisabledRules,
|
|
||||||
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
|
|
||||||
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
|
|
||||||
Apps: cfg.Apps,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Core Rule Set: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
params.CoreRuleSet = coreRuleSet
|
|
||||||
}
|
|
||||||
|
|
||||||
// addCounters adds to params the metrics, the table of clients, the ledger
|
// addCounters adds to params the metrics, the table of clients, the ledger
|
||||||
// and the anomaly counters, as sww builds them from params' settings. The
|
// and the anomaly counters, as sww builds them from params' settings. The
|
||||||
// metrics read the ledger, the table of clients and the rules.
|
// metrics read the ledger, the table of clients and the rules.
|
||||||
@@ -562,17 +540,14 @@ func addCounters(params *waf.Params) {
|
|||||||
cfg.MetricsBusiestCountriesAndASNs,
|
cfg.MetricsBusiestCountriesAndASNs,
|
||||||
cfg.InstanceName,
|
cfg.InstanceName,
|
||||||
)
|
)
|
||||||
params.Limiter = ratelimit.New(ratelimit.Params{
|
params.Limiter = ratelimit.New(ratelimit.Limits{
|
||||||
Limits: ratelimit.Limits{
|
PerMinute: cfg.RateLimitPerMinute,
|
||||||
PerMinute: cfg.RateLimitPerMinute,
|
PerHour: cfg.RateLimitPerHour,
|
||||||
PerHour: cfg.RateLimitPerHour,
|
PerDay: cfg.RateLimitPerDay,
|
||||||
PerDay: cfg.RateLimitPerDay,
|
BytesPerMinute: cfg.BytesLimitPerMinute,
|
||||||
BytesPerMinute: cfg.BytesLimitPerMinute,
|
BytesPerHour: cfg.BytesLimitPerHour,
|
||||||
BytesPerHour: cfg.BytesLimitPerHour,
|
BytesPerDay: cfg.BytesLimitPerDay,
|
||||||
BytesPerDay: cfg.BytesLimitPerDay,
|
}, cfg.MaxTrackedClients)
|
||||||
},
|
|
||||||
MaxClients: cfg.MaxTrackedClients,
|
|
||||||
})
|
|
||||||
params.Ledger = bans.New(bans.Params{
|
params.Ledger = bans.New(bans.Params{
|
||||||
LimitBanDuration: cfg.LimitBanDuration,
|
LimitBanDuration: cfg.LimitBanDuration,
|
||||||
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
|
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
|
||||||
|
|||||||
@@ -7,6 +7,10 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// deny is the SWW_WAF_BLOCKLIST_ACTION and the SWW_WAF_DNSBL_ABUSEIPDB_ACTION that
|
||||||
|
// refuses the requests of a client a source lists.
|
||||||
|
const deny = "deny"
|
||||||
|
|
||||||
// noteListed notes each of sources, the URLs of the blocklists or the
|
// noteListed notes each of sources, the URLs of the blocklists or the
|
||||||
// DNSBL zones, their keys masked, that list the client, as noteHit does,
|
// DNSBL zones, their keys masked, that list the client, as noteHit does,
|
||||||
// with reason.
|
// with reason.
|
||||||
|
|||||||
@@ -421,7 +421,7 @@ func (rq *request) answerAtTheEnd() (geojs.Answer, bool) {
|
|||||||
return geojs.Answer{}, false
|
return geojs.Answer{}, false
|
||||||
}
|
}
|
||||||
|
|
||||||
if rq.h.config.LookupSource == config.LookupSourceFile {
|
if rq.h.config.LookupSource == "file" {
|
||||||
return rq.lookupAnswer, true
|
return rq.lookupAnswer, true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user