From 478c3affad4f3ecf15dfafccdb22fa0c2f393e34 Mon Sep 17 00:00:00 2001 From: clawbot Date: Fri, 13 Nov 2026 19:08:21 +0000 Subject: [PATCH] Remove the CrowdSec decision feed (closes #315) sneak ruled to remove it. Gone: the heuristic crowdsec, the ban cause crowdsec and its earlier_bans count, the tag crowdsec, the settings SWW_WAF_CROWDSEC_LAPI_URL and SWW_WAF_CROWDSEC_LAPI_KEY, the decision list fetch and parse in internal/reputation, their tests, and every mention in the README, SPEC.md and docs/. EVALUATION.md keeps its comparison of tools. The ban ledger's internal ban function loses its expires parameter, which only CrowdSec bans used. Model: opus-5-5 --- README.md | 8 +- SPEC.md | 51 +-- docs/alerts.md | 8 +- docs/request-log.md | 5 +- docs/settings.md | 15 +- docs/state-files.md | 2 +- internal/alerts/alerts.go | 5 +- internal/bans/bans.go | 143 ++---- internal/bans/crowdsec_test.go | 136 ------ internal/config/waf.go | 82 +--- internal/config/waf_test.go | 153 +------ internal/metrics/metrics.go | 23 +- internal/reputation/crowdsec_test.go | 622 --------------------------- internal/reputation/export_test.go | 1 - internal/reputation/reputation.go | 191 +------- internal/requestlog/requestlog.go | 8 +- internal/state/state.go | 7 +- internal/state/state_test.go | 26 +- internal/sww/sww_test.go | 86 ---- internal/waf/answertime_test.go | 20 - internal/waf/bans.go | 52 --- internal/waf/crowdsec.go | 54 --- internal/waf/crowdsec_test.go | 249 ----------- internal/waf/heuristics.go | 1 - internal/waf/proxy.go | 16 +- internal/waf/reputation.go | 3 +- internal/waf/waf.go | 9 +- 27 files changed, 125 insertions(+), 1851 deletions(-) delete mode 100644 internal/bans/crowdsec_test.go delete mode 100644 internal/reputation/crowdsec_test.go delete mode 100644 internal/waf/crowdsec.go delete mode 100644 internal/waf/crowdsec_test.go diff --git a/README.md b/README.md index ee52a25..24544b2 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,7 @@ It protects against: - give the clients of chosen networks and countries a fraction of every limit, so that they are banned sooner; - refuse or limit clients on a blocklist, a DNS blocklist, or with a high - AbuseIPDB score; - - ban clients named by your own CrowdSec engine. + AbuseIPDB score. - Slow and oversized requests. Every request and every answer has a time limit and a size limit. @@ -290,8 +289,6 @@ Every setting the code reads, with its default: | `SWW_WAF_ABUSEIPDB_KEY` | unset | The key of your AbuseIPDB account; set, a client that has committed an offence is checked in the background. | | `SWW_WAF_ABUSEIPDB_MIN_SCORE` | `75` | The least AbuseIPDB abuse confidence score, from 0 to 100, that is a hit. | | `SWW_WAF_ABUSEIPDB_DAILY_BUDGET` | `900` | The most AbuseIPDB checks made in a day. | -| `SWW_WAF_CROWDSEC_LAPI_URL` | unset | The local API of a CrowdSec engine you run, such as `http://172.17.0.1:8080`; its decision list is fetched every minute, and a listed client is banned. | -| `SWW_WAF_CROWDSEC_LAPI_KEY` | unset | The key the engine gave for `smallwebwaf`, as `cscli bouncers add smallwebwaf` prints it; set with the URL, never alone. | | `SWW_WAF_DNSBL_ABUSEIPDB_ACTION` | `limit:25` | What a client listed by a DNS blocklist, or scored a hit by AbuseIPDB, gets: `deny`, `limit:` or `log`. | | `SWW_WAF_IP_REPUTATION_CACHE_TTL` | `24h` | How long a DNS blocklist verdict or an AbuseIPDB score is used. | | `SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS` | `2000` | How long a DNS blocklist query or an AbuseIPDB check may take, in milliseconds. | @@ -570,8 +567,7 @@ Each request gets tags from what `waf` knows of its client, written in the - `asn=` and `country=`, from the lookup; - one tag per list of `SWW_WAF_TAG_LISTS` that holds the client, such as `torexit` or `vpn=mullvad`; -- `crowdsec`, `dnsbl=` and `abuseipdb`, when that reputation source lists - the client; +- `dnsbl=` and `abuseipdb`, when that reputation source lists the client; - `exploitsignature`, when a rule file or the Core Rule Set matched the request; - `asn=` and `prefix=`, from a routewatch instance, while `SWW_WAF_ROUTEWATCH_URL` is set. diff --git a/SPEC.md b/SPEC.md index ad75904..f25cb5e 100644 --- a/SPEC.md +++ b/SPEC.md @@ -176,7 +176,6 @@ field gives that name. `refusal_limit` and `byte_limits` give theirs in 8. The reputation sources (see "Reputation"), in this order: - A client on a fetched blocklist gets `SWW_WAF_BLOCKLIST_ACTION` (`blocklists`). - - A client on the CrowdSec decision list is banned (`crowdsec`). - A client listed by a DNS blocklist gets `SWW_WAF_DNSBL_ABUSEIPDB_ACTION` (`dnsbl`). - A client scored at least `SWW_WAF_ABUSEIPDB_MIN_SCORE` by AbuseIPDB gets @@ -301,14 +300,6 @@ without a ban, because the Core Rule Set has false positives. A `block` rule does the same. Those refusals count toward the refusal limit, so a client that keeps setting them off is banned for a broken limit. -A client on the CrowdSec decision list is banned until the decision ends, with -the cause `crowdsec`. Only a client that sends a request is banned, so a long -list does not fill `bans.json`. Such a ban is never made permanent. It does not -make the netblock's next ban for a broken limit longer. A ban lifted while the -decision is in force is made again at the client's next request. To lift it for -good, delete the decision in CrowdSec, then lift the ban once the list has been -fetched again. - These refuse each request they cover, record nothing in `bans.json`, and count no offence: @@ -338,10 +329,10 @@ toward `SWW_WAF_MAX_BANS`, so such bans can never fill the table. A request refused under a ban that runs out is logged with the action `temporarily_banned`. Under a permanent ban the action is `permanently_banned`. -A request refused for a clear sign of attack or a CrowdSec decision takes the -action for the ban it made. A request that makes a ban permanent is logged as -`permanently_banned`. A request over a rate limit is logged as -`client_is_rate_limited`, whichever ban it makes. +A request refused for a clear sign of attack takes the action for the ban it +made. A request that makes a ban permanent is logged as `permanently_banned`. A +request over a rate limit is logged as `client_is_rate_limited`, whichever ban +it makes. ## Settings @@ -575,9 +566,9 @@ Each tag and where it comes from: - One tag per list of `SWW_WAF_TAG_LISTS` that holds the client (`tag_lists`), the tag being the list's name, such as `torexit` or `vpn=mullvad`. These lists are fetched and kept like a blocklist. `waf` ships none of its own. -- `crowdsec`, `dnsbl=` and `abuseipdb`, when that reputation source lists - the client, whatever the source's action. A zone with a key is named with the - key masked, as the `reputation` field gives it. +- `dnsbl=` and `abuseipdb`, when that reputation source lists the client, + whatever the source's action. A zone with a key is named with the key masked, + as the `reputation` field gives it. - `exploitsignature`, when a rule file or the Core Rule Set matched the request. For the Core Rule Set a match is a score at or over `SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD`. @@ -1960,12 +1951,11 @@ to the surrounding netblock. The four ban lengths are under "Bans". ### Reputation No source is on by default. A source acts on a list kept elsewhere; the defaults -judge a client only by what it does here. AbuseIPDB and CrowdSec also need an -account or an engine of the operator's own. The list best suited to be on by -default, Spamhaus DROP, has terms, below, that forbid frequent fetching, and -Spamhaus may block an address that breaks them. Each `waf` fetches its own copy, -so several of them on one host, sharing one address, would often fetch too close -together. +judge a client only by what it does here. AbuseIPDB also needs an account of the +operator's own. The list best suited to be on by default, Spamhaus DROP, has +terms, below, that forbid frequent fetching, and Spamhaus may block an address +that breaks them. Each `waf` fetches its own copy, so several of them on one +host, sharing one address, would often fetch too close together. - Blocklists (`SWW_WAF_BLOCKLIST_URLS`) are text files of addresses and netblocks, one to a line, as the Spamhaus DROP list is written. Anything after @@ -1998,18 +1988,6 @@ together. percentage of every request and byte limit, and suits a list of addresses shared with ordinary visitors, such as Tor exits. `log` only notes the hit in the log line and raises an alert. -- A CrowdSec engine on the host (`SWW_WAF_CROWDSEC_LAPI_URL`) has its decision - list fetched every minute, with `SWW_WAF_CROWDSEC_LAPI_KEY` in the `X-Api-Key` - header. The list is kept like a fetched blocklist, and a client on it is - banned (see "Bans"). Only decisions of the type `ban` on an address or a - netblock are used; any other, such as a captcha or a country, is left out. A - decision lasts for the `duration` it had left when the engine answered. This - is how a fleet-wide blocklist can arrive without `waf` depending on CrowdSec. - A fetch fails when: - - the engine answers other than `200`, a redirect included; - - the fetch does not finish within a minute; - - the answer is longer than 16 MiB; - - the answer is not a list of decisions. - Each DNS blocklist (`SWW_WAF_DNSBL_ZONES`) is asked about a client's own address in the background, the first time the client is seen. No request waits for the answer. The name asked about is the one RFC 5782 gives: the numbers of @@ -2437,14 +2415,13 @@ client without an answer by then counts as unknown. No question waits on these, which run in the background: - the DNS blocklists and AbuseIPDB; -- fetching the blocklists, the tag lists, the CrowdSec decision list and the - file `SWW_WAF_ASN_LIMIT_PERCENT_URL` names. +- fetching the blocklists, the tag lists and the file + `SWW_WAF_ASN_LIMIT_PERCENT_URL` names. In `enforce` mode, a question does not wait for GeoJS when it is a request: - from a client on a fetched blocklist, while `SWW_WAF_BLOCKLIST_ACTION` is `deny`; -- from a client on the CrowdSec decision list; - that would take its client over a rate limit at 100% of it, since a percentage can only lower a limit. diff --git a/docs/alerts.md b/docs/alerts.md index 5b7bb26..11918c5 100644 --- a/docs/alerts.md +++ b/docs/alerts.md @@ -13,9 +13,8 @@ back or dropped. threshold. The request was refused unless the alert's `detail` gives `mode`: `detect` or `observe`. - `anomaly`: a count over an anomaly threshold. -- `reputation_hit`: a request from a client on a blocklist, the CrowdSec - decision list or a DNS blocklist, or with an AbuseIPDB score that is a hit. - One alert per source. +- `reputation_hit`: a request from a client on a blocklist or a DNS blocklist, + or with an AbuseIPDB score that is a hit. One alert per source. - `source_failure`: one of: - GeoJS failing; - a list that cannot be fetched; @@ -69,8 +68,7 @@ on one line: "earlier_bans": { "limit": 0, "attack": 0, - "admin": 0, - "crowdsec": 0 + "admin": 0 } } }, diff --git a/docs/request-log.md b/docs/request-log.md index f3a1209..64452b4 100644 --- a/docs/request-log.md +++ b/docs/request-log.md @@ -97,7 +97,7 @@ Every one of these lines has: - `client_denied`: refused for `SWW_WAF_DENY_NETS`, for a reputation source with the action `deny`, or for a tag in `SWW_WAF_TAG_DENY`; - `temporarily_banned`: refused under a ban that runs out, or for a `ban` - rule, a trap path or a CrowdSec decision that made such a ban; + rule or a trap path that made such a ban; - `permanently_banned`: refused under a ban that never runs out, including a ban this request made or made permanent; - `country_denied`, `client_is_rate_limited`, `blocked_by_block_rule` and @@ -118,8 +118,6 @@ Every one of these lines has: - `countries`: `country`, and `setting`, the country list that refused it; - `tag_deny`: `tag`, the tag of `SWW_WAF_TAG_DENY` the request carries; - `blocklists`: `lists`, the blocklist URLs that list the client; - - `crowdsec`: `scenario`, the decision's scenario, and `expires`, when the - decision ends; - `dnsbl`: `zones`, the DNS blocklist zones that list the client, their keys masked; - `abuseipdb`: `score`, and `min_score`, `SWW_WAF_ABUSEIPDB_MIN_SCORE`; @@ -171,7 +169,6 @@ Every one of these lines has: and the rules that matched, for a request it inspected. - `reputation`: the sources the client is on, in this order: - blocklist URLs; - - the CrowdSec decision list's URL; - DNS blocklist zones; - `abuseipdb`. - `tags`: the request's tags, sorted, left out when it has none. Each tag and diff --git a/docs/settings.md b/docs/settings.md index 3a08ca1..765c1bf 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -180,8 +180,8 @@ in [`SPEC.md`](../SPEC.md). `name=URL` pairs, such as `torexit=https://example.com/tor.txt`. A name is the tag, and may hold `=`, such as `vpn=mullvad`. Each list is an `http` or `https` URL fetched like a blocklist. A name or URL listed twice is refused. - So is a URL that is also in `SWW_WAF_BLOCKLIST_URLS`, - `SWW_WAF_ASN_LIMIT_PERCENT_URL`, or the CrowdSec decision list. + So is a URL that is also in `SWW_WAF_BLOCKLIST_URLS` or is + `SWW_WAF_ASN_LIMIT_PERCENT_URL`. - `SWW_WAF_TAG_DENY` (default empty): tags whose requests are refused, such as `torexit,country=CN`. A tag listed twice, an `asn=` tag without a number, or a `country=` tag without a country, is refused. @@ -214,9 +214,6 @@ tag that no request can get stops the start. A request can get only these: `SWW_WAF_CORE_RULE_SET_MODE` nor `SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD` is `off`. -`crowdsec` stops the start in both settings, because a CrowdSec decision refuses -the request first. - ## Biased thresholds - `SWW_WAF_ASN_BYTES_LIMIT_PERCENT` and `SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT` @@ -333,14 +330,6 @@ the request first. score, from 0 to 100, that is a hit. - `SWW_WAF_ABUSEIPDB_DAILY_BUDGET` (default `900`): the most checks made in a day, a whole number above zero. AbuseIPDB's free accounts may make 1,000. -- `SWW_WAF_CROWDSEC_LAPI_URL` (default unset): the local API of a CrowdSec - engine you run, such as `http://172.17.0.1:8080`. An `http` or `https` URL - without a user or a fragment; its decision list is at `v1/decisions` under it. - Set without the key, it is refused. So is a decision list URL that is also in - `SWW_WAF_BLOCKLIST_URLS`, or that is `SWW_WAF_ASN_LIMIT_PERCENT_URL`. -- `SWW_WAF_CROWDSEC_LAPI_KEY` (default unset): the key the engine gave for - `smallwebwaf`, as `cscli bouncers add smallwebwaf` prints it. Set without the - URL, it is refused. - `SWW_WAF_DNSBL_ABUSEIPDB_ACTION` (default `limit:25`): `deny`, `limit:` or `log`, for DNS blocklist and AbuseIPDB hits. - `SWW_WAF_IP_REPUTATION_CACHE_TTL` (default `24h`): how long a zone's verdict diff --git a/docs/state-files.md b/docs/state-files.md index b7c358f..af6643d 100644 --- a/docs/state-files.md +++ b/docs/state-files.md @@ -25,7 +25,7 @@ Active and past bans: every ban an admin made, and up to `SWW_WAF_MAX_BANS` of those `waf` made. Per entry: - `netblock`, `start` and `expires` (`null` for a permanent ban); -- `cause`: `attack`, `limit`, `admin` or `crowdsec`; +- `cause`: `attack`, `limit` or `admin`; - `reason`: a short reason; - `lifted`: when an admin lifted it, if one did; - `notes`: the notes below. diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go index 3b1b308..452c77b 100644 --- a/internal/alerts/alerts.go +++ b/internal/alerts/alerts.go @@ -48,9 +48,8 @@ const ( // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD, refused in block mode, let through in // detect mode. EventCoreRuleSetMatch = "core_rule_set_match" - // EventReputationHit is a request whose client a blocklist, the - // CrowdSec decision list or a DNSBL zone lists, or whose AbuseIPDB score - // is a hit. + // EventReputationHit is a request whose client a blocklist or a DNSBL + // zone lists, or whose AbuseIPDB score is a hit. EventReputationHit = "reputation_hit" // EventSourceFailure is GeoJS failing or refusing sww, a fetch // of a list failing, a query to a DNSBL zone or a check with AbuseIPDB diff --git a/internal/bans/bans.go b/internal/bans/bans.go index 3579399..b581382 100644 --- a/internal/bans/bans.go +++ b/internal/bans/bans.go @@ -1,10 +1,8 @@ // Package bans is the ban ledger: the bans sww makes on the // netblocks of clients that break a rate limit, a byte limit or the refusal -// limit, show a clear sign of attack or are listed by the CrowdSec -// decision list, and -// those an admin makes, with their notes, as the "Bans" section of SPEC.md -// describes. The bans are kept in memory, and written to bans.json and -// read from it by the state package. +// limit or show a clear sign of attack, and those an admin makes, with their +// notes, as the "Bans" section of SPEC.md describes. The bans are kept in +// memory, and written to bans.json and read from it by the state package. package bans import ( @@ -28,9 +26,6 @@ const ( // CauseAdmin is a ban an admin made, or one sww made that an // admin keeps. It is never dropped. CauseAdmin = "admin" - // CauseCrowdSec is a ban sww made for a client the CrowdSec - // decision list lists. It ends when CrowdSec's decision does. - CauseCrowdSec = "crowdsec" ) // repeatFactor is how many times as long as the netblock's last ban a ban @@ -45,8 +40,8 @@ type Params struct { // LimitBanDuration is how long a first ban for a broken limit lasts. LimitBanDuration time.Duration // LimitBanRepeatWindow is how soon after the end of the netblock's - // ban that ended last, other than one for a clear sign of attack or for - // CrowdSec's decision, a broken limit counts as a repeat, which bans for + // ban that ended last, other than one for a clear sign of attack, a + // broken limit counts as a repeat, which bans for // repeatFactor times as long as that ban. LimitBanRepeatWindow time.Duration // A ban for a broken limit that would be longer than @@ -68,11 +63,10 @@ type Ban struct { Start time.Time // Expires is when the ban ends, zero for a permanent ban. Expires time.Time - // Cause is CauseLimit, CauseAttack, CauseAdmin or CauseCrowdSec. + // Cause is CauseLimit, CauseAttack or CauseAdmin. Cause string - // Reason is a short text: for a ban sww made, the limit broken, - // the rule that matched or the scenario of CrowdSec's decision; for an - // admin's, what the admin wrote. + // Reason is a short text: for a ban sww made, the limit broken or + // the rule that matched; for an admin's, what the admin wrote. Reason string // Lifted is when an admin lifted the ban, zero while no admin has. A // lifted ban refuses nothing, and does not make the netblock's next @@ -133,8 +127,7 @@ type Notes struct { // log's reputation names them. It is left out when none did. Reputation []ReputationHit `json:"reputation,omitempty"` // Request is the request that broke the limit, or whose bytes broke - // it, that was the clear sign of attack, or that came from a client the - // CrowdSec decision list lists. + // it, or that was the clear sign of attack. Request Request `json:"request"` // Requests is how many requests the netblock has sent since it was // first seen, and Refused how many of them the ban has refused so @@ -148,9 +141,8 @@ type Notes struct { // ReputationHit is a reputation source that listed a client, as a // reputation_hit alert's detail gives it: Source is the URL of the -// blocklist or of the CrowdSec decision list, the DNSBL zone with its key -// masked, or "abuseipdb", and Score, for AbuseIPDB alone, its score of the -// client. +// blocklist, the DNSBL zone with its key masked, or "abuseipdb", and Score, +// for AbuseIPDB alone, its score of the client. type ReputationHit struct { Source string `json:"source"` Score *int64 `json:"score,omitempty"` @@ -158,10 +150,9 @@ type ReputationHit struct { // EarlierBans counts a netblock's bans before a ban, by cause. type EarlierBans struct { - Limit int `json:"limit"` - Attack int `json:"attack"` - Admin int `json:"admin"` - CrowdSec int `json:"crowdsec"` + Limit int `json:"limit"` + Attack int `json:"attack"` + Admin int `json:"admin"` } // Request is a request in a ban's notes. Each text is cut to 256 bytes. @@ -303,8 +294,7 @@ func activeBan(bans []Ban, now time.Time) *Ban { // BanForLimit bans netblock at now for a broken limit, with notes, and // returns the ban, and true. A first ban lasts LimitBanDuration. A ban // made within LimitBanRepeatWindow after the netblock's ban that ended -// last, other than one for a clear sign of attack or for CrowdSec's -// decision, or a lifted one, lasts +// last, other than one for a clear sign of attack or a lifted one, lasts // repeatFactor times as long as that one. A ban that would be longer // than LimitBanPermanentAbove is permanent instead. If a ban on netblock is still // active, as when two of its requests break a limit at once, that ban is @@ -318,7 +308,6 @@ func (l *Ledger) BanForLimit( return l.ban( netblock, now, - time.Time{}, CauseLimit, limitReason(notes), notes, @@ -334,7 +323,6 @@ func (l *Ledger) WouldBanForLimit( return l.ban( netblock, now, - time.Time{}, CauseLimit, limitReason(notes), notes, @@ -353,7 +341,6 @@ func (l *Ledger) BanForAttack( return l.ban( netblock, now, - time.Time{}, CauseAttack, attackReason(notes), notes, @@ -369,7 +356,6 @@ func (l *Ledger) WouldBanForAttack( return l.ban( netblock, now, - time.Time{}, CauseAttack, attackReason(notes), notes, @@ -377,51 +363,9 @@ func (l *Ledger) WouldBanForAttack( ) } -// BanForCrowdSec bans netblock at now until expires, when CrowdSec's -// decision on the client ends, with notes, and returns the ban, and -// whether it made it, as BanForLimit does. Its reason is "CrowdSec's -// decision for ", the scenario that made the decision. -func (l *Ledger) BanForCrowdSec( - netblock netip.Prefix, - now, expires time.Time, - scenario string, - notes Notes, -) (Ban, bool) { - return l.ban( - netblock, - now, - expires, - CauseCrowdSec, - crowdSecReason(scenario), - notes, - true, - ) -} - -// WouldBanForCrowdSec returns what BanForCrowdSec would, without making -// the ban: what observe mode would have done. -func (l *Ledger) WouldBanForCrowdSec( - netblock netip.Prefix, - now, expires time.Time, - scenario string, - notes Notes, -) (Ban, bool) { - return l.ban( - netblock, - now, - expires, - CauseCrowdSec, - crowdSecReason(scenario), - notes, - false, - ) -} - -// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit, -// CauseAttack or CauseCrowdSec, made at now would be permanent, as -// BanForLimit, BanForAttack or BanForCrowdSec would make it. It works out -// nothing else of the ban. A ban for CrowdSec's decision is never -// permanent: it ends with the decision. +// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit +// or CauseAttack, made at now would be permanent, as BanForLimit or +// BanForAttack would make it. It works out nothing else of the ban. func (l *Ledger) WouldBePermanent( netblock netip.Prefix, now time.Time, cause string, ) bool { @@ -433,14 +377,11 @@ func (l *Ledger) WouldBePermanent( held = *bans } - switch cause { - case CauseAttack: + if cause == CauseAttack { return l.attackExpiry(held, now).IsZero() - case CauseLimit: - return l.limitExpiry(held, now).IsZero() - default: // CauseCrowdSec - return false } + + return l.limitExpiry(held, now).IsZero() } // limitReason is the reason of a ban for a broken limit, with notes. @@ -459,12 +400,6 @@ func attackReason(notes Notes) string { return "matched the rule " + notes.RuleID } -// crowdSecReason is the reason of a ban for CrowdSec's decision, which -// scenario made. -func crowdSecReason(scenario string) string { - return "CrowdSec's decision for " + scenario -} - // BanForAdmin bans netblock at now for an admin, with reason, until // expires, or for good when expires is zero, and returns the ban, whose // cause is CauseAdmin. Unlike BanForLimit and BanForAttack, it makes the @@ -704,15 +639,13 @@ func (l *Ledger) holds(netblock netip.Prefix, start time.Time) bool { }) } -// ban bans netblock at now for cause, with reason and notes, as -// BanForLimit, BanForAttack and BanForCrowdSec describe, and returns the -// ban, and whether it made it. expires is when a ban for CauseCrowdSec -// ends, and zero for the others, whose end the ledger works out. Unless -// keep is true, the ban is not made, only returned: it is the ban that -// would have been made. +// ban bans netblock at now for cause, CauseLimit or CauseAttack, with +// reason and notes, as BanForLimit and BanForAttack describe, and returns +// the ban, and whether it made it. Unless keep is true, the ban is not +// made, only returned: it is the ban that would have been made. func (l *Ledger) ban( netblock netip.Prefix, - now, expires time.Time, + now time.Time, cause, reason string, notes Notes, keep bool, @@ -743,13 +676,10 @@ func (l *Ledger) ban( Notes: notes, } - switch cause { - case CauseAttack: + if cause == CauseAttack { ban.Expires = l.attackExpiry(held, now) - case CauseLimit: + } else { ban.Expires = l.limitExpiry(held, now) - default: // CauseCrowdSec - ban.Expires = expires } if !keep { @@ -784,16 +714,13 @@ func earlierBans(held []Ban) EarlierBans { fromHere.Attack++ case CauseAdmin: fromHere.Admin++ - case CauseCrowdSec: - fromHere.CrowdSec++ } notes := ban.Notes.EarlierBans count := EarlierBans{ - Limit: notes.Limit + fromHere.Limit, - Attack: notes.Attack + fromHere.Attack, - Admin: notes.Admin + fromHere.Admin, - CrowdSec: notes.CrowdSec + fromHere.CrowdSec, + Limit: notes.Limit + fromHere.Limit, + Attack: notes.Attack + fromHere.Attack, + Admin: notes.Admin + fromHere.Admin, } // Of counts that name as many bans, the earlier ban's is taken: it @@ -808,7 +735,7 @@ func earlierBans(held []Ban) EarlierBans { // total returns how many bans e counts, of all causes. func (e EarlierBans) total() int { - return e.Limit + e.Attack + e.Admin + e.CrowdSec + return e.Limit + e.Attack + e.Admin } // markChanged has Changed receive a value, unless one is waiting already. @@ -913,9 +840,9 @@ func (l *Ledger) add(ban Ban) { // limitExpiry returns when a ban for a broken limit made at now ends, or // zero when it is permanent. held are the netblock's bans, none of them // active, of which the one that ended last, other than a ban for a clear -// sign of attack or for CrowdSec's decision, or a lifted one, can make the -// new ban longer. A ban an admin adds to bans.json can start after -// another and end before it, so that one is looked for among them all. +// sign of attack or a lifted one, can make the new ban longer. A ban an +// admin adds to bans.json can start after another and end before it, so +// that one is looked for among them all. func (l *Ledger) limitExpiry(held []Ban, now time.Time) time.Time { length := l.params.LimitBanDuration diff --git a/internal/bans/crowdsec_test.go b/internal/bans/crowdsec_test.go deleted file mode 100644 index e936308..0000000 --- a/internal/bans/crowdsec_test.go +++ /dev/null @@ -1,136 +0,0 @@ -package bans_test - -import ( - "net/netip" - "reflect" - "testing" - "time" - - "sneak.berlin/go/smallwebwaf/internal/bans" -) - -// scenario is the scenario of the tests' CrowdSec decisions. -const scenario = "crowdsecurity/ssh-bf" - -func TestCrowdSecBanLastsUntilTheDecisionEnds(t *testing.T) { - t.Parallel() - - ledger := bans.New(defaultParams()) - netblock := netip.MustParsePrefix("203.0.113.9/32") - expires := midnight().Add(4 * time.Hour) - - // The ban that would be made is not made. - would, wouldBan := ledger.WouldBanForCrowdSec( - netblock, - midnight(), - expires, - scenario, - bans.Notes{}, - ) - if !wouldBan || len(ledger.Bans(netblock)) != 0 { - t.Errorf( - "would ban %t, and the ledger holds %+v, want true and nothing", - wouldBan, - ledger.Bans(netblock), - ) - } - - const reason = "CrowdSec's decision for " + scenario - - ban, made := ledger.BanForCrowdSec( - netblock, - midnight(), - expires, - scenario, - bans.Notes{}, - ) - if !made || !reflect.DeepEqual(ban, would) || - ban.Cause != bans.CauseCrowdSec || - !ban.Expires.Equal(expires) || - ban.Reason != reason || - ledger.Made(bans.CauseCrowdSec) != 1 { - t.Errorf( - "made %t the ban %+v, want the one that would be made, %+v, for "+ - "crowdsec until %s", - made, - ban, - would, - expires, - ) - } - - // A second decision on the netblock while the ban lasts makes no other. - again, made := ledger.BanForCrowdSec(netblock, midnight().Add(time.Hour), - expires.Add(time.Hour), scenario, bans.Notes{}) - if made || !again.Expires.Equal(expires) || - ledger.Made(bans.CauseCrowdSec) != 1 { - t.Errorf( - "made %t the ban %+v while the first lasts, want none", - made, - again, - ) - } -} - -func TestCrowdSecBanIsNeverMadePermanent(t *testing.T) { - t.Parallel() - - ledger := bans.New(defaultParams()) - netblock := netip.MustParsePrefix("203.0.113.9/32") - expires := midnight().Add(4 * time.Hour) - ledger.BanForCrowdSec( - netblock, - midnight(), - expires, - scenario, - bans.Notes{}, - ) - - // A request as the ban ends is refused, and leaves it as it is. - last := expires.Add(-time.Nanosecond) - - held, banned, madePermanent := ledger.Check(netblock.Addr(), last) - if !banned || madePermanent || !held.Expires.Equal(expires) || - ledger.WouldBePermanent(netblock, last, bans.CauseCrowdSec) { - t.Errorf( - "as the ban ends, banned %t with %+v, made permanent %t, want "+ - "refused under the ban as it was", - banned, - held, - madePermanent, - ) - } - - if _, banned, _ := ledger.Check(netblock.Addr(), expires); banned { - t.Error("the ban refuses a request once the decision has ended") - } -} - -func TestCrowdSecBanIsCountedAndDoesNotLengthenTheNextBanForALimit( - t *testing.T, -) { - t.Parallel() - - ledger := bans.New(defaultParams()) - netblock := netip.MustParsePrefix("203.0.113.9/32") - - // Three times the three days would be permanent; a limit broken as the - // ban for CrowdSec's decision ends bans for an hour, as a first broken - // limit does. - crowdSec, _ := ledger.BanForCrowdSec( - netblock, - midnight(), - midnight().Add(3*day), - scenario, - bans.Notes{}, - ) - limit, _ := ledger.BanForLimit(netblock, crowdSec.Expires, bans.Notes{}) - - if limit.Expires.Sub(limit.Start) != time.Hour || - limit.Notes.EarlierBans != (bans.EarlierBans{CrowdSec: 1}) { - t.Errorf( - "the ban for a limit is %+v, want one of an hour after one for crowdsec", - limit, - ) - } -} diff --git a/internal/config/waf.go b/internal/config/waf.go index dfe5e72..29e522c 100644 --- a/internal/config/waf.go +++ b/internal/config/waf.go @@ -157,13 +157,6 @@ type WAF struct { DNSBLAbuseIPDBLimitPercent int64 IPReputationCacheTTL time.Duration DNSBLAbuseIPDBTimeout time.Duration - // CrowdSecDecisionsURL is where the decision list of the CrowdSec - // engine whose local API SWW_WAF_CROWDSEC_LAPI_URL names is fetched from: - // that URL with v1/decisions added to its path, "" while it is unset and - // none is. CrowdSecKey is the key the engine is asked with - // (SWW_WAF_CROWDSEC_LAPI_KEY). - CrowdSecDecisionsURL string - CrowdSecKey string // BanResponse is the status a refused client is answered with, 403 // or 429, or 0 to close the connection without an answer // (SWW_WAF_BAN_RESPONSE). It answers a banned client, a request that @@ -473,21 +466,12 @@ var ( "is not an IP address with an optional port, " + "such as 192.0.2.53 or [2001:db8::53]:5353", ) - errNotLAPIURL = errors.New( - "is not an http or https URL without a user or a fragment, " + - "such as http://172.17.0.1:8080") errNotRouteWatchURL = errors.New( "is not an http or https URL without a user or a fragment, " + "such as https://routewatch.example") - errNeedsLAPIKey = errors.New("the engine answers no request without it") - errLAPIKeyUnused = errors.New( - "it is sent only to the engine at that URL", - ) - errAnotherList = errors.New( + errFetchedOtherwise = errors.New( "is in SWW_WAF_BLOCKLIST_URLS or is SWW_WAF_ASN_LIMIT_PERCENT_URL too", ) - errFetchedOtherwise = errors.New("is in SWW_WAF_BLOCKLIST_URLS, is " + - "SWW_WAF_ASN_LIMIT_PERCENT_URL or is the CrowdSec decision list too") errNotTagList = errors.New( "is not a name, = and an http or https URL, such as " + "torexit=https://example.com/tor.txt") @@ -502,8 +486,6 @@ var ( errExploitTag = errors.New( "cannot lower limits: the rate limits count a request before its " + "rules are matched") - errCrowdSecTag = errors.New( - "cannot be used: a CrowdSec decision refuses the request first") errNoRequestGetsTag = errors.New( "is a tag no request can get: it is not a name in SWW_WAF_TAG_LISTS, an " + "asn= or country= tag, a dnsbl= tag of a zone in SWW_WAF_DNSBL_ZONES, " + @@ -605,10 +587,6 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) { "SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS", "2000", ), - CrowdSecDecisionsURL: env.crowdSecDecisionsURL( - "SWW_WAF_CROWDSEC_LAPI_URL", - ), - CrowdSecKey: env.secret("SWW_WAF_CROWDSEC_LAPI_KEY"), BanResponse: env.banResponse("SWW_WAF_BAN_RESPONSE", "403"), LimitBanDuration: env.durationNotOff( "SWW_WAF_LIMIT_BAN_DURATION", @@ -755,7 +733,6 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) { env.checkLookupDBPath(cfg) env.checkCountriesAndLookups(cfg) env.checkASNLimitPercentURL(cfg) - env.checkCrowdSec(cfg) env.checkTagLists(cfg) env.checkTags(cfg) @@ -1061,26 +1038,6 @@ func (e *environment) resolver(name string) netip.AddrPort { return resolver } -// crowdSecDecisionsURL reads the setting that is the URL of the CrowdSec -// engine's local API, such as http://172.17.0.1:8080, and returns the URL -// its decision list is fetched from, that URL with v1/decisions added to -// its path, "" while it is unset or empty. -func (e *environment) crowdSecDecisionsURL(name string) string { - value := e.value(name, "") - if value == "" { - return "" - } - - lapi, err := url.Parse(value) - if err != nil || !isHTTPURL(lapi) { - e.check(name, fmt.Errorf("%q %w", value, errNotLAPIURL)) - - return "" - } - - return lapi.JoinPath("v1", "decisions").String() -} - // routeWatchURL reads the setting that is the URL of a routewatch instance, // "" while it is unset or empty. func (e *environment) routeWatchURL(name string) string { @@ -1190,39 +1147,12 @@ func (e *environment) checkASNLimitPercentURL(cfg *WAF) { } } -// checkCrowdSec refuses SWW_WAF_CROWDSEC_LAPI_URL without -// SWW_WAF_CROWDSEC_LAPI_KEY, the key without the URL, and a decision list -// that is fetched as another list too. -func (e *environment) checkCrowdSec(cfg *WAF) { - decisionsURL := cfg.CrowdSecDecisionsURL - - switch { - case decisionsURL != "" && cfg.CrowdSecKey == "": - e.check("SWW_WAF_CROWDSEC_LAPI_URL", fmt.Errorf( - "is set while SWW_WAF_CROWDSEC_LAPI_KEY is unset; %w", - errNeedsLAPIKey, - )) - case decisionsURL == "" && cfg.CrowdSecKey != "": - e.check("SWW_WAF_CROWDSEC_LAPI_KEY", fmt.Errorf( - "is set while SWW_WAF_CROWDSEC_LAPI_URL is unset; %w", - errLAPIKeyUnused, - )) - case decisionsURL != "" && (slices.Contains(cfg.BlocklistURLs, decisionsURL) || - decisionsURL == cfg.ASNLimitPercentURL): - e.check( - "SWW_WAF_CROWDSEC_LAPI_URL", - fmt.Errorf("gives the decision list %q, which %w", - decisionsURL, errAnotherList), - ) - } -} - // checkTagLists refuses a list of SWW_WAF_TAG_LISTS whose URL is fetched as // another list too: the file at a URL is read as one kind of list. func (e *environment) checkTagLists(cfg *WAF) { for _, list := range cfg.TagLists { if slices.Contains(cfg.BlocklistURLs, list.URL) || - list.URL == cfg.ASNLimitPercentURL || list.URL == cfg.CrowdSecDecisionsURL { + list.URL == cfg.ASNLimitPercentURL { e.check( "SWW_WAF_TAG_LISTS", fmt.Errorf("%q %w", list.URL, errFetchedOtherwise), @@ -1232,8 +1162,7 @@ func (e *environment) checkTagLists(cfg *WAF) { } // checkTags refuses a tag of SWW_WAF_TAG_DENY or SWW_WAF_TAG_LIMIT_PERCENT that would -// never apply: crowdsec, whose request is refused before tags are acted on, and -// any tag these settings cannot give a request. +// never apply: any tag these settings cannot give a request. func (e *environment) checkTags(cfg *WAF) { for _, tag := range cfg.TagDeny { e.check("SWW_WAF_TAG_DENY", checkTag(cfg, tag)) @@ -1246,10 +1175,6 @@ func (e *environment) checkTags(cfg *WAF) { // checkTag returns an error when tag would never apply, as checkTags says. func checkTag(cfg *WAF, tag string) error { - if tag == CrowdSecTag { - return fmt.Errorf("%q %w", tag, errCrowdSecTag) - } - if !requestCanGetTag(cfg, tag) { return fmt.Errorf("%q %w", MaskZoneKey(tag), errNoRequestGetsTag) } @@ -2010,7 +1935,6 @@ const ( DNSBLTagPrefix = "dnsbl=" PrefixTagPrefix = "prefix=" AbuseIPDBTag = "abuseipdb" - CrowdSecTag = "crowdsec" // ExploitSignatureTag is the tag of a request a rule file or the Core // Rule Set matched. ExploitSignatureTag = "exploitsignature" diff --git a/internal/config/waf_test.go b/internal/config/waf_test.go index 5dc4c17..31dda14 100644 --- a/internal/config/waf_test.go +++ b/internal/config/waf_test.go @@ -65,8 +65,6 @@ const ( ipReputationCacheTTL = "SWW_WAF_IP_REPUTATION_CACHE_TTL" dnsblAbuseIPDBTimeout = "SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS" routeWatchURL = "SWW_WAF_ROUTEWATCH_URL" - crowdSecURL = "SWW_WAF_CROWDSEC_LAPI_URL" - crowdSecKey = "SWW_WAF_CROWDSEC_LAPI_KEY" banResponse = "SWW_WAF_BAN_RESPONSE" limitBanDuration = "SWW_WAF_LIMIT_BAN_DURATION" limitBanRepeatWindow = "SWW_WAF_LIMIT_BAN_REPEAT_WINDOW" @@ -1813,9 +1811,6 @@ func TestTagThatNoRequestCanGetIsToldApartFromOneItCan(t *testing.T) { "abuseipdb with a key", environment{abuseIPDBKey: "k", tagLimitPercent: "abuseipdb:50"}, "", }, - {"crowdsec, though a list has its name", environment{ - tagLists: "crowdsec=https://example.com/c.txt", tagDeny: "crowdsec", - }, `"crowdsec" cannot be used`}, {exploitTag, environment{tagDeny: exploitTag}, ""}, { exploitTag + " with rule files off", @@ -1887,26 +1882,20 @@ func TestErrorOfATagSettingLeavesOutAZoneKey(t *testing.T) { func TestTagListURLFetchedAsAnotherListStopsTheStart(t *testing.T) { t.Parallel() - const lapi = "http://172.17.0.1:8080" - for _, tc := range []struct { name, listURL string env environment }{ {"a blocklist", dropURL, environment{blocklistURLs: dropURL}}, {"the file of AS:percent lines", asnURL, environment{asnLimitPercentURL: asnURL}}, - { - "the CrowdSec decision list", lapi + "/v1/decisions", - environment{"SWW_WAF_CROWDSEC_LAPI_URL": lapi, "SWW_WAF_CROWDSEC_LAPI_KEY": "key"}, - }, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() tc.env[tagLists] = "dup=" + tc.listURL - want := tagLists + `: "` + tc.listURL + `" is in SWW_WAF_BLOCKLIST_URLS, is ` + - "SWW_WAF_ASN_LIMIT_PERCENT_URL or is the CrowdSec decision list too" + want := tagLists + `: "` + tc.listURL + `" is in SWW_WAF_BLOCKLIST_URLS ` + + "or is SWW_WAF_ASN_LIMIT_PERCENT_URL too" _, err := config.WAFFromEnvironment(tc.env.viper()) if err == nil || err.Error() != want { @@ -2456,138 +2445,6 @@ func TestAbuseIPDBKeyIsLoggedMasked(t *testing.T) { } } -func TestCrowdSecSettingsGiveTheDecisionListAndTheKey(t *testing.T) { - t.Parallel() - - cfg := wafFromEnvironment(t, environment{}) - if cfg.CrowdSecDecisionsURL != "" || cfg.CrowdSecKey != "" { - t.Errorf( - "by default, the decision list %q and the key %q, want neither", - cfg.CrowdSecDecisionsURL, - cfg.CrowdSecKey, - ) - } - - for lapi, want := range map[string]string{ - "http://172.17.0.1:8080": "http://172.17.0.1:8080/v1/decisions", - "http://172.17.0.1:8080/": "http://172.17.0.1:8080/v1/decisions", - "https://crowdsec.example/lapi/": "https://crowdsec.example/lapi/v1/decisions", - "https://crowdsec.example:8443/x": "https://crowdsec.example:8443/x/v1/decisions", - } { - cfg := wafFromEnvironment( - t, - environment{crowdSecURL: lapi, crowdSecKey: token}, - ) - if cfg.CrowdSecDecisionsURL != want || cfg.CrowdSecKey != token { - t.Errorf( - "%s=%s gave the decision list %q and the key %q, want %s and %s", - crowdSecURL, - lapi, - cfg.CrowdSecDecisionsURL, - cfg.CrowdSecKey, - want, - token, - ) - } - } -} - -func TestInvalidCrowdSecSettingStopsTheStartSayingWhatIsWrong(t *testing.T) { - t.Parallel() - - const ( - lapi = "http://172.17.0.1:8080" - notLAPIURL = " is not an http or https URL without a user or a fragment, " + - "such as http://172.17.0.1:8080" - anotherList = `gives the decision list "` + lapi + `/v1/decisions", which is in ` + - `SWW_WAF_BLOCKLIST_URLS or is SWW_WAF_ASN_LIMIT_PERCENT_URL too` - ) - - for _, tc := range []struct { - name string - env environment - want string - }{ - { - "a URL that is not http", - environment{crowdSecURL: "ftp://172.17.0.1", crowdSecKey: token}, - crowdSecURL + `: "ftp://172.17.0.1"` + notLAPIURL, - }, - { - "a URL with a user", - environment{crowdSecURL: "http://bouncer@172.17.0.1:8080", crowdSecKey: token}, - crowdSecURL + `: "http://bouncer@172.17.0.1:8080"` + notLAPIURL, - }, - { - "the URL without the key", - environment{crowdSecURL: lapi}, - crowdSecURL + ": is set while " + crowdSecKey + " is unset; the engine " + - "answers no request without it", - }, - { - "the key without the URL", - environment{crowdSecKey: token}, - crowdSecKey + ": is set while " + crowdSecURL + " is unset; it is sent only " + - "to the engine at that URL", - }, - { - "the decision list as a blocklist too", - environment{ - crowdSecURL: lapi, crowdSecKey: token, - blocklistURLs: "https://lists.example/drop.txt," + lapi + "/v1/decisions", - }, - crowdSecURL + ": " + anotherList, - }, - { - "the decision list as the file of AS:percent lines too", - environment{ - crowdSecURL: lapi + "/", crowdSecKey: token, - asnLimitPercentURL: lapi + "/v1/decisions", - }, - crowdSecURL + ": " + anotherList, - }, - // The key itself is never shown. - { - "a key with a control character", - environment{crowdSecURL: lapi, crowdSecKey: token + "\r"}, - crowdSecKey + ": holds a control character, such as the carriage return " + - "of a Windows line end", - }, - } { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - _, err := config.WAFFromEnvironment(tc.env.viper()) - if err == nil || err.Error() != tc.want { - t.Errorf("error %v, want %s", err, tc.want) - } - }) - } -} - -func TestCrowdSecKeyIsLoggedMasked(t *testing.T) { - t.Parallel() - - cfg := wafFromEnvironment(t, environment{ - crowdSecURL: "http://172.17.0.1:8080", crowdSecKey: token, - }) - - var out bytes.Buffer - - slog.New(slog.NewJSONHandler(&out, nil)). - Info("starting", "settings", cfg) - - logged := out.String() - if strings.Contains(logged, token) || - !strings.Contains(logged, `"`+crowdSecKey+`":"********"`) || - !strings.Contains( - logged, - `"`+crowdSecURL+`":"http://172.17.0.1:8080"`, - ) { - t.Errorf("the key is not logged masked beside the URL: %s", logged) - } -} - func TestSizesAndOff(t *testing.T) { t.Parallel() @@ -2714,10 +2571,10 @@ func TestInvalidLookupTagOrLogValueStopsTheStart(t *testing.T) { {tagDeny, "a,,b"}, {tagDeny, "country=nk"}, {tagDeny, "asn=abc"}, {tagDeny, "asn="}, {tagDeny, "torexit,TOREXIT"}, {tagDeny, "torexti"}, - {tagDeny, zenTag}, {tagDeny, "abuseipdb"}, {tagDeny, "crowdsec"}, + {tagDeny, zenTag}, {tagDeny, "abuseipdb"}, {tagLimitPercent, "torexti:50"}, {tagLimitPercent, "dnsbl=zen.example:50"}, - {tagLimitPercent, "abuseipdb:50"}, {tagLimitPercent, "crowdsec:50"}, + {tagLimitPercent, "abuseipdb:50"}, {tagLimitPercent, "vpn :50"}, {tagLimitPercent, "vpn"}, {tagLimitPercent, ":50"}, {tagLimitPercent, "vpn:"}, @@ -3091,8 +2948,6 @@ func TestLogsEachSettingWithItsValue(t *testing.T) { ipReputationCacheTTL: defaultIPReputationCacheTTL, dnsblAbuseIPDBTimeout: "2000", routeWatchURL: "", - crowdSecURL: "", - crowdSecKey: "", banResponse: "403", limitBanDuration: "1h", limitBanRepeatWindow: "24h", diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 832d988..24e92f3 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -185,7 +185,7 @@ func (m *Metrics) AddBansAndClients( ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time, ) { for _, cause := range []string{ - bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseCrowdSec, + bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, } { m.registry.MustRegister( prometheus.NewCounterFunc(prometheus.CounterOpts{ @@ -302,20 +302,20 @@ const sourceLabel = "source" // AddReputation adds the metrics of the lists fetched from URLs and of the // DNSBL zones, by source, each list's URL or each zone, its key masked as -// config.MaskZoneKey masks it: the requests whose client a blocklist, the -// CrowdSec decision list, a zone's verdict or AbuseIPDB's score lists, -// which ReputationHit counts, and, read from lists and dnsbl as the -// metrics are asked for, for a list, the fetches that failed and when the -// copy in use was fetched, and for a zone, the queries made and those that -// failed. It is called once, before ReputationHit. +// config.MaskZoneKey masks it: the requests whose client a blocklist, a +// zone's verdict or AbuseIPDB's score lists, which ReputationHit counts, +// and, read from lists and dnsbl as the metrics are asked for, for a list, +// the fetches that failed and when the copy in use was fetched, and for a +// zone, the queries made and those that failed. It is called once, before +// ReputationHit. func (m *Metrics) AddReputation( lists *reputation.Lists, dnsbl *reputation.DNSBL, ) { m.reputationHits = counterVec( "sww_reputation_hits_total", - "Requests whose client a blocklist, the CrowdSec decision list, a DNSBL "+ - "zone or AbuseIPDB lists, by the list's URL, the zone, or abuseipdb.", + "Requests whose client a blocklist, a DNSBL zone or AbuseIPDB lists, "+ + "by the list's URL, the zone, or abuseipdb.", []string{sourceLabel}, ) m.registry.MustRegister(m.reputationHits) @@ -378,9 +378,8 @@ func (m *Metrics) AddAbuseIPDB(abuseIPDB *reputation.AbuseIPDB) { ) } -// ReputationHit counts a request whose client source lists: a blocklist -// or the CrowdSec decision list, by its URL, a DNSBL zone, its key masked, -// or AbuseIPDB, abuseipdb. +// ReputationHit counts a request whose client source lists: a blocklist, +// by its URL, a DNSBL zone, its key masked, or AbuseIPDB, abuseipdb. func (m *Metrics) ReputationHit(source string) { m.reputationHits.WithLabelValues(source).Inc() } diff --git a/internal/reputation/crowdsec_test.go b/internal/reputation/crowdsec_test.go deleted file mode 100644 index 5fa7157..0000000 --- a/internal/reputation/crowdsec_test.go +++ /dev/null @@ -1,622 +0,0 @@ -package reputation_test - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "log/slog" - "net/http" - "net/netip" - "reflect" - "strings" - "sync" - "testing" - "testing/synctest" - "time" - - "sneak.berlin/go/smallwebwaf/internal/alerts" - "sneak.berlin/go/smallwebwaf/internal/reputation" -) - -// The tests run in a synctest bubble, as those of the blocklists do, and -// fetch the decision list from engine, a stand-in for a CrowdSec engine -// that answers without the network. - -const ( - // decisionsURL is the decision list of the tests' engine, and engineKey - // the key it answers. - decisionsURL = "http://crowdsec.example:8080/v1/decisions" - engineKey = "crowdsec-key-0123456789abcdef" - // sshBF and probing are scenarios of the engine's decisions. - sshBF = "crowdsecurity/ssh-bf" - probing = "crowdsecurity/http-probing" - // ban is the type of a decision to ban, and rangeScope the scope of a - // decision on a netblock, as CrowdSec names them. - ban = "ban" - rangeScope = "Range" -) - -func TestCrowdSecDecisionBansItsNetblockUntilItEndsEvenWithTheEngineDown( - t *testing.T, -) { - t.Parallel() - - synctest.Test(t, func(t *testing.T) { - began := time.Now() - manual := "manual 'ban' from 'localhost'" - e := &engine{key: engineKey, decisions: []decision{ - {"Ip", suspect, ban, manual, began.Add(6 * time.Hour)}, - // A shorter decision on the same address, which is not the one - // used. - {"Ip", suspect, ban, sshBF, began.Add(4 * time.Hour)}, - { - rangeScope, - "198.51.100.0/24", - ban, - probing, - began.Add(time.Hour), - }, - {"Ip", "2001:db8::1", ban, sshBF, began.Add(2 * time.Hour)}, - // Left out: a decision to show a captcha, and one on a country. - {"Ip", "192.0.2.50", "captcha", probing, began.Add(time.Hour)}, - {"Country", "KP", ban, manual, began.Add(time.Hour)}, - }} - lists := start(t, e, crowdSecParams()) - - for addr, want := range map[string]reputation.Decision{ - suspect: {Expires: began.Add(6 * time.Hour), Scenario: manual}, - "198.51.100.0": {Expires: began.Add(time.Hour), Scenario: probing}, - "198.51.100.255": {Expires: began.Add(time.Hour), Scenario: probing}, - "2001:db8::1": {Expires: began.Add(2 * time.Hour), Scenario: sshBF}, - "203.0.113.10": {}, - "198.51.101.0": {}, - "2001:db8::2": {}, - "192.0.2.50": {}, - } { - wantDecision(t, lists, addr, want) - } - - // With the engine down, the copy kept still holds the decision on - // 198.51.100.0/24, which no longer bans once it has ended. - e.set(func(e *engine) { e.failing = true }) - time.Sleep(time.Hour - time.Nanosecond) - synctest.Wait() - wantDecision( - t, - lists, - "198.51.100.7", - reputation.Decision{ - Expires: began.Add(time.Hour), - Scenario: probing, - }, - ) - - time.Sleep(time.Nanosecond) - synctest.Wait() - wantDecision(t, lists, "198.51.100.7", reputation.Decision{}) - wantDecision( - t, - lists, - suspect, - reputation.Decision{ - Expires: began.Add(6 * time.Hour), - Scenario: manual, - }, - ) - }) -} - -func TestCrowdSecDecisionListFetchedAgainEveryMinute(t *testing.T) { - t.Parallel() - - synctest.Test(t, func(t *testing.T) { - began := time.Now() - e := &engine{key: engineKey, decisions: []decision{ - {"Ip", suspect, ban, sshBF, began.Add(4 * time.Hour)}, - }} - lists := start(t, e, crowdSecParams()) - wantEngineFetches(t, e, 1) - - added := reputation.Decision{ - Expires: began.Add(2 * time.Hour), - Scenario: probing, - } - - e.set(func(e *engine) { - e.decisions = append(e.decisions, - decision{"Ip", "203.0.113.10", ban, probing, added.Expires}) - }) - - time.Sleep(time.Minute - time.Nanosecond) - wantEngineFetches(t, e, 1) - wantDecision(t, lists, "203.0.113.10", reputation.Decision{}) - - time.Sleep(time.Nanosecond) - wantEngineFetches(t, e, 2) - wantDecision(t, lists, "203.0.113.10", added) - }) -} - -func TestCrowdSecDecisionOnAClientIsTheOneThatEndsLast(t *testing.T) { - t.Parallel() - - synctest.Test(t, func(t *testing.T) { - began := time.Now() - e := &engine{key: engineKey, decisions: []decision{ - // Two decisions on one address, the shorter listed first. - {"Ip", suspect, ban, sshBF, began.Add(2 * time.Hour)}, - {"Ip", suspect, ban, probing, began.Add(4 * time.Hour)}, - // 198.51.100.130 is held by a decision on its address that ends - // after the one on its netblock, and 192.0.2.20 by one that ends - // before. - { - rangeScope, - "198.51.100.128/25", - ban, - sshBF, - began.Add(time.Hour), - }, - {"Ip", "198.51.100.130", ban, probing, began.Add(3 * time.Hour)}, - { - rangeScope, - "192.0.2.0/24", - ban, - probing, - began.Add(5 * time.Hour), - }, - {"Ip", "192.0.2.20", ban, sshBF, began.Add(2 * time.Hour)}, - }} - lists := start(t, e, crowdSecParams()) - - wantDecision( - t, - lists, - suspect, - reputation.Decision{ - Expires: began.Add(4 * time.Hour), - Scenario: probing, - }, - ) - wantDecision( - t, - lists, - "198.51.100.130", - reputation.Decision{ - Expires: began.Add(3 * time.Hour), - Scenario: probing, - }, - ) - wantDecision( - t, - lists, - "192.0.2.20", - reputation.Decision{ - Expires: began.Add(5 * time.Hour), - Scenario: probing, - }, - ) - }) -} - -func TestCrowdSecAnswerOfNoDecisionIsAGoodCopyThatListsNoClient( - t *testing.T, -) { - t.Parallel() - - synctest.Test(t, func(t *testing.T) { - began := time.Now() - e := &engine{key: engineKey, decisions: []decision{ - {"Ip", suspect, ban, sshBF, began.Add(4 * time.Hour)}, - }} - lists := start(t, e, crowdSecParams()) - - // With its decision deleted, the engine answers null. - e.set(func(e *engine) { e.decisions = nil }) - time.Sleep(time.Minute) - wantEngineFetches(t, e, 2) - - want := []reputation.List{{ - URL: decisionsURL, Tried: time.Now(), Fetched: time.Now(), Lines: []string{"null"}, - }} - if got := lists.Snapshot(); !reflect.DeepEqual(got, want) { - t.Errorf("lists %+v, want %+v", got, want) - } - - if lists.Failures(decisionsURL) != 0 { - t.Errorf("%d failures, want 0", lists.Failures(decisionsURL)) - } - - wantDecision(t, lists, suspect, reputation.Decision{}) - }) -} - -func TestCrowdSecFailureKeepsTheLastGoodCopyAlertsOncePerCooldownAndHidesTheKey( - t *testing.T, -) { - t.Parallel() - - for _, tc := range crowdSecFailures() { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - synctest.Test(t, func(t *testing.T) { - var log bytes.Buffer - - began := time.Now() - e := &engine{key: engineKey, decisions: []decision{ - {"Ip", suspect, ban, sshBF, began.Add(4 * time.Hour)}, - }} - queue := newQueue() - p := crowdSecParams() - p.ProcessLog = slog.New(slog.NewJSONHandler(&log, nil)) - p.Alerts = queue - lists := start(t, e, p) - kept := lists.Snapshot() - - e.set(tc.fail) - - // Each failure is tried again a minute after it. - for range 2 { - time.Sleep(time.Minute) - synctest.Wait() - } - - wantEngineFetches(t, e, 3) - wantDecision( - t, - lists, - suspect, - reputation.Decision{ - Expires: began.Add(4 * time.Hour), - Scenario: sshBF, - }, - ) - - want := kept[0] - want.Tried = time.Now() - - got := lists.Snapshot() - if !reflect.DeepEqual(got, []reputation.List{want}) { - t.Errorf( - "lists %+v, want the first copy, last tried now, %+v", - got, - want, - ) - } - - if n := lists.Failures(decisionsURL); n != 2 { - t.Errorf("%d failures, want 2", n) - } - - // One alert for the first failure; the cooldown holds back the - // second. - failed := time.Now().Add(-time.Minute) - alert := fetchFailure(failed, decisionsURL, tc.error) - wantAlert(t, queue, alert) - - if !strings.Contains( - log.String(), - `"msg":"fetching a list failed",`+ - `"url":"`+decisionsURL+`","error":"`+tc.error, - ) { - t.Errorf("logged\n%s\nwant the failures", log.String()) - } - - wantKeyNotShown(t, e, log.String(), lists, queue) - }) - }) - } -} - -// crowdSecFailure is a way for the engine to fail: fail has it answer the -// fetches after the first so that they fail with error. -type crowdSecFailure struct { - name string - fail func(e *engine) - error string -} - -// crowdSecFailures returns the ways the engine can fail. -func crowdSecFailures() []crowdSecFailure { - const notDecision = " does not give an address or a netblock and a duration, " + - "such as 4h0m0s" - - return []crowdSecFailure{ - { - "an answer other than 200", - func(e *engine) { e.failing = true }, - "the server answered 503 Service Unavailable", - }, - { - "a key the engine refuses", - func(e *engine) { e.key = "another-key-0123456789abcdef" }, - "the server answered 403 Forbidden", - }, - { - "a redirect", - func(e *engine) { e.redirect = "http://elsewhere.example/v1/decisions" }, - "the server answered 302 Found", - }, - { - "an answer that does not read", - func(e *engine) { e.answer = "" }, - "read the answer: invalid character '<' looking for beginning of value", - }, - { - "a decision to ban whose value does not read", - func(e *engine) { - e.answer = `[{"duration": "4h", "scenario": "` + sshBF + `", ` + - `"scope": "Ip", "type": "ban", "value": "203.0.113.300"}]` - }, - "decision 1" + notDecision, - }, - { - "a decision to ban whose duration does not read", - func(e *engine) { - e.answer = `[{"duration": "4h", "scope": "Country", "type": "ban", ` + - `"value": "KP"}, {"duration": "four hours", "scope": "Range", ` + - `"type": "ban", "value": "198.51.100.0/24"}]` - }, - "decision 2" + notDecision, - }, - } -} - -// wantKeyNotShown checks that no fetch carried the engine's key to a URL -// other than its decision list, such as the one a redirect names, and that -// the key is in none of what the fetches leave behind: log, the process -// log, the alerts waiting in queue, and the copies of lists, which -// lists.json keeps. -func wantKeyNotShown( - t *testing.T, - e *engine, - log string, - lists *reputation.Lists, - queue *alerts.Queue, -) { - t.Helper() - - e.mu.Lock() - keySentTo := e.keySentTo - e.mu.Unlock() - - if len(keySentTo) != 0 { - t.Errorf("the key was sent to %v", keySentTo) - } - - shown, err := json.Marshal([]any{lists.Snapshot(), waiting(queue)}) - if err != nil { - t.Fatalf("encode: %v", err) - } - - if strings.Contains(log+string(shown), engineKey) { - t.Errorf("the key is shown in\n%s\n%s", log, shown) - } -} - -func TestCrowdSecDecisionListKeptAcrossARestartEndsWhenItsDecisionsDo( - t *testing.T, -) { - t.Parallel() - - synctest.Test(t, func(t *testing.T) { - began := time.Now() - e := &engine{key: engineKey, decisions: []decision{ - { - rangeScope, - "198.51.100.0/24", - ban, - probing, - began.Add(time.Hour), - }, - }} - lists := start(t, e, crowdSecParams()) - kept := lists.Snapshot() - - // Restarted half an hour later with what lists.json keeps, and - // the engine down, the decision still bans, until the end it had at - // the fetch, half an hour on. - time.Sleep(30 * time.Minute) - - down := &engine{key: engineKey, failing: true} - again := reputation.New(crowdSecParams()) - again.SetTransport(down) - - err := again.Load(kept) - if err != nil { - t.Fatalf("load: %v", err) - } - - run(t, again) - - want := reputation.Decision{ - Expires: began.Add(time.Hour), - Scenario: probing, - } - wantDecision(t, again, "198.51.100.7", want) - - time.Sleep(30*time.Minute - time.Nanosecond) - synctest.Wait() - wantDecision(t, again, "198.51.100.7", want) - - time.Sleep(time.Nanosecond) - synctest.Wait() - wantDecision(t, again, "198.51.100.7", reputation.Decision{}) - }) -} - -func TestLoadTakesACrowdSecListNeverFetchedAndRefusesACopyThatDoesNotRead( - t *testing.T, -) { - t.Parallel() - - now := time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC) - lists := reputation.New(crowdSecParams()) - - // Tried, and never fetched: there is no copy to read. - err := lists.Load([]reputation.List{{URL: decisionsURL, Tried: now}}) - if err != nil { - t.Errorf("load the list never fetched: %v", err) - } - - err = lists.Load([]reputation.List{{ - URL: decisionsURL, Tried: now, Fetched: now, Lines: []string{ - `[{"duration": "4h", "scope": "Range", "type": "ban", ` + - `"value": "198.51.100.0/33"}]`, - }, - }}) - - const want = "the copy of " + decisionsURL + ": decision 1 does not give an " + - "address or a netblock and a duration, such as 4h0m0s" - if err == nil || err.Error() != want { - t.Errorf("error %v, want %s", err, want) - } -} - -// engine is a stand-in for the local API of a CrowdSec engine. It answers -// a fetch of the decision list that carries its key in X-Api-Key with its -// decisions still in force, each with the time it has left as it answers, -// by the bubble's clock, as an engine does, or with answer while that is -// not "". It answers 403 to a fetch without its key, as an engine does, -// with a redirect to redirect while that is not "", and 503 while failing. -// It counts the fetches, and notes in keySentTo the URL of each fetch of -// another URL that carries a key, as one following a redirect would. -type engine struct { - mu sync.Mutex - key string - decisions []decision - answer string - redirect string - failing bool - fetches int - keySentTo []string -} - -// decision is a decision of the engine, which ends at expires. -type decision struct { - scope, value, kind, scenario string - expires time.Time -} - -// RoundTrip has the engine answer req, in place of the network. -func (e *engine) RoundTrip(req *http.Request) (*http.Response, error) { - e.mu.Lock() - defer e.mu.Unlock() - - e.fetches++ - - if req.URL.String() != decisionsURL && - req.Header.Get("X-Api-Key") != "" { - e.keySentTo = append(e.keySentTo, req.URL.String()) - } - - status, header, body := http.StatusOK, http.Header{}, e.answer - - switch { - case req.URL.String() != decisionsURL || req.Header.Get("X-Api-Key") != e.key: - status, body = http.StatusForbidden, `{"message":"access forbidden"}` - case e.redirect != "": - status, header = http.StatusFound, http.Header{ - "Location": {e.redirect}, - } - case e.failing: - status, body = http.StatusServiceUnavailable, "" - case body == "": - body = e.inForce(time.Now()) - } - - return &http.Response{ - StatusCode: status, - Status: fmt.Sprintf("%d %s", status, http.StatusText(status)), - Header: header, - Body: io.NopCloser(strings.NewReader(body)), - Request: req, - }, nil -} - -// inForce returns the decisions in force at now, as the engine answers -// them: a JSON list, null for none. -func (e *engine) inForce(now time.Time) string { - var answer []map[string]string - - for _, d := range e.decisions { - if now.Before(d.expires) { - answer = append(answer, map[string]string{ - "duration": d.expires.Sub(now). - String(), - "origin": "crowdsec", - "scenario": d.scenario, "scope": d.scope, "type": d.kind, "value": d.value, - }) - } - } - - body, err := json.Marshal(answer) - if err != nil { - panic(err) // a list of maps of strings always encodes - } - - return string(body) -} - -// set changes the engine with change. -func (e *engine) set(change func(e *engine)) { - e.mu.Lock() - defer e.mu.Unlock() - - change(e) -} - -// crowdSecParams returns the Params of the decision list of the tests' -// engine, fetched with its key, by the bubble's clock, with alerts to a -// queue that sends none. -func crowdSecParams() reputation.Params { - p := params() - p.CrowdSecDecisionsURL = decisionsURL - p.CrowdSecKey = engineKey - - return p -} - -// wantEngineFetches waits until Run has made the fetches due, and checks -// how many the engine has had. -func wantEngineFetches(t *testing.T, e *engine, want int) { - t.Helper() - - synctest.Wait() - - e.mu.Lock() - got := e.fetches - e.mu.Unlock() - - if got != want { - t.Errorf("%d fetches, want %d", got, want) - } -} - -// wantDecision checks the decision lists says is in force on addr now, -// the zero Decision for none. -func wantDecision( - t *testing.T, - lists *reputation.Lists, - addr string, - want reputation.Decision, -) { - t.Helper() - - got, listed := lists.CrowdSecDecision( - netip.MustParseAddr(addr), - time.Now(), - ) - if listed != !want.Expires.IsZero() || - listed && - (!got.Expires.Equal(want.Expires) || got.Scenario != want.Scenario) { - t.Errorf( - "%s has the decision %+v in force %t, want %+v", - addr, - got, - listed, - want, - ) - } -} diff --git a/internal/reputation/export_test.go b/internal/reputation/export_test.go index 578f604..322a2d3 100644 --- a/internal/reputation/export_test.go +++ b/internal/reputation/export_test.go @@ -11,7 +11,6 @@ import ( // network. func (l *Lists) SetTransport(transport http.RoundTripper) { l.httpClient.Transport = transport - l.crowdSecClient.Transport = transport } // SetTransport has a's checks go through transport instead of the diff --git a/internal/reputation/reputation.go b/internal/reputation/reputation.go index 6672d04..2b22cf7 100644 --- a/internal/reputation/reputation.go +++ b/internal/reputation/reputation.go @@ -1,8 +1,7 @@ // Package reputation fetches the lists the settings name by URL: the // blocklists of SWW_WAF_BLOCKLIST_URLS, the file of AS:percent lines -// SWW_WAF_ASN_LIMIT_PERCENT_URL names, the address lists of SWW_WAF_TAG_LISTS, and -// the decision list of the CrowdSec engine SWW_WAF_CROWDSEC_LAPI_URL names. It -// keeps the last good copy of +// SWW_WAF_ASN_LIMIT_PERCENT_URL names, and the address lists of +// SWW_WAF_TAG_LISTS. It keeps the last good copy of // each, whole, comment lines included, which is used while a fetch fails, // and when each was last tried. It also asks the DNSBL zones of // SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks @@ -13,7 +12,6 @@ package reputation import ( "context" - "encoding/json" "errors" "fmt" "io" @@ -35,10 +33,6 @@ const ( maxListBytes = 16 << 20 // fetchTimeout bounds one fetch of a list. fetchTimeout = time.Minute - // crowdSecRefresh is how long after the CrowdSec decision list was last - // fetched or tried it is fetched again: the engine is the operator's - // own, and makes and ends decisions all the time. - crowdSecRefresh = time.Minute // mappedBits is the length of ::ffff:0.0.0.0/96, the netblock of every // IPv4-mapped address. mappedBits = 96 @@ -52,9 +46,6 @@ var ( ) errNotASNPercent = errors.New( "is not an AS number, : and a percentage, such as AS64496:50") - errNotDecision = errors.New( - "does not give an address or a netblock and a duration, such as 4h0m0s", - ) ) // List is a list as lists.json holds it: the URL it is fetched from, @@ -77,14 +68,8 @@ type Params struct { ASNLimitPercentURL string // TagLists are the address lists that add a tag (SWW_WAF_TAG_LISTS). TagLists []config.TagList - // CrowdSecDecisionsURL is the CrowdSec decision list, "" while - // SWW_WAF_CROWDSEC_LAPI_URL is unset, fetched with CrowdSecKey - // (SWW_WAF_CROWDSEC_LAPI_KEY). - CrowdSecDecisionsURL string - CrowdSecKey string // Refresh is how long after a list was last fetched or tried it is - // fetched again (SWW_WAF_BLOCKLIST_REFRESH), but for the CrowdSec decision - // list, which is fetched again crowdSecRefresh after. + // fetched again (SWW_WAF_BLOCKLIST_REFRESH). Refresh time.Duration // Now tells the time, normally time.Now in UTC. Now func() time.Time @@ -99,10 +84,6 @@ type Params struct { type Lists struct { params Params httpClient *http.Client - // crowdSecClient fetches the CrowdSec decision list. It follows no - // redirect, so that the key goes to the engine alone: a redirect is a - // failure. - crowdSecClient *http.Client mu sync.Mutex // lists are by URL, one for each URL Params names. @@ -119,22 +100,12 @@ type list struct { } // entries are what the lines of a copy say: for a blocklist or a tag list, the -// netblocks it names, with the lengths among them, for the file of AS:percent -// lines, the percentage it gives each AS number, and for the CrowdSec decision -// list, the decision on each netblock that ends last, with the lengths among -// them. +// netblocks it names, with the lengths among them, and for the file of +// AS:percent lines, the percentage it gives each AS number. type entries struct { netblocks map[netip.Prefix]bool lengths []int percents map[string]int64 - decisions map[netip.Prefix]Decision -} - -// Decision is a decision of the CrowdSec engine to ban a netblock: when -// it ends, and the scenario that made it, such as crowdsecurity/ssh-bf. -type Decision struct { - Expires time.Time - Scenario string } // New returns the lists, without a copy of any yet. @@ -142,12 +113,7 @@ func New(params Params) *Lists { l := &Lists{ params: params, httpClient: &http.Client{}, - crowdSecClient: &http.Client{ - CheckRedirect: func(*http.Request, []*http.Request) error { - return http.ErrUseLastResponse - }, - }, - lists: map[string]*list{}, + lists: map[string]*list{}, } for _, listURL := range l.URLs() { @@ -159,7 +125,7 @@ func New(params Params) *Lists { // URLs returns the URL of every list: the blocklists' in the order // SWW_WAF_BLOCKLIST_URLS names them, then SWW_WAF_ASN_LIMIT_PERCENT_URL, then the -// tag lists', then the CrowdSec decision list's. +// tag lists'. func (l *Lists) URLs() []string { urls := slices.Clone(l.params.BlocklistURLs) if l.params.ASNLimitPercentURL != "" { @@ -170,10 +136,6 @@ func (l *Lists) URLs() []string { urls = append(urls, list.URL) } - if l.params.CrowdSecDecisionsURL != "" { - urls = append(urls, l.params.CrowdSecDecisionsURL) - } - return urls } @@ -225,40 +187,6 @@ func (l *Lists) ASNLimitPercent(asn string) (int64, bool) { return percent, listed } -// CrowdSecDecision returns the decision of the copy of the CrowdSec -// decision list on a netblock that holds addr and that ends last, and -// whether it is still in force at now. A decision that has ended no -// longer bans, even before the next fetch drops it. -func (l *Lists) CrowdSecDecision( - addr netip.Addr, - now time.Time, -) (Decision, bool) { - if l.params.CrowdSecDecisionsURL == "" { - return Decision{}, false - } - - l.mu.Lock() - defer l.mu.Unlock() - - kept := l.lists[l.params.CrowdSecDecisionsURL].entries - - var last Decision - - for _, length := range kept.lengths { - netblock, err := addr.Prefix(length) - if err != nil { - continue // an IPv6 netblock's length, past an IPv4 address's 32 bits - } - - decision := kept.decisions[netblock] - if decision.Expires.After(last.Expires) { - last = decision - } - } - - return last, now.Before(last.Expires) -} - // Fetched returns when the copy in use of the list at listURL was // fetched, or zero while there is none. func (l *Lists) Fetched(listURL string) time.Time { @@ -331,7 +259,7 @@ func (l *Lists) Load(lists []List) error { continue // dropped, or a list tried but never fetched, without a copy } - read, err := l.parse(kept.URL, kept.Lines, kept.Fetched) + read, err := l.parse(kept.URL, kept.Lines) if err != nil { return fmt.Errorf("the copy of %s: %w", kept.URL, err) } @@ -377,8 +305,7 @@ func (l *Lists) fetchDue(ctx context.Context) time.Time { } // due returns when the list at listURL is to be fetched: Refresh after it -// was last fetched or tried, the later of the two, or crowdSecRefresh -// after for the CrowdSec decision list. +// was last fetched or tried, the later of the two. func (l *Lists) due(listURL string) time.Time { l.mu.Lock() defer l.mu.Unlock() @@ -390,10 +317,6 @@ func (l *Lists) due(listURL string) time.Time { last = held.kept.Tried } - if listURL == l.params.CrowdSecDecisionsURL { - return last.Add(crowdSecRefresh) - } - return last.Add(l.params.Refresh) } @@ -408,7 +331,7 @@ func (l *Lists) fetch(ctx context.Context, listURL string) { var found entries if err == nil { - found, err = l.parse(listURL, lines, now) + found, err = l.parse(listURL, lines) } cutOff := err != nil && ctx.Err() != nil @@ -468,11 +391,8 @@ func raiseFailure(queue *alerts.Queue, reason, source string, err error) { }) } -// get fetches the list at listURL, and returns its lines. The CrowdSec -// decision list is fetched with CrowdSecKey in the header X-Api-Key, where -// the engine looks for it, by crowdSecClient, which follows no redirect. -// An answer other than 200, or a list longer than maxListBytes, is a -// failure. +// get fetches the list at listURL, and returns its lines. An answer other +// than 200, or a list longer than maxListBytes, is a failure. func (l *Lists) get(ctx context.Context, listURL string) ([]string, error) { ctx, cancel := context.WithTimeout(ctx, fetchTimeout) defer cancel() @@ -487,14 +407,7 @@ func (l *Lists) get(ctx context.Context, listURL string) ([]string, error) { return nil, fmt.Errorf("make the request: %w", err) } - client := l.httpClient - - if listURL == l.params.CrowdSecDecisionsURL { - req.Header.Set("X-Api-Key", l.params.CrowdSecKey) - client = l.crowdSecClient - } - - res, err := client.Do(req) + res, err := l.httpClient.Do(req) if err != nil { // Do's error names the URL, which the log line and the alert name // already: only what went wrong is kept. @@ -526,22 +439,16 @@ func (l *Lists) get(ctx context.Context, listURL string) ([]string, error) { return lines, nil } -// parse reads the lines of the list at listURL, fetched at fetched: those -// of a blocklist or a tag list, of the file of AS:percent lines, or of the -// CrowdSec decision list. In all but the decision list, anything after a ; or -// a # on a line is left out, and so is a line left blank. Any other line that +// parse reads the lines of the list at listURL: those of a blocklist or a +// tag list, or of the file of AS:percent lines. Anything after a ; or a # +// on a line is left out, and so is a line left blank. Any other line that // does not read is an error naming it by its number. -func (l *Lists) parse( - listURL string, lines []string, fetched time.Time, -) (entries, error) { - switch listURL { - case l.params.ASNLimitPercentURL: +func (l *Lists) parse(listURL string, lines []string) (entries, error) { + if listURL == l.params.ASNLimitPercentURL { return parsePercents(lines) - case l.params.CrowdSecDecisionsURL: - return parseDecisions(lines, fetched) - default: - return parseNetblocks(lines) } + + return parseNetblocks(lines) } // parseNetblocks reads the lines of a blocklist or a tag list, each an address @@ -628,64 +535,6 @@ func parsePercents(lines []string) (entries, error) { return found, nil } -// parseDecisions reads the lines of the CrowdSec decision list fetched at -// fetched: the engine's answer, a JSON list of its decisions in force, -// null while it has none. A decision of the type ban whose scope is Ip or -// Range, as CrowdSec names them, bans its value, an address or a netblock -// as parseNetblock reads it, until its duration, the time it had left as -// the engine answered, has passed since fetched. Any other decision, such -// as one to show a captcha or one on a country, is left out. A decision -// to ban whose value or duration does not read is an error naming it by -// its number. -func parseDecisions(lines []string, fetched time.Time) (entries, error) { - var answer []struct { - Duration string `json:"duration"` - Scenario string `json:"scenario"` - Scope string `json:"scope"` - Type string `json:"type"` - Value string `json:"value"` - } - - err := json.Unmarshal([]byte(strings.Join(lines, "\n")), &answer) - if err != nil { - return entries{}, fmt.Errorf("read the answer: %w", err) - } - - found := entries{decisions: map[netip.Prefix]Decision{}} - - for i, decision := range answer { - if decision.Type != "ban" || - (decision.Scope != "Ip" && decision.Scope != "Range") { - continue - } - - netblock, ok := parseNetblock(decision.Value) - - duration, err := time.ParseDuration(decision.Duration) - if !ok || err != nil { - return entries{}, fmt.Errorf( - "decision %d %w", - i+1, - errNotDecision, - ) - } - - expires := fetched.Add(duration) - if expires.After(found.decisions[netblock].Expires) { - found.decisions[netblock] = Decision{ - Expires: expires, - Scenario: decision.Scenario, - } - } - - if !slices.Contains(found.lengths, netblock.Bits()) { - found.lengths = append(found.lengths, netblock.Bits()) - } - } - - return found, nil -} - // withoutComment returns line without anything after a ; or a #, and // without the spaces around what is left. func withoutComment(line string) string { diff --git a/internal/requestlog/requestlog.go b/internal/requestlog/requestlog.go index c8676f5..6039087 100644 --- a/internal/requestlog/requestlog.go +++ b/internal/requestlog/requestlog.go @@ -29,9 +29,8 @@ const ( ActionClientIsRateLimited = "client_is_rate_limited" // ActionTemporarilyBanned is a request refused because a ban that runs - // out covers its client, or because it matched a ban rule, asked for a - // trap path or the CrowdSec decision list lists its client, and so made - // such a ban. + // out covers its client, or because it matched a ban rule or asked for a + // trap path, and so made such a ban. ActionTemporarilyBanned = "temporarily_banned" // ActionPermanentlyBanned is a request refused because a ban that never @@ -184,8 +183,7 @@ type Line struct { // LimitHitMinuteRefusals for the refusal limit. LimitHit string `json:"limit_hit,omitempty"` // Reputation are the URLs of the blocklists that list the client, then - // that of the CrowdSec decision list when it does, then the DNSBL zones - // whose verdict lists it, their keys masked, then abuseipdb when its + // the DNSBL zones whose verdict lists it, their keys masked, then abuseipdb when its // score is a hit. Reputation []string `json:"reputation,omitempty"` // Tags are the request's tags, sorted. diff --git a/internal/state/state.go b/internal/state/state.go index 8e8d9b2..9e9d4f4 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -104,7 +104,7 @@ var ( errVersion = errors.New("unknown version") // errMissing is for an entry without a field it needs. errMissing = errors.New("has no") - errCause = errors.New("is not limit, attack, admin or crowdsec") + errCause = errors.New("is not limit, attack or admin") errExpires = errors.New("is not after its start") errDestination = errors.New("is not webhook, slack or ntfy") errScope = errors.New( @@ -967,7 +967,7 @@ func (e BanEntry) ban() bans.Ban { // each expires is read again as written. An expires not after the start // is refused too, since the netblock's next ban for a broken limit, three // times as long as this one, would then refuse nothing; and so is a cause -// other than limit, attack, admin or crowdsec, most likely misspelt. +// other than limit, attack or admin, most likely misspelt. func (f *bansFile) check(data []byte) error { var written struct { Bans []struct { @@ -991,8 +991,7 @@ func (f *bansFile) check(data []byte) error { case entry.Expires != nil && !entry.Expires.After(entry.Start): return fmt.Errorf("entry %d's expires %w", i+1, errExpires) case entry.Cause != "" && entry.Cause != bans.CauseLimit && - entry.Cause != bans.CauseAttack && entry.Cause != bans.CauseAdmin && - entry.Cause != bans.CauseCrowdSec: + entry.Cause != bans.CauseAttack && entry.Cause != bans.CauseAdmin: return fmt.Errorf( "entry %d's cause %q %w", i+1, diff --git a/internal/state/state_test.go b/internal/state/state_test.go index 8dd50e5..af3c840 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -114,8 +114,7 @@ const permanentBansJSON = `{ "earlier_bans": { "limit": 3, "attack": 1, - "admin": 1, - "crowdsec": 2 + "admin": 1 } } } @@ -1188,10 +1187,10 @@ func TestBanWithAnotherCauseStopsTheStart(t *testing.T) { `{"netblock": "203.0.113.10/32", "start": "2026-10-06T00:00:00Z", `+ `"expires": null, "cause": "admin"}, `+ `{"netblock": "203.0.113.11/32", "start": "2026-10-06T00:00:00Z", `+ - `"expires": "2026-10-06T04:00:00Z", "cause": "crowdsec"}, `+ + `"expires": "2026-10-06T04:00:00Z", "cause": "limit"}, `+ `{"netblock": "203.0.113.12/32", "start": "2026-10-06T00:00:00Z", `+ `"expires": null, "cause": "atack"}]}`, - `: entry 4's cause "atack" is not limit, attack, admin or crowdsec`) + `: entry 4's cause "atack" is not limit, attack or admin`) } func TestBanThatDoesNotEndAfterItsStartStopsTheStart(t *testing.T) { @@ -2466,9 +2465,8 @@ func office() netip.Prefix { return netip.MustParsePrefix("203.0.113.0/24") } -// fill puts a permanent ban an admin made, a ban for a broken limit, one -// for a clear sign of attack and one for CrowdSec's decision, clients -// with counts and histories, +// fill puts a permanent ban an admin made, a ban for a broken limit and one +// for a clear sign of attack, clients with counts and histories, // GeoJS answers, the blocklists' last tries and the copy of one, two // verdicts of a DNSBL zone, the AbuseIPDB checks spent today with two // scores, and two answers of routewatch, as filledListsJSON, @@ -2485,13 +2483,6 @@ func fill(params state.Params) { }) params.Ledger.BanForAttack(netip.MustParsePrefix("192.0.2.1/32"), now, bans.Notes{RuleID: "env-file", Target: "path"}) - params.Ledger.BanForCrowdSec( - netip.MustParsePrefix("198.51.100.9/32"), - now, - now.Add(4*time.Hour), - "crowdsecurity/ssh-bf", - bans.Notes{}, - ) for _, c := range []string{"2001:db8::/64", "203.0.113.9/32", "192.0.2.1/32"} { params.Limiter.Count(netip.MustParsePrefix(c), now, whole) @@ -2634,10 +2625,9 @@ func permanentBan() bans.Ban { Requests: 1500, Refused: 3, EarlierBans: bans.EarlierBans{ - Limit: 3, - Attack: 1, - Admin: 1, - CrowdSec: 2, + Limit: 3, + Attack: 1, + Admin: 1, }, }, } diff --git a/internal/sww/sww_test.go b/internal/sww/sww_test.go index 20f014e..42c4a52 100644 --- a/internal/sww/sww_test.go +++ b/internal/sww/sww_test.go @@ -847,92 +847,6 @@ func TestBlocklistTriesAndCopiesKeptInListsJSONAcrossRestarts(t *testing.T) { } } -func TestCrowdSecDecisionListKeptInListsJSONAcrossARestart(t *testing.T) { - t.Parallel() - - const bouncerKey = "crowdsec-key-0123456789abcdef" - - // A stand-in for the engine's local API, which bans 203.0.113.0/24 for - // four hours, and answers only a request with its key, while it is up. - down := new(atomic.Bool) - engine := httptest.NewServer(http.HandlerFunc( - func(w http.ResponseWriter, r *http.Request) { - switch { - case r.URL.Path != "/v1/decisions" || r.Header.Get("X-Api-Key") != bouncerKey: - w.WriteHeader(http.StatusForbidden) - case down.Load(): - w.WriteHeader(http.StatusServiceUnavailable) - default: - _, _ = io.WriteString( - w, - `[{"duration": "4h0m0s", "origin": "crowdsec", `+ - `"scenario": "crowdsecurity/http-probing", "scope": "Range", `+ - `"type": "ban", "value": "203.0.113.0/24"}]`, - ) - } - })) - t.Cleanup(engine.Close) - - dir, cache := t.TempDir(), t.TempDir() - env := map[string]string{ - listenAddr: localhost + ":0", - stateDir: dir, - cacheDir: cache, - ruleFilesDir: t.TempDir(), - "SWW_WAF_CROWDSEC_LAPI_URL": engine.URL, - "SWW_WAF_CROWDSEC_LAPI_KEY": bouncerKey, - } - - // Once the list is fetched, the client's request bans it. - first := runUntilStopped(t, env, func(url string) { - for statusFrom(t, url, placed) != http.StatusForbidden { - time.Sleep(pollInterval) - } - }) - - ban := onlyBan(t, dir) - if ban["netblock"] != placed+"/32" || ban["cause"] != "crowdsec" || - ban["reason"] != "CrowdSec's decision for crowdsecurity/http-probing" { - t.Errorf( - "bans.json holds %v, want the ban for crowdsec on %s", - ban, - placed, - ) - } - - // Restarted with the engine down, the copy kept in lists.json bans - // another client in the netblock from its first request. - down.Store(true) - - second := runUntilStopped(t, env, func(url string) { - wantStatus(t, url, "203.0.113.10", http.StatusForbidden) - }) - - // The key is in neither run's output, nor in a state or cache file. - files := []string{ - filepath.Join(dir, "bans.json"), filepath.Join(dir, "clients.json"), - filepath.Join(cache, "lists.json"), - } - shown := make([]string, 0, len(files)+2) - shown = append(shown, first.text(), second.text()) - - for _, path := range files { - data, err := os.ReadFile(path) //nolint:gosec // the test's - if err != nil { - t.Fatalf("read %s: %v", path, err) - } - - shown = append(shown, string(data)) - } - - if all := strings.Join(shown, "\n"); strings.Contains(all, bouncerKey) { - t.Errorf( - "the key is shown in the output or the state and cache files:\n%s", - all, - ) - } -} - // wantDeniedByList checks that the request log line is of a request the // blocklist at listURL refused. func wantDeniedByList(t *testing.T, line map[string]any, listURL string) { diff --git a/internal/waf/answertime_test.go b/internal/waf/answertime_test.go index 197b3d2..ba963e3 100644 --- a/internal/waf/answertime_test.go +++ b/internal/waf/answertime_test.go @@ -48,9 +48,6 @@ const ( silentAbuseIPDBURL = "silent://abuseipdb/api/v2/check" // wholeLimit is the percentage of a rate limit that lowers it not at all. wholeLimit = 100 - // clientsDecision is the CrowdSec decision list holding client alone. - clientsDecision = `[{"duration": "4h0m0s", "scenario": "crowdsecurity/ssh-bf", ` + - `"scope": "Ip", "type": "ban", "value": "` + client + `"}]` ) // cookies returns a Cookie header's value with count cookies. @@ -296,10 +293,6 @@ func TestTheHeaderLimitsComeBeforeEveryCheckThatRefuses(t *testing.T) { "/", leaveAsItIs, }, {"a blocklist", map[string]string{blocklistURLs: dropURL}, "/", listClient}, - { - "the CrowdSec decision list", - map[string]string{crowdSecURL: lapi, crowdSecKey: bouncerKey}, "/", decideOnClient, - }, { "a DNS blocklist", map[string]string{ dnsblZones: dnsblZone, dnsblResolver: noResolver, dnsblAbuseIPDBAction: actionDeny, @@ -592,12 +585,6 @@ func listClient(t *testing.T, server *waf.Server, now time.Time) { } } -// decideOnClient puts client on the CrowdSec decision list. -func decideOnClient(t *testing.T, server *waf.Server, now time.Time) { - t.Helper() - loadDecisions(t, server, now, clientsDecision) -} - // leaveAsItIs prepares nothing. func leaveAsItIs(*testing.T, *waf.Server, time.Time) {} @@ -641,12 +628,6 @@ func TestARefusedRequestDoesNotWaitForASlowGeoJS(t *testing.T) { name: "a blocklist", env: map[string]string{blocklistURLs: dropURL}, prepare: listClient, status: forbidden, action: requestlog.ActionClientDenied, }, - { - name: "the CrowdSec decision list", - env: map[string]string{crowdSecURL: lapi, crowdSecKey: bouncerKey}, - prepare: decideOnClient, status: forbidden, - action: requestlog.ActionTemporarilyBanned, - }, { name: "the header limits", cookie: cookies(cookieLimit + 1), prepare: leaveAsItIs, status: tooLarge, @@ -774,7 +755,6 @@ func TestWAFAnswersWithinTheTimeLimitWhileEveryOutsideSourceHangs( dnsblZones: dnsblZone, dnsblResolver: resolver, abuseIPDBKey: accountKey, blocklistURLs: held + "/drop.txt", asnLimitPercentURL: held + "/asn.txt", - crowdSecURL: held, crowdSecKey: bouncerKey, }) // The lists are fetched outside the synctest bubble below: a wait on the diff --git a/internal/waf/bans.go b/internal/waf/bans.go index 7e16880..2da3361 100644 --- a/internal/waf/bans.go +++ b/internal/waf/bans.go @@ -11,7 +11,6 @@ import ( "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/ratelimit" - "sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/requestlog" ) @@ -206,57 +205,6 @@ func (rq *request) banForAttack(now time.Time, notes bans.Notes) string { return banAction(ban.Permanent()) } -// banForCrowdSec bans the client's netblock at now until decision, -// CrowdSec's decision on the client, ends. In observe mode it makes no -// ban, and raises the alert for the ban it would have made, if that alert -// would be sent. It returns the action for that ban. -func (rq *request) banForCrowdSec( - now time.Time, - decision reputation.Decision, -) string { - netblock := rq.h.netblock(rq.client) - if rq.h.config.Observe && - !rq.wouldAlertBan(netblock, now, bans.CauseCrowdSec) { - return banAction( - rq.h.ledger.WouldBePermanent(netblock, now, bans.CauseCrowdSec), - ) - } - - notes := bans.Notes{ - ASN: rq.line.ASN, - ASName: rq.line.ASName, - Country: rq.line.Country, - Reputation: rq.reputation, - Request: rq.noted(now, rq.h.config.BanResponse), - Requests: rq.netblockRequests(netblock), - } - - if rq.h.config.Observe { - ban, wouldBan := rq.h.ledger.WouldBanForCrowdSec( - netblock, - now, - decision.Expires, - decision.Scenario, - notes, - ) - if wouldBan { - rq.alertBan(ban) - } - - return banAction(ban.Permanent()) - } - - ban, made := rq.h.ledger.BanForCrowdSec(netblock, now, decision.Expires, - decision.Scenario, notes) - rq.line.BanExpires = banExpires(ban) - - if made { - rq.alertBan(ban) - } - - return banAction(ban.Permanent()) -} - // wouldAlertBan reports whether the alert for a ban on netblock for cause // made at now would be sent. In observe mode the ban the request would // have made is worked out only then, at most once per diff --git a/internal/waf/crowdsec.go b/internal/waf/crowdsec.go deleted file mode 100644 index 7d8ae5f..0000000 --- a/internal/waf/crowdsec.go +++ /dev/null @@ -1,54 +0,0 @@ -package waf - -// The heuristic crowdsec. It reads SWW_WAF_CROWDSEC_LAPI_URL, and the decisions -// through the reputation lists. - -import ( - "context" - "time" - - "sneak.berlin/go/smallwebwaf/internal/bans" - "sneak.berlin/go/smallwebwaf/internal/config" - "sneak.berlin/go/smallwebwaf/internal/reputation" - "sneak.berlin/go/smallwebwaf/internal/requestlog" -) - -// crowdSec refuses a request when a decision of the CrowdSec decision list on -// the client is in force at now. It then adds the tag crowdsec, notes the -// list, as noteHit does, and bans the client until that decision ends. Its -// figures are the decision's scenario and when it ends. -type crowdSec struct{} - -func (crowdSec) name() string { return "crowdsec" } - -func (crowdSec) check( - _ context.Context, - rq *request, - now time.Time, -) *verdict { - decision, listed := rq.crowdSecDecisionInForce(now) - if !listed { - return nil - } - - rq.addTag(config.CrowdSecTag) - rq.noteHit(bans.ReputationHit{Source: rq.h.config.CrowdSecDecisionsURL}, - "listed by the CrowdSec decision list") - - return &verdict{ - action: rq.banForCrowdSec(now, decision), - figures: map[string]any{ - "scenario": decision.Scenario, - "expires": requestlog.FormatTime(decision.Expires), - }, - } -} - -// crowdSecDecisionInForce returns the decision of the CrowdSec decision list on -// the client that is in force at now, and reports whether there is one, which -// the heuristic crowdsec refuses the request for. -func (rq *request) crowdSecDecisionInForce( - now time.Time, -) (reputation.Decision, bool) { - return rq.h.lists.CrowdSecDecision(rq.client, now) -} diff --git a/internal/waf/crowdsec_test.go b/internal/waf/crowdsec_test.go deleted file mode 100644 index a2617ea..0000000 --- a/internal/waf/crowdsec_test.go +++ /dev/null @@ -1,249 +0,0 @@ -package waf_test - -import ( - "net/http" - "net/netip" - "reflect" - "testing" - "time" - - "sneak.berlin/go/smallwebwaf/internal/alerts" - "sneak.berlin/go/smallwebwaf/internal/bans" - "sneak.berlin/go/smallwebwaf/internal/reputation" - "sneak.berlin/go/smallwebwaf/internal/requestlog" - "sneak.berlin/go/smallwebwaf/internal/waf" -) - -// The CrowdSec settings, and the tests' engine, which is never asked: each -// test puts in the copy of its decision list, at decisionsURL, that it -// needs, as lists.json would at start. -const ( - crowdSecURL = "SWW_WAF_CROWDSEC_LAPI_URL" - crowdSecKey = "SWW_WAF_CROWDSEC_LAPI_KEY" - lapi = "http://crowdsec.example:8080" - decisionsURL = lapi + "/v1/decisions" - bouncerKey = "crowdsec-key-0123456789abcdef" -) - -func TestClientTheCrowdSecDecisionListListsIsBannedUntilTheDecisionEnds( - t *testing.T, -) { - t.Parallel() - - s, clk, server, queue := startWithAlerts(t, map[string]string{ - crowdSecURL: lapi, crowdSecKey: bouncerKey, metricsToken: token, - }) - // client had four hours left on its decision as the engine answered. - fetched := clk.Now() - loadDecisions(t, server, fetched, `[{"duration": "4h0m0s", `+ - `"scenario": "crowdsecurity/ssh-bf", "scope": "Ip", "type": "ban", `+ - `"value": "`+client+`"}]`) - expires := requestlog.FormatTime(fetched.Add(4 * time.Hour)) - - // Its first request is refused, and bans it until the decision ends. - banned := requestlog.ActionTemporarilyBanned - line := s.get(client, http.StatusForbidden, banned) - wantReputation(t, line, decisionsURL) - - if line.BanExpires != expires { - t.Errorf("log line has ban_expires %q, want %s", - line.BanExpires, expires) - } - - listed := []bans.ReputationHit{{Source: decisionsURL}} - - held := server.Ledger.Bans(netip.MustParsePrefix(client + "/32")) - if len(held) != 1 || held[0].Cause != bans.CauseCrowdSec || - !held[0].Start.Equal(fetched) || - !held[0].Expires.Equal(fetched.Add(4*time.Hour)) || - held[0].Reason != "CrowdSec's decision for crowdsecurity/ssh-bf" || - !reflect.DeepEqual(held[0].Notes.Reputation, listed) || - held[0].Notes.Request.Path != "/" || held[0].Notes.Requests != 1 { - t.Fatalf( - "bans %+v, want one for crowdsec of four hours, with the list and "+ - "the request in its notes", - held, - ) - } - - // The listing raises a reputation_hit alert, and the ban its own. - waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook] - if len(waiting) != 2 || waiting[0].Event != alerts.EventReputationHit || - waiting[0].Reason != "listed by the CrowdSec decision list" || - !reflect.DeepEqual( - waiting[1], - banAlert(alerts.EventBan, fetched, client, held[0], - expires), - ) { - t.Errorf( - "alerts waiting %+v, want a reputation_hit alert, then the ban's", - waiting, - ) - } - - // Each request while the ban lasts is refused under it, as under any - // ban, and once it has ended the client is let through. - clk.advance(4*time.Hour - time.Second) - - line = s.get(client, http.StatusForbidden, banned) - wantReputation(t, line) - - if line.BanExpires != expires { - t.Errorf("log line has ban_expires %q, want %s", - line.BanExpires, expires) - } - - clk.advance(time.Second) - s.get(client, http.StatusOK, requestlog.ActionForward) - - // The ban and the hit are counted, and the list has the metrics of any - // list fetched from a URL. - metrics := s.scrape() - labels := `{instance="` + alertInstance + `",source="` + decisionsURL + `"}` - - wantMetric(t, metrics, `sww_bans_made_total{cause="crowdsec",`+ - `instance="`+alertInstance+`"}`, 1) - wantMetric(t, metrics, "sww_reputation_hits_total"+labels, 1) - wantMetric(t, metrics, "sww_reputation_failures_total"+labels, 0) - wantMetric( - t, - metrics, - "sww_reputation_last_fetch_timestamp_seconds"+labels, - float64(fetched.Unix()), - ) -} - -func TestCrowdSecBanNamesItsHeuristicScenarioAndEnd(t *testing.T) { - t.Parallel() - - s, clk, server, _ := startWithAlerts(t, map[string]string{ - crowdSecURL: lapi, crowdSecKey: bouncerKey, - }) - loadDecisions(t, server, clk.Now(), `[{"duration": "4h0m0s", `+ - `"scenario": "crowdsecurity/ssh-bf", "scope": "Ip", "type": "ban", `+ - `"value": "`+client+`"}]`) - - line := s.get( - client, - http.StatusForbidden, - requestlog.ActionTemporarilyBanned, - ) - wantHeuristic( - t, - line, - "crowdsec", - `{"scenario": "crowdsecurity/ssh-bf", `+ - `"expires": "`+requestlog.FormatTime(clk.Now().Add(4*time.Hour))+`"}`, - ) - wantTags(t, line, "crowdsec") -} - -func TestEndedCrowdSecDecisionNoLongerBansThoughTheCopyStillHoldsIt( - t *testing.T, -) { - t.Parallel() - - s, clk, server := startWithClock(t, "", map[string]string{ - crowdSecURL: lapi, crowdSecKey: bouncerKey, - }) - // 198.51.100.0/24 and 2001:db8::9 had a minute left as the engine - // answered. - fetched := clk.Now() - loadDecisions(t, server, fetched, `[{"duration": "1m0s", `+ - `"scenario": "crowdsecurity/http-probing", "scope": "Range", "type": "ban", `+ - `"value": "198.51.100.0/24"}, {"duration": "1m0s", `+ - `"scenario": "crowdsecurity/http-probing", "scope": "Ip", "type": "ban", `+ - `"value": "2001:db8::9"}]`) - - // Just before its end, the decision bans a client in the netblock, and - // one on an IPv6 address bans the address's group, the /64. - clk.advance(time.Minute - time.Nanosecond) - s.get( - "198.51.100.7", - http.StatusForbidden, - requestlog.ActionTemporarilyBanned, - ) - s.get( - "2001:db8::9", - http.StatusForbidden, - requestlog.ActionTemporarilyBanned, - ) - s.get( - "2001:db8::5", - http.StatusForbidden, - requestlog.ActionTemporarilyBanned, - ) - - // Once it has ended, it bans no other client, and the bans it made end - // with it. - clk.advance(time.Nanosecond) - - for _, from := range []string{"198.51.100.8", "198.51.100.7", "2001:db8::5"} { - wantReputation( - t, - s.get(from, http.StatusOK, requestlog.ActionForward), - ) - } - - if made := server.Ledger.Made(bans.CauseCrowdSec); made != 2 { - t.Errorf( - "%d bans made for crowdsec, want 2, on 198.51.100.7/32 and "+ - "2001:db8::/64", - made, - ) - } - - if held := server.Ledger.Bans(netip.MustParsePrefix("2001:db8::/64")); len( - held, - ) != 1 { - t.Errorf("bans of 2001:db8::/64 %+v, want one", held) - } -} - -func TestObserveModeForwardsAClientTheCrowdSecDecisionListListsAndAlertsTheBan( - t *testing.T, -) { - t.Parallel() - - s, clk, server, queue := startWithAlerts(t, map[string]string{ - crowdSecURL: lapi, crowdSecKey: bouncerKey, mode: observe, - }) - loadDecisions(t, server, clk.Now(), `[{"duration": "4h0m0s", `+ - `"scenario": "crowdsecurity/ssh-bf", "scope": "Ip", "type": "ban", `+ - `"value": "`+client+`"}]`) - - line := s.get(client, http.StatusOK, requestlog.ActionForward) - wantWouldAction(t, line, requestlog.ActionTemporarilyBanned) - wantReputation(t, line, decisionsURL) - - if held := server.Ledger.Snapshot(); len(held) != 0 { - t.Errorf("bans %+v, want none", held) - } - - waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook] - if len(waiting) != 2 || waiting[1].Event != alerts.EventBan || - waiting[1].Detail["cause"] != bans.CauseCrowdSec || - waiting[1].Detail["mode"] != observe { - t.Errorf( - "alerts waiting %+v, want a reputation_hit alert, then the ban alert "+ - "marked observe", - waiting, - ) - } -} - -// loadDecisions puts into server's lists the copy of the decision list at -// decisionsURL, answer, the engine's answer, fetched at fetched, as -// lists.json would at start. -func loadDecisions( - t *testing.T, server *waf.Server, fetched time.Time, answer string, -) { - t.Helper() - - err := server.Lists.Load([]reputation.List{{ - URL: decisionsURL, Tried: fetched, Fetched: fetched, Lines: []string{answer}, - }}) - if err != nil { - t.Fatalf("load the decision list: %v", err) - } -} diff --git a/internal/waf/heuristics.go b/internal/waf/heuristics.go index b928eab..920410d 100644 --- a/internal/waf/heuristics.go +++ b/internal/waf/heuristics.go @@ -60,7 +60,6 @@ func afterLookup() []heuristic { return []heuristic{ countries{}, blocklists{}, - crowdSec{}, dnsbl{}, abuseIPDB{}, tagDeny{}, diff --git a/internal/waf/proxy.go b/internal/waf/proxy.go index 370c6d3..59500fe 100644 --- a/internal/waf/proxy.go +++ b/internal/waf/proxy.go @@ -75,8 +75,8 @@ type Params struct { Rules *rules.Files // Alerts receive the alert for each ban waf makes or makes // permanent, for each count over an anomaly threshold, for each request - // whose client a blocklist, the CrowdSec decision list, a DNSBL zone or - // AbuseIPDB lists, for each request the Core Rule Set scores at or over + // whose client a blocklist, a DNSBL zone or AbuseIPDB lists, for each + // request the Core Rule Set scores at or over // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD, and for GeoJS failing, a fetch of a list // failing, a query to a DNSBL zone or a check with AbuseIPDB failing, // or the day's AbuseIPDB checks used up. @@ -194,11 +194,13 @@ func newReputation( ) (*reputation.Lists, *reputation.DNSBL, *reputation.AbuseIPDB) { cfg := params.Config lists := reputation.New(reputation.Params{ - BlocklistURLs: cfg.BlocklistURLs, Refresh: cfg.BlocklistRefresh, - ASNLimitPercentURL: cfg.ASNLimitPercentURL, - TagLists: cfg.TagLists, - CrowdSecDecisionsURL: cfg.CrowdSecDecisionsURL, CrowdSecKey: cfg.CrowdSecKey, - Now: params.Now, ProcessLog: params.ProcessLog, Alerts: params.Alerts, + BlocklistURLs: cfg.BlocklistURLs, + Refresh: cfg.BlocklistRefresh, + ASNLimitPercentURL: cfg.ASNLimitPercentURL, + TagLists: cfg.TagLists, + Now: params.Now, + ProcessLog: params.ProcessLog, + Alerts: params.Alerts, }) dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{ Zones: cfg.DNSBLZones, Resolver: cfg.DNSBLResolver, diff --git a/internal/waf/reputation.go b/internal/waf/reputation.go index c6692d7..06bd4f5 100644 --- a/internal/waf/reputation.go +++ b/internal/waf/reputation.go @@ -1,7 +1,6 @@ package waf -// What the reputation heuristics share: blocklists, crowdsec, dnsbl and -// abuseipdb. +// What the reputation heuristics share: blocklists, dnsbl and abuseipdb. import ( "sneak.berlin/go/smallwebwaf/internal/alerts" diff --git a/internal/waf/waf.go b/internal/waf/waf.go index b5372ad..ad12c76 100644 --- a/internal/waf/waf.go +++ b/internal/waf/waf.go @@ -146,18 +146,15 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // refusedWhateverTheAnswer reports whether the request is one that does not wait // for GeoJS, since the checks will refuse it at now whatever GeoJS answers about -// its client: the heuristic crowdsec, blocklists or rate_limits will refuse it, -// as a function in that heuristic's file says. In observe mode, which refuses +// its client: the heuristic blocklists or rate_limits will refuse it, as a +// function in that heuristic's file says. In observe mode, which refuses // nothing, none is. rproxy lets a request through when waf answers too late. func (rq *request) refusedWhateverTheAnswer(now time.Time) bool { if rq.h.config.Observe { return false } - _, decided := rq.crowdSecDecisionInForce(now) - - return decided || - rq.refusedByBlocklists(rq.h.lists.ListedBy(rq.client)) || + return rq.refusedByBlocklists(rq.h.lists.ListedBy(rq.client)) || rq.overAWholeRateLimit(now) }