CrowdSec decision list fetched, kept, and its clients banned until the decision ends (closes #106)
check / check (push) Waiting to run
check / check (push) Waiting to run
SWWAF_CROWDSEC_LAPI_URL and SWWAF_CROWDSEC_LAPI_KEY name an engine whose decision list, <url>/v1/decisions, is fetched every minute with the key in X-Api-Key, following no redirect, and kept as a blocklist is: used while a fetch fails, and across restarts through reputation.json. Ban decisions on an Ip or a Range end at the fetch time plus their duration. A listed client's request is refused and bans its netblock with the cause crowdsec until the decision ends; bans.json, ban notes and metrics take the cause. Judgement call: fetched every minute, not a setting. Judgement call: a crowdsec ban never lengthens a limit ban. Judgement call: a lifted crowdsec ban is remade while its decision lasts. Model: opus-5-5
This commit is contained in:
@@ -46,8 +46,9 @@ const (
|
||||
EventAnomaly = "anomaly"
|
||||
// EventWAFBlock comes with the Core Rule Set; nothing raises it yet.
|
||||
EventWAFBlock = "waf_block"
|
||||
// EventReputationHit is a request whose client a blocklist or a DNSBL
|
||||
// zone lists, or whose AbuseIPDB score is a hit.
|
||||
// EventReputationHit is a request whose client a blocklist, the
|
||||
// CrowdSec decision list or a DNSBL zone lists, or whose AbuseIPDB score
|
||||
// is a hit.
|
||||
EventReputationHit = "reputation_hit"
|
||||
// EventSourceFailure is GeoJS failing or refusing smallwebwaf, a fetch
|
||||
// of a list failing, a query to a DNSBL zone or a check with AbuseIPDB
|
||||
|
||||
+87
-38
@@ -1,8 +1,9 @@
|
||||
// Package bans is the ban ledger: the bans smallwebwaf makes on the
|
||||
// netblocks of clients that break a rate limit or a byte limit or show a
|
||||
// clear sign of attack, and those an admin makes, with their notes, as
|
||||
// the "Bans" section of SPEC.md describes. The bans are kept in memory,
|
||||
// and written to bans.json and read from it by the state package.
|
||||
// netblocks of clients that break a rate limit or a byte limit, show a
|
||||
// clear sign of attack or are listed by the CrowdSec decision list, and
|
||||
// those an admin makes, with their notes, as the "Bans" section of SPEC.md
|
||||
// describes. The bans are kept in memory, and written to bans.json and
|
||||
// read from it by the state package.
|
||||
package bans
|
||||
|
||||
import (
|
||||
@@ -26,6 +27,9 @@ const (
|
||||
// CauseAdmin is a ban an admin made, or one smallwebwaf made that an
|
||||
// admin keeps. It is never dropped.
|
||||
CauseAdmin = "admin"
|
||||
// CauseCrowdSec is a ban smallwebwaf made for a client the CrowdSec
|
||||
// decision list lists. It ends when CrowdSec's decision does.
|
||||
CauseCrowdSec = "crowdsec"
|
||||
)
|
||||
|
||||
// repeatFactor is how many times as long as the netblock's last ban a ban
|
||||
@@ -40,9 +44,9 @@ type Rules struct {
|
||||
// LimitBanDuration is how long a first ban for a broken limit lasts.
|
||||
LimitBanDuration time.Duration
|
||||
// LimitBanRepeatWindow is how soon after the end of the netblock's
|
||||
// ban that ended last, other than one for a clear sign of attack, a
|
||||
// broken limit counts as a repeat, which bans for repeatFactor times as
|
||||
// long as that ban.
|
||||
// ban that ended last, other than one for a clear sign of attack or for
|
||||
// CrowdSec's decision, a broken limit counts as a repeat, which bans for
|
||||
// repeatFactor times as long as that ban.
|
||||
LimitBanRepeatWindow time.Duration
|
||||
// MaxBanDuration is the longest ban for a broken limit; one that would
|
||||
// be longer is permanent instead.
|
||||
@@ -63,10 +67,11 @@ type Ban struct {
|
||||
Start time.Time
|
||||
// Expires is when the ban ends, zero for a permanent ban.
|
||||
Expires time.Time
|
||||
// Cause is CauseLimit, CauseAttack or CauseAdmin.
|
||||
// Cause is CauseLimit, CauseAttack, CauseAdmin or CauseCrowdSec.
|
||||
Cause string
|
||||
// Reason is a short text: for a ban smallwebwaf made, the limit broken
|
||||
// or the rule that matched; for an admin's, what the admin wrote.
|
||||
// Reason is a short text: for a ban smallwebwaf made, the limit broken,
|
||||
// the rule that matched or the scenario of CrowdSec's decision; for an
|
||||
// admin's, what the admin wrote.
|
||||
Reason string
|
||||
// Lifted is when an admin lifted the ban, zero while no admin has. A
|
||||
// lifted ban refuses nothing, and does not make the netblock's next
|
||||
@@ -123,7 +128,8 @@ type Notes struct {
|
||||
// log's reputation names them. It is left out when none did.
|
||||
Reputation []ReputationHit `json:"reputation,omitempty"`
|
||||
// Request is the request that broke the limit, or whose bytes broke
|
||||
// it, or that was the clear sign of attack.
|
||||
// it, that was the clear sign of attack, or that came from a client the
|
||||
// CrowdSec decision list lists.
|
||||
Request Request `json:"request"`
|
||||
// Requests is how many requests the netblock has sent since it was
|
||||
// first seen, and Refused how many of them the ban has refused so
|
||||
@@ -136,9 +142,10 @@ type Notes struct {
|
||||
}
|
||||
|
||||
// ReputationHit is a reputation source that listed a client, as a
|
||||
// reputation_hit alert's detail gives it: Source is the blocklist's URL,
|
||||
// the DNSBL zone with its key masked, or "abuseipdb", and Score, for
|
||||
// AbuseIPDB alone, its score of the client.
|
||||
// reputation_hit alert's detail gives it: Source is the URL of the
|
||||
// blocklist or of the CrowdSec decision list, the DNSBL zone with its key
|
||||
// masked, or "abuseipdb", and Score, for AbuseIPDB alone, its score of the
|
||||
// client.
|
||||
type ReputationHit struct {
|
||||
Source string `json:"source"`
|
||||
Score *int64 `json:"score,omitempty"`
|
||||
@@ -146,9 +153,10 @@ type ReputationHit struct {
|
||||
|
||||
// EarlierBans counts a netblock's bans before a ban, by cause.
|
||||
type EarlierBans struct {
|
||||
Limit int `json:"limit"`
|
||||
Attack int `json:"attack"`
|
||||
Admin int `json:"admin"`
|
||||
Limit int `json:"limit"`
|
||||
Attack int `json:"attack"`
|
||||
Admin int `json:"admin"`
|
||||
CrowdSec int `json:"crowdsec"`
|
||||
}
|
||||
|
||||
// Request is a request in a ban's notes. Each text is cut to 256 bytes.
|
||||
@@ -276,7 +284,8 @@ func activeBan(bans []Ban, now time.Time) *Ban {
|
||||
// BanForLimit bans netblock at now for a broken limit, with notes, and
|
||||
// returns the ban, and true. A first ban lasts LimitBanDuration. A ban
|
||||
// made within LimitBanRepeatWindow after the netblock's ban that ended
|
||||
// last, other than one for a clear sign of attack or a lifted one, lasts
|
||||
// last, other than one for a clear sign of attack or for CrowdSec's
|
||||
// decision, or a lifted one, lasts
|
||||
// repeatFactor times as long as that one. A ban that would be longer
|
||||
// than MaxBanDuration is permanent instead. If a ban on netblock is still
|
||||
// active, as when two of its requests break a limit at once, that ban is
|
||||
@@ -287,7 +296,7 @@ func activeBan(bans []Ban, now time.Time) *Ban {
|
||||
func (l *Ledger) BanForLimit(
|
||||
netblock netip.Prefix, now time.Time, notes Notes,
|
||||
) (Ban, bool) {
|
||||
return l.ban(netblock, now, CauseLimit, limitReason(notes), notes, true)
|
||||
return l.ban(netblock, now, time.Time{}, CauseLimit, limitReason(notes), notes, true)
|
||||
}
|
||||
|
||||
// WouldBanForLimit returns what BanForLimit would, without making the ban:
|
||||
@@ -295,7 +304,7 @@ func (l *Ledger) BanForLimit(
|
||||
func (l *Ledger) WouldBanForLimit(
|
||||
netblock netip.Prefix, now time.Time, notes Notes,
|
||||
) (Ban, bool) {
|
||||
return l.ban(netblock, now, CauseLimit, limitReason(notes), notes, false)
|
||||
return l.ban(netblock, now, time.Time{}, CauseLimit, limitReason(notes), notes, false)
|
||||
}
|
||||
|
||||
// BanForAttack bans netblock at now for a clear sign of attack, with
|
||||
@@ -306,7 +315,7 @@ func (l *Ledger) WouldBanForLimit(
|
||||
func (l *Ledger) BanForAttack(
|
||||
netblock netip.Prefix, now time.Time, notes Notes,
|
||||
) (Ban, bool) {
|
||||
return l.ban(netblock, now, CauseAttack, attackReason(notes), notes, true)
|
||||
return l.ban(netblock, now, time.Time{}, CauseAttack, attackReason(notes), notes, true)
|
||||
}
|
||||
|
||||
// WouldBanForAttack returns what BanForAttack would, without making the
|
||||
@@ -314,12 +323,35 @@ func (l *Ledger) BanForAttack(
|
||||
func (l *Ledger) WouldBanForAttack(
|
||||
netblock netip.Prefix, now time.Time, notes Notes,
|
||||
) (Ban, bool) {
|
||||
return l.ban(netblock, now, CauseAttack, attackReason(notes), notes, false)
|
||||
return l.ban(netblock, now, time.Time{}, CauseAttack, attackReason(notes), notes,
|
||||
false)
|
||||
}
|
||||
|
||||
// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit
|
||||
// or CauseAttack, made at now would be permanent, as BanForLimit or
|
||||
// BanForAttack would make it. It works out nothing else of the ban.
|
||||
// BanForCrowdSec bans netblock at now until expires, when CrowdSec's
|
||||
// decision on the client ends, with notes, and returns the ban, and
|
||||
// whether it made it, as BanForLimit does. Its reason is "CrowdSec's
|
||||
// decision for <scenario>", the scenario that made the decision.
|
||||
func (l *Ledger) BanForCrowdSec(
|
||||
netblock netip.Prefix, now, expires time.Time, scenario string, notes Notes,
|
||||
) (Ban, bool) {
|
||||
return l.ban(netblock, now, expires, CauseCrowdSec, crowdSecReason(scenario), notes,
|
||||
true)
|
||||
}
|
||||
|
||||
// WouldBanForCrowdSec returns what BanForCrowdSec would, without making
|
||||
// the ban: what observe mode would have done.
|
||||
func (l *Ledger) WouldBanForCrowdSec(
|
||||
netblock netip.Prefix, now, expires time.Time, scenario string, notes Notes,
|
||||
) (Ban, bool) {
|
||||
return l.ban(netblock, now, expires, CauseCrowdSec, crowdSecReason(scenario), notes,
|
||||
false)
|
||||
}
|
||||
|
||||
// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit,
|
||||
// CauseAttack or CauseCrowdSec, made at now would be permanent, as
|
||||
// BanForLimit, BanForAttack or BanForCrowdSec would make it. It works out
|
||||
// nothing else of the ban. A ban for CrowdSec's decision is never
|
||||
// permanent: it ends with the decision.
|
||||
func (l *Ledger) WouldBePermanent(
|
||||
netblock netip.Prefix, now time.Time, cause string,
|
||||
) bool {
|
||||
@@ -331,11 +363,14 @@ func (l *Ledger) WouldBePermanent(
|
||||
held = *bans
|
||||
}
|
||||
|
||||
if cause == CauseAttack {
|
||||
switch cause {
|
||||
case CauseAttack:
|
||||
return l.attackExpiry(held, now).IsZero()
|
||||
case CauseLimit:
|
||||
return l.limitExpiry(held, now).IsZero()
|
||||
default: // CauseCrowdSec
|
||||
return false
|
||||
}
|
||||
|
||||
return l.limitExpiry(held, now).IsZero()
|
||||
}
|
||||
|
||||
// limitReason is the reason of a ban for a broken limit, with notes.
|
||||
@@ -350,6 +385,12 @@ func attackReason(notes Notes) string {
|
||||
return "matched the rule " + notes.RuleID
|
||||
}
|
||||
|
||||
// crowdSecReason is the reason of a ban for CrowdSec's decision, which
|
||||
// scenario made.
|
||||
func crowdSecReason(scenario string) string {
|
||||
return "CrowdSec's decision for " + scenario
|
||||
}
|
||||
|
||||
// BanForAdmin bans netblock at now for an admin, with reason, until
|
||||
// expires, or for good when expires is zero, and returns the ban, whose
|
||||
// cause is CauseAdmin. Unlike BanForLimit and BanForAttack, it makes the
|
||||
@@ -587,11 +628,14 @@ func (l *Ledger) holds(netblock netip.Prefix, start time.Time) bool {
|
||||
}
|
||||
|
||||
// ban bans netblock at now for cause, with reason and notes, as
|
||||
// BanForLimit and BanForAttack describe, and returns the ban, and whether
|
||||
// it made it. Unless keep is true, the ban is not made, only returned: it
|
||||
// is the ban that would have been made.
|
||||
// BanForLimit, BanForAttack and BanForCrowdSec describe, and returns the
|
||||
// ban, and whether it made it. expires is when a ban for CauseCrowdSec
|
||||
// ends, and zero for the others, whose end the ledger works out. Unless
|
||||
// keep is true, the ban is not made, only returned: it is the ban that
|
||||
// would have been made.
|
||||
func (l *Ledger) ban(
|
||||
netblock netip.Prefix, now time.Time, cause, reason string, notes Notes, keep bool,
|
||||
netblock netip.Prefix, now, expires time.Time, cause, reason string, notes Notes,
|
||||
keep bool,
|
||||
) (Ban, bool) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
@@ -613,10 +657,13 @@ func (l *Ledger) ban(
|
||||
notes.Request = notes.Request.cut()
|
||||
ban := Ban{Netblock: netblock, Start: now, Cause: cause, Reason: reason, Notes: notes}
|
||||
|
||||
if cause == CauseAttack {
|
||||
switch cause {
|
||||
case CauseAttack:
|
||||
ban.Expires = l.attackExpiry(held, now)
|
||||
} else {
|
||||
case CauseLimit:
|
||||
ban.Expires = l.limitExpiry(held, now)
|
||||
default: // CauseCrowdSec
|
||||
ban.Expires = expires
|
||||
}
|
||||
|
||||
if !keep {
|
||||
@@ -645,6 +692,8 @@ func earlierBans(held []Ban) EarlierBans {
|
||||
earlier.Attack++
|
||||
case CauseAdmin:
|
||||
earlier.Admin++
|
||||
case CauseCrowdSec:
|
||||
earlier.CrowdSec++
|
||||
}
|
||||
}
|
||||
|
||||
@@ -738,16 +787,16 @@ func (l *Ledger) add(ban Ban) {
|
||||
// limitExpiry returns when a ban for a broken limit made at now ends, or
|
||||
// zero when it is permanent. held are the netblock's bans, none of them
|
||||
// active, of which the one that ended last, other than a ban for a clear
|
||||
// sign of attack or a lifted one, can make the new ban longer. A ban an
|
||||
// admin adds to bans.json can start after another and end before it, so
|
||||
// that one is looked for among them all.
|
||||
// sign of attack or for CrowdSec's decision, or a lifted one, can make the
|
||||
// new ban longer. A ban an admin adds to bans.json can start after
|
||||
// another and end before it, so that one is looked for among them all.
|
||||
func (l *Ledger) limitExpiry(held []Ban, now time.Time) time.Time {
|
||||
length := l.rules.LimitBanDuration
|
||||
|
||||
var last *Ban
|
||||
|
||||
for i, ban := range held {
|
||||
if ban.Cause != CauseAttack && ban.Lifted.IsZero() &&
|
||||
if (ban.Cause == CauseLimit || ban.Cause == CauseAdmin) && ban.Lifted.IsZero() &&
|
||||
(last == nil || ban.Expires.After(last.Expires)) {
|
||||
last = &held[i]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package bans_test
|
||||
|
||||
import (
|
||||
"net/netip"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
)
|
||||
|
||||
// scenario is the scenario of the tests' CrowdSec decisions.
|
||||
const scenario = "crowdsecurity/ssh-bf"
|
||||
|
||||
func TestCrowdSecBanLastsUntilTheDecisionEnds(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ledger := bans.New(defaultRules())
|
||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||
expires := midnight().Add(4 * time.Hour)
|
||||
|
||||
// The ban that would be made is not made.
|
||||
would, wouldBan := ledger.WouldBanForCrowdSec(netblock, midnight(), expires,
|
||||
scenario, bans.Notes{})
|
||||
if !wouldBan || len(ledger.Bans(netblock)) != 0 {
|
||||
t.Errorf("would ban %t, and the ledger holds %+v, want true and nothing",
|
||||
wouldBan, ledger.Bans(netblock))
|
||||
}
|
||||
|
||||
const reason = "CrowdSec's decision for " + scenario
|
||||
|
||||
ban, made := ledger.BanForCrowdSec(netblock, midnight(), expires, scenario,
|
||||
bans.Notes{})
|
||||
if !made || !reflect.DeepEqual(ban, would) || ban.Cause != bans.CauseCrowdSec ||
|
||||
!ban.Expires.Equal(expires) || ban.Reason != reason ||
|
||||
ledger.Made(bans.CauseCrowdSec) != 1 {
|
||||
t.Errorf("made %t the ban %+v, want the one that would be made, %+v, for "+
|
||||
"crowdsec until %s", made, ban, would, expires)
|
||||
}
|
||||
|
||||
// A second decision on the netblock while the ban lasts makes no other.
|
||||
again, made := ledger.BanForCrowdSec(netblock, midnight().Add(time.Hour),
|
||||
expires.Add(time.Hour), scenario, bans.Notes{})
|
||||
if made || !again.Expires.Equal(expires) || ledger.Made(bans.CauseCrowdSec) != 1 {
|
||||
t.Errorf("made %t the ban %+v while the first lasts, want none", made, again)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrowdSecBanIsNeverMadePermanent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ledger := bans.New(defaultRules())
|
||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||
expires := midnight().Add(4 * time.Hour)
|
||||
ledger.BanForCrowdSec(netblock, midnight(), expires, scenario, bans.Notes{})
|
||||
|
||||
// A request as the ban ends is refused, and leaves it as it is.
|
||||
last := expires.Add(-time.Nanosecond)
|
||||
|
||||
held, banned, madePermanent := ledger.Check(netblock.Addr(), last)
|
||||
if !banned || madePermanent || !held.Expires.Equal(expires) ||
|
||||
ledger.WouldBePermanent(netblock, last, bans.CauseCrowdSec) {
|
||||
t.Errorf("as the ban ends, banned %t with %+v, made permanent %t, want "+
|
||||
"refused under the ban as it was", banned, held, madePermanent)
|
||||
}
|
||||
|
||||
if _, banned, _ := ledger.Check(netblock.Addr(), expires); banned {
|
||||
t.Error("the ban refuses a request once the decision has ended")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrowdSecBanIsCountedAndDoesNotLengthenTheNextBanForALimit(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ledger := bans.New(defaultRules())
|
||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||
|
||||
// Three times the three days would be permanent; a limit broken as the
|
||||
// ban for CrowdSec's decision ends bans for an hour, as a first broken
|
||||
// limit does.
|
||||
crowdSec, _ := ledger.BanForCrowdSec(netblock, midnight(), midnight().Add(3*day),
|
||||
scenario, bans.Notes{})
|
||||
limit, _ := ledger.BanForLimit(netblock, crowdSec.Expires, bans.Notes{})
|
||||
|
||||
if limit.Expires.Sub(limit.Start) != time.Hour ||
|
||||
limit.Notes.EarlierBans != (bans.EarlierBans{CrowdSec: 1}) {
|
||||
t.Errorf("the ban for a limit is %+v, want one of an hour after one for crowdsec",
|
||||
limit)
|
||||
}
|
||||
}
|
||||
@@ -181,6 +181,13 @@ type Config struct {
|
||||
ReputationLimitPercent int64
|
||||
ReputationCacheTTL time.Duration
|
||||
ReputationTimeout time.Duration
|
||||
// CrowdSecDecisionsURL is where the decision list of the CrowdSec
|
||||
// engine whose local API SWWAF_CROWDSEC_LAPI_URL names is fetched from:
|
||||
// that URL with v1/decisions added to its path, "" while it is unset and
|
||||
// none is. CrowdSecKey is the key the engine is asked with
|
||||
// (SWWAF_CROWDSEC_LAPI_KEY).
|
||||
CrowdSecDecisionsURL string
|
||||
CrowdSecKey string
|
||||
// BanResponse is the status a refused client is answered with, 403
|
||||
// or 429, or 0 to close the connection without an answer
|
||||
// (SWWAF_BAN_RESPONSE). It answers a banned client, a request that
|
||||
@@ -410,6 +417,13 @@ var (
|
||||
"names IPv6 clients are asked about by")
|
||||
errNotResolver = errors.New("is not an IP address with an optional port, " +
|
||||
"such as 192.0.2.53 or [2001:db8::53]:5353")
|
||||
errNotLAPIURL = errors.New(
|
||||
"is not an http or https URL without a user or a fragment, " +
|
||||
"such as http://172.17.0.1:8080")
|
||||
errNeedsLAPIKey = errors.New("the engine answers no request without it")
|
||||
errLAPIKeyUnused = errors.New("it is sent only to the engine at that URL")
|
||||
errAnotherList = errors.New(
|
||||
"is in SWWAF_BLOCKLIST_URLS or is SWWAF_ASN_LIMIT_PERCENT_URL too")
|
||||
)
|
||||
|
||||
// FromEnvironment reads the settings with lookupEnv, normally
|
||||
@@ -471,6 +485,8 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
|
||||
AbuseIPDBDailyBudget: env.numberNotOff("SWWAF_ABUSEIPDB_DAILY_BUDGET", "900"),
|
||||
ReputationCacheTTL: env.durationNotOff("SWWAF_REPUTATION_CACHE_TTL", "24h"),
|
||||
ReputationTimeout: env.durationNotOff("SWWAF_REPUTATION_TIMEOUT", "2s"),
|
||||
CrowdSecDecisionsURL: env.crowdSecDecisionsURL("SWWAF_CROWDSEC_LAPI_URL"),
|
||||
CrowdSecKey: env.secret("SWWAF_CROWDSEC_LAPI_KEY"),
|
||||
BanResponse: env.banResponse("SWWAF_BAN_RESPONSE", "403"),
|
||||
LimitBanDuration: env.durationNotOff("SWWAF_LIMIT_BAN_DURATION", "1h"),
|
||||
LimitBanRepeatWindow: env.durationNotOff("SWWAF_LIMIT_BAN_REPEAT_WINDOW", "24h"),
|
||||
@@ -523,6 +539,7 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
|
||||
env.checkLookupDBPath(cfg)
|
||||
env.checkCountriesAndLookups(cfg)
|
||||
env.checkASNLimitPercentURL(cfg)
|
||||
env.checkCrowdSec(cfg)
|
||||
|
||||
if env.err != nil {
|
||||
return nil, env.err
|
||||
@@ -835,6 +852,26 @@ func (e *environment) resolver(name string) netip.AddrPort {
|
||||
return resolver
|
||||
}
|
||||
|
||||
// crowdSecDecisionsURL reads the setting that is the URL of the CrowdSec
|
||||
// engine's local API, such as http://172.17.0.1:8080, and returns the URL
|
||||
// its decision list is fetched from, that URL with v1/decisions added to
|
||||
// its path, "" while it is unset or empty.
|
||||
func (e *environment) crowdSecDecisionsURL(name string) string {
|
||||
value := e.value(name, "")
|
||||
if value == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
lapi, err := url.Parse(value)
|
||||
if err != nil || !isHTTPURL(lapi) {
|
||||
e.check(name, fmt.Errorf("%q %w", value, errNotLAPIURL))
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
return lapi.JoinPath("v1", "decisions").String()
|
||||
}
|
||||
|
||||
// lookupSource reads the setting that is where clients are looked up:
|
||||
// geojs, file, or off.
|
||||
func (e *environment) lookupSource(name, defaultValue string) string {
|
||||
@@ -912,6 +949,26 @@ func (e *environment) checkASNLimitPercentURL(cfg *Config) {
|
||||
}
|
||||
}
|
||||
|
||||
// checkCrowdSec refuses SWWAF_CROWDSEC_LAPI_URL without
|
||||
// SWWAF_CROWDSEC_LAPI_KEY, the key without the URL, and a decision list
|
||||
// that is fetched as another list too.
|
||||
func (e *environment) checkCrowdSec(cfg *Config) {
|
||||
decisionsURL := cfg.CrowdSecDecisionsURL
|
||||
|
||||
switch {
|
||||
case decisionsURL != "" && cfg.CrowdSecKey == "":
|
||||
e.check("SWWAF_CROWDSEC_LAPI_URL", fmt.Errorf(
|
||||
"is set while SWWAF_CROWDSEC_LAPI_KEY is unset; %w", errNeedsLAPIKey))
|
||||
case decisionsURL == "" && cfg.CrowdSecKey != "":
|
||||
e.check("SWWAF_CROWDSEC_LAPI_KEY", fmt.Errorf(
|
||||
"is set while SWWAF_CROWDSEC_LAPI_URL is unset; %w", errLAPIKeyUnused))
|
||||
case decisionsURL != "" && (slices.Contains(cfg.BlocklistURLs, decisionsURL) ||
|
||||
decisionsURL == cfg.ASNLimitPercentURL):
|
||||
e.check("SWWAF_CROWDSEC_LAPI_URL", fmt.Errorf("gives the decision list %q, which %w",
|
||||
decisionsURL, errAnotherList))
|
||||
}
|
||||
}
|
||||
|
||||
// headerNames reads a setting that is a list of header names, and
|
||||
// returns them in lower case.
|
||||
func (e *environment) headerNames(name, defaultValue string) []string {
|
||||
|
||||
@@ -71,6 +71,8 @@ const (
|
||||
reputationAction = "SWWAF_REPUTATION_ACTION"
|
||||
reputationCacheTTL = "SWWAF_REPUTATION_CACHE_TTL"
|
||||
reputationTimeout = "SWWAF_REPUTATION_TIMEOUT"
|
||||
crowdSecURL = "SWWAF_CROWDSEC_LAPI_URL"
|
||||
crowdSecKey = "SWWAF_CROWDSEC_LAPI_KEY"
|
||||
banResponse = "SWWAF_BAN_RESPONSE"
|
||||
limitBanDuration = "SWWAF_LIMIT_BAN_DURATION"
|
||||
limitBanRepeatWindow = "SWWAF_LIMIT_BAN_REPEAT_WINDOW"
|
||||
@@ -1653,6 +1655,121 @@ func TestAbuseIPDBKeyIsLoggedMasked(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrowdSecSettingsGiveTheDecisionListAndTheKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg := fromEnvironment(t, environment{})
|
||||
if cfg.CrowdSecDecisionsURL != "" || cfg.CrowdSecKey != "" {
|
||||
t.Errorf("by default, the decision list %q and the key %q, want neither",
|
||||
cfg.CrowdSecDecisionsURL, cfg.CrowdSecKey)
|
||||
}
|
||||
|
||||
for lapi, want := range map[string]string{
|
||||
"http://172.17.0.1:8080": "http://172.17.0.1:8080/v1/decisions",
|
||||
"http://172.17.0.1:8080/": "http://172.17.0.1:8080/v1/decisions",
|
||||
"https://crowdsec.example/lapi/": "https://crowdsec.example/lapi/v1/decisions",
|
||||
"https://crowdsec.example:8443/x": "https://crowdsec.example:8443/x/v1/decisions",
|
||||
} {
|
||||
cfg := fromEnvironment(t, environment{crowdSecURL: lapi, crowdSecKey: token})
|
||||
if cfg.CrowdSecDecisionsURL != want || cfg.CrowdSecKey != token {
|
||||
t.Errorf("%s=%s gave the decision list %q and the key %q, want %s and %s",
|
||||
crowdSecURL, lapi, cfg.CrowdSecDecisionsURL, cfg.CrowdSecKey, want, token)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidCrowdSecSettingStopsTheStartSayingWhatIsWrong(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const (
|
||||
lapi = "http://172.17.0.1:8080"
|
||||
notLAPIURL = " is not an http or https URL without a user or a fragment, " +
|
||||
"such as http://172.17.0.1:8080"
|
||||
anotherList = `gives the decision list "` + lapi + `/v1/decisions", which is in ` +
|
||||
`SWWAF_BLOCKLIST_URLS or is SWWAF_ASN_LIMIT_PERCENT_URL too`
|
||||
)
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
env environment
|
||||
want string
|
||||
}{
|
||||
{
|
||||
"a URL that is not http",
|
||||
environment{crowdSecURL: "ftp://172.17.0.1", crowdSecKey: token},
|
||||
crowdSecURL + `: "ftp://172.17.0.1"` + notLAPIURL,
|
||||
},
|
||||
{
|
||||
"a URL with a user",
|
||||
environment{crowdSecURL: "http://bouncer@172.17.0.1:8080", crowdSecKey: token},
|
||||
crowdSecURL + `: "http://bouncer@172.17.0.1:8080"` + notLAPIURL,
|
||||
},
|
||||
{
|
||||
"the URL without the key",
|
||||
environment{crowdSecURL: lapi},
|
||||
crowdSecURL + ": is set while " + crowdSecKey + " is unset; the engine " +
|
||||
"answers no request without it",
|
||||
},
|
||||
{
|
||||
"the key without the URL",
|
||||
environment{crowdSecKey: token},
|
||||
crowdSecKey + ": is set while " + crowdSecURL + " is unset; it is sent only " +
|
||||
"to the engine at that URL",
|
||||
},
|
||||
{
|
||||
"the decision list as a blocklist too",
|
||||
environment{
|
||||
crowdSecURL: lapi, crowdSecKey: token,
|
||||
blocklistURLs: "https://lists.example/drop.txt," + lapi + "/v1/decisions",
|
||||
},
|
||||
crowdSecURL + ": " + anotherList,
|
||||
},
|
||||
{
|
||||
"the decision list as the file of AS:percent lines too",
|
||||
environment{
|
||||
crowdSecURL: lapi + "/", crowdSecKey: token,
|
||||
asnLimitPercentURL: lapi + "/v1/decisions",
|
||||
},
|
||||
crowdSecURL + ": " + anotherList,
|
||||
},
|
||||
// The key itself is never shown.
|
||||
{
|
||||
"a key with a control character",
|
||||
environment{crowdSecURL: lapi, crowdSecKey: token + "\r"},
|
||||
crowdSecKey + ": holds a control character, such as the carriage return " +
|
||||
"of a Windows line end",
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := config.FromEnvironment(tc.env.lookupEnv)
|
||||
if err == nil || err.Error() != tc.want {
|
||||
t.Errorf("error %v, want %s", err, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrowdSecKeyIsLoggedMasked(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg := fromEnvironment(t, environment{
|
||||
crowdSecURL: "http://172.17.0.1:8080", crowdSecKey: token,
|
||||
})
|
||||
|
||||
var out bytes.Buffer
|
||||
|
||||
slog.New(slog.NewJSONHandler(&out, nil)).Info("starting", "settings", cfg)
|
||||
|
||||
logged := out.String()
|
||||
if strings.Contains(logged, token) ||
|
||||
!strings.Contains(logged, `"`+crowdSecKey+`":"********"`) ||
|
||||
!strings.Contains(logged, `"`+crowdSecURL+`":"http://172.17.0.1:8080"`) {
|
||||
t.Errorf("the key is not logged masked beside the URL: %s", logged)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSizesAndOff(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -2096,6 +2213,8 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
|
||||
reputationAction: "limit:25",
|
||||
reputationCacheTTL: defaultReputationCacheTTL,
|
||||
reputationTimeout: "2s",
|
||||
crowdSecURL: "",
|
||||
crowdSecKey: "",
|
||||
banResponse: "403",
|
||||
limitBanDuration: "1h",
|
||||
limitBanRepeatWindow: "24h",
|
||||
|
||||
+14
-11
@@ -154,7 +154,9 @@ func New(topN int, instanceName string) *Metrics {
|
||||
func (m *Metrics) AddBansAndClients(
|
||||
ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time,
|
||||
) {
|
||||
for _, cause := range []string{bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin} {
|
||||
for _, cause := range []string{
|
||||
bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseCrowdSec,
|
||||
} {
|
||||
m.registry.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||
Name: "smallwebwaf_bans_made_total",
|
||||
Help: "Bans made, by cause.",
|
||||
@@ -263,16 +265,16 @@ const sourceLabel = "source"
|
||||
|
||||
// AddReputation adds the metrics of the lists fetched from URLs and of the
|
||||
// DNSBL zones, by source, each list's URL or each zone, its key masked as
|
||||
// config.MaskZoneKey masks it: the requests whose client a blocklist, a
|
||||
// zone's verdict or AbuseIPDB's score lists, which ReputationHit counts,
|
||||
// and, read from lists and dnsbl as the metrics are asked for, for a list,
|
||||
// the fetches that failed and when the copy in use was fetched, and for a
|
||||
// zone, the queries made and those that failed. It is called once, before
|
||||
// ReputationHit.
|
||||
// config.MaskZoneKey masks it: the requests whose client a blocklist, the
|
||||
// CrowdSec decision list, a zone's verdict or AbuseIPDB's score lists,
|
||||
// which ReputationHit counts, and, read from lists and dnsbl as the
|
||||
// metrics are asked for, for a list, the fetches that failed and when the
|
||||
// copy in use was fetched, and for a zone, the queries made and those that
|
||||
// failed. It is called once, before ReputationHit.
|
||||
func (m *Metrics) AddReputation(lists *reputation.Lists, dnsbl *reputation.DNSBL) {
|
||||
m.reputationHits = counterVec("smallwebwaf_reputation_hits_total",
|
||||
"Requests whose client a blocklist, a DNSBL zone or AbuseIPDB lists, by "+
|
||||
"the blocklist's URL, the zone, or abuseipdb.",
|
||||
"Requests whose client a blocklist, the CrowdSec decision list, a DNSBL "+
|
||||
"zone or AbuseIPDB lists, by the list's URL, the zone, or abuseipdb.",
|
||||
[]string{sourceLabel})
|
||||
m.registry.MustRegister(m.reputationHits)
|
||||
|
||||
@@ -325,8 +327,9 @@ func (m *Metrics) AddAbuseIPDB(abuseIPDB *reputation.AbuseIPDB) {
|
||||
)
|
||||
}
|
||||
|
||||
// ReputationHit counts a request whose client source lists: a blocklist,
|
||||
// by its URL, a DNSBL zone, its key masked, or AbuseIPDB, abuseipdb.
|
||||
// ReputationHit counts a request whose client source lists: a blocklist
|
||||
// or the CrowdSec decision list, by its URL, a DNSBL zone, its key masked,
|
||||
// or AbuseIPDB, abuseipdb.
|
||||
func (m *Metrics) ReputationHit(source string) {
|
||||
m.reputationHits.WithLabelValues(source).Inc()
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||
)
|
||||
@@ -202,6 +203,44 @@ func (rq *request) banForAttack(now time.Time, rule rules.Rule) {
|
||||
}
|
||||
}
|
||||
|
||||
// banForCrowdSec bans the client's netblock at now until decision,
|
||||
// CrowdSec's decision on the client, ends. In observe mode it makes no
|
||||
// ban, and raises the alert for the ban it would have made, if that alert
|
||||
// would be sent.
|
||||
func (rq *request) banForCrowdSec(now time.Time, decision reputation.Decision) {
|
||||
netblock := rq.h.netblock(rq.client)
|
||||
if rq.h.config.Observe && !rq.wouldAlertBan(netblock, now, bans.CauseCrowdSec) {
|
||||
return
|
||||
}
|
||||
|
||||
notes := bans.Notes{
|
||||
ASN: rq.line.ASN,
|
||||
ASName: rq.line.ASName,
|
||||
Country: rq.line.Country,
|
||||
Reputation: rq.reputation,
|
||||
Request: rq.noted(now, rq.h.config.BanResponse),
|
||||
Requests: rq.netblockRequests(netblock),
|
||||
}
|
||||
|
||||
if rq.h.config.Observe {
|
||||
ban, wouldBan := rq.h.ledger.WouldBanForCrowdSec(netblock, now, decision.Expires,
|
||||
decision.Scenario, notes)
|
||||
if wouldBan {
|
||||
rq.alertBan(ban)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
ban, made := rq.h.ledger.BanForCrowdSec(netblock, now, decision.Expires,
|
||||
decision.Scenario, notes)
|
||||
rq.line.BanExpires = banExpires(ban)
|
||||
|
||||
if made {
|
||||
rq.alertBan(ban)
|
||||
}
|
||||
}
|
||||
|
||||
// wouldAlertBan reports whether the alert for a ban on netblock for cause
|
||||
// made at now would be sent. In observe mode the ban the request would
|
||||
// have made is worked out only then, at most once per
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
package proxy_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/proxy"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// The CrowdSec settings, and the tests' engine, which is never asked: each
|
||||
// test puts in the copy of its decision list, at decisionsURL, that it
|
||||
// needs, as reputation.json would at start.
|
||||
const (
|
||||
crowdSecURL = "SWWAF_CROWDSEC_LAPI_URL"
|
||||
crowdSecKey = "SWWAF_CROWDSEC_LAPI_KEY"
|
||||
lapi = "http://crowdsec.example:8080"
|
||||
decisionsURL = lapi + "/v1/decisions"
|
||||
bouncerKey = "crowdsec-key-0123456789abcdef"
|
||||
)
|
||||
|
||||
func TestClientTheCrowdSecDecisionListListsIsBannedUntilTheDecisionEnds(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, clk, server, queue := startWithAlerts(t, map[string]string{
|
||||
crowdSecURL: lapi, crowdSecKey: bouncerKey, metricsToken: token,
|
||||
})
|
||||
// client had four hours left on its decision as the engine answered.
|
||||
fetched := clk.Now()
|
||||
loadDecisions(t, server, fetched, `[{"duration": "4h0m0s", `+
|
||||
`"scenario": "crowdsecurity/ssh-bf", "scope": "Ip", "type": "ban", `+
|
||||
`"value": "`+client+`"}]`)
|
||||
expires := requestlog.FormatTime(fetched.Add(4 * time.Hour))
|
||||
|
||||
// Its first request is refused, and bans it until the decision ends.
|
||||
line := s.get(client, http.StatusForbidden, requestlog.ActionBanned)
|
||||
wantReputation(t, line, decisionsURL)
|
||||
|
||||
if line.BanExpires != expires {
|
||||
t.Errorf("log line has ban_expires %q, want %s", line.BanExpires, expires)
|
||||
}
|
||||
|
||||
listed := []bans.ReputationHit{{Source: decisionsURL}}
|
||||
|
||||
held := server.Ledger.Bans(netip.MustParsePrefix(client + "/32"))
|
||||
if len(held) != 1 || held[0].Cause != bans.CauseCrowdSec ||
|
||||
!held[0].Start.Equal(fetched) || !held[0].Expires.Equal(fetched.Add(4*time.Hour)) ||
|
||||
held[0].Reason != "CrowdSec's decision for crowdsecurity/ssh-bf" ||
|
||||
!reflect.DeepEqual(held[0].Notes.Reputation, listed) ||
|
||||
held[0].Notes.Request.Path != "/" || held[0].Notes.Requests != 1 {
|
||||
t.Fatalf("bans %+v, want one for crowdsec of four hours, with the list and "+
|
||||
"the request in its notes", held)
|
||||
}
|
||||
|
||||
// The listing raises a reputation_hit alert, and the ban its own.
|
||||
waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
|
||||
if len(waiting) != 2 || waiting[0].Event != alerts.EventReputationHit ||
|
||||
waiting[0].Reason != "listed by the CrowdSec decision list" ||
|
||||
!reflect.DeepEqual(waiting[1], banAlert(alerts.EventBan, fetched, client, held[0],
|
||||
expires)) {
|
||||
t.Errorf("alerts waiting %+v, want a reputation_hit alert, then the ban's",
|
||||
waiting)
|
||||
}
|
||||
|
||||
// Each request while the ban lasts is refused under it, as under any
|
||||
// ban, and once it has ended the client is let through.
|
||||
clk.advance(4*time.Hour - time.Second)
|
||||
|
||||
line = s.get(client, http.StatusForbidden, requestlog.ActionBanned)
|
||||
wantReputation(t, line)
|
||||
|
||||
if line.BanExpires != expires {
|
||||
t.Errorf("log line has ban_expires %q, want %s", line.BanExpires, expires)
|
||||
}
|
||||
|
||||
clk.advance(time.Second)
|
||||
s.get(client, http.StatusOK, requestlog.ActionForward)
|
||||
|
||||
// The ban and the hit are counted, and the list has the metrics of any
|
||||
// list fetched from a URL.
|
||||
metrics := s.scrape(unplaced)
|
||||
labels := `{instance="` + alertInstance + `",source="` + decisionsURL + `"}`
|
||||
|
||||
wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="crowdsec",`+
|
||||
`instance="`+alertInstance+`"}`, 1)
|
||||
wantMetric(t, metrics, "smallwebwaf_reputation_hits_total"+labels, 1)
|
||||
wantMetric(t, metrics, "smallwebwaf_reputation_failures_total"+labels, 0)
|
||||
wantMetric(t, metrics, "smallwebwaf_reputation_last_fetch_timestamp_seconds"+labels,
|
||||
float64(fetched.Unix()))
|
||||
}
|
||||
|
||||
func TestEndedCrowdSecDecisionNoLongerBansThoughTheCopyStillHoldsIt(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, clk, server := startWithClock(t, "", map[string]string{
|
||||
crowdSecURL: lapi, crowdSecKey: bouncerKey,
|
||||
})
|
||||
// 198.51.100.0/24 and 2001:db8::9 had a minute left as the engine
|
||||
// answered.
|
||||
fetched := clk.Now()
|
||||
loadDecisions(t, server, fetched, `[{"duration": "1m0s", `+
|
||||
`"scenario": "crowdsecurity/http-probing", "scope": "Range", "type": "ban", `+
|
||||
`"value": "198.51.100.0/24"}, {"duration": "1m0s", `+
|
||||
`"scenario": "crowdsecurity/http-probing", "scope": "Ip", "type": "ban", `+
|
||||
`"value": "2001:db8::9"}]`)
|
||||
|
||||
// Just before its end, the decision bans a client in the netblock, and
|
||||
// one on an IPv6 address bans the address's group, the /64.
|
||||
clk.advance(time.Minute - time.Nanosecond)
|
||||
s.get("198.51.100.7", http.StatusForbidden, requestlog.ActionBanned)
|
||||
s.get("2001:db8::9", http.StatusForbidden, requestlog.ActionBanned)
|
||||
s.get("2001:db8::5", http.StatusForbidden, requestlog.ActionBanned)
|
||||
|
||||
// Once it has ended, it bans no other client, and the bans it made end
|
||||
// with it.
|
||||
clk.advance(time.Nanosecond)
|
||||
|
||||
for _, from := range []string{"198.51.100.8", "198.51.100.7", "2001:db8::5"} {
|
||||
wantReputation(t, s.get(from, http.StatusOK, requestlog.ActionForward))
|
||||
}
|
||||
|
||||
if made := server.Ledger.Made(bans.CauseCrowdSec); made != 2 {
|
||||
t.Errorf("%d bans made for crowdsec, want 2, on 198.51.100.7/32 and "+
|
||||
"2001:db8::/64", made)
|
||||
}
|
||||
|
||||
if held := server.Ledger.Bans(netip.MustParsePrefix("2001:db8::/64")); len(held) != 1 {
|
||||
t.Errorf("bans of 2001:db8::/64 %+v, want one", held)
|
||||
}
|
||||
}
|
||||
|
||||
func TestObserveModeForwardsAClientTheCrowdSecDecisionListListsAndAlertsTheBan(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
s, clk, server, queue := startWithAlerts(t, map[string]string{
|
||||
crowdSecURL: lapi, crowdSecKey: bouncerKey, mode: observe,
|
||||
})
|
||||
loadDecisions(t, server, clk.Now(), `[{"duration": "4h0m0s", `+
|
||||
`"scenario": "crowdsecurity/ssh-bf", "scope": "Ip", "type": "ban", `+
|
||||
`"value": "`+client+`"}]`)
|
||||
|
||||
line := s.get(client, http.StatusOK, requestlog.ActionForward)
|
||||
wantWouldAction(t, line, requestlog.ActionBanned)
|
||||
wantReputation(t, line, decisionsURL)
|
||||
|
||||
if held := server.Ledger.Snapshot(); len(held) != 0 {
|
||||
t.Errorf("bans %+v, want none", held)
|
||||
}
|
||||
|
||||
waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
|
||||
if len(waiting) != 2 || waiting[1].Event != alerts.EventBan ||
|
||||
waiting[1].Detail["cause"] != bans.CauseCrowdSec ||
|
||||
waiting[1].Detail["mode"] != observe {
|
||||
t.Errorf("alerts waiting %+v, want a reputation_hit alert, then the ban alert "+
|
||||
"marked observe", waiting)
|
||||
}
|
||||
}
|
||||
|
||||
// loadDecisions puts into server's lists the copy of the decision list at
|
||||
// decisionsURL, answer, the engine's answer, fetched at fetched, as
|
||||
// reputation.json would at start.
|
||||
func loadDecisions(
|
||||
t *testing.T, server *proxy.Server, fetched time.Time, answer string,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
err := server.Lists.Load([]reputation.List{{
|
||||
URL: decisionsURL, Tried: fetched, Fetched: fetched, Lines: []string{answer},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("load the decision list: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -74,9 +74,10 @@ type Params struct {
|
||||
Rules *rules.Files
|
||||
// Alerts receive the alert for each ban the proxy makes or makes
|
||||
// permanent, for each count over an anomaly threshold, for each request
|
||||
// whose client a blocklist, a DNSBL zone or AbuseIPDB lists, and for
|
||||
// GeoJS failing, a fetch of a list failing, a query to a DNSBL zone or
|
||||
// a check with AbuseIPDB failing, or the day's AbuseIPDB checks used up.
|
||||
// whose client a blocklist, the CrowdSec decision list, a DNSBL zone or
|
||||
// AbuseIPDB lists, and for GeoJS failing, a fetch of a list failing, a
|
||||
// query to a DNSBL zone or a check with AbuseIPDB failing, or the day's
|
||||
// AbuseIPDB checks used up.
|
||||
Alerts *alerts.Queue
|
||||
}
|
||||
|
||||
@@ -202,8 +203,9 @@ func newReputation(
|
||||
cfg := params.Config
|
||||
lists := reputation.New(reputation.Params{
|
||||
BlocklistURLs: cfg.BlocklistURLs, Refresh: cfg.BlocklistRefresh,
|
||||
ASNLimitPercentURL: cfg.ASNLimitPercentURL, Now: params.Now,
|
||||
ProcessLog: params.ProcessLog, Alerts: params.Alerts,
|
||||
ASNLimitPercentURL: cfg.ASNLimitPercentURL,
|
||||
CrowdSecDecisionsURL: cfg.CrowdSecDecisionsURL, CrowdSecKey: cfg.CrowdSecKey,
|
||||
Now: params.Now, ProcessLog: params.ProcessLog, Alerts: params.Alerts,
|
||||
})
|
||||
dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{
|
||||
Zones: cfg.DNSBLZones, Resolver: cfg.DNSBLResolver, CacheTTL: cfg.ReputationCacheTTL,
|
||||
|
||||
@@ -2,6 +2,7 @@ package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
@@ -25,6 +26,22 @@ func (rq *request) blocklistDenied() bool {
|
||||
return rq.blocklisted && rq.h.config.BlocklistAction == deny
|
||||
}
|
||||
|
||||
// crowdSecBanned reports whether a decision of the CrowdSec decision list
|
||||
// on the client is in force at now. If one is, it notes the list, as
|
||||
// noteHit does, and bans the client until that decision ends.
|
||||
func (rq *request) crowdSecBanned(now time.Time) bool {
|
||||
decision, listed := rq.h.lists.CrowdSecDecision(rq.client, now)
|
||||
if !listed {
|
||||
return false
|
||||
}
|
||||
|
||||
rq.noteHit(bans.ReputationHit{Source: rq.h.config.CrowdSecDecisionsURL},
|
||||
"listed by the CrowdSec decision list")
|
||||
rq.banForCrowdSec(now, decision)
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// dnsblDenied notes the DNSBL zones whose verdict lists the client, as
|
||||
// noteListed does, and reports whether SWWAF_REPUTATION_ACTION, being
|
||||
// deny, refuses the request. Being limit, it lowers the client's limits
|
||||
|
||||
@@ -226,8 +226,9 @@ func (rq *request) check(ctx context.Context) *refusal {
|
||||
// other client, SWWAF_DENY_NETS comes first, then a ban on its netblock,
|
||||
// so that a client either refuses is not looked up, then the lookup of
|
||||
// its AS number and country, then the country lists, then the blocklists,
|
||||
// then the DNSBL zones' verdicts, and then AbuseIPDB's score; a request
|
||||
// any of them refuses is not counted for the rate limits. Then come the
|
||||
// then the CrowdSec decision list, which bans the client it lists, then
|
||||
// the DNSBL zones' verdicts, and then AbuseIPDB's score; a request any of
|
||||
// them refuses is not counted for the rate limits. Then come the
|
||||
// rate limits, unless the client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the
|
||||
// request's path is exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that
|
||||
// every other request is counted, each of them by the client's limit
|
||||
@@ -260,6 +261,10 @@ func (rq *request) checkClient(ctx context.Context) string {
|
||||
return requestlog.ActionDenied
|
||||
}
|
||||
|
||||
if rq.crowdSecBanned(now) {
|
||||
return requestlog.ActionBanned
|
||||
}
|
||||
|
||||
if rq.dnsblDenied(ctx) || rq.abuseIPDBDenied(ctx) {
|
||||
return requestlog.ActionDenied
|
||||
}
|
||||
|
||||
@@ -0,0 +1,513 @@
|
||||
package reputation_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
)
|
||||
|
||||
// The tests run in a synctest bubble, as those of the blocklists do, and
|
||||
// fetch the decision list from engine, a stand-in for a CrowdSec engine
|
||||
// that answers without the network.
|
||||
|
||||
const (
|
||||
// decisionsURL is the decision list of the tests' engine, and engineKey
|
||||
// the key it answers.
|
||||
decisionsURL = "http://crowdsec.example:8080/v1/decisions"
|
||||
engineKey = "crowdsec-key-0123456789abcdef"
|
||||
// sshBF and probing are scenarios of the engine's decisions.
|
||||
sshBF = "crowdsecurity/ssh-bf"
|
||||
probing = "crowdsecurity/http-probing"
|
||||
// ban is the type of a decision to ban, and rangeScope the scope of a
|
||||
// decision on a netblock, as CrowdSec names them.
|
||||
ban = "ban"
|
||||
rangeScope = "Range"
|
||||
)
|
||||
|
||||
func TestCrowdSecDecisionBansItsNetblockUntilItEndsEvenWithTheEngineDown(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
began := time.Now()
|
||||
manual := "manual 'ban' from 'localhost'"
|
||||
e := &engine{key: engineKey, decisions: []decision{
|
||||
{"Ip", suspect, ban, manual, began.Add(6 * time.Hour)},
|
||||
// A shorter decision on the same address, which is not the one
|
||||
// used.
|
||||
{"Ip", suspect, ban, sshBF, began.Add(4 * time.Hour)},
|
||||
{rangeScope, "198.51.100.0/24", ban, probing, began.Add(time.Hour)},
|
||||
{"Ip", "2001:db8::1", ban, sshBF, began.Add(2 * time.Hour)},
|
||||
// Left out: a decision to show a captcha, and one on a country.
|
||||
{"Ip", "192.0.2.50", "captcha", probing, began.Add(time.Hour)},
|
||||
{"Country", "KP", ban, manual, began.Add(time.Hour)},
|
||||
}}
|
||||
lists := start(t, e, crowdSecParams())
|
||||
|
||||
for addr, want := range map[string]reputation.Decision{
|
||||
suspect: {Expires: began.Add(6 * time.Hour), Scenario: manual},
|
||||
"198.51.100.0": {Expires: began.Add(time.Hour), Scenario: probing},
|
||||
"198.51.100.255": {Expires: began.Add(time.Hour), Scenario: probing},
|
||||
"2001:db8::1": {Expires: began.Add(2 * time.Hour), Scenario: sshBF},
|
||||
"203.0.113.10": {},
|
||||
"198.51.101.0": {},
|
||||
"2001:db8::2": {},
|
||||
"192.0.2.50": {},
|
||||
} {
|
||||
wantDecision(t, lists, addr, want)
|
||||
}
|
||||
|
||||
// With the engine down, the copy kept still holds the decision on
|
||||
// 198.51.100.0/24, which no longer bans once it has ended.
|
||||
e.set(func(e *engine) { e.failing = true })
|
||||
time.Sleep(time.Hour - time.Nanosecond)
|
||||
synctest.Wait()
|
||||
wantDecision(t, lists, "198.51.100.7",
|
||||
reputation.Decision{Expires: began.Add(time.Hour), Scenario: probing})
|
||||
|
||||
time.Sleep(time.Nanosecond)
|
||||
synctest.Wait()
|
||||
wantDecision(t, lists, "198.51.100.7", reputation.Decision{})
|
||||
wantDecision(t, lists, suspect,
|
||||
reputation.Decision{Expires: began.Add(6 * time.Hour), Scenario: manual})
|
||||
})
|
||||
}
|
||||
|
||||
func TestCrowdSecDecisionListFetchedAgainEveryMinute(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
began := time.Now()
|
||||
e := &engine{key: engineKey, decisions: []decision{
|
||||
{"Ip", suspect, ban, sshBF, began.Add(4 * time.Hour)},
|
||||
}}
|
||||
lists := start(t, e, crowdSecParams())
|
||||
wantEngineFetches(t, e, 1)
|
||||
|
||||
added := reputation.Decision{Expires: began.Add(2 * time.Hour), Scenario: probing}
|
||||
|
||||
e.set(func(e *engine) {
|
||||
e.decisions = append(e.decisions,
|
||||
decision{"Ip", "203.0.113.10", ban, probing, added.Expires})
|
||||
})
|
||||
|
||||
time.Sleep(time.Minute - time.Nanosecond)
|
||||
wantEngineFetches(t, e, 1)
|
||||
wantDecision(t, lists, "203.0.113.10", reputation.Decision{})
|
||||
|
||||
time.Sleep(time.Nanosecond)
|
||||
wantEngineFetches(t, e, 2)
|
||||
wantDecision(t, lists, "203.0.113.10", added)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCrowdSecDecisionOnAClientIsTheOneThatEndsLast(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
began := time.Now()
|
||||
e := &engine{key: engineKey, decisions: []decision{
|
||||
// Two decisions on one address, the shorter listed first.
|
||||
{"Ip", suspect, ban, sshBF, began.Add(2 * time.Hour)},
|
||||
{"Ip", suspect, ban, probing, began.Add(4 * time.Hour)},
|
||||
// 198.51.100.130 is held by a decision on its address that ends
|
||||
// after the one on its netblock, and 192.0.2.20 by one that ends
|
||||
// before.
|
||||
{rangeScope, "198.51.100.128/25", ban, sshBF, began.Add(time.Hour)},
|
||||
{"Ip", "198.51.100.130", ban, probing, began.Add(3 * time.Hour)},
|
||||
{rangeScope, "192.0.2.0/24", ban, probing, began.Add(5 * time.Hour)},
|
||||
{"Ip", "192.0.2.20", ban, sshBF, began.Add(2 * time.Hour)},
|
||||
}}
|
||||
lists := start(t, e, crowdSecParams())
|
||||
|
||||
wantDecision(t, lists, suspect,
|
||||
reputation.Decision{Expires: began.Add(4 * time.Hour), Scenario: probing})
|
||||
wantDecision(t, lists, "198.51.100.130",
|
||||
reputation.Decision{Expires: began.Add(3 * time.Hour), Scenario: probing})
|
||||
wantDecision(t, lists, "192.0.2.20",
|
||||
reputation.Decision{Expires: began.Add(5 * time.Hour), Scenario: probing})
|
||||
})
|
||||
}
|
||||
|
||||
func TestCrowdSecAnswerOfNoDecisionIsAGoodCopyThatListsNoClient(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
began := time.Now()
|
||||
e := &engine{key: engineKey, decisions: []decision{
|
||||
{"Ip", suspect, ban, sshBF, began.Add(4 * time.Hour)},
|
||||
}}
|
||||
lists := start(t, e, crowdSecParams())
|
||||
|
||||
// With its decision deleted, the engine answers null.
|
||||
e.set(func(e *engine) { e.decisions = nil })
|
||||
time.Sleep(time.Minute)
|
||||
wantEngineFetches(t, e, 2)
|
||||
|
||||
want := []reputation.List{{
|
||||
URL: decisionsURL, Tried: time.Now(), Fetched: time.Now(), Lines: []string{"null"},
|
||||
}}
|
||||
if got := lists.Snapshot(); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("lists %+v, want %+v", got, want)
|
||||
}
|
||||
|
||||
if lists.Failures(decisionsURL) != 0 {
|
||||
t.Errorf("%d failures, want 0", lists.Failures(decisionsURL))
|
||||
}
|
||||
|
||||
wantDecision(t, lists, suspect, reputation.Decision{})
|
||||
})
|
||||
}
|
||||
|
||||
func TestCrowdSecFailureKeepsTheLastGoodCopyAlertsOncePerCooldownAndHidesTheKey(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
for _, tc := range crowdSecFailures() {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
var log bytes.Buffer
|
||||
|
||||
began := time.Now()
|
||||
e := &engine{key: engineKey, decisions: []decision{
|
||||
{"Ip", suspect, ban, sshBF, began.Add(4 * time.Hour)},
|
||||
}}
|
||||
queue := newQueue()
|
||||
p := crowdSecParams()
|
||||
p.ProcessLog = slog.New(slog.NewJSONHandler(&log, nil))
|
||||
p.Alerts = queue
|
||||
lists := start(t, e, p)
|
||||
kept := lists.Snapshot()
|
||||
|
||||
e.set(tc.fail)
|
||||
|
||||
// Each failure is tried again a minute after it.
|
||||
for range 2 {
|
||||
time.Sleep(time.Minute)
|
||||
synctest.Wait()
|
||||
}
|
||||
|
||||
wantEngineFetches(t, e, 3)
|
||||
wantDecision(t, lists, suspect,
|
||||
reputation.Decision{Expires: began.Add(4 * time.Hour), Scenario: sshBF})
|
||||
|
||||
want := kept[0]
|
||||
want.Tried = time.Now()
|
||||
|
||||
if got := lists.Snapshot(); !reflect.DeepEqual(got, []reputation.List{want}) {
|
||||
t.Errorf("lists %+v, want the first copy, last tried now, %+v", got, want)
|
||||
}
|
||||
|
||||
if lists.Failures(decisionsURL) != 2 {
|
||||
t.Errorf("%d failures, want 2", lists.Failures(decisionsURL))
|
||||
}
|
||||
|
||||
// One alert for the first failure; the cooldown holds back the
|
||||
// second.
|
||||
wantAlert(t, queue,
|
||||
fetchFailure(time.Now().Add(-time.Minute), decisionsURL, tc.error))
|
||||
|
||||
if !strings.Contains(log.String(), `"msg":"fetching a list failed",`+
|
||||
`"url":"`+decisionsURL+`","error":"`+tc.error) {
|
||||
t.Errorf("logged\n%s\nwant the failures", log.String())
|
||||
}
|
||||
|
||||
wantKeyNotShown(t, e, log.String(), lists, queue)
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// crowdSecFailure is a way for the engine to fail: fail has it answer the
|
||||
// fetches after the first so that they fail with error.
|
||||
type crowdSecFailure struct {
|
||||
name string
|
||||
fail func(e *engine)
|
||||
error string
|
||||
}
|
||||
|
||||
// crowdSecFailures returns the ways the engine can fail.
|
||||
func crowdSecFailures() []crowdSecFailure {
|
||||
const notDecision = " does not give an address or a netblock and a duration, " +
|
||||
"such as 4h0m0s"
|
||||
|
||||
return []crowdSecFailure{
|
||||
{
|
||||
"an answer other than 200",
|
||||
func(e *engine) { e.failing = true },
|
||||
"the server answered 503 Service Unavailable",
|
||||
},
|
||||
{
|
||||
"a key the engine refuses",
|
||||
func(e *engine) { e.key = "another-key-0123456789abcdef" },
|
||||
"the server answered 403 Forbidden",
|
||||
},
|
||||
{
|
||||
"a redirect",
|
||||
func(e *engine) { e.redirect = "http://elsewhere.example/v1/decisions" },
|
||||
"the server answered 302 Found",
|
||||
},
|
||||
{
|
||||
"an answer that does not read",
|
||||
func(e *engine) { e.answer = "<html>" },
|
||||
"read the answer: invalid character '<' looking for beginning of value",
|
||||
},
|
||||
{
|
||||
"a decision to ban whose value does not read",
|
||||
func(e *engine) {
|
||||
e.answer = `[{"duration": "4h", "scenario": "` + sshBF + `", ` +
|
||||
`"scope": "Ip", "type": "ban", "value": "203.0.113.300"}]`
|
||||
},
|
||||
"decision 1" + notDecision,
|
||||
},
|
||||
{
|
||||
"a decision to ban whose duration does not read",
|
||||
func(e *engine) {
|
||||
e.answer = `[{"duration": "4h", "scope": "Country", "type": "ban", ` +
|
||||
`"value": "KP"}, {"duration": "four hours", "scope": "Range", ` +
|
||||
`"type": "ban", "value": "198.51.100.0/24"}]`
|
||||
},
|
||||
"decision 2" + notDecision,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// wantKeyNotShown checks that no fetch carried the engine's key to a URL
|
||||
// other than its decision list, such as the one a redirect names, and that
|
||||
// the key is in none of what the fetches leave behind: log, the process
|
||||
// log, the alerts waiting in queue, and the copies of lists, which
|
||||
// reputation.json keeps.
|
||||
func wantKeyNotShown(
|
||||
t *testing.T, e *engine, log string, lists *reputation.Lists, queue *alerts.Queue,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
e.mu.Lock()
|
||||
keySentTo := e.keySentTo
|
||||
e.mu.Unlock()
|
||||
|
||||
if len(keySentTo) != 0 {
|
||||
t.Errorf("the key was sent to %v", keySentTo)
|
||||
}
|
||||
|
||||
shown, err := json.Marshal([]any{lists.Snapshot(), waiting(queue)})
|
||||
if err != nil {
|
||||
t.Fatalf("encode: %v", err)
|
||||
}
|
||||
|
||||
if strings.Contains(log+string(shown), engineKey) {
|
||||
t.Errorf("the key is shown in\n%s\n%s", log, shown)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrowdSecDecisionListKeptAcrossARestartEndsWhenItsDecisionsDo(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
began := time.Now()
|
||||
e := &engine{key: engineKey, decisions: []decision{
|
||||
{rangeScope, "198.51.100.0/24", ban, probing, began.Add(time.Hour)},
|
||||
}}
|
||||
lists := start(t, e, crowdSecParams())
|
||||
kept := lists.Snapshot()
|
||||
|
||||
// Restarted half an hour later with what reputation.json keeps, and
|
||||
// the engine down, the decision still bans, until the end it had at
|
||||
// the fetch, half an hour on.
|
||||
time.Sleep(30 * time.Minute)
|
||||
|
||||
down := &engine{key: engineKey, failing: true}
|
||||
again := reputation.New(crowdSecParams())
|
||||
again.SetTransport(down)
|
||||
|
||||
err := again.Load(kept)
|
||||
if err != nil {
|
||||
t.Fatalf("load: %v", err)
|
||||
}
|
||||
|
||||
run(t, again)
|
||||
|
||||
want := reputation.Decision{Expires: began.Add(time.Hour), Scenario: probing}
|
||||
wantDecision(t, again, "198.51.100.7", want)
|
||||
|
||||
time.Sleep(30*time.Minute - time.Nanosecond)
|
||||
synctest.Wait()
|
||||
wantDecision(t, again, "198.51.100.7", want)
|
||||
|
||||
time.Sleep(time.Nanosecond)
|
||||
synctest.Wait()
|
||||
wantDecision(t, again, "198.51.100.7", reputation.Decision{})
|
||||
})
|
||||
}
|
||||
|
||||
func TestLoadTakesACrowdSecListNeverFetchedAndRefusesACopyThatDoesNotRead(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
now := time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)
|
||||
lists := reputation.New(crowdSecParams())
|
||||
|
||||
// Tried, and never fetched: there is no copy to read.
|
||||
err := lists.Load([]reputation.List{{URL: decisionsURL, Tried: now}})
|
||||
if err != nil {
|
||||
t.Errorf("load the list never fetched: %v", err)
|
||||
}
|
||||
|
||||
err = lists.Load([]reputation.List{{
|
||||
URL: decisionsURL, Tried: now, Fetched: now, Lines: []string{
|
||||
`[{"duration": "4h", "scope": "Range", "type": "ban", ` +
|
||||
`"value": "198.51.100.0/33"}]`,
|
||||
},
|
||||
}})
|
||||
|
||||
const want = "the copy of " + decisionsURL + ": decision 1 does not give an " +
|
||||
"address or a netblock and a duration, such as 4h0m0s"
|
||||
if err == nil || err.Error() != want {
|
||||
t.Errorf("error %v, want %s", err, want)
|
||||
}
|
||||
}
|
||||
|
||||
// engine is a stand-in for the local API of a CrowdSec engine. It answers
|
||||
// a fetch of the decision list that carries its key in X-Api-Key with its
|
||||
// decisions still in force, each with the time it has left as it answers,
|
||||
// by the bubble's clock, as an engine does, or with answer while that is
|
||||
// not "". It answers 403 to a fetch without its key, as an engine does,
|
||||
// with a redirect to redirect while that is not "", and 503 while failing.
|
||||
// It counts the fetches, and notes in keySentTo the URL of each fetch of
|
||||
// another URL that carries a key, as one following a redirect would.
|
||||
type engine struct {
|
||||
mu sync.Mutex
|
||||
key string
|
||||
decisions []decision
|
||||
answer string
|
||||
redirect string
|
||||
failing bool
|
||||
fetches int
|
||||
keySentTo []string
|
||||
}
|
||||
|
||||
// decision is a decision of the engine, which ends at expires.
|
||||
type decision struct {
|
||||
scope, value, kind, scenario string
|
||||
expires time.Time
|
||||
}
|
||||
|
||||
// RoundTrip has the engine answer req, in place of the network.
|
||||
func (e *engine) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
|
||||
e.fetches++
|
||||
|
||||
if req.URL.String() != decisionsURL && req.Header.Get("X-Api-Key") != "" {
|
||||
e.keySentTo = append(e.keySentTo, req.URL.String())
|
||||
}
|
||||
|
||||
status, header, body := http.StatusOK, http.Header{}, e.answer
|
||||
|
||||
switch {
|
||||
case req.URL.String() != decisionsURL || req.Header.Get("X-Api-Key") != e.key:
|
||||
status, body = http.StatusForbidden, `{"message":"access forbidden"}`
|
||||
case e.redirect != "":
|
||||
status, header = http.StatusFound, http.Header{"Location": {e.redirect}}
|
||||
case e.failing:
|
||||
status, body = http.StatusServiceUnavailable, ""
|
||||
case body == "":
|
||||
body = e.inForce(time.Now())
|
||||
}
|
||||
|
||||
return &http.Response{
|
||||
StatusCode: status,
|
||||
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
|
||||
Header: header,
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
Request: req,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// inForce returns the decisions in force at now, as the engine answers
|
||||
// them: a JSON list, null for none.
|
||||
func (e *engine) inForce(now time.Time) string {
|
||||
var answer []map[string]string
|
||||
|
||||
for _, d := range e.decisions {
|
||||
if now.Before(d.expires) {
|
||||
answer = append(answer, map[string]string{
|
||||
"duration": d.expires.Sub(now).String(), "origin": "crowdsec",
|
||||
"scenario": d.scenario, "scope": d.scope, "type": d.kind, "value": d.value,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
body, err := json.Marshal(answer)
|
||||
if err != nil {
|
||||
panic(err) // a list of maps of strings always encodes
|
||||
}
|
||||
|
||||
return string(body)
|
||||
}
|
||||
|
||||
// set changes the engine with change.
|
||||
func (e *engine) set(change func(e *engine)) {
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
|
||||
change(e)
|
||||
}
|
||||
|
||||
// crowdSecParams returns the Params of the decision list of the tests'
|
||||
// engine, fetched with its key, by the bubble's clock, with alerts to a
|
||||
// queue that sends none.
|
||||
func crowdSecParams() reputation.Params {
|
||||
p := params()
|
||||
p.CrowdSecDecisionsURL = decisionsURL
|
||||
p.CrowdSecKey = engineKey
|
||||
|
||||
return p
|
||||
}
|
||||
|
||||
// wantEngineFetches waits until Run has made the fetches due, and checks
|
||||
// how many the engine has had.
|
||||
func wantEngineFetches(t *testing.T, e *engine, want int) {
|
||||
t.Helper()
|
||||
|
||||
synctest.Wait()
|
||||
|
||||
e.mu.Lock()
|
||||
got := e.fetches
|
||||
e.mu.Unlock()
|
||||
|
||||
if got != want {
|
||||
t.Errorf("%d fetches, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// wantDecision checks the decision lists says is in force on addr now,
|
||||
// the zero Decision for none.
|
||||
func wantDecision(
|
||||
t *testing.T, lists *reputation.Lists, addr string, want reputation.Decision,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
got, listed := lists.CrowdSecDecision(netip.MustParseAddr(addr), time.Now())
|
||||
if listed != !want.Expires.IsZero() ||
|
||||
listed && (!got.Expires.Equal(want.Expires) || got.Scenario != want.Scenario) {
|
||||
t.Errorf("%s has the decision %+v in force %t, want %+v", addr, got, listed, want)
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
// network.
|
||||
func (l *Lists) SetTransport(transport http.RoundTripper) {
|
||||
l.httpClient.Transport = transport
|
||||
l.crowdSecClient.Transport = transport
|
||||
}
|
||||
|
||||
// SetTransport has a's checks go through transport instead of the
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
// Package reputation fetches the lists the settings name by URL: the
|
||||
// blocklists of SWWAF_BLOCKLIST_URLS, and the file of AS:percent lines
|
||||
// SWWAF_ASN_LIMIT_PERCENT_URL names. It keeps the last good copy of each,
|
||||
// whole, comment lines included, which is used while a fetch fails, and
|
||||
// when each was last tried. It also asks the DNSBL zones of
|
||||
// blocklists of SWWAF_BLOCKLIST_URLS, the file of AS:percent lines
|
||||
// SWWAF_ASN_LIMIT_PERCENT_URL names, and the decision list of the CrowdSec
|
||||
// engine SWWAF_CROWDSEC_LAPI_URL names. It keeps the last good copy of
|
||||
// each, whole, comment lines included, which is used while a fetch fails,
|
||||
// and when each was last tried. It also asks the DNSBL zones of
|
||||
// SWWAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks
|
||||
// clients with AbuseIPDB, and keeps their scores and the checks spent
|
||||
// today. The state package writes all of these to reputation.json and
|
||||
@@ -11,6 +12,7 @@ package reputation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -32,6 +34,10 @@ const (
|
||||
maxListBytes = 16 << 20
|
||||
// fetchTimeout bounds one fetch of a list.
|
||||
fetchTimeout = time.Minute
|
||||
// crowdSecRefresh is how long after the CrowdSec decision list was last
|
||||
// fetched or tried it is fetched again: the engine is the operator's
|
||||
// own, and makes and ends decisions all the time.
|
||||
crowdSecRefresh = time.Minute
|
||||
// mappedBits is the length of ::ffff:0.0.0.0/96, the netblock of every
|
||||
// IPv4-mapped address.
|
||||
mappedBits = 96
|
||||
@@ -43,6 +49,8 @@ var (
|
||||
errNotNetblock = errors.New("is not an address or a netblock, such as 192.0.2.0/24")
|
||||
errNotASNPercent = errors.New(
|
||||
"is not an AS number, : and a percentage, such as AS64496:50")
|
||||
errNotDecision = errors.New(
|
||||
"does not give an address or a netblock and a duration, such as 4h0m0s")
|
||||
)
|
||||
|
||||
// List is a list as reputation.json holds it: the URL it is fetched from,
|
||||
@@ -63,8 +71,14 @@ type Params struct {
|
||||
// (SWWAF_ASN_LIMIT_PERCENT_URL), "" while it is unset.
|
||||
BlocklistURLs []string
|
||||
ASNLimitPercentURL string
|
||||
// CrowdSecDecisionsURL is the CrowdSec decision list, "" while
|
||||
// SWWAF_CROWDSEC_LAPI_URL is unset, fetched with CrowdSecKey
|
||||
// (SWWAF_CROWDSEC_LAPI_KEY).
|
||||
CrowdSecDecisionsURL string
|
||||
CrowdSecKey string
|
||||
// Refresh is how long after a list was last fetched or tried it is
|
||||
// fetched again (SWWAF_BLOCKLIST_REFRESH).
|
||||
// fetched again (SWWAF_BLOCKLIST_REFRESH), but for the CrowdSec decision
|
||||
// list, which is fetched again crowdSecRefresh after.
|
||||
Refresh time.Duration
|
||||
// Now tells the time, normally time.Now in UTC.
|
||||
Now func() time.Time
|
||||
@@ -79,6 +93,10 @@ type Params struct {
|
||||
type Lists struct {
|
||||
params Params
|
||||
httpClient *http.Client
|
||||
// crowdSecClient fetches the CrowdSec decision list. It follows no
|
||||
// redirect, so that the key goes to the engine alone: a redirect is a
|
||||
// failure.
|
||||
crowdSecClient *http.Client
|
||||
|
||||
mu sync.Mutex
|
||||
// lists are by URL, one for each URL Params names.
|
||||
@@ -95,17 +113,36 @@ type list struct {
|
||||
}
|
||||
|
||||
// entries are what the lines of a copy say: for a blocklist, the netblocks
|
||||
// it names, with the lengths among them, and for the file of AS:percent
|
||||
// lines, the percentage it gives each AS number.
|
||||
// it names, with the lengths among them, for the file of AS:percent lines,
|
||||
// the percentage it gives each AS number, and for the CrowdSec decision
|
||||
// list, the decision on each netblock that ends last, with the lengths
|
||||
// among them.
|
||||
type entries struct {
|
||||
netblocks map[netip.Prefix]bool
|
||||
lengths []int
|
||||
percents map[string]int64
|
||||
decisions map[netip.Prefix]Decision
|
||||
}
|
||||
|
||||
// Decision is a decision of the CrowdSec engine to ban a netblock: when
|
||||
// it ends, and the scenario that made it, such as crowdsecurity/ssh-bf.
|
||||
type Decision struct {
|
||||
Expires time.Time
|
||||
Scenario string
|
||||
}
|
||||
|
||||
// New returns the lists, without a copy of any yet.
|
||||
func New(params Params) *Lists {
|
||||
l := &Lists{params: params, httpClient: &http.Client{}, lists: map[string]*list{}}
|
||||
l := &Lists{
|
||||
params: params,
|
||||
httpClient: &http.Client{},
|
||||
crowdSecClient: &http.Client{
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||
return http.ErrUseLastResponse
|
||||
},
|
||||
},
|
||||
lists: map[string]*list{},
|
||||
}
|
||||
|
||||
for _, listURL := range l.URLs() {
|
||||
l.lists[listURL] = &list{kept: List{URL: listURL}}
|
||||
@@ -115,13 +152,18 @@ func New(params Params) *Lists {
|
||||
}
|
||||
|
||||
// URLs returns the URL of every list: the blocklists' in the order
|
||||
// SWWAF_BLOCKLIST_URLS names them, then SWWAF_ASN_LIMIT_PERCENT_URL.
|
||||
// SWWAF_BLOCKLIST_URLS names them, then SWWAF_ASN_LIMIT_PERCENT_URL, then
|
||||
// the CrowdSec decision list's.
|
||||
func (l *Lists) URLs() []string {
|
||||
urls := slices.Clone(l.params.BlocklistURLs)
|
||||
if l.params.ASNLimitPercentURL != "" {
|
||||
urls = append(urls, l.params.ASNLimitPercentURL)
|
||||
}
|
||||
|
||||
if l.params.CrowdSecDecisionsURL != "" {
|
||||
urls = append(urls, l.params.CrowdSecDecisionsURL)
|
||||
}
|
||||
|
||||
return urls
|
||||
}
|
||||
|
||||
@@ -157,6 +199,37 @@ func (l *Lists) ASNLimitPercent(asn string) (int64, bool) {
|
||||
return percent, listed
|
||||
}
|
||||
|
||||
// CrowdSecDecision returns the decision of the copy of the CrowdSec
|
||||
// decision list on a netblock that holds addr and that ends last, and
|
||||
// whether it is still in force at now. A decision that has ended no
|
||||
// longer bans, even before the next fetch drops it.
|
||||
func (l *Lists) CrowdSecDecision(addr netip.Addr, now time.Time) (Decision, bool) {
|
||||
if l.params.CrowdSecDecisionsURL == "" {
|
||||
return Decision{}, false
|
||||
}
|
||||
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
kept := l.lists[l.params.CrowdSecDecisionsURL].entries
|
||||
|
||||
var last Decision
|
||||
|
||||
for _, length := range kept.lengths {
|
||||
netblock, err := addr.Prefix(length)
|
||||
if err != nil {
|
||||
continue // an IPv6 netblock's length, past an IPv4 address's 32 bits
|
||||
}
|
||||
|
||||
decision := kept.decisions[netblock]
|
||||
if decision.Expires.After(last.Expires) {
|
||||
last = decision
|
||||
}
|
||||
}
|
||||
|
||||
return last, now.Before(last.Expires)
|
||||
}
|
||||
|
||||
// Fetched returns when the copy in use of the list at listURL was
|
||||
// fetched, or zero while there is none.
|
||||
func (l *Lists) Fetched(listURL string) time.Time {
|
||||
@@ -174,10 +247,9 @@ func (l *Lists) Failures(listURL string) int {
|
||||
return l.lists[listURL].failures
|
||||
}
|
||||
|
||||
// Run fetches each list once Refresh has passed since it was last fetched
|
||||
// or tried, the later of the two, until ctx is done. A list never tried is
|
||||
// fetched at once, and so is one whose last try or copy, read from
|
||||
// reputation.json, is that old.
|
||||
// Run fetches each list once it is due, as due tells, until ctx is done. A
|
||||
// list never tried is fetched at once, and so is one that is due by its
|
||||
// last try or copy read from reputation.json.
|
||||
func (l *Lists) Run(ctx context.Context) {
|
||||
if len(l.lists) == 0 {
|
||||
return
|
||||
@@ -225,11 +297,12 @@ func (l *Lists) Load(lists []List) error {
|
||||
found := make(map[string]entries, len(lists))
|
||||
|
||||
for _, kept := range lists {
|
||||
if _, named := l.lists[kept.URL]; !named {
|
||||
continue
|
||||
_, named := l.lists[kept.URL]
|
||||
if !named || kept.Fetched.IsZero() {
|
||||
continue // dropped, or a list tried but never fetched, without a copy
|
||||
}
|
||||
|
||||
read, err := l.parse(kept.URL, kept.Lines)
|
||||
read, err := l.parse(kept.URL, kept.Lines, kept.Fetched)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the copy of %s: %w", kept.URL, err)
|
||||
}
|
||||
@@ -245,9 +318,8 @@ func (l *Lists) Load(lists []List) error {
|
||||
}
|
||||
|
||||
for _, kept := range lists {
|
||||
read, named := found[kept.URL]
|
||||
if named {
|
||||
l.lists[kept.URL].kept, l.lists[kept.URL].entries = kept, read
|
||||
if _, named := l.lists[kept.URL]; named {
|
||||
l.lists[kept.URL].kept, l.lists[kept.URL].entries = kept, found[kept.URL]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -276,7 +348,8 @@ func (l *Lists) fetchDue(ctx context.Context) time.Time {
|
||||
}
|
||||
|
||||
// due returns when the list at listURL is to be fetched: Refresh after it
|
||||
// was last fetched or tried, the later of the two.
|
||||
// was last fetched or tried, the later of the two, or crowdSecRefresh
|
||||
// after for the CrowdSec decision list.
|
||||
func (l *Lists) due(listURL string) time.Time {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
@@ -288,6 +361,10 @@ func (l *Lists) due(listURL string) time.Time {
|
||||
last = held.kept.Tried
|
||||
}
|
||||
|
||||
if listURL == l.params.CrowdSecDecisionsURL {
|
||||
return last.Add(crowdSecRefresh)
|
||||
}
|
||||
|
||||
return last.Add(l.params.Refresh)
|
||||
}
|
||||
|
||||
@@ -298,14 +375,14 @@ func (l *Lists) due(listURL string) time.Time {
|
||||
// so that a restart waits for it: the server may have had its request.
|
||||
func (l *Lists) fetch(ctx context.Context, listURL string) {
|
||||
lines, err := l.get(ctx, listURL)
|
||||
now := l.params.Now()
|
||||
|
||||
var found entries
|
||||
if err == nil {
|
||||
found, err = l.parse(listURL, lines)
|
||||
found, err = l.parse(listURL, lines, now)
|
||||
}
|
||||
|
||||
cutOff := err != nil && ctx.Err() != nil
|
||||
now := l.params.Now()
|
||||
|
||||
l.mu.Lock()
|
||||
|
||||
@@ -350,8 +427,11 @@ func raiseFailure(queue *alerts.Queue, reason, source string, err error) {
|
||||
})
|
||||
}
|
||||
|
||||
// get fetches the list at listURL, and returns its lines. An answer other
|
||||
// than 200, or a list longer than maxListBytes, is a failure.
|
||||
// get fetches the list at listURL, and returns its lines. The CrowdSec
|
||||
// decision list is fetched with CrowdSecKey in the header X-Api-Key, where
|
||||
// the engine looks for it, by crowdSecClient, which follows no redirect.
|
||||
// An answer other than 200, or a list longer than maxListBytes, is a
|
||||
// failure.
|
||||
func (l *Lists) get(ctx context.Context, listURL string) ([]string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, fetchTimeout)
|
||||
defer cancel()
|
||||
@@ -361,7 +441,14 @@ func (l *Lists) get(ctx context.Context, listURL string) ([]string, error) {
|
||||
return nil, fmt.Errorf("make the request: %w", err)
|
||||
}
|
||||
|
||||
res, err := l.httpClient.Do(req)
|
||||
client := l.httpClient
|
||||
|
||||
if listURL == l.params.CrowdSecDecisionsURL {
|
||||
req.Header.Set("X-Api-Key", l.params.CrowdSecKey)
|
||||
client = l.crowdSecClient
|
||||
}
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
// Do's error names the URL, which the log line and the alert name
|
||||
// already: only what went wrong is kept.
|
||||
@@ -393,16 +480,22 @@ func (l *Lists) get(ctx context.Context, listURL string) ([]string, error) {
|
||||
return lines, nil
|
||||
}
|
||||
|
||||
// parse reads the lines of the list at listURL: those of a blocklist, or
|
||||
// of the file of AS:percent lines. Anything after a ; or a # on a line is
|
||||
// parse reads the lines of the list at listURL, fetched at fetched: those
|
||||
// of a blocklist, of the file of AS:percent lines, or of the CrowdSec
|
||||
// decision list. In the first two, anything after a ; or a # on a line is
|
||||
// left out, and so is a line left blank. Any other line that does not read
|
||||
// is an error naming it by its number.
|
||||
func (l *Lists) parse(listURL string, lines []string) (entries, error) {
|
||||
if listURL == l.params.ASNLimitPercentURL {
|
||||
func (l *Lists) parse(
|
||||
listURL string, lines []string, fetched time.Time,
|
||||
) (entries, error) {
|
||||
switch listURL {
|
||||
case l.params.ASNLimitPercentURL:
|
||||
return parsePercents(lines)
|
||||
case l.params.CrowdSecDecisionsURL:
|
||||
return parseDecisions(lines, fetched)
|
||||
default:
|
||||
return parseNetblocks(lines)
|
||||
}
|
||||
|
||||
return parseNetblocks(lines)
|
||||
}
|
||||
|
||||
// parseNetblocks reads a blocklist's lines, each an address or a netblock
|
||||
@@ -486,6 +579,56 @@ func parsePercents(lines []string) (entries, error) {
|
||||
return found, nil
|
||||
}
|
||||
|
||||
// parseDecisions reads the lines of the CrowdSec decision list fetched at
|
||||
// fetched: the engine's answer, a JSON list of its decisions in force,
|
||||
// null while it has none. A decision of the type ban whose scope is Ip or
|
||||
// Range, as CrowdSec names them, bans its value, an address or a netblock
|
||||
// as parseNetblock reads it, until its duration, the time it had left as
|
||||
// the engine answered, has passed since fetched. Any other decision, such
|
||||
// as one to show a captcha or one on a country, is left out. A decision
|
||||
// to ban whose value or duration does not read is an error naming it by
|
||||
// its number.
|
||||
func parseDecisions(lines []string, fetched time.Time) (entries, error) {
|
||||
var answer []struct {
|
||||
Duration string `json:"duration"`
|
||||
Scenario string `json:"scenario"`
|
||||
Scope string `json:"scope"`
|
||||
Type string `json:"type"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
err := json.Unmarshal([]byte(strings.Join(lines, "\n")), &answer)
|
||||
if err != nil {
|
||||
return entries{}, fmt.Errorf("read the answer: %w", err)
|
||||
}
|
||||
|
||||
found := entries{decisions: map[netip.Prefix]Decision{}}
|
||||
|
||||
for i, decision := range answer {
|
||||
if decision.Type != "ban" || (decision.Scope != "Ip" && decision.Scope != "Range") {
|
||||
continue
|
||||
}
|
||||
|
||||
netblock, ok := parseNetblock(decision.Value)
|
||||
|
||||
duration, err := time.ParseDuration(decision.Duration)
|
||||
if !ok || err != nil {
|
||||
return entries{}, fmt.Errorf("decision %d %w", i+1, errNotDecision)
|
||||
}
|
||||
|
||||
expires := fetched.Add(duration)
|
||||
if expires.After(found.decisions[netblock].Expires) {
|
||||
found.decisions[netblock] = Decision{Expires: expires, Scenario: decision.Scenario}
|
||||
}
|
||||
|
||||
if !slices.Contains(found.lengths, netblock.Bits()) {
|
||||
found.lengths = append(found.lengths, netblock.Bits())
|
||||
}
|
||||
}
|
||||
|
||||
return found, nil
|
||||
}
|
||||
|
||||
// withoutComment returns line without anything after a ; or a #, and
|
||||
// without the spaces around what is left.
|
||||
func withoutComment(line string) string {
|
||||
|
||||
@@ -215,12 +215,7 @@ func TestFailedFetchKeepsTheLastGoodCopyAndAlertsOncePerCooldown(t *testing.T) {
|
||||
|
||||
// One alert for the first failure; the cooldown holds back the
|
||||
// second.
|
||||
wantAlert(t, queue, alerts.Alert{
|
||||
Time: time.Now().Add(-refresh),
|
||||
Event: alerts.EventSourceFailure,
|
||||
Reason: "fetching a list failed",
|
||||
Detail: map[string]any{"source": dropURL, "error": tc.error},
|
||||
})
|
||||
wantAlert(t, queue, fetchFailure(time.Now().Add(-refresh), dropURL, tc.error))
|
||||
|
||||
if !strings.Contains(log.String(), `"msg":"fetching a list failed",`+
|
||||
`"url":"`+dropURL+`","error":"`+tc.error) {
|
||||
@@ -535,7 +530,9 @@ func newQueue() *alerts.Queue {
|
||||
|
||||
// start returns the lists of p, fetched through servers by Run, which runs
|
||||
// until the test ends, once Run has fetched those due at start.
|
||||
func start(t *testing.T, servers *standIn, p reputation.Params) *reputation.Lists {
|
||||
func start(
|
||||
t *testing.T, servers http.RoundTripper, p reputation.Params,
|
||||
) *reputation.Lists {
|
||||
t.Helper()
|
||||
|
||||
lists := reputation.New(p)
|
||||
@@ -597,6 +594,17 @@ func waiting(queue *alerts.Queue) []alerts.Alert {
|
||||
return queue.Snapshot().Waiting[alerts.DestinationWebhook]
|
||||
}
|
||||
|
||||
// fetchFailure is the source_failure alert raised at the time raised for
|
||||
// a fetch of the list at listURL that failed with err.
|
||||
func fetchFailure(raised time.Time, listURL, err string) alerts.Alert {
|
||||
return alerts.Alert{
|
||||
Time: raised,
|
||||
Event: alerts.EventSourceFailure,
|
||||
Reason: "fetching a list failed",
|
||||
Detail: map[string]any{"source": listURL, "error": err},
|
||||
}
|
||||
}
|
||||
|
||||
// wantAlert checks that want is the one alert waiting in queue, and that
|
||||
// the cooldown has held back one repeat of it.
|
||||
func wantAlert(t *testing.T, queue *alerts.Queue, want alerts.Alert) {
|
||||
|
||||
@@ -29,7 +29,8 @@ const (
|
||||
// over a rate limit, which bans the client.
|
||||
ActionRateLimited = "rate_limited"
|
||||
// ActionBanned is a request refused because a ban covers its client,
|
||||
// or because it matched a ban rule, which bans the client.
|
||||
// or because it matched a ban rule or the CrowdSec decision list lists
|
||||
// its client, either of which bans the client.
|
||||
ActionBanned = "banned"
|
||||
// ActionRuleBlocked is a request refused because it matched a block
|
||||
// rule.
|
||||
@@ -140,8 +141,9 @@ type Line struct {
|
||||
// minute_bytes, hour_bytes or day_bytes for a byte limit.
|
||||
LimitHit string `json:"limit_hit,omitempty"`
|
||||
// Reputation are the URLs of the blocklists that list the client, then
|
||||
// the DNSBL zones whose verdict lists it, their keys masked, then
|
||||
// abuseipdb when its score is a hit.
|
||||
// that of the CrowdSec decision list when it does, then the DNSBL zones
|
||||
// whose verdict lists it, their keys masked, then abuseipdb when its
|
||||
// score is a hit.
|
||||
Reputation []string `json:"reputation,omitempty"`
|
||||
// Offence is the offence the request was held as, OffenceLimit.
|
||||
Offence string `json:"offence,omitempty"`
|
||||
|
||||
@@ -716,6 +716,80 @@ func TestBlocklistTriesAndCopiesKeptInReputationJSONAcrossRestarts(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrowdSecDecisionListKeptInReputationJSONAcrossARestart(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const bouncerKey = "crowdsec-key-0123456789abcdef"
|
||||
|
||||
// A stand-in for the engine's local API, which bans 203.0.113.0/24 for
|
||||
// four hours, and answers only a request with its key, while it is up.
|
||||
down := new(atomic.Bool)
|
||||
engine := httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.URL.Path != "/v1/decisions" || r.Header.Get("X-Api-Key") != bouncerKey:
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
case down.Load():
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
default:
|
||||
_, _ = io.WriteString(w, `[{"duration": "4h0m0s", "origin": "crowdsec", `+
|
||||
`"scenario": "crowdsecurity/http-probing", "scope": "Range", `+
|
||||
`"type": "ban", "value": "203.0.113.0/24"}]`)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(engine.Close)
|
||||
|
||||
dir := t.TempDir()
|
||||
env := map[string]string{
|
||||
listenAddr: localhost + ":0",
|
||||
upstreamURL: startApp(t),
|
||||
stateDir: dir,
|
||||
rulesDir: t.TempDir(),
|
||||
trustedProxies: localhost + "/32",
|
||||
"SWWAF_CROWDSEC_LAPI_URL": engine.URL,
|
||||
"SWWAF_CROWDSEC_LAPI_KEY": bouncerKey,
|
||||
}
|
||||
|
||||
// Once the list is fetched, the client's request bans it.
|
||||
first := runUntilStopped(t, env, func(url string) {
|
||||
for statusFrom(t, url, placed) != http.StatusForbidden {
|
||||
time.Sleep(pollInterval)
|
||||
}
|
||||
})
|
||||
|
||||
ban := onlyBan(t, dir)
|
||||
if ban["netblock"] != placed+"/32" || ban["cause"] != "crowdsec" ||
|
||||
ban["reason"] != "CrowdSec's decision for crowdsecurity/http-probing" {
|
||||
t.Errorf("bans.json holds %v, want the ban for crowdsec on %s", ban, placed)
|
||||
}
|
||||
|
||||
// Restarted with the engine down, the copy kept in reputation.json bans
|
||||
// another client in the netblock from its first request.
|
||||
down.Store(true)
|
||||
|
||||
second := runUntilStopped(t, env, func(url string) {
|
||||
wantStatus(t, url, "203.0.113.10", http.StatusForbidden)
|
||||
})
|
||||
|
||||
// The key is in neither run's output, nor in a state file.
|
||||
files := []string{"bans.json", "reputation.json", "clients.json"}
|
||||
shown := make([]string, 0, len(files)+2)
|
||||
shown = append(shown, first.text(), second.text())
|
||||
|
||||
for _, name := range files {
|
||||
data, err := os.ReadFile(filepath.Join(dir, name)) //nolint:gosec // the test's
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", name, err)
|
||||
}
|
||||
|
||||
shown = append(shown, string(data))
|
||||
}
|
||||
|
||||
if all := strings.Join(shown, "\n"); strings.Contains(all, bouncerKey) {
|
||||
t.Errorf("the key is shown in the output or the state files:\n%s", all)
|
||||
}
|
||||
}
|
||||
|
||||
// wantDeniedByList checks that the request log line is of a request the
|
||||
// blocklist at listURL refused.
|
||||
func wantDeniedByList(t *testing.T, line map[string]any, listURL string) {
|
||||
|
||||
@@ -60,7 +60,7 @@ var (
|
||||
errVersion = errors.New("unknown version")
|
||||
// errMissing is for an entry without a field it needs.
|
||||
errMissing = errors.New("has no")
|
||||
errCause = errors.New("is not limit, attack or admin")
|
||||
errCause = errors.New("is not limit, attack, admin or crowdsec")
|
||||
errDestination = errors.New("is not webhook, slack or ntfy")
|
||||
errScope = errors.New("is not client, net, asn, total or watch")
|
||||
errWaitingList = errors.New(`waiting is a list, but now lists the alerts by ` +
|
||||
@@ -647,7 +647,7 @@ func (e BanEntry) ban() bans.Ban {
|
||||
// worked out, or an expires, which would make it permanent. A permanent
|
||||
// ban's expires is null, which Bans cannot tell from a missing one, so
|
||||
// each expires is read again as written. A cause other than limit,
|
||||
// attack or admin, most likely misspelt, is refused too.
|
||||
// attack, admin or crowdsec, most likely misspelt, is refused too.
|
||||
func (f *bansFile) check(data []byte) error {
|
||||
var written struct {
|
||||
Bans []struct {
|
||||
@@ -669,7 +669,8 @@ func (f *bansFile) check(data []byte) error {
|
||||
case written.Bans[i].Expires == nil:
|
||||
return missing(i, "expires")
|
||||
case entry.Cause != "" && entry.Cause != bans.CauseLimit &&
|
||||
entry.Cause != bans.CauseAttack && entry.Cause != bans.CauseAdmin:
|
||||
entry.Cause != bans.CauseAttack && entry.Cause != bans.CauseAdmin &&
|
||||
entry.Cause != bans.CauseCrowdSec:
|
||||
return fmt.Errorf("entry %d's cause %q %w", i+1, entry.Cause, errCause)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,7 +97,8 @@ const permanentBansJSON = `{
|
||||
"earlier_bans": {
|
||||
"limit": 3,
|
||||
"attack": 1,
|
||||
"admin": 1
|
||||
"admin": 1,
|
||||
"crowdsec": 2
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -849,8 +850,10 @@ func TestBanWithAnotherCauseStopsTheStart(t *testing.T) {
|
||||
`{"netblock": "203.0.113.10/32", "start": "2026-10-06T00:00:00Z", `+
|
||||
`"expires": null, "cause": "admin"}, `+
|
||||
`{"netblock": "203.0.113.11/32", "start": "2026-10-06T00:00:00Z", `+
|
||||
`"expires": "2026-10-06T04:00:00Z", "cause": "crowdsec"}, `+
|
||||
`{"netblock": "203.0.113.12/32", "start": "2026-10-06T00:00:00Z", `+
|
||||
`"expires": null, "cause": "atack"}]}`,
|
||||
`: entry 3's cause "atack" is not limit, attack or admin`)
|
||||
`: entry 4's cause "atack" is not limit, attack, admin or crowdsec`)
|
||||
}
|
||||
|
||||
func TestUnknownVersionStopsTheStart(t *testing.T) {
|
||||
@@ -1726,8 +1729,9 @@ func office() netip.Prefix {
|
||||
return netip.MustParsePrefix("203.0.113.0/24")
|
||||
}
|
||||
|
||||
// fill puts a permanent ban an admin made, a ban for a broken limit and
|
||||
// one for a clear sign of attack, clients with counts and histories,
|
||||
// fill puts a permanent ban an admin made, a ban for a broken limit, one
|
||||
// for a clear sign of attack and one for CrowdSec's decision, clients
|
||||
// with counts and histories,
|
||||
// GeoJS answers, the blocklists' last tries and the copy of one, two
|
||||
// verdicts of a DNSBL zone, and the AbuseIPDB checks spent today with two
|
||||
// scores, as filledReputationJSON holds them, and alerts
|
||||
@@ -1743,6 +1747,8 @@ func fill(params state.Params) {
|
||||
})
|
||||
params.Ledger.BanForAttack(netip.MustParsePrefix("192.0.2.1/32"), now,
|
||||
bans.Notes{RuleID: "env-file", Target: "path"})
|
||||
params.Ledger.BanForCrowdSec(netip.MustParsePrefix("198.51.100.9/32"), now,
|
||||
now.Add(4*time.Hour), "crowdsecurity/ssh-bf", bans.Notes{})
|
||||
|
||||
for _, c := range []string{"2001:db8::/64", "203.0.113.9/32", "192.0.2.1/32"} {
|
||||
params.Limiter.Count(netip.MustParsePrefix(c), now, whole)
|
||||
@@ -1844,7 +1850,7 @@ func permanentBan() bans.Ban {
|
||||
},
|
||||
Requests: 1500,
|
||||
Refused: 3,
|
||||
EarlierBans: bans.EarlierBans{Limit: 3, Attack: 1, Admin: 1},
|
||||
EarlierBans: bans.EarlierBans{Limit: 3, Attack: 1, Admin: 1, CrowdSec: 2},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user