CrowdSec decision list fetched, kept, and its clients banned until the decision ends (closes #106)

SWWAF_CROWDSEC_LAPI_URL and SWWAF_CROWDSEC_LAPI_KEY name an engine whose
decision list, <url>/v1/decisions, is fetched every minute with the key in
X-Api-Key and kept as a blocklist is: used while a fetch fails, and across
restarts through reputation.json. Ban decisions on an Ip or a Range end at
the fetch time plus their duration. A listed client's request is refused and
bans its netblock with the cause crowdsec until the decision ends; bans.json,
ban notes and metrics take the cause.

Judgement call: fetched every minute, not a setting.
Judgement call: a crowdsec ban never lengthens a limit ban.
Judgement call: a lifted crowdsec ban is remade while its decision lasts.

Model: opus-5-5

internal/bans/bans.go

@@ -1,8 +1,9 @@
 // Package bans is the ban ledger: the bans smallwebwaf makes on the
-// netblocks of clients that break a rate limit or a byte limit or show a
-// clear sign of attack, and those an admin makes, with their notes, as
-// the "Bans" section of SPEC.md describes. The bans are kept in memory,
-// and written to bans.json and read from it by the state package.
+// netblocks of clients that break a rate limit or a byte limit, show a
+// clear sign of attack or are listed by the CrowdSec decision list, and
+// those an admin makes, with their notes, as the "Bans" section of SPEC.md
+// describes. The bans are kept in memory, and written to bans.json and
+// read from it by the state package.
 package bans
 
 import (
@@ -26,6 +27,9 @@ const (
 	// CauseAdmin is a ban an admin made, or one smallwebwaf made that an
 	// admin keeps. It is never dropped.
 	CauseAdmin = "admin"
+	// CauseCrowdSec is a ban smallwebwaf made for a client the CrowdSec
+	// decision list lists. It ends when CrowdSec's decision does.
+	CauseCrowdSec = "crowdsec"
 )
 
 // repeatFactor is how many times as long as the netblock's last ban a ban
@@ -40,9 +44,9 @@ type Rules struct {
 	// LimitBanDuration is how long a first ban for a broken limit lasts.
 	LimitBanDuration time.Duration
 	// LimitBanRepeatWindow is how soon after the end of the netblock's
-	// ban that ended last, other than one for a clear sign of attack, a
-	// broken limit counts as a repeat, which bans for repeatFactor times as
-	// long as that ban.
+	// ban that ended last, other than one for a clear sign of attack or for
+	// CrowdSec's decision, a broken limit counts as a repeat, which bans for
+	// repeatFactor times as long as that ban.
 	LimitBanRepeatWindow time.Duration
 	// MaxBanDuration is the longest ban for a broken limit; one that would
 	// be longer is permanent instead.
@@ -63,10 +67,11 @@ type Ban struct {
 	Start    time.Time
 	// Expires is when the ban ends, zero for a permanent ban.
 	Expires time.Time
-	// Cause is CauseLimit, CauseAttack or CauseAdmin.
+	// Cause is CauseLimit, CauseAttack, CauseAdmin or CauseCrowdSec.
 	Cause string
-	// Reason is a short text: for a ban smallwebwaf made, the limit broken
-	// or the rule that matched; for an admin's, what the admin wrote.
+	// Reason is a short text: for a ban smallwebwaf made, the limit broken,
+	// the rule that matched or the scenario of CrowdSec's decision; for an
+	// admin's, what the admin wrote.
 	Reason string
 	// Lifted is when an admin lifted the ban, zero while no admin has. A
 	// lifted ban refuses nothing, and does not make the netblock's next
@@ -123,7 +128,8 @@ type Notes struct {
 	// log's reputation names them. It is left out when none did.
 	Reputation []ReputationHit `json:"reputation,omitempty"`
 	// Request is the request that broke the limit, or whose bytes broke
-	// it, or that was the clear sign of attack.
+	// it, that was the clear sign of attack, or that came from a client the
+	// CrowdSec decision list lists.
 	Request Request `json:"request"`
 	// Requests is how many requests the netblock has sent since it was
 	// first seen, and Refused how many of them the ban has refused so
@@ -136,9 +142,10 @@ type Notes struct {
 }
 
 // ReputationHit is a reputation source that listed a client, as a
-// reputation_hit alert's detail gives it: Source is the blocklist's URL,
-// the DNSBL zone with its key masked, or "abuseipdb", and Score, for
-// AbuseIPDB alone, its score of the client.
+// reputation_hit alert's detail gives it: Source is the URL of the
+// blocklist or of the CrowdSec decision list, the DNSBL zone with its key
+// masked, or "abuseipdb", and Score, for AbuseIPDB alone, its score of the
+// client.
 type ReputationHit struct {
 	Source string `json:"source"`
 	Score  *int64 `json:"score,omitempty"`
@@ -146,9 +153,10 @@ type ReputationHit struct {
 
 // EarlierBans counts a netblock's bans before a ban, by cause.
 type EarlierBans struct {
-	Limit  int `json:"limit"`
-	Attack int `json:"attack"`
-	Admin  int `json:"admin"`
+	Limit    int `json:"limit"`
+	Attack   int `json:"attack"`
+	Admin    int `json:"admin"`
+	CrowdSec int `json:"crowdsec"`
 }
 
 // Request is a request in a ban's notes. Each text is cut to 256 bytes.
@@ -276,7 +284,8 @@ func activeBan(bans []Ban, now time.Time) *Ban {
 // BanForLimit bans netblock at now for a broken limit, with notes, and
 // returns the ban, and true. A first ban lasts LimitBanDuration. A ban
 // made within LimitBanRepeatWindow after the netblock's ban that ended
-// last, other than one for a clear sign of attack or a lifted one, lasts
+// last, other than one for a clear sign of attack or for CrowdSec's
+// decision, or a lifted one, lasts
 // repeatFactor times as long as that one. A ban that would be longer
 // than MaxBanDuration is permanent instead. If a ban on netblock is still
 // active, as when two of its requests break a limit at once, that ban is
@@ -287,7 +296,7 @@ func activeBan(bans []Ban, now time.Time) *Ban {
 func (l *Ledger) BanForLimit(
 	netblock netip.Prefix, now time.Time, notes Notes,
 ) (Ban, bool) {
-	return l.ban(netblock, now, CauseLimit, limitReason(notes), notes, true)
+	return l.ban(netblock, now, time.Time{}, CauseLimit, limitReason(notes), notes, true)
 }
 
 // WouldBanForLimit returns what BanForLimit would, without making the ban:
@@ -295,7 +304,7 @@ func (l *Ledger) BanForLimit(
 func (l *Ledger) WouldBanForLimit(
 	netblock netip.Prefix, now time.Time, notes Notes,
 ) (Ban, bool) {
-	return l.ban(netblock, now, CauseLimit, limitReason(notes), notes, false)
+	return l.ban(netblock, now, time.Time{}, CauseLimit, limitReason(notes), notes, false)
 }
 
 // BanForAttack bans netblock at now for a clear sign of attack, with
@@ -306,7 +315,7 @@ func (l *Ledger) WouldBanForLimit(
 func (l *Ledger) BanForAttack(
 	netblock netip.Prefix, now time.Time, notes Notes,
 ) (Ban, bool) {
-	return l.ban(netblock, now, CauseAttack, attackReason(notes), notes, true)
+	return l.ban(netblock, now, time.Time{}, CauseAttack, attackReason(notes), notes, true)
 }
 
 // WouldBanForAttack returns what BanForAttack would, without making the
@@ -314,12 +323,35 @@ func (l *Ledger) BanForAttack(
 func (l *Ledger) WouldBanForAttack(
 	netblock netip.Prefix, now time.Time, notes Notes,
 ) (Ban, bool) {
-	return l.ban(netblock, now, CauseAttack, attackReason(notes), notes, false)
+	return l.ban(netblock, now, time.Time{}, CauseAttack, attackReason(notes), notes,
+		false)
 }
 
-// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit
-// or CauseAttack, made at now would be permanent, as BanForLimit or
-// BanForAttack would make it. It works out nothing else of the ban.
+// BanForCrowdSec bans netblock at now until expires, when CrowdSec's
+// decision on the client ends, with notes, and returns the ban, and
+// whether it made it, as BanForLimit does. Its reason is "CrowdSec's
+// decision for <scenario>", the scenario that made the decision.
+func (l *Ledger) BanForCrowdSec(
+	netblock netip.Prefix, now, expires time.Time, scenario string, notes Notes,
+) (Ban, bool) {
+	return l.ban(netblock, now, expires, CauseCrowdSec, crowdSecReason(scenario), notes,
+		true)
+}
+
+// WouldBanForCrowdSec returns what BanForCrowdSec would, without making
+// the ban: what observe mode would have done.
+func (l *Ledger) WouldBanForCrowdSec(
+	netblock netip.Prefix, now, expires time.Time, scenario string, notes Notes,
+) (Ban, bool) {
+	return l.ban(netblock, now, expires, CauseCrowdSec, crowdSecReason(scenario), notes,
+		false)
+}
+
+// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit,
+// CauseAttack or CauseCrowdSec, made at now would be permanent, as
+// BanForLimit, BanForAttack or BanForCrowdSec would make it. It works out
+// nothing else of the ban. A ban for CrowdSec's decision is never
+// permanent: it ends with the decision.
 func (l *Ledger) WouldBePermanent(
 	netblock netip.Prefix, now time.Time, cause string,
 ) bool {
@@ -331,11 +363,14 @@ func (l *Ledger) WouldBePermanent(
 		held = *bans
 	}
 
-	if cause == CauseAttack {
+	switch cause {
+	case CauseAttack:
 		return l.attackExpiry(held, now).IsZero()
+	case CauseLimit:
+		return l.limitExpiry(held, now).IsZero()
+	default: // CauseCrowdSec
+		return false
 	}
-
-	return l.limitExpiry(held, now).IsZero()
 }
 
 // limitReason is the reason of a ban for a broken limit, with notes.
@@ -350,6 +385,12 @@ func attackReason(notes Notes) string {
 	return "matched the rule " + notes.RuleID
 }
 
+// crowdSecReason is the reason of a ban for CrowdSec's decision, which
+// scenario made.
+func crowdSecReason(scenario string) string {
+	return "CrowdSec's decision for " + scenario
+}
+
 // BanForAdmin bans netblock at now for an admin, with reason, until
 // expires, or for good when expires is zero, and returns the ban, whose
 // cause is CauseAdmin. Unlike BanForLimit and BanForAttack, it makes the
@@ -587,11 +628,14 @@ func (l *Ledger) holds(netblock netip.Prefix, start time.Time) bool {
 }
 
 // ban bans netblock at now for cause, with reason and notes, as
-// BanForLimit and BanForAttack describe, and returns the ban, and whether
-// it made it. Unless keep is true, the ban is not made, only returned: it
-// is the ban that would have been made.
+// BanForLimit, BanForAttack and BanForCrowdSec describe, and returns the
+// ban, and whether it made it. expires is when a ban for CauseCrowdSec
+// ends, and zero for the others, whose end the ledger works out. Unless
+// keep is true, the ban is not made, only returned: it is the ban that
+// would have been made.
 func (l *Ledger) ban(
-	netblock netip.Prefix, now time.Time, cause, reason string, notes Notes, keep bool,
+	netblock netip.Prefix, now, expires time.Time, cause, reason string, notes Notes,
+	keep bool,
 ) (Ban, bool) {
 	l.mu.Lock()
 	defer l.mu.Unlock()
@@ -613,10 +657,13 @@ func (l *Ledger) ban(
 	notes.Request = notes.Request.cut()
 	ban := Ban{Netblock: netblock, Start: now, Cause: cause, Reason: reason, Notes: notes}
 
-	if cause == CauseAttack {
+	switch cause {
+	case CauseAttack:
 		ban.Expires = l.attackExpiry(held, now)
-	} else {
+	case CauseLimit:
 		ban.Expires = l.limitExpiry(held, now)
+	default: // CauseCrowdSec
+		ban.Expires = expires
 	}
 
 	if !keep {
@@ -645,6 +692,8 @@ func earlierBans(held []Ban) EarlierBans {
 			earlier.Attack++
 		case CauseAdmin:
 			earlier.Admin++
+		case CauseCrowdSec:
+			earlier.CrowdSec++
 		}
 	}
 
@@ -738,16 +787,16 @@ func (l *Ledger) add(ban Ban) {
 // limitExpiry returns when a ban for a broken limit made at now ends, or
 // zero when it is permanent. held are the netblock's bans, none of them
 // active, of which the one that ended last, other than a ban for a clear
-// sign of attack or a lifted one, can make the new ban longer. A ban an
-// admin adds to bans.json can start after another and end before it, so
-// that one is looked for among them all.
+// sign of attack or for CrowdSec's decision, or a lifted one, can make the
+// new ban longer. A ban an admin adds to bans.json can start after
+// another and end before it, so that one is looked for among them all.
 func (l *Ledger) limitExpiry(held []Ban, now time.Time) time.Time {
 	length := l.rules.LimitBanDuration
 
 	var last *Ban
 
 	for i, ban := range held {
-		if ban.Cause != CauseAttack && ban.Lifted.IsZero() &&
+		if (ban.Cause == CauseLimit || ban.Cause == CauseAdmin) && ban.Lifted.IsZero() &&
 			(last == nil || ban.Expires.After(last.Expires)) {
 			last = &held[i]
 		}

internal/bans/crowdsec_test.go

@@ -0,0 +1,90 @@
+package bans_test
+
+import (
+	"net/netip"
+	"reflect"
+	"testing"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/bans"
+)
+
+// scenario is the scenario of the tests' CrowdSec decisions.
+const scenario = "crowdsecurity/ssh-bf"
+
+func TestCrowdSecBanLastsUntilTheDecisionEnds(t *testing.T) {
+	t.Parallel()
+
+	ledger := bans.New(defaultRules())
+	netblock := netip.MustParsePrefix("203.0.113.9/32")
+	expires := midnight().Add(4 * time.Hour)
+
+	// The ban that would be made is not made.
+	would, wouldBan := ledger.WouldBanForCrowdSec(netblock, midnight(), expires,
+		scenario, bans.Notes{})
+	if !wouldBan || len(ledger.Bans(netblock)) != 0 {
+		t.Errorf("would ban %t, and the ledger holds %+v, want true and nothing",
+			wouldBan, ledger.Bans(netblock))
+	}
+
+	const reason = "CrowdSec's decision for " + scenario
+
+	ban, made := ledger.BanForCrowdSec(netblock, midnight(), expires, scenario,
+		bans.Notes{})
+	if !made || !reflect.DeepEqual(ban, would) || ban.Cause != bans.CauseCrowdSec ||
+		!ban.Expires.Equal(expires) || ban.Reason != reason ||
+		ledger.Made(bans.CauseCrowdSec) != 1 {
+		t.Errorf("made %t the ban %+v, want the one that would be made, %+v, for "+
+			"crowdsec until %s", made, ban, would, expires)
+	}
+
+	// A second decision on the netblock while the ban lasts makes no other.
+	again, made := ledger.BanForCrowdSec(netblock, midnight().Add(time.Hour),
+		expires.Add(time.Hour), scenario, bans.Notes{})
+	if made || !again.Expires.Equal(expires) || ledger.Made(bans.CauseCrowdSec) != 1 {
+		t.Errorf("made %t the ban %+v while the first lasts, want none", made, again)
+	}
+}
+
+func TestCrowdSecBanIsNeverMadePermanent(t *testing.T) {
+	t.Parallel()
+
+	ledger := bans.New(defaultRules())
+	netblock := netip.MustParsePrefix("203.0.113.9/32")
+	expires := midnight().Add(4 * time.Hour)
+	ledger.BanForCrowdSec(netblock, midnight(), expires, scenario, bans.Notes{})
+
+	// A request as the ban ends is refused, and leaves it as it is.
+	last := expires.Add(-time.Nanosecond)
+
+	held, banned, madePermanent := ledger.Check(netblock.Addr(), last)
+	if !banned || madePermanent || !held.Expires.Equal(expires) ||
+		ledger.WouldBePermanent(netblock, last, bans.CauseCrowdSec) {
+		t.Errorf("as the ban ends, banned %t with %+v, made permanent %t, want "+
+			"refused under the ban as it was", banned, held, madePermanent)
+	}
+
+	if _, banned, _ := ledger.Check(netblock.Addr(), expires); banned {
+		t.Error("the ban refuses a request once the decision has ended")
+	}
+}
+
+func TestCrowdSecBanIsCountedAndDoesNotLengthenTheNextBanForALimit(t *testing.T) {
+	t.Parallel()
+
+	ledger := bans.New(defaultRules())
+	netblock := netip.MustParsePrefix("203.0.113.9/32")
+
+	// Three times the three days would be permanent; a limit broken as the
+	// ban for CrowdSec's decision ends bans for an hour, as a first broken
+	// limit does.
+	crowdSec, _ := ledger.BanForCrowdSec(netblock, midnight(), midnight().Add(3*day),
+		scenario, bans.Notes{})
+	limit, _ := ledger.BanForLimit(netblock, crowdSec.Expires, bans.Notes{})
+
+	if limit.Expires.Sub(limit.Start) != time.Hour ||
+		limit.Notes.EarlierBans != (bans.EarlierBans{CrowdSec: 1}) {
+		t.Errorf("the ban for a limit is %+v, want one of an hour after one for crowdsec",
+			limit)
+	}
+}