waf: ban the clients on AbuseIPDB's blacklist (closes #411)
check / check (push) Failing after 8m52s

While SWW_WAF_ABUSEIPDB_KEY is set, waf fetches AbuseIPDB's blacklist at
the start and every SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS
(60), cut at SWW_WAF_ABUSEIPDB_THRESHOLD (90, from 25 to 100). Each
listed client's netblock gets a ban of SWW_WAF_ABUSEIPDB_BAN_DAYS (30)
with the new cause abuseipdb. The blacklist itself is not kept.

These bans do not count toward SWW_WAF_MAX_BANS, so a 10,000-address
blacklist cannot push out the bans for limits and attacks. An ended
abuseipdb ban is forgotten at each fetch and each read of bans.json. A
netblock with an active ban is left alone, and so is one whose abuseipdb
ban an admin lifted, until that ban would have ended. Observe mode makes
no ban.

Model: opus-5-5
This commit is contained in:
clawbot committed 2026-11-19 10:03:42 +00:00
1 parent 9f42460f70
commit dfffeb73ec
22 files changed
+1304 -205

No files matched your search

+161 -34
View File
@@ -1,8 +1,9 @@
// Package bans is the ban ledger: the bans sww makes on the
// netblocks of clients that break a rate limit, a byte limit or the refusal
// limit or show a clear sign of attack, and those an admin makes, with their
// notes, as the "Bans" section of SPEC.md describes. The bans are kept in
// memory, and written to bans.json and read from it by the state package.
// limit or show a clear sign of attack, or are on AbuseIPDB's blacklist, and
// those an admin makes, with their notes, as the "Bans" section of SPEC.md
// describes. The bans are kept in memory, and written to bans.json and read
// from it by the state package.
package bans
import (
@@ -26,6 +27,9 @@ const (
// CauseAdmin is a ban an admin made, or one sww made that an
// admin keeps. It is never dropped.
CauseAdmin = "admin"
// CauseAbuseIPDB is a ban sww made for a client on AbuseIPDB's
// blacklist. It is forgotten once it has ended.
CauseAbuseIPDB = "abuseipdb"
)
// repeatFactor is how many times as long as the netblock's last ban a ban
@@ -50,10 +54,11 @@ type Params struct {
// AttackBanDuration is how long a first ban for a clear sign of attack
// lasts.
AttackBanDuration time.Duration
// MaxBans is the most bans held whose cause is not CauseAdmin, at
// least one. Past it, the earliest such ban of the netblock that has
// gone longest without a request is dropped. Bans whose cause is
// CauseAdmin are held besides, and never dropped.
// MaxBans is the most bans held whose cause is CauseLimit or
// CauseAttack, at least one. Past it, the earliest such ban of the
// netblock that has gone longest without a request is dropped. Bans
// whose cause is CauseAdmin or CauseAbuseIPDB are held besides, and
// never dropped this way.
MaxBans int
}
@@ -63,10 +68,11 @@ type Ban struct {
Start time.Time
// Expires is when the ban ends, zero for a permanent ban.
Expires time.Time
// Cause is CauseLimit, CauseAttack or CauseAdmin.
// Cause is CauseLimit, CauseAttack, CauseAdmin or CauseAbuseIPDB.
Cause string
// Reason is a short text: for a ban sww made, the limit broken or
// the rule that matched; for an admin's, what the admin wrote.
// Reason is a short text: for a ban sww made, the limit broken, the
// rule that matched or AbuseIPDB's score; for an admin's, what the
// admin wrote.
Reason string
// Lifted is when an admin lifted the ban, zero while no admin has. A
// lifted ban refuses nothing, and does not make the netblock's next
@@ -127,7 +133,8 @@ type Notes struct {
// log's reputation names them. It is left out when none did.
Reputation []ReputationHit `json:"reputation,omitempty"`
// Request is the request that broke the limit, or whose bytes broke
// it, or that was the clear sign of attack.
// it, or that was the clear sign of attack. A ban for AbuseIPDB's
// blacklist has none, and leaves it zero.
Request Request `json:"request"`
// Requests is how many requests the netblock has sent since it was
// first seen, and Refused how many of them the ban has refused so
@@ -150,9 +157,10 @@ type ReputationHit struct {
// EarlierBans counts a netblock's bans before a ban, by cause.
type EarlierBans struct {
Limit int `json:"limit"`
Attack int `json:"attack"`
Admin int `json:"admin"`
Limit int `json:"limit"`
Attack int `json:"attack"`
Admin int `json:"admin"`
AbuseIPDB int `json:"abuseipdb"`
}
// Request is a request in a ban's notes. Each text is cut to 256 bytes.
@@ -180,8 +188,8 @@ type Ledger struct {
// netblocks holds each banned netblock's bans, oldest first. Check and
// Find make each netblock they find the most recently seen.
netblocks *simplelru.LRU[netip.Prefix, *[]Ban]
// held is how many bans netblocks holds whose cause is not CauseAdmin,
// at most params.MaxBans.
// held is how many bans netblocks holds whose cause is CauseLimit or
// CauseAttack, at most params.MaxBans.
held int
// made is how many bans have been made since the start, by cause: by
// the ledger, and by an admin, through BanForAdmin or in an edit of
@@ -198,8 +206,9 @@ type Ledger struct {
// New returns a Ledger with no ban yet.
func New(params Params) *Ledger {
// The ledger drops bans itself, and never those whose cause is
// CauseAdmin, however many there are, so the LRU has no limit of its
// own: it keeps the netblocks in the order they were last seen.
// CauseAdmin or CauseAbuseIPDB to make room, however many there are, so
// the LRU has no limit of its own: it keeps the netblocks in the order
// they were last seen.
netblocks, err := simplelru.NewLRU[netip.Prefix, *[]Ban](
math.MaxInt,
nil,
@@ -429,6 +438,88 @@ func (l *Ledger) BanForAdmin(
return ban
}
// BanForAbuseIPDB bans each netblock in scores at now until expires, and
// returns how many bans it made. scores are the netblocks of the clients
// on AbuseIPDB's blacklist, each with the highest score of its clients.
// A netblock is left as it is while a ban on it is active, or while a ban
// for the blacklist that an admin lifted has not reached its end: the
// admin's lift holds that long. Each ban whose cause is CauseAbuseIPDB
// and that has ended is forgotten first, so that the bans held for the
// blacklist are those of the clients listed within one ban's length.
func (l *Ledger) BanForAbuseIPDB(
now, expires time.Time, scores map[netip.Prefix]int64,
) int {
l.mu.Lock()
defer l.mu.Unlock()
changed := l.forgetEndedAbuseIPDBBans(now)
made := 0
for netblock, score := range scores {
netblock = netblock.Masked()
var notes Notes
if held, found := l.netblocks.Peek(netblock); found {
if activeBan(*held, now) != nil ||
liftedAbuseIPDBBan(*held, now) {
continue
}
notes.EarlierBans = earlierBans(*held)
}
// The source is named as the request log names AbuseIPDB.
notes.Reputation = []ReputationHit{
{Source: CauseAbuseIPDB, Score: &score},
}
l.add(Ban{
Netblock: netblock,
Start: now,
Expires: expires,
Cause: CauseAbuseIPDB,
Reason: fmt.Sprintf(
"on AbuseIPDB's blacklist with a score of %d", score,
),
Notes: notes,
})
made++
}
l.made[CauseAbuseIPDB] += made
if changed || made > 0 {
l.markChanged()
}
return made
}
// ForgetEndedAbuseIPDBBans forgets the bans whose cause is CauseAbuseIPDB
// and that have ended at now, as BanForAbuseIPDB does first. The state
// package calls it each time it reads bans.json, so that these bans are
// also forgotten while no fetch of the blacklist reaches the ledger.
func (l *Ledger) ForgetEndedAbuseIPDBBans(now time.Time) {
l.mu.Lock()
defer l.mu.Unlock()
if l.forgetEndedAbuseIPDBBans(now) {
l.markChanged()
}
}
// liftedAbuseIPDBBan reports whether bans, a netblock's, hold a ban whose
// cause is CauseAbuseIPDB that an admin lifted, and whose end is not
// reached at now.
func liftedAbuseIPDBBan(bans []Ban, now time.Time) bool {
return slices.ContainsFunc(bans, func(ban Ban) bool {
return ban.Cause == CauseAbuseIPDB && !ban.Lifted.IsZero() &&
(ban.Permanent() || now.Before(ban.Expires))
})
}
// Lift lifts, at now, every ban active then on a netblock client is in,
// as an admin does, and returns those bans. A lifted ban is kept, refuses
// nothing, and does not make the netblock's next ban longer.
@@ -515,9 +606,9 @@ func (l *Ledger) AddLookup(
}
// Made returns how many bans for cause have been made since the start:
// for CauseLimit and CauseAttack, by the ledger; for CauseAdmin, by an
// admin, with BanForAdmin or in an edit of bans.json, as LoadEdit counts
// them. The bans read from bans.json at the start are not among them.
// for CauseLimit, CauseAttack and CauseAbuseIPDB, by the ledger; for
// CauseAdmin, by an admin, with BanForAdmin or in an edit of bans.json, as
// LoadEdit counts them. The bans read from bans.json at the start are not among them.
func (l *Ledger) Made(cause string) int {
l.mu.Lock()
defer l.mu.Unlock()
@@ -574,8 +665,8 @@ func (l *Ledger) Snapshot() []Ban {
// seen. A ban without a cause is an admin's, and gets CauseAdmin. Each
// netblock is masked to its length, so that 203.0.113.9/24 is
// 203.0.113.0/24, and each text in the notes is cut to 256 bytes. Past
// MaxBans the earliest bans whose cause is not CauseAdmin are dropped, as
// when they are made.
// MaxBans the earliest bans that count toward it are dropped, as when they
// are made.
func (l *Ledger) Load(bans []Ban) {
l.mu.Lock()
defer l.mu.Unlock()
@@ -639,6 +730,35 @@ func (l *Ledger) holds(netblock netip.Prefix, start time.Time) bool {
})
}
// forgetEndedAbuseIPDBBans forgets each ban whose cause is CauseAbuseIPDB
// and whose end is reached at now, lifted or not, and each netblock left
// without a ban. It reports whether it forgot one.
func (l *Ledger) forgetEndedAbuseIPDBBans(now time.Time) bool {
forgot := false
for _, netblock := range l.netblocks.Keys() {
bans, _ := l.netblocks.Peek(netblock)
kept := slices.DeleteFunc(*bans, func(ban Ban) bool {
return ban.Cause == CauseAbuseIPDB && !ban.Permanent() &&
!now.Before(ban.Expires)
})
if len(kept) == len(*bans) {
continue
}
forgot = true
if len(kept) == 0 {
l.netblocks.Remove(netblock)
} else {
*bans = kept
}
}
return forgot
}
// ban bans netblock at now for cause, CauseLimit or CauseAttack, with
// reason and notes, as BanForLimit and BanForAttack describe, and returns
// the ban, and whether it made it. Unless keep is true, the ban is not
@@ -714,13 +834,16 @@ func earlierBans(held []Ban) EarlierBans {
fromHere.Attack++
case CauseAdmin:
fromHere.Admin++
case CauseAbuseIPDB:
fromHere.AbuseIPDB++
}
notes := ban.Notes.EarlierBans
count := EarlierBans{
Limit: notes.Limit + fromHere.Limit,
Attack: notes.Attack + fromHere.Attack,
Admin: notes.Admin + fromHere.Admin,
Limit: notes.Limit + fromHere.Limit,
Attack: notes.Attack + fromHere.Attack,
Admin: notes.Admin + fromHere.Admin,
AbuseIPDB: notes.AbuseIPDB + fromHere.AbuseIPDB,
}
// Of counts that name as many bans, the earlier ban's is taken: it
@@ -735,7 +858,7 @@ func earlierBans(held []Ban) EarlierBans {
// total returns how many bans e counts, of all causes.
func (e EarlierBans) total() int {
return e.Limit + e.Attack + e.Admin
return e.Limit + e.Attack + e.Admin + e.AbuseIPDB
}
// markChanged has Changed receive a value, unless one is waiting already.
@@ -806,9 +929,9 @@ func (l *Ledger) covering(client netip.Addr) []*[]Ban {
// add adds ban to its netblock's bans, after the last, and makes its
// netblock the most recently seen. With MaxBans held, it drops one first,
// unless ban's cause is CauseAdmin, which does not count toward MaxBans.
// unless ban does not count toward MaxBans.
func (l *Ledger) add(ban Ban) {
counted := ban.Cause != CauseAdmin
counted := countsTowardMaxBans(ban)
if counted && l.held == l.params.MaxBans {
l.dropOne()
}
@@ -889,7 +1012,13 @@ func (l *Ledger) attackExpiry(held []Ban, now time.Time) time.Time {
return now.Add(l.params.AttackBanDuration)
}
// dropOne drops the earliest ban whose cause is not CauseAdmin of the
// countsTowardMaxBans reports whether ban counts toward MaxBans: its cause
// is CauseLimit or CauseAttack.
func countsTowardMaxBans(ban Ban) bool {
return ban.Cause == CauseLimit || ban.Cause == CauseAttack
}
// dropOne drops the earliest ban that counts toward MaxBans of the
// netblock that has gone longest without a request, of those that hold
// such a ban, and the netblock with it if that was its only ban. It is
// called with at least one such ban held. It looks at each netblock once
@@ -898,11 +1027,9 @@ func (l *Ledger) dropOne() {
for range l.netblocks.Len() {
netblock, bans, _ := l.netblocks.GetOldest()
i := slices.IndexFunc(*bans, func(ban Ban) bool {
return ban.Cause != CauseAdmin
})
i := slices.IndexFunc(*bans, countsTowardMaxBans)
if i < 0 {
// Its bans are all an admin's, and never dropped. Get makes
// None of its bans is dropped to make room. Get makes
// it the most recently seen, so that the next netblock is
// looked at; when it was seen matters only for dropping a
// ban, and a ban added to it makes it the most recently seen