waf: ban the clients on AbuseIPDB's blacklist (closes #411)
check / check (push) Failing after 8m52s
check / check (push) Failing after 8m52s
While SWW_WAF_ABUSEIPDB_KEY is set, waf fetches AbuseIPDB's blacklist at the start and every SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS (60), cut at SWW_WAF_ABUSEIPDB_THRESHOLD (90, from 25 to 100). Each listed client's netblock gets a ban of SWW_WAF_ABUSEIPDB_BAN_DAYS (30) with the new cause abuseipdb. The blacklist itself is not kept. These bans do not count toward SWW_WAF_MAX_BANS, so a 10,000-address blacklist cannot push out the bans for limits and attacks. An ended abuseipdb ban is forgotten at each fetch and each read of bans.json. A netblock with an active ban is left alone, and so is one whose abuseipdb ban an admin lifted, until that ban would have ended. Observe mode makes no ban. Model: opus-5-5
This commit is contained in:
1 parent
9f42460f70
commit
dfffeb73ec
22 files changed
+1304
-205
No files matched your search
@@ -0,0 +1,165 @@
|
||||
package bans_test
|
||||
|
||||
import (
|
||||
"net/netip"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
)
|
||||
|
||||
func TestBlacklistedNetblockIsBannedUntilTheGivenEnd(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ledger := bans.New(defaultParams())
|
||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||
now := midnight()
|
||||
expires := now.Add(30 * day)
|
||||
|
||||
made := ledger.BanForAbuseIPDB(
|
||||
now, expires, map[netip.Prefix]int64{netblock: 95},
|
||||
)
|
||||
if made != 1 || ledger.Made(bans.CauseAbuseIPDB) != 1 {
|
||||
t.Fatalf("made %d bans, %d counted, want 1 and 1",
|
||||
made, ledger.Made(bans.CauseAbuseIPDB))
|
||||
}
|
||||
|
||||
ban, banned, madePermanent := ledger.Check(netblock.Addr(), now)
|
||||
if !banned || madePermanent || !ban.Expires.Equal(expires) ||
|
||||
ban.Cause != bans.CauseAbuseIPDB ||
|
||||
ban.Reason != "on AbuseIPDB's blacklist with a score of 95" ||
|
||||
len(ban.Notes.Reputation) != 1 ||
|
||||
ban.Notes.Reputation[0].Source != "abuseipdb" ||
|
||||
*ban.Notes.Reputation[0].Score != 95 {
|
||||
t.Errorf(
|
||||
"ban %+v, banned %t, made permanent %t, want the blacklist's "+
|
||||
"ban until %s",
|
||||
ban,
|
||||
banned,
|
||||
madePermanent,
|
||||
expires,
|
||||
)
|
||||
}
|
||||
|
||||
if _, banned, _ := ledger.Check(netblock.Addr(), expires); banned {
|
||||
t.Error("the ban refuses a request at its end")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlacklistLeavesANetblockWithAnActiveBanAsItIs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ledger := bans.New(defaultParams())
|
||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||
now := midnight()
|
||||
scores := map[netip.Prefix]int64{netblock: 100}
|
||||
|
||||
limitBan, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
|
||||
|
||||
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 0 {
|
||||
t.Errorf("made %d bans over a ban for a limit, want none", made)
|
||||
}
|
||||
|
||||
// Once that ban has ended the netblock is banned, and the blacklist
|
||||
// fetched again makes no second ban while the first lasts.
|
||||
later := limitBan.Expires
|
||||
if made := ledger.BanForAbuseIPDB(later, later.Add(30*day), scores); made != 1 {
|
||||
t.Errorf("made %d bans after the ban for a limit, want 1", made)
|
||||
}
|
||||
|
||||
later = later.Add(time.Hour)
|
||||
if made := ledger.BanForAbuseIPDB(later, later.Add(30*day), scores); made != 0 {
|
||||
t.Errorf("made %d bans while the blacklist's lasts, want none", made)
|
||||
}
|
||||
|
||||
wantBans(t, ledger, map[netip.Prefix]int{netblock: 2})
|
||||
|
||||
if got := ledger.Bans(netblock)[1].Notes.EarlierBans; got != (bans.EarlierBans{
|
||||
Limit: 1,
|
||||
}) {
|
||||
t.Errorf("earlier bans %+v, want 1 for a limit", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLiftedBlacklistBanHoldsUntilItsEnd(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ledger := bans.New(defaultParams())
|
||||
netblock := netip.MustParsePrefix("203.0.113.9/32")
|
||||
now := midnight()
|
||||
expires := now.Add(30 * day)
|
||||
scores := map[netip.Prefix]int64{netblock: 100}
|
||||
|
||||
ledger.BanForAbuseIPDB(now, expires, scores)
|
||||
ledger.Lift(netblock.Addr(), now)
|
||||
|
||||
// The admin lifted it: the blacklist fetched again until its end bans
|
||||
// nothing.
|
||||
later := expires.Add(-time.Hour)
|
||||
if made := ledger.BanForAbuseIPDB(later, later.Add(30*day), scores); made != 0 {
|
||||
t.Errorf("made %d bans before the lifted ban's end, want none", made)
|
||||
}
|
||||
|
||||
// From its end on, the blacklist bans the netblock again, and the
|
||||
// lifted ban, ended, is forgotten.
|
||||
if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), scores); made != 1 {
|
||||
t.Errorf("made %d bans at the lifted ban's end, want 1", made)
|
||||
}
|
||||
|
||||
wantBans(t, ledger, map[netip.Prefix]int{netblock: 1})
|
||||
}
|
||||
|
||||
func TestEndedBlacklistBansAreForgottenAtTheNextFetch(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ledger := bans.New(defaultParams())
|
||||
listed := netip.MustParsePrefix("203.0.113.9/32")
|
||||
limited := netip.MustParsePrefix("203.0.113.10/32")
|
||||
now := midnight()
|
||||
expires := now.Add(30 * day)
|
||||
|
||||
ledger.BanForAbuseIPDB(now, expires, map[netip.Prefix]int64{listed: 100})
|
||||
ledger.BanForLimit(limited, now, bans.Notes{})
|
||||
|
||||
// listed is off the blacklist when it is fetched after the ban's end.
|
||||
if made := ledger.BanForAbuseIPDB(expires, expires.Add(30*day), nil); made != 0 {
|
||||
t.Errorf("made %d bans for an empty blacklist, want none", made)
|
||||
}
|
||||
|
||||
wantBans(t, ledger, map[netip.Prefix]int{listed: 0, limited: 1})
|
||||
}
|
||||
|
||||
func TestBlacklistBansDoNotCountTowardMaxBans(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
params := defaultParams()
|
||||
params.MaxBans = 1
|
||||
ledger := bans.New(params)
|
||||
limited := netip.MustParsePrefix("203.0.113.1/32")
|
||||
attacker := netip.MustParsePrefix("203.0.113.2/32")
|
||||
now := midnight()
|
||||
|
||||
scores := map[netip.Prefix]int64{}
|
||||
for i := range 3 {
|
||||
scores[netip.PrefixFrom(netip.AddrFrom4([4]byte{198, 51, 100, byte(i)}), 32)] = 100
|
||||
}
|
||||
|
||||
ledger.BanForLimit(limited, now, bans.Notes{})
|
||||
|
||||
if made := ledger.BanForAbuseIPDB(now, now.Add(30*day), scores); made != 3 {
|
||||
t.Errorf("made %d bans for 3 netblocks, want 3", made)
|
||||
}
|
||||
|
||||
wantBans(t, ledger, map[netip.Prefix]int{limited: 1})
|
||||
|
||||
// The ban for an attack takes the place of the one for a limit, and no
|
||||
// blacklist ban goes.
|
||||
ledger.BanForAttack(attacker, now, bans.Notes{})
|
||||
|
||||
want := map[netip.Prefix]int{limited: 0, attacker: 1}
|
||||
for netblock := range scores {
|
||||
want[netblock] = 1
|
||||
}
|
||||
|
||||
wantBans(t, ledger, want)
|
||||
}
|
||||
+161
-34
@@ -1,8 +1,9 @@
|
||||
// Package bans is the ban ledger: the bans sww makes on the
|
||||
// netblocks of clients that break a rate limit, a byte limit or the refusal
|
||||
// limit or show a clear sign of attack, and those an admin makes, with their
|
||||
// notes, as the "Bans" section of SPEC.md describes. The bans are kept in
|
||||
// memory, and written to bans.json and read from it by the state package.
|
||||
// limit or show a clear sign of attack, or are on AbuseIPDB's blacklist, and
|
||||
// those an admin makes, with their notes, as the "Bans" section of SPEC.md
|
||||
// describes. The bans are kept in memory, and written to bans.json and read
|
||||
// from it by the state package.
|
||||
package bans
|
||||
|
||||
import (
|
||||
@@ -26,6 +27,9 @@ const (
|
||||
// CauseAdmin is a ban an admin made, or one sww made that an
|
||||
// admin keeps. It is never dropped.
|
||||
CauseAdmin = "admin"
|
||||
// CauseAbuseIPDB is a ban sww made for a client on AbuseIPDB's
|
||||
// blacklist. It is forgotten once it has ended.
|
||||
CauseAbuseIPDB = "abuseipdb"
|
||||
)
|
||||
|
||||
// repeatFactor is how many times as long as the netblock's last ban a ban
|
||||
@@ -50,10 +54,11 @@ type Params struct {
|
||||
// AttackBanDuration is how long a first ban for a clear sign of attack
|
||||
// lasts.
|
||||
AttackBanDuration time.Duration
|
||||
// MaxBans is the most bans held whose cause is not CauseAdmin, at
|
||||
// least one. Past it, the earliest such ban of the netblock that has
|
||||
// gone longest without a request is dropped. Bans whose cause is
|
||||
// CauseAdmin are held besides, and never dropped.
|
||||
// MaxBans is the most bans held whose cause is CauseLimit or
|
||||
// CauseAttack, at least one. Past it, the earliest such ban of the
|
||||
// netblock that has gone longest without a request is dropped. Bans
|
||||
// whose cause is CauseAdmin or CauseAbuseIPDB are held besides, and
|
||||
// never dropped this way.
|
||||
MaxBans int
|
||||
}
|
||||
|
||||
@@ -63,10 +68,11 @@ type Ban struct {
|
||||
Start time.Time
|
||||
// Expires is when the ban ends, zero for a permanent ban.
|
||||
Expires time.Time
|
||||
// Cause is CauseLimit, CauseAttack or CauseAdmin.
|
||||
// Cause is CauseLimit, CauseAttack, CauseAdmin or CauseAbuseIPDB.
|
||||
Cause string
|
||||
// Reason is a short text: for a ban sww made, the limit broken or
|
||||
// the rule that matched; for an admin's, what the admin wrote.
|
||||
// Reason is a short text: for a ban sww made, the limit broken, the
|
||||
// rule that matched or AbuseIPDB's score; for an admin's, what the
|
||||
// admin wrote.
|
||||
Reason string
|
||||
// Lifted is when an admin lifted the ban, zero while no admin has. A
|
||||
// lifted ban refuses nothing, and does not make the netblock's next
|
||||
@@ -127,7 +133,8 @@ type Notes struct {
|
||||
// log's reputation names them. It is left out when none did.
|
||||
Reputation []ReputationHit `json:"reputation,omitempty"`
|
||||
// Request is the request that broke the limit, or whose bytes broke
|
||||
// it, or that was the clear sign of attack.
|
||||
// it, or that was the clear sign of attack. A ban for AbuseIPDB's
|
||||
// blacklist has none, and leaves it zero.
|
||||
Request Request `json:"request"`
|
||||
// Requests is how many requests the netblock has sent since it was
|
||||
// first seen, and Refused how many of them the ban has refused so
|
||||
@@ -150,9 +157,10 @@ type ReputationHit struct {
|
||||
|
||||
// EarlierBans counts a netblock's bans before a ban, by cause.
|
||||
type EarlierBans struct {
|
||||
Limit int `json:"limit"`
|
||||
Attack int `json:"attack"`
|
||||
Admin int `json:"admin"`
|
||||
Limit int `json:"limit"`
|
||||
Attack int `json:"attack"`
|
||||
Admin int `json:"admin"`
|
||||
AbuseIPDB int `json:"abuseipdb"`
|
||||
}
|
||||
|
||||
// Request is a request in a ban's notes. Each text is cut to 256 bytes.
|
||||
@@ -180,8 +188,8 @@ type Ledger struct {
|
||||
// netblocks holds each banned netblock's bans, oldest first. Check and
|
||||
// Find make each netblock they find the most recently seen.
|
||||
netblocks *simplelru.LRU[netip.Prefix, *[]Ban]
|
||||
// held is how many bans netblocks holds whose cause is not CauseAdmin,
|
||||
// at most params.MaxBans.
|
||||
// held is how many bans netblocks holds whose cause is CauseLimit or
|
||||
// CauseAttack, at most params.MaxBans.
|
||||
held int
|
||||
// made is how many bans have been made since the start, by cause: by
|
||||
// the ledger, and by an admin, through BanForAdmin or in an edit of
|
||||
@@ -198,8 +206,9 @@ type Ledger struct {
|
||||
// New returns a Ledger with no ban yet.
|
||||
func New(params Params) *Ledger {
|
||||
// The ledger drops bans itself, and never those whose cause is
|
||||
// CauseAdmin, however many there are, so the LRU has no limit of its
|
||||
// own: it keeps the netblocks in the order they were last seen.
|
||||
// CauseAdmin or CauseAbuseIPDB to make room, however many there are, so
|
||||
// the LRU has no limit of its own: it keeps the netblocks in the order
|
||||
// they were last seen.
|
||||
netblocks, err := simplelru.NewLRU[netip.Prefix, *[]Ban](
|
||||
math.MaxInt,
|
||||
nil,
|
||||
@@ -429,6 +438,88 @@ func (l *Ledger) BanForAdmin(
|
||||
return ban
|
||||
}
|
||||
|
||||
// BanForAbuseIPDB bans each netblock in scores at now until expires, and
|
||||
// returns how many bans it made. scores are the netblocks of the clients
|
||||
// on AbuseIPDB's blacklist, each with the highest score of its clients.
|
||||
// A netblock is left as it is while a ban on it is active, or while a ban
|
||||
// for the blacklist that an admin lifted has not reached its end: the
|
||||
// admin's lift holds that long. Each ban whose cause is CauseAbuseIPDB
|
||||
// and that has ended is forgotten first, so that the bans held for the
|
||||
// blacklist are those of the clients listed within one ban's length.
|
||||
func (l *Ledger) BanForAbuseIPDB(
|
||||
now, expires time.Time, scores map[netip.Prefix]int64,
|
||||
) int {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
changed := l.forgetEndedAbuseIPDBBans(now)
|
||||
made := 0
|
||||
|
||||
for netblock, score := range scores {
|
||||
netblock = netblock.Masked()
|
||||
|
||||
var notes Notes
|
||||
|
||||
if held, found := l.netblocks.Peek(netblock); found {
|
||||
if activeBan(*held, now) != nil ||
|
||||
liftedAbuseIPDBBan(*held, now) {
|
||||
continue
|
||||
}
|
||||
|
||||
notes.EarlierBans = earlierBans(*held)
|
||||
}
|
||||
|
||||
// The source is named as the request log names AbuseIPDB.
|
||||
notes.Reputation = []ReputationHit{
|
||||
{Source: CauseAbuseIPDB, Score: &score},
|
||||
}
|
||||
|
||||
l.add(Ban{
|
||||
Netblock: netblock,
|
||||
Start: now,
|
||||
Expires: expires,
|
||||
Cause: CauseAbuseIPDB,
|
||||
Reason: fmt.Sprintf(
|
||||
"on AbuseIPDB's blacklist with a score of %d", score,
|
||||
),
|
||||
Notes: notes,
|
||||
})
|
||||
|
||||
made++
|
||||
}
|
||||
|
||||
l.made[CauseAbuseIPDB] += made
|
||||
|
||||
if changed || made > 0 {
|
||||
l.markChanged()
|
||||
}
|
||||
|
||||
return made
|
||||
}
|
||||
|
||||
// ForgetEndedAbuseIPDBBans forgets the bans whose cause is CauseAbuseIPDB
|
||||
// and that have ended at now, as BanForAbuseIPDB does first. The state
|
||||
// package calls it each time it reads bans.json, so that these bans are
|
||||
// also forgotten while no fetch of the blacklist reaches the ledger.
|
||||
func (l *Ledger) ForgetEndedAbuseIPDBBans(now time.Time) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
if l.forgetEndedAbuseIPDBBans(now) {
|
||||
l.markChanged()
|
||||
}
|
||||
}
|
||||
|
||||
// liftedAbuseIPDBBan reports whether bans, a netblock's, hold a ban whose
|
||||
// cause is CauseAbuseIPDB that an admin lifted, and whose end is not
|
||||
// reached at now.
|
||||
func liftedAbuseIPDBBan(bans []Ban, now time.Time) bool {
|
||||
return slices.ContainsFunc(bans, func(ban Ban) bool {
|
||||
return ban.Cause == CauseAbuseIPDB && !ban.Lifted.IsZero() &&
|
||||
(ban.Permanent() || now.Before(ban.Expires))
|
||||
})
|
||||
}
|
||||
|
||||
// Lift lifts, at now, every ban active then on a netblock client is in,
|
||||
// as an admin does, and returns those bans. A lifted ban is kept, refuses
|
||||
// nothing, and does not make the netblock's next ban longer.
|
||||
@@ -515,9 +606,9 @@ func (l *Ledger) AddLookup(
|
||||
}
|
||||
|
||||
// Made returns how many bans for cause have been made since the start:
|
||||
// for CauseLimit and CauseAttack, by the ledger; for CauseAdmin, by an
|
||||
// admin, with BanForAdmin or in an edit of bans.json, as LoadEdit counts
|
||||
// them. The bans read from bans.json at the start are not among them.
|
||||
// for CauseLimit, CauseAttack and CauseAbuseIPDB, by the ledger; for
|
||||
// CauseAdmin, by an admin, with BanForAdmin or in an edit of bans.json, as
|
||||
// LoadEdit counts them. The bans read from bans.json at the start are not among them.
|
||||
func (l *Ledger) Made(cause string) int {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
@@ -574,8 +665,8 @@ func (l *Ledger) Snapshot() []Ban {
|
||||
// seen. A ban without a cause is an admin's, and gets CauseAdmin. Each
|
||||
// netblock is masked to its length, so that 203.0.113.9/24 is
|
||||
// 203.0.113.0/24, and each text in the notes is cut to 256 bytes. Past
|
||||
// MaxBans the earliest bans whose cause is not CauseAdmin are dropped, as
|
||||
// when they are made.
|
||||
// MaxBans the earliest bans that count toward it are dropped, as when they
|
||||
// are made.
|
||||
func (l *Ledger) Load(bans []Ban) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
@@ -639,6 +730,35 @@ func (l *Ledger) holds(netblock netip.Prefix, start time.Time) bool {
|
||||
})
|
||||
}
|
||||
|
||||
// forgetEndedAbuseIPDBBans forgets each ban whose cause is CauseAbuseIPDB
|
||||
// and whose end is reached at now, lifted or not, and each netblock left
|
||||
// without a ban. It reports whether it forgot one.
|
||||
func (l *Ledger) forgetEndedAbuseIPDBBans(now time.Time) bool {
|
||||
forgot := false
|
||||
|
||||
for _, netblock := range l.netblocks.Keys() {
|
||||
bans, _ := l.netblocks.Peek(netblock)
|
||||
|
||||
kept := slices.DeleteFunc(*bans, func(ban Ban) bool {
|
||||
return ban.Cause == CauseAbuseIPDB && !ban.Permanent() &&
|
||||
!now.Before(ban.Expires)
|
||||
})
|
||||
if len(kept) == len(*bans) {
|
||||
continue
|
||||
}
|
||||
|
||||
forgot = true
|
||||
|
||||
if len(kept) == 0 {
|
||||
l.netblocks.Remove(netblock)
|
||||
} else {
|
||||
*bans = kept
|
||||
}
|
||||
}
|
||||
|
||||
return forgot
|
||||
}
|
||||
|
||||
// ban bans netblock at now for cause, CauseLimit or CauseAttack, with
|
||||
// reason and notes, as BanForLimit and BanForAttack describe, and returns
|
||||
// the ban, and whether it made it. Unless keep is true, the ban is not
|
||||
@@ -714,13 +834,16 @@ func earlierBans(held []Ban) EarlierBans {
|
||||
fromHere.Attack++
|
||||
case CauseAdmin:
|
||||
fromHere.Admin++
|
||||
case CauseAbuseIPDB:
|
||||
fromHere.AbuseIPDB++
|
||||
}
|
||||
|
||||
notes := ban.Notes.EarlierBans
|
||||
count := EarlierBans{
|
||||
Limit: notes.Limit + fromHere.Limit,
|
||||
Attack: notes.Attack + fromHere.Attack,
|
||||
Admin: notes.Admin + fromHere.Admin,
|
||||
Limit: notes.Limit + fromHere.Limit,
|
||||
Attack: notes.Attack + fromHere.Attack,
|
||||
Admin: notes.Admin + fromHere.Admin,
|
||||
AbuseIPDB: notes.AbuseIPDB + fromHere.AbuseIPDB,
|
||||
}
|
||||
|
||||
// Of counts that name as many bans, the earlier ban's is taken: it
|
||||
@@ -735,7 +858,7 @@ func earlierBans(held []Ban) EarlierBans {
|
||||
|
||||
// total returns how many bans e counts, of all causes.
|
||||
func (e EarlierBans) total() int {
|
||||
return e.Limit + e.Attack + e.Admin
|
||||
return e.Limit + e.Attack + e.Admin + e.AbuseIPDB
|
||||
}
|
||||
|
||||
// markChanged has Changed receive a value, unless one is waiting already.
|
||||
@@ -806,9 +929,9 @@ func (l *Ledger) covering(client netip.Addr) []*[]Ban {
|
||||
|
||||
// add adds ban to its netblock's bans, after the last, and makes its
|
||||
// netblock the most recently seen. With MaxBans held, it drops one first,
|
||||
// unless ban's cause is CauseAdmin, which does not count toward MaxBans.
|
||||
// unless ban does not count toward MaxBans.
|
||||
func (l *Ledger) add(ban Ban) {
|
||||
counted := ban.Cause != CauseAdmin
|
||||
counted := countsTowardMaxBans(ban)
|
||||
if counted && l.held == l.params.MaxBans {
|
||||
l.dropOne()
|
||||
}
|
||||
@@ -889,7 +1012,13 @@ func (l *Ledger) attackExpiry(held []Ban, now time.Time) time.Time {
|
||||
return now.Add(l.params.AttackBanDuration)
|
||||
}
|
||||
|
||||
// dropOne drops the earliest ban whose cause is not CauseAdmin of the
|
||||
// countsTowardMaxBans reports whether ban counts toward MaxBans: its cause
|
||||
// is CauseLimit or CauseAttack.
|
||||
func countsTowardMaxBans(ban Ban) bool {
|
||||
return ban.Cause == CauseLimit || ban.Cause == CauseAttack
|
||||
}
|
||||
|
||||
// dropOne drops the earliest ban that counts toward MaxBans of the
|
||||
// netblock that has gone longest without a request, of those that hold
|
||||
// such a ban, and the netblock with it if that was its only ban. It is
|
||||
// called with at least one such ban held. It looks at each netblock once
|
||||
@@ -898,11 +1027,9 @@ func (l *Ledger) dropOne() {
|
||||
for range l.netblocks.Len() {
|
||||
netblock, bans, _ := l.netblocks.GetOldest()
|
||||
|
||||
i := slices.IndexFunc(*bans, func(ban Ban) bool {
|
||||
return ban.Cause != CauseAdmin
|
||||
})
|
||||
i := slices.IndexFunc(*bans, countsTowardMaxBans)
|
||||
if i < 0 {
|
||||
// Its bans are all an admin's, and never dropped. Get makes
|
||||
// None of its bans is dropped to make room. Get makes
|
||||
// it the most recently seen, so that the next netblock is
|
||||
// looked at; when it was seen matters only for dropping a
|
||||
// ban, and a ban added to it makes it the most recently seen
|
||||
|
||||
@@ -158,6 +158,14 @@ type WAF struct {
|
||||
DNSBLAbuseIPDBLimitPercent int64
|
||||
IPReputationCacheTTL time.Duration
|
||||
DNSBLAbuseIPDBTimeout time.Duration
|
||||
// While AbuseIPDBKey is set, AbuseIPDB's blacklist of the clients it
|
||||
// scores AbuseIPDBThreshold or more (SWW_WAF_ABUSEIPDB_THRESHOLD), from
|
||||
// 25 to 100, is fetched at the start and every AbuseIPDBBlacklistRefresh
|
||||
// (SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS). Each client on it
|
||||
// is banned for AbuseIPDBBanDuration (SWW_WAF_ABUSEIPDB_BAN_DAYS).
|
||||
AbuseIPDBThreshold int64
|
||||
AbuseIPDBBlacklistRefresh time.Duration
|
||||
AbuseIPDBBanDuration time.Duration
|
||||
// BanResponse is the status a refused client is answered with, 403
|
||||
// or 429, or 0 to close the connection without an answer
|
||||
// (SWW_WAF_BAN_RESPONSE). It answers a banned client, a request that
|
||||
@@ -330,6 +338,9 @@ const (
|
||||
// netblock a provider is usually given: a shorter one would make one
|
||||
// client of the customers of several providers.
|
||||
minIPv6GroupPrefix = 32
|
||||
// minAbuseIPDBThreshold is the lowest score AbuseIPDB's blacklist can
|
||||
// be cut at.
|
||||
minAbuseIPDBThreshold = 25
|
||||
// minTokenLength is the fewest characters a token may have.
|
||||
minTokenLength = 32
|
||||
// maxParanoiaLevel is the Core Rule Set's highest paranoia level.
|
||||
@@ -446,6 +457,13 @@ var (
|
||||
errNotPercent = errors.New(
|
||||
"is not a percentage, a whole number from 0 to 100",
|
||||
)
|
||||
errNotAbuseIPDBThreshold = errors.New(
|
||||
"is not a whole number from 25 to 100, the scores AbuseIPDB's " +
|
||||
"blacklist can be cut at",
|
||||
)
|
||||
errNotWholeUnits = errors.New(
|
||||
"is not a whole number above zero, such as 30, that fits a duration",
|
||||
)
|
||||
errListedTwice = errors.New("is listed twice")
|
||||
errNotListURL = errors.New(
|
||||
"is not an http or https URL without a user or a fragment, " +
|
||||
@@ -587,6 +605,20 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
||||
"SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS",
|
||||
"2000",
|
||||
),
|
||||
AbuseIPDBThreshold: env.abuseIPDBThreshold(
|
||||
"SWW_WAF_ABUSEIPDB_THRESHOLD",
|
||||
"90",
|
||||
),
|
||||
AbuseIPDBBlacklistRefresh: env.wholeUnits(
|
||||
"SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS",
|
||||
"60",
|
||||
time.Minute,
|
||||
),
|
||||
AbuseIPDBBanDuration: env.wholeUnits(
|
||||
"SWW_WAF_ABUSEIPDB_BAN_DAYS",
|
||||
"30",
|
||||
day,
|
||||
),
|
||||
BanResponse: env.banResponse("SWW_WAF_BAN_RESPONSE", "403"),
|
||||
LimitBanDuration: env.durationNotOff(
|
||||
"SWW_WAF_LIMIT_BAN_DURATION",
|
||||
@@ -957,6 +989,35 @@ func (e *environment) percent(name, defaultValue string) int64 {
|
||||
return percent
|
||||
}
|
||||
|
||||
// abuseIPDBThreshold reads the setting that is the least score of the
|
||||
// clients on AbuseIPDB's blacklist: a whole number from 25 to 100, the
|
||||
// range AbuseIPDB takes.
|
||||
func (e *environment) abuseIPDBThreshold(name, defaultValue string) int64 {
|
||||
value := e.value(name, defaultValue)
|
||||
|
||||
threshold, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil || threshold < minAbuseIPDBThreshold || threshold > 100 {
|
||||
e.check(name, fmt.Errorf("%q %w", value, errNotAbuseIPDBThreshold))
|
||||
}
|
||||
|
||||
return threshold
|
||||
}
|
||||
|
||||
// wholeUnits reads a setting that is a whole number above zero of unit,
|
||||
// such as minutes or days, and returns it as a duration.
|
||||
func (e *environment) wholeUnits(
|
||||
name, defaultValue string, unit time.Duration,
|
||||
) time.Duration {
|
||||
value := e.value(name, defaultValue)
|
||||
|
||||
n, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil || n <= 0 || n > math.MaxInt64/int64(unit) {
|
||||
e.check(name, fmt.Errorf("%q %w", value, errNotWholeUnits))
|
||||
}
|
||||
|
||||
return time.Duration(n) * unit
|
||||
}
|
||||
|
||||
// listURL reads a setting that is the URL a list is fetched from, "" while
|
||||
// it is unset or empty.
|
||||
func (e *environment) listURL(name string) string {
|
||||
|
||||
@@ -61,6 +61,9 @@ const (
|
||||
abuseIPDBKey = "SWW_WAF_ABUSEIPDB_KEY"
|
||||
abuseIPDBMinScore = "SWW_WAF_ABUSEIPDB_MIN_SCORE"
|
||||
abuseIPDBDailyBudget = "SWW_WAF_ABUSEIPDB_DAILY_BUDGET"
|
||||
abuseIPDBThreshold = "SWW_WAF_ABUSEIPDB_THRESHOLD"
|
||||
abuseIPDBBlacklistRefresh = "SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS"
|
||||
abuseIPDBBanDays = "SWW_WAF_ABUSEIPDB_BAN_DAYS"
|
||||
dnsblAbuseIPDBAction = "SWW_WAF_DNSBL_ABUSEIPDB_ACTION"
|
||||
ipReputationCacheTTL = "SWW_WAF_IP_REPUTATION_CACHE_TTL"
|
||||
dnsblAbuseIPDBTimeout = "SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS"
|
||||
@@ -2399,6 +2402,75 @@ func TestAbuseIPDBSettingsAsSet(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAbuseIPDBBlacklistSettingsAsSet(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg := wafFromEnvironment(t, environment{})
|
||||
if cfg.AbuseIPDBThreshold != 90 ||
|
||||
cfg.AbuseIPDBBlacklistRefresh != time.Hour ||
|
||||
cfg.AbuseIPDBBanDuration != 30*24*time.Hour {
|
||||
t.Errorf(
|
||||
"by default, the threshold %d, the refresh %s and the ban %s, "+
|
||||
"want 90, 1h and 720h",
|
||||
cfg.AbuseIPDBThreshold,
|
||||
cfg.AbuseIPDBBlacklistRefresh,
|
||||
cfg.AbuseIPDBBanDuration,
|
||||
)
|
||||
}
|
||||
|
||||
cfg = wafFromEnvironment(t, environment{
|
||||
abuseIPDBThreshold: "25", abuseIPDBBlacklistRefresh: "1440",
|
||||
abuseIPDBBanDays: "1",
|
||||
})
|
||||
if cfg.AbuseIPDBThreshold != 25 ||
|
||||
cfg.AbuseIPDBBlacklistRefresh != 24*time.Hour ||
|
||||
cfg.AbuseIPDBBanDuration != 24*time.Hour {
|
||||
t.Errorf(
|
||||
"set, the threshold %d, the refresh %s and the ban %s, "+
|
||||
"want 25, 24h and 24h",
|
||||
cfg.AbuseIPDBThreshold,
|
||||
cfg.AbuseIPDBBlacklistRefresh,
|
||||
cfg.AbuseIPDBBanDuration,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidAbuseIPDBBlacklistSettingStopsTheStartSayingWhatIsWrong(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
const (
|
||||
notThreshold = " is not a whole number from 25 to 100, the scores " +
|
||||
"AbuseIPDB's blacklist can be cut at"
|
||||
notWhole = " is not a whole number above zero, such as 30, that fits " +
|
||||
"a duration"
|
||||
)
|
||||
|
||||
for _, tc := range []struct{ name, value, want string }{
|
||||
{abuseIPDBThreshold, "24", `"24"` + notThreshold},
|
||||
{abuseIPDBThreshold, "101", `"101"` + notThreshold},
|
||||
{abuseIPDBThreshold, off, `"off"` + notThreshold},
|
||||
{abuseIPDBBlacklistRefresh, "0", `"0"` + notWhole},
|
||||
{abuseIPDBBlacklistRefresh, "1h", `"1h"` + notWhole},
|
||||
{abuseIPDBBanDays, "-1", `"-1"` + notWhole},
|
||||
{abuseIPDBBanDays, "200000", `"200000"` + notWhole},
|
||||
} {
|
||||
t.Run(tc.name+"="+tc.value, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := config.WAFFromEnvironment(
|
||||
environment{tc.name: tc.value}.viper(),
|
||||
)
|
||||
|
||||
want := tc.name + ": " + tc.want
|
||||
if err == nil || err.Error() != want {
|
||||
t.Errorf("error %v, want %s", err, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidAbuseIPDBSettingStopsTheStartSayingWhatIsWrong(
|
||||
t *testing.T,
|
||||
) {
|
||||
@@ -2951,6 +3023,9 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
|
||||
abuseIPDBKey: "",
|
||||
abuseIPDBMinScore: "75",
|
||||
abuseIPDBDailyBudget: "900",
|
||||
abuseIPDBThreshold: "90",
|
||||
abuseIPDBBlacklistRefresh: "60",
|
||||
abuseIPDBBanDays: "30",
|
||||
dnsblAbuseIPDBAction: "limit:25",
|
||||
ipReputationCacheTTL: defaultIPReputationCacheTTL,
|
||||
dnsblAbuseIPDBTimeout: "2000",
|
||||
|
||||
@@ -185,7 +185,7 @@ func (m *Metrics) AddBansAndClients(
|
||||
ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time,
|
||||
) {
|
||||
for _, cause := range []string{
|
||||
bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin,
|
||||
bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin, bans.CauseAbuseIPDB,
|
||||
} {
|
||||
m.registry.MustRegister(
|
||||
prometheus.NewCounterFunc(prometheus.CounterOpts{
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -23,6 +24,9 @@ const (
|
||||
// AbuseIPDBURL is where clients are checked: the check endpoint of
|
||||
// AbuseIPDB's API.
|
||||
AbuseIPDBURL = "https://api.abuseipdb.com/api/v2/check"
|
||||
// AbuseIPDBBlacklistURL is where AbuseIPDB's blacklist is fetched from:
|
||||
// the blacklist endpoint of AbuseIPDB's API.
|
||||
AbuseIPDBBlacklistURL = "https://api.abuseipdb.com/api/v2/blacklist"
|
||||
// AbuseIPDBSource is how the request log, the alerts and the metrics
|
||||
// name AbuseIPDB.
|
||||
AbuseIPDBSource = "abuseipdb"
|
||||
@@ -32,6 +36,8 @@ const (
|
||||
|
||||
var (
|
||||
errNoScore = errors.New("the answer gives no abuseConfidenceScore")
|
||||
errNoBlacklist = errors.New("the answer gives no data")
|
||||
errNotAddress = errors.New("is not an address")
|
||||
errBudgetUsedUp = errors.New(
|
||||
"checks spent; none is made until the day ends at 00:00 UTC")
|
||||
)
|
||||
@@ -53,6 +59,13 @@ type ChecksSpent struct {
|
||||
Spent int
|
||||
}
|
||||
|
||||
// BlacklistEntry is a client on AbuseIPDB's blacklist: its address and its
|
||||
// abuse confidence score.
|
||||
type BlacklistEntry struct {
|
||||
Addr netip.Addr
|
||||
Score int64
|
||||
}
|
||||
|
||||
// AbuseIPDBParams are what NewAbuseIPDB needs.
|
||||
type AbuseIPDBParams struct {
|
||||
// URL is where clients are checked, normally AbuseIPDBURL, with Key,
|
||||
@@ -69,10 +82,19 @@ type AbuseIPDBParams struct {
|
||||
// take (SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS).
|
||||
CacheTTL time.Duration
|
||||
Timeout time.Duration
|
||||
// BlacklistURL is where the blacklist is fetched from, normally
|
||||
// AbuseIPDBBlacklistURL, with the clients scored Threshold or more
|
||||
// (SWW_WAF_ABUSEIPDB_THRESHOLD), every BlacklistRefresh
|
||||
// (SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS). Blacklisted
|
||||
// receives the clients on each blacklist fetched.
|
||||
BlacklistURL string
|
||||
Threshold int64
|
||||
BlacklistRefresh time.Duration
|
||||
Blacklisted func([]BlacklistEntry)
|
||||
// Now tells the time, normally time.Now in UTC.
|
||||
Now func() time.Time
|
||||
// ProcessLog receives each check that fails, and why, and the day's
|
||||
// budget used up.
|
||||
// ProcessLog receives each check or fetch of the blacklist that fails,
|
||||
// and why, and the day's budget used up.
|
||||
ProcessLog *slog.Logger
|
||||
// Alerts receive a source_failure alert for each.
|
||||
Alerts *alerts.Queue
|
||||
@@ -256,6 +278,40 @@ func (a *AbuseIPDB) LoadChecksSpent(checks ChecksSpent) {
|
||||
a.day, a.spent = checks.Day, checks.Spent
|
||||
}
|
||||
|
||||
// RunBlacklist fetches AbuseIPDB's blacklist at once, and again every
|
||||
// BlacklistRefresh, until ctx is done, and gives the clients on each copy
|
||||
// fetched to Blacklisted. While Key is unset it does nothing. A fetch that
|
||||
// fails is logged and raised as a source_failure alert, unless it was cut
|
||||
// off as ctx ended.
|
||||
func (a *AbuseIPDB) RunBlacklist(ctx context.Context) {
|
||||
if a.params.Key == "" {
|
||||
return
|
||||
}
|
||||
|
||||
for {
|
||||
listed, err := a.fetchBlacklist(ctx)
|
||||
|
||||
switch {
|
||||
case ctx.Err() != nil:
|
||||
return
|
||||
case err != nil:
|
||||
a.alert("fetching AbuseIPDB's blacklist failed", err)
|
||||
default:
|
||||
a.params.Blacklisted(listed)
|
||||
}
|
||||
|
||||
timer := time.NewTimer(a.params.BlacklistRefresh)
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
|
||||
return
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
|
||||
// the score as client's, and notes the check as no longer under way. A
|
||||
// check that fails gives no score: it is counted, logged and raised as a
|
||||
@@ -344,6 +400,94 @@ func (a *AbuseIPDB) ask(
|
||||
return *answer.Data.AbuseConfidenceScore, nil
|
||||
}
|
||||
|
||||
// fetchBlacklist fetches the clients on AbuseIPDB's blacklist scored
|
||||
// Threshold or more, sending the key in the header Key. An answer other
|
||||
// than 200, one longer than maxListBytes or without the list, an entry
|
||||
// whose address does not read, and none within fetchTimeout, fail.
|
||||
func (a *AbuseIPDB) fetchBlacklist(
|
||||
ctx context.Context,
|
||||
) ([]BlacklistEntry, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, fetchTimeout)
|
||||
defer cancel()
|
||||
|
||||
query := url.Values{
|
||||
"confidenceMinimum": {strconv.FormatInt(a.params.Threshold, 10)},
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet,
|
||||
a.params.BlacklistURL+"?"+query.Encode(), http.NoBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("make the request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Key", a.params.Key)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
res, err := a.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetch the blacklist: %w", errors.Unwrap(err))
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = res.Body.Close()
|
||||
}()
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("%w %s", errStatus, res.Status)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(res.Body, maxListBytes+1))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read the blacklist: %w", err)
|
||||
}
|
||||
|
||||
if len(body) > maxListBytes {
|
||||
return nil, errTooLong
|
||||
}
|
||||
|
||||
return parseBlacklist(body)
|
||||
}
|
||||
|
||||
// parseBlacklist reads body, AbuseIPDB's answer with its blacklist.
|
||||
func parseBlacklist(body []byte) ([]BlacklistEntry, error) {
|
||||
var answer struct {
|
||||
Data *[]struct {
|
||||
IPAddress string `json:"ipAddress"`
|
||||
AbuseConfidenceScore int64 `json:"abuseConfidenceScore"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
err := json.Unmarshal(body, &answer)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read the blacklist: %w", err)
|
||||
}
|
||||
|
||||
if answer.Data == nil {
|
||||
return nil, errNoBlacklist
|
||||
}
|
||||
|
||||
listed := make([]BlacklistEntry, 0, len(*answer.Data))
|
||||
|
||||
for i, entry := range *answer.Data {
|
||||
addr, err := netip.ParseAddr(entry.IPAddress)
|
||||
if err != nil || addr.Zone() != "" {
|
||||
// The error leaves out what the entry holds, as it does a
|
||||
// client's address.
|
||||
return nil, fmt.Errorf(
|
||||
"entry %d's ipAddress %w",
|
||||
i+1,
|
||||
errNotAddress,
|
||||
)
|
||||
}
|
||||
|
||||
listed = append(listed, BlacklistEntry{
|
||||
Addr: addr.Unmap(), Score: entry.AbuseConfidenceScore,
|
||||
})
|
||||
}
|
||||
|
||||
return listed, nil
|
||||
}
|
||||
|
||||
// alert raises a source_failure alert from AbuseIPDB with reason and err,
|
||||
// and logs them.
|
||||
func (a *AbuseIPDB) alert(reason string, err error) {
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
package reputation_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
)
|
||||
|
||||
// The tests of the blacklist run in synctest bubbles, as those of the
|
||||
// checks do, with AbuseIPDB a stand-in reached without the network.
|
||||
|
||||
// blacklistURL is where the tests fetch the blacklist from.
|
||||
const blacklistURL = "https://abuseipdb.example/api/v2/blacklist"
|
||||
|
||||
func TestBlacklistIsFetchedAtOnceAndEveryRefresh(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
abuseIPDB := &blacklistStandIn{body: `{"meta":{},"data":[` +
|
||||
`{"ipAddress":"203.0.113.9","abuseConfidenceScore":100},` +
|
||||
`{"ipAddress":"::ffff:198.51.100.7","abuseConfidenceScore":95},` +
|
||||
`{"ipAddress":"2001:db8::9","abuseConfidenceScore":90}]}`}
|
||||
got := runBlacklist(t, abuseIPDB, key, newQueue())
|
||||
|
||||
want := []reputation.BlacklistEntry{
|
||||
{Addr: netip.MustParseAddr("203.0.113.9"), Score: 100},
|
||||
{Addr: netip.MustParseAddr("198.51.100.7"), Score: 95},
|
||||
{Addr: netip.MustParseAddr("2001:db8::9"), Score: 90},
|
||||
}
|
||||
|
||||
synctest.Wait()
|
||||
got.want(t, want)
|
||||
|
||||
time.Sleep(time.Hour - time.Nanosecond)
|
||||
synctest.Wait()
|
||||
got.want(t, want)
|
||||
|
||||
time.Sleep(time.Nanosecond)
|
||||
synctest.Wait()
|
||||
got.want(t, want, want)
|
||||
|
||||
// Each fetch asks for the clients scored 90 or more, with the key
|
||||
// in the header Key alone.
|
||||
abuseIPDB.mu.Lock()
|
||||
defer abuseIPDB.mu.Unlock()
|
||||
|
||||
for _, req := range abuseIPDB.asked {
|
||||
if req.URL.String() != blacklistURL+"?confidenceMinimum=90" ||
|
||||
req.Header.Get("Key") != key {
|
||||
t.Errorf(
|
||||
"asked %s with the key %q, want %s?confidenceMinimum=90 "+
|
||||
"with %q",
|
||||
req.URL,
|
||||
req.Header.Get("Key"),
|
||||
blacklistURL,
|
||||
key,
|
||||
)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestFailedBlacklistFetchIsAlertedAndTriedAgainAtTheNextRefresh(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
for _, tc := range []struct{ name, body, error string }{
|
||||
{
|
||||
"a refusal", "",
|
||||
"the server answered 429 Too Many Requests",
|
||||
},
|
||||
{
|
||||
"an answer without the list", `{"errors":[]}`,
|
||||
"the answer gives no data",
|
||||
},
|
||||
{
|
||||
"an entry that is not an address",
|
||||
`{"data":[{"ipAddress":"203.0.113.9"},{"ipAddress":"example"}]}`,
|
||||
"entry 2's ipAddress is not an address",
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
abuseIPDB := &blacklistStandIn{body: tc.body}
|
||||
if tc.body == "" {
|
||||
abuseIPDB.status = http.StatusTooManyRequests
|
||||
}
|
||||
|
||||
queue := newQueue()
|
||||
got := runBlacklist(t, abuseIPDB, key, queue)
|
||||
|
||||
synctest.Wait()
|
||||
|
||||
got.want(t)
|
||||
|
||||
alerted := waiting(queue)
|
||||
if len(alerted) != 1 ||
|
||||
alerted[0].Event != alerts.EventSourceFailure ||
|
||||
alerted[0].Reason != "fetching AbuseIPDB's blacklist failed" ||
|
||||
alerted[0].Detail["source"] != reputation.AbuseIPDBSource ||
|
||||
alerted[0].Detail["error"] != tc.error {
|
||||
t.Errorf(
|
||||
"alerts %+v, want the failure %q",
|
||||
alerted,
|
||||
tc.error,
|
||||
)
|
||||
}
|
||||
|
||||
abuseIPDB.answer(`{"data":[]}`)
|
||||
time.Sleep(time.Hour)
|
||||
synctest.Wait()
|
||||
got.want(t, []reputation.BlacklistEntry{})
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlacklistIsNotFetchedWithoutAKey(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
abuseIPDB := &blacklistStandIn{body: `{"data":[]}`}
|
||||
got := runBlacklist(t, abuseIPDB, "", newQueue())
|
||||
|
||||
time.Sleep(time.Hour)
|
||||
synctest.Wait()
|
||||
got.want(t)
|
||||
|
||||
if len(abuseIPDB.asked) != 0 {
|
||||
t.Errorf("fetched the blacklist %d times, want none",
|
||||
len(abuseIPDB.asked))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// blacklistStandIn is a stand-in for AbuseIPDB that answers each fetch of
|
||||
// the blacklist with status, 200 when it is zero, and body, and keeps the
|
||||
// requests.
|
||||
type blacklistStandIn struct {
|
||||
mu sync.Mutex
|
||||
status int
|
||||
body string
|
||||
asked []*http.Request
|
||||
}
|
||||
|
||||
// RoundTrip has the stand-in answer req, in place of the network.
|
||||
func (s *blacklistStandIn) RoundTrip(
|
||||
req *http.Request,
|
||||
) (*http.Response, error) {
|
||||
s.mu.Lock()
|
||||
s.asked = append(s.asked, req)
|
||||
status, body := s.status, s.body
|
||||
s.mu.Unlock()
|
||||
|
||||
if status == 0 {
|
||||
status = http.StatusOK
|
||||
}
|
||||
|
||||
return &http.Response{
|
||||
StatusCode: status,
|
||||
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
|
||||
Header: http.Header{},
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
Request: req,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// answer has the stand-in answer each fetch with 200 and body.
|
||||
func (s *blacklistStandIn) answer(body string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
s.status, s.body = 0, body
|
||||
}
|
||||
|
||||
// blacklisted holds the clients on each blacklist given to Blacklisted.
|
||||
type blacklisted struct {
|
||||
mu sync.Mutex
|
||||
fetched [][]reputation.BlacklistEntry
|
||||
}
|
||||
|
||||
// want checks the clients on each blacklist given so far.
|
||||
func (b *blacklisted) want(
|
||||
t *testing.T,
|
||||
want ...[]reputation.BlacklistEntry,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
|
||||
if !slices.EqualFunc(b.fetched, want, slices.Equal) {
|
||||
t.Errorf("blacklists %v, want %v", b.fetched, want)
|
||||
}
|
||||
}
|
||||
|
||||
// runBlacklist runs RunBlacklist, until the test ends, with the key
|
||||
// accountKey, the threshold 90 and a refresh of an hour, fetching from
|
||||
// abuseIPDB and raising alerts into queue, and returns what it gives
|
||||
// Blacklisted.
|
||||
func runBlacklist(
|
||||
t *testing.T, abuseIPDB *blacklistStandIn, accountKey string,
|
||||
queue *alerts.Queue,
|
||||
) *blacklisted {
|
||||
t.Helper()
|
||||
|
||||
got := &blacklisted{}
|
||||
p := abuseIPDBParams()
|
||||
p.Key = accountKey
|
||||
p.BlacklistURL = blacklistURL
|
||||
p.Threshold = 90
|
||||
p.BlacklistRefresh = time.Hour
|
||||
p.Alerts = queue
|
||||
p.Blacklisted = func(listed []reputation.BlacklistEntry) {
|
||||
got.mu.Lock()
|
||||
defer got.mu.Unlock()
|
||||
|
||||
got.fetched = append(got.fetched, listed)
|
||||
}
|
||||
|
||||
checker := reputation.NewAbuseIPDB(p)
|
||||
checker.SetTransport(abuseIPDB)
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
done := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
checker.RunBlacklist(ctx)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
t.Cleanup(func() {
|
||||
cancel()
|
||||
<-done
|
||||
})
|
||||
|
||||
return got
|
||||
}
|
||||
@@ -7,7 +7,8 @@
|
||||
// SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks
|
||||
// clients with AbuseIPDB, and keeps their scores and the checks spent
|
||||
// today. The state package writes all of these to files and reads them
|
||||
// from there, so that a restart keeps them too.
|
||||
// from there, so that a restart keeps them too. Last, it fetches
|
||||
// AbuseIPDB's blacklist, which it does not keep.
|
||||
package reputation
|
||||
|
||||
import (
|
||||
|
||||
@@ -104,7 +104,7 @@ var (
|
||||
errVersion = errors.New("unknown version")
|
||||
// errMissing is for an entry without a field it needs.
|
||||
errMissing = errors.New("has no")
|
||||
errCause = errors.New("is not limit, attack or admin")
|
||||
errCause = errors.New("is not limit, attack, admin or abuseipdb")
|
||||
errExpires = errors.New("is not after its start")
|
||||
errDestination = errors.New("is not webhook, slack or ntfy")
|
||||
errScope = errors.New(
|
||||
@@ -638,6 +638,8 @@ func (f *Files) takeInState(
|
||||
f.params.Ledger.Load(held)
|
||||
}
|
||||
|
||||
f.params.Ledger.ForgetEndedAbuseIPDBBans(f.params.Now())
|
||||
|
||||
entries = len(held)
|
||||
case clientsJSON:
|
||||
var file clientsFile
|
||||
@@ -967,7 +969,7 @@ func (e BanEntry) ban() bans.Ban {
|
||||
// each expires is read again as written. An expires not after the start
|
||||
// is refused too, since the netblock's next ban for a broken limit, three
|
||||
// times as long as this one, would then refuse nothing; and so is a cause
|
||||
// other than limit, attack or admin, most likely misspelt.
|
||||
// other than limit, attack, admin or abuseipdb, most likely misspelt.
|
||||
func (f *bansFile) check(data []byte) error {
|
||||
var written struct {
|
||||
Bans []struct {
|
||||
@@ -991,7 +993,8 @@ func (f *bansFile) check(data []byte) error {
|
||||
case entry.Expires != nil && !entry.Expires.After(entry.Start):
|
||||
return fmt.Errorf("entry %d's expires %w", i+1, errExpires)
|
||||
case entry.Cause != "" && entry.Cause != bans.CauseLimit &&
|
||||
entry.Cause != bans.CauseAttack && entry.Cause != bans.CauseAdmin:
|
||||
entry.Cause != bans.CauseAttack && entry.Cause != bans.CauseAdmin &&
|
||||
entry.Cause != bans.CauseAbuseIPDB:
|
||||
return fmt.Errorf(
|
||||
"entry %d's cause %q %w",
|
||||
i+1,
|
||||
|
||||
@@ -114,7 +114,8 @@ const permanentBansJSON = `{
|
||||
"earlier_bans": {
|
||||
"limit": 3,
|
||||
"attack": 1,
|
||||
"admin": 1
|
||||
"admin": 1,
|
||||
"abuseipdb": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1190,7 +1191,7 @@ func TestBanWithAnotherCauseStopsTheStart(t *testing.T) {
|
||||
`"expires": "2026-10-06T04:00:00Z", "cause": "limit"}, `+
|
||||
`{"netblock": "203.0.113.12/32", "start": "2026-10-06T00:00:00Z", `+
|
||||
`"expires": null, "cause": "atack"}]}`,
|
||||
`: entry 4's cause "atack" is not limit, attack or admin`)
|
||||
`: entry 4's cause "atack" is not limit, attack, admin or abuseipdb`)
|
||||
}
|
||||
|
||||
func TestBanThatDoesNotEndAfterItsStartStopsTheStart(t *testing.T) {
|
||||
@@ -2090,6 +2091,39 @@ func TestLiftedBanReadAtTheStart(t *testing.T) {
|
||||
wantLiftedBanKept(t, load(t, params), dir, params.Ledger)
|
||||
}
|
||||
|
||||
func TestEndedAbuseIPDBBanForgottenWhenBansJSONIsReadAtTheStart(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
ended := netip.MustParsePrefix("203.0.113.9/32")
|
||||
lasting := netip.MustParsePrefix("203.0.113.10/32")
|
||||
limited := netip.MustParsePrefix("203.0.113.11/32")
|
||||
|
||||
// At midnight, the start's time, the first ban has ended and the second
|
||||
// lasts. The third has ended too, but is for a limit.
|
||||
dir := t.TempDir()
|
||||
edit(t, dir, bansJSON, `{"version": 1, "bans": [`+
|
||||
`{"netblock": "203.0.113.9/32", "start": "2026-09-06T00:00:00Z", `+
|
||||
`"expires": "2026-10-06T00:00:00Z", "cause": "abuseipdb"}, `+
|
||||
`{"netblock": "203.0.113.10/32", "start": "2026-09-07T00:00:00Z", `+
|
||||
`"expires": "2026-10-07T00:00:00Z", "cause": "abuseipdb"}, `+
|
||||
`{"netblock": "203.0.113.11/32", "start": "2026-10-05T00:00:00Z", `+
|
||||
`"expires": "2026-10-05T01:00:00Z", "cause": "limit"}]}`)
|
||||
|
||||
params := newParams(dir)
|
||||
load(t, params)
|
||||
|
||||
for netblock, want := range map[netip.Prefix]int{
|
||||
ended: 0, lasting: 1, limited: 1,
|
||||
} {
|
||||
if got := len(params.Ledger.Bans(netblock)); got != want {
|
||||
t.Errorf("the ledger holds %d bans on %s, want %d",
|
||||
got, netblock, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBanLiftedByAnEditWhileRunning(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
+7
-1
@@ -284,6 +284,7 @@ func newServer(
|
||||
Alerts: alertQueue,
|
||||
|
||||
BesideLookupTimeLimit: besideLookupTimeLimit,
|
||||
AbuseIPDBBlacklistURL: reputation.AbuseIPDBBlacklistURL,
|
||||
})
|
||||
server.Metrics.AddAlerts(alertQueue)
|
||||
|
||||
@@ -383,7 +384,8 @@ func startSending(
|
||||
// serve serves requests on listener, writes the state and cache files as they
|
||||
// are due, takes in an admin's edits of them, reads the rule files again as
|
||||
// they change, and the lookup database when it is replaced, fetches the
|
||||
// lists the settings name by URL as they are due, and sends the alerts,
|
||||
// lists the settings name by URL as they are due, bans the clients on
|
||||
// AbuseIPDB's blacklist as it is fetched, and sends the alerts,
|
||||
// until ctx is done. Then it gives the requests in progress
|
||||
// shutdownTimeout to finish, and writes every state and cache file,
|
||||
// alerts.json with the alerts still waiting.
|
||||
@@ -410,6 +412,9 @@ func serve(
|
||||
}
|
||||
})
|
||||
listsFetched := inBackground(func() { server.Lists.Run(writing) })
|
||||
blacklistFetched := inBackground(func() {
|
||||
server.AbuseIPDB.RunBlacklist(writing)
|
||||
})
|
||||
alertsSent := inBackground(func() { alertQueue.Run(writing) })
|
||||
|
||||
select {
|
||||
@@ -452,6 +457,7 @@ func serve(
|
||||
<-rulesWatched
|
||||
<-lookupFileWatched
|
||||
<-listsFetched
|
||||
<-blacklistFetched
|
||||
<-alertsSent
|
||||
|
||||
err = files.WriteAll()
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
package waf
|
||||
|
||||
// The bans for AbuseIPDB's blacklist. They read SWW_WAF_ABUSEIPDB_BAN_DAYS and
|
||||
// SWW_WAF_MODE; the blacklist is fetched by the reputation package.
|
||||
|
||||
import (
|
||||
"net/netip"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
)
|
||||
|
||||
// banAbuseIPDBBlacklist bans the netblock of each client on AbuseIPDB's
|
||||
// blacklist, listed, for SWW_WAF_ABUSEIPDB_BAN_DAYS, as the ledger's
|
||||
// BanForAbuseIPDB describes, and logs how many clients were listed and how
|
||||
// many bans were made. Clients in one netblock make one ban, with the
|
||||
// highest of their scores. In observe mode it makes no ban.
|
||||
func (h *handler) banAbuseIPDBBlacklist(listed []reputation.BlacklistEntry) {
|
||||
scores := make(map[netip.Prefix]int64, len(listed))
|
||||
|
||||
for _, client := range listed {
|
||||
netblock := h.netblock(client.Addr)
|
||||
scores[netblock] = max(scores[netblock], client.Score)
|
||||
}
|
||||
|
||||
made := 0
|
||||
|
||||
if !h.config.Observe {
|
||||
now := h.now()
|
||||
made = h.ledger.BanForAbuseIPDB(
|
||||
now, now.Add(h.config.AbuseIPDBBanDuration), scores,
|
||||
)
|
||||
}
|
||||
|
||||
h.processLog.Info(
|
||||
"fetched AbuseIPDB's blacklist",
|
||||
"clients", len(listed),
|
||||
"bans_made", made,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package waf_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// abuseIPDBBanDays is the setting of how long a ban for AbuseIPDB's
|
||||
// blacklist lasts, in days.
|
||||
const abuseIPDBBanDays = "SWW_WAF_ABUSEIPDB_BAN_DAYS"
|
||||
|
||||
func TestClientOnTheFetchedAbuseIPDBBlacklistIsRefused(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, clk, server := startWithClock(t, "", map[string]string{
|
||||
abuseIPDBKey: accountKey,
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
stopped := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
server.AbuseIPDB.RunBlacklist(ctx)
|
||||
close(stopped)
|
||||
}()
|
||||
|
||||
t.Cleanup(func() {
|
||||
cancel()
|
||||
<-stopped
|
||||
})
|
||||
|
||||
waitUntil(func() bool {
|
||||
active, _ := server.Ledger.Count(clk.Now())
|
||||
|
||||
return active == 1
|
||||
})
|
||||
|
||||
s.get(client, http.StatusForbidden, requestlog.ActionTemporarilyBanned)
|
||||
s.get(otherClient, http.StatusOK, requestlog.ActionForward)
|
||||
}
|
||||
|
||||
func TestClientsOnTheAbuseIPDBBlacklistAreBannedForTheBanDays(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, clk, server := startWithClock(t, "", map[string]string{
|
||||
abuseIPDBBanDays: "2", banScopeV4Prefix: "24",
|
||||
})
|
||||
|
||||
// Two clients in one netblock make one ban, with the higher score.
|
||||
server.BanAbuseIPDBBlacklist([]reputation.BlacklistEntry{
|
||||
{Addr: netip.MustParseAddr(client), Score: 95},
|
||||
{Addr: netip.MustParseAddr(otherClient), Score: 100},
|
||||
{Addr: netip.MustParseAddr("2001:db8:5::1"), Score: 90},
|
||||
})
|
||||
|
||||
held := server.Ledger.Bans(netip.MustParsePrefix(clientsNet))
|
||||
if len(held) != 1 || held[0].Cause != bans.CauseAbuseIPDB ||
|
||||
*held[0].Notes.Reputation[0].Score != 100 {
|
||||
t.Errorf("bans on %s %+v, want one for the blacklist, with 100",
|
||||
clientsNet, held)
|
||||
}
|
||||
|
||||
// Every client of a banned netblock is refused, the IPv6 address's /64
|
||||
// included.
|
||||
for _, from := range []string{"203.0.113.200", "2001:db8:5::ffff:1"} {
|
||||
s.get(from, http.StatusForbidden, requestlog.ActionTemporarilyBanned)
|
||||
}
|
||||
|
||||
s.get("2001:db8:5:1::1", http.StatusOK, requestlog.ActionForward)
|
||||
|
||||
clk.advance(2 * 24 * time.Hour)
|
||||
s.get(client, http.StatusOK, requestlog.ActionForward)
|
||||
}
|
||||
|
||||
func TestAbuseIPDBBlacklistMakesNoBanInObserveMode(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, clk, server := startWithClock(
|
||||
t,
|
||||
"",
|
||||
map[string]string{mode: "observe"},
|
||||
)
|
||||
|
||||
server.BanAbuseIPDBBlacklist([]reputation.BlacklistEntry{
|
||||
{Addr: netip.MustParseAddr(client), Score: 100},
|
||||
})
|
||||
|
||||
if active, _ := server.Ledger.Count(clk.Now()); active != 0 {
|
||||
t.Errorf("%d bans active, want none", active)
|
||||
}
|
||||
|
||||
s.get(client, http.StatusOK, requestlog.ActionForward)
|
||||
}
|
||||
@@ -1,4 +1,13 @@
|
||||
package waf
|
||||
|
||||
import "sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
|
||||
// BesideLookupTimeLimit is besideLookupTimeLimit.
|
||||
const BesideLookupTimeLimit = besideLookupTimeLimit
|
||||
|
||||
// BanAbuseIPDBBlacklist does on s what a fetch of AbuseIPDB's blacklist with
|
||||
// the clients on listed does.
|
||||
func (s *Server) BanAbuseIPDBBlacklist(listed []reputation.BlacklistEntry) {
|
||||
h := s.Handler.(*handler) //nolint:forcetypeassert // New makes it one
|
||||
h.banAbuseIPDBBlacklist(listed)
|
||||
}
|
||||
+14
-6
@@ -63,6 +63,10 @@ type Params struct {
|
||||
// AbuseIPDBURL is where clients are checked with AbuseIPDB while
|
||||
// SWW_WAF_ABUSEIPDB_KEY is set, normally reputation.AbuseIPDBURL.
|
||||
AbuseIPDBURL string
|
||||
// AbuseIPDBBlacklistURL is where AbuseIPDB's blacklist is fetched from
|
||||
// while SWW_WAF_ABUSEIPDB_KEY is set, normally
|
||||
// reputation.AbuseIPDBBlacklistURL.
|
||||
AbuseIPDBBlacklistURL string
|
||||
// LookupFile is the lookup database they are looked up in while
|
||||
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
|
||||
LookupFile *lookup.File
|
||||
@@ -126,7 +130,6 @@ func newHandler(params Params) *handler {
|
||||
params.Config.MetricsBusiestCountriesAndASNs,
|
||||
params.Config.InstanceName,
|
||||
)
|
||||
lists, dnsbl, abuseIPDB := newReputation(params, m)
|
||||
|
||||
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
|
||||
if err != nil {
|
||||
@@ -167,9 +170,6 @@ func newHandler(params Params) *handler {
|
||||
Alerts: params.Alerts,
|
||||
}),
|
||||
lookupFile: params.LookupFile,
|
||||
lists: lists,
|
||||
dnsbl: dnsbl,
|
||||
abuseIPDB: abuseIPDB,
|
||||
rules: params.Rules,
|
||||
coreRuleSet: params.CoreRuleSet,
|
||||
alerts: params.Alerts,
|
||||
@@ -179,6 +179,10 @@ func newHandler(params Params) *handler {
|
||||
besideLookupTimeLimit,
|
||||
),
|
||||
}
|
||||
h.lists, h.dnsbl, h.abuseIPDB = newReputation(
|
||||
params, m, h.banAbuseIPDBBlacklist,
|
||||
)
|
||||
|
||||
if params.Config.RouteWatchURL != "" {
|
||||
h.routeWatch = routewatch.New(routewatch.Params{
|
||||
URL: params.Config.RouteWatchURL, Now: params.Now, ProcessLog: params.ProcessLog,
|
||||
@@ -208,9 +212,11 @@ func newHandler(params Params) *handler {
|
||||
// newReputation returns the lists fetched from URLs, the DNSBL zones'
|
||||
// verdicts and AbuseIPDB's scores, as the settings in params name them,
|
||||
// with none fetched, asked for or checked yet, and adds their metrics to
|
||||
// m, AbuseIPDB's while SWW_WAF_ABUSEIPDB_KEY is set.
|
||||
// m, AbuseIPDB's while SWW_WAF_ABUSEIPDB_KEY is set. blacklisted receives
|
||||
// the clients on each AbuseIPDB blacklist fetched.
|
||||
func newReputation(
|
||||
params Params, m *metrics.Metrics,
|
||||
blacklisted func([]reputation.BlacklistEntry),
|
||||
) (*reputation.Lists, *reputation.DNSBL, *reputation.AbuseIPDB) {
|
||||
cfg := params.Config
|
||||
lists := reputation.New(reputation.Params{
|
||||
@@ -231,7 +237,9 @@ func newReputation(
|
||||
abuseIPDB := reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
|
||||
URL: params.AbuseIPDBURL, Key: cfg.AbuseIPDBKey, MinScore: cfg.AbuseIPDBMinScore,
|
||||
DailyBudget: cfg.AbuseIPDBDailyBudget, CacheTTL: cfg.IPReputationCacheTTL,
|
||||
Timeout: cfg.DNSBLAbuseIPDBTimeout, Now: params.Now, ProcessLog: params.ProcessLog,
|
||||
Timeout: cfg.DNSBLAbuseIPDBTimeout, BlacklistURL: params.AbuseIPDBBlacklistURL,
|
||||
Threshold: cfg.AbuseIPDBThreshold, BlacklistRefresh: cfg.AbuseIPDBBlacklistRefresh,
|
||||
Blacklisted: blacklisted, Now: params.Now, ProcessLog: params.ProcessLog,
|
||||
Alerts: params.Alerts,
|
||||
})
|
||||
|
||||
|
||||
@@ -424,7 +424,8 @@ func startRProxy(
|
||||
// With no geojsURL, there is no stand-in for GeoJS to look clients up at, and
|
||||
// SWW_WAF_LOOKUP_SOURCE is off unless env sets it. While it is file, the lookup
|
||||
// database SWW_WAF_LOOKUP_DB_PATH names is read. Clients are checked with AbuseIPDB
|
||||
// at abuseIPDBURL while env sets SWW_WAF_ABUSEIPDB_KEY.
|
||||
// at abuseIPDBURL while env sets SWW_WAF_ABUSEIPDB_KEY, and its blacklist is
|
||||
// fetched from abuseIPDBBlacklistURL.
|
||||
// SWW_WAF_CORE_RULE_SET_MODE is off unless env sets it, so that only the tests of the
|
||||
// Core Rule Set have their requests inspected by it.
|
||||
func newWAF(
|
||||
@@ -513,6 +514,7 @@ func newWAFWith(
|
||||
// The tests of the limit set it back, and hold the Core Rule Set
|
||||
// with a stand-in.
|
||||
BesideLookupTimeLimit: time.Hour,
|
||||
AbuseIPDBBlacklistURL: abuseIPDBBlacklistURL,
|
||||
})
|
||||
|
||||
return server, out, alertQueue
|
||||
|
||||
@@ -822,22 +822,39 @@ const abuseipdb = reputation.AbuseIPDBSource
|
||||
// through which AbuseIPDB is asked.
|
||||
const abuseIPDBURL = "abuseipdb://stand-in/api/v2/check"
|
||||
|
||||
// abuseIPDBBlacklistURL is where newWAF has AbuseIPDB's blacklist fetched
|
||||
// from: at abuseIPDBStandIn too.
|
||||
const abuseIPDBBlacklistURL = "abuseipdb://stand-in/api/v2/blacklist"
|
||||
|
||||
// abuseIPDBStandIn is a stand-in for AbuseIPDB that gives every client the
|
||||
// score 100, at once and without the network.
|
||||
// score 100, at once and without the network. Its blacklist lists client
|
||||
// alone, with the score 100, to a fetch with accountKey that asks for the
|
||||
// clients scored 90 or more, the default threshold; it refuses any other
|
||||
// fetch of it with 400.
|
||||
type abuseIPDBStandIn struct{}
|
||||
|
||||
// RoundTrip answers req with the score 100.
|
||||
// RoundTrip answers req with the score 100, or with the blacklist.
|
||||
func (abuseIPDBStandIn) RoundTrip(
|
||||
req *http.Request,
|
||||
) (*http.Response, error) {
|
||||
status, body := http.StatusOK, `{"data":{"abuseConfidenceScore":100}}`
|
||||
|
||||
if req.URL.Path == "/api/v2/blacklist" {
|
||||
body = `{"data":[{"ipAddress":"` + client +
|
||||
`","abuseConfidenceScore":100}]}`
|
||||
|
||||
if req.Header.Get("Key") != accountKey ||
|
||||
req.URL.RawQuery != "confidenceMinimum=90" {
|
||||
status, body = http.StatusBadRequest, `{"errors":[]}`
|
||||
}
|
||||
}
|
||||
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Status: "200 OK",
|
||||
StatusCode: status,
|
||||
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
|
||||
Header: http.Header{},
|
||||
Body: io.NopCloser(
|
||||
strings.NewReader(`{"data":{"abuseConfidenceScore":100}}`),
|
||||
),
|
||||
Request: req,
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
Request: req,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user