waf: ban the clients on AbuseIPDB's blacklist (closes #411)
check / check (push) Successful in 8m16s

While SWW_WAF_ABUSEIPDB_KEY is set, waf fetches AbuseIPDB's blacklist at
the start and every SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS
(60), cut at SWW_WAF_ABUSEIPDB_THRESHOLD (90, from 25 to 100). Each
listed client's netblock gets a ban of SWW_WAF_ABUSEIPDB_BAN_DAYS (30)
with the new cause abuseipdb. The blacklist itself is not kept.

These bans do not count toward SWW_WAF_MAX_BANS, so a 10,000-address
blacklist cannot push out the bans for limits and attacks. An ended
abuseipdb ban is forgotten at each fetch and each read of bans.json. A
netblock with an active ban is left alone, and so is one whose abuseipdb
ban an admin lifted, until that ban would have ended. Observe mode makes
no ban.

Model: opus-5-5
This commit was merged in pull request #416.
This commit is contained in:
clawbot committed 2026-11-19 12:42:41 +01:00
1 parent 01e6b5d47e
commit 809d2664d7
22 files changed
+1338 -222

No files matched your search

+46
View File
@@ -0,0 +1,46 @@
package waf
// The bans for AbuseIPDB's blacklist. They read SWW_WAF_ABUSEIPDB_BAN_DAYS and
// SWW_WAF_MODE; the blacklist is fetched by the reputation package.
import (
"log/slog"
"net/netip"
"time"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/reputation"
)
// BanAbuseIPDBBlacklist bans in ledger the netblock of each client on
// AbuseIPDB's blacklist, listed, for SWW_WAF_ABUSEIPDB_BAN_DAYS from now, as
// the ledger's BanForAbuseIPDB describes. It logs to processLog how many
// clients were listed and how many bans were made. Clients in one netblock
// make one ban, with the highest of their scores. In observe mode it makes
// no ban.
func BanAbuseIPDBBlacklist(
cfg *config.WAF, ledger *bans.Ledger, processLog *slog.Logger,
now time.Time, listed []reputation.BlacklistEntry,
) {
scores := make(map[netip.Prefix]int64, len(listed))
for _, client := range listed {
netblock := banNetblock(cfg, client.Addr)
scores[netblock] = max(scores[netblock], client.Score)
}
made := 0
if !cfg.Observe {
made = ledger.BanForAbuseIPDB(
now, now.Add(cfg.AbuseIPDBBanDuration), scores,
)
}
processLog.Info(
"fetched AbuseIPDB's blacklist",
"clients", len(listed),
"bans_made", made,
)
}
+100
View File
@@ -0,0 +1,100 @@
package waf_test
import (
"context"
"net/http"
"net/netip"
"testing"
"time"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
// abuseIPDBBanDays is the setting of how long a ban for AbuseIPDB's
// blacklist lasts, in days.
const abuseIPDBBanDays = "SWW_WAF_ABUSEIPDB_BAN_DAYS"
func TestClientOnTheFetchedAbuseIPDBBlacklistIsRefused(t *testing.T) {
t.Parallel()
s, clk, server := startWithClock(t, "", map[string]string{
abuseIPDBKey: accountKey,
})
ctx, cancel := context.WithCancel(t.Context())
stopped := make(chan struct{})
go func() {
server.AbuseIPDB.RunBlacklist(ctx)
close(stopped)
}()
t.Cleanup(func() {
cancel()
<-stopped
})
waitUntil(func() bool {
active, _ := server.Ledger.Count(clk.Now())
return active == 1
})
s.get(client, http.StatusForbidden, requestlog.ActionTemporarilyBanned)
s.get(otherClient, http.StatusOK, requestlog.ActionForward)
}
func TestClientsOnTheAbuseIPDBBlacklistAreBannedForTheBanDays(t *testing.T) {
t.Parallel()
s, clk, server := startWithClock(t, "", map[string]string{
abuseIPDBBanDays: "2", banScopeV4Prefix: "24",
})
// Two clients in one netblock make one ban, with the higher score.
server.BanAbuseIPDBBlacklist([]reputation.BlacklistEntry{
{Addr: netip.MustParseAddr(client), Score: 95},
{Addr: netip.MustParseAddr(otherClient), Score: 100},
{Addr: netip.MustParseAddr("2001:db8:5::1"), Score: 90},
})
held := server.Ledger.Bans(netip.MustParsePrefix(clientsNet))
if len(held) != 1 || held[0].Cause != bans.CauseAbuseIPDB ||
*held[0].Notes.Reputation[0].Score != 100 {
t.Errorf("bans on %s %+v, want one for the blacklist, with 100",
clientsNet, held)
}
// Every client of a banned netblock is refused, the IPv6 address's /64
// included.
for _, from := range []string{"203.0.113.200", "2001:db8:5::ffff:1"} {
s.get(from, http.StatusForbidden, requestlog.ActionTemporarilyBanned)
}
s.get("2001:db8:5:1::1", http.StatusOK, requestlog.ActionForward)
clk.advance(2 * 24 * time.Hour)
s.get(client, http.StatusOK, requestlog.ActionForward)
}
func TestAbuseIPDBBlacklistMakesNoBanInObserveMode(t *testing.T) {
t.Parallel()
s, clk, server := startWithClock(
t,
"",
map[string]string{mode: "observe"},
)
server.BanAbuseIPDBBlacklist([]reputation.BlacklistEntry{
{Addr: netip.MustParseAddr(client), Score: 100},
})
if active, _ := server.Ledger.Count(clk.Now()); active != 0 {
t.Errorf("%d bans active, want none", active)
}
s.get(client, http.StatusOK, requestlog.ActionForward)
}
+9
View File
@@ -1,4 +1,13 @@
package waf
import "sneak.berlin/go/smallwebwaf/internal/reputation"
// BesideLookupTimeLimit is besideLookupTimeLimit.
const BesideLookupTimeLimit = besideLookupTimeLimit
// BanAbuseIPDBBlacklist does on s what a fetch of AbuseIPDB's blacklist with
// the clients on listed does.
func (s *Server) BanAbuseIPDBBlacklist(listed []reputation.BlacklistEntry) {
h := s.Handler.(*handler) //nolint:forcetypeassert // New makes it one
BanAbuseIPDBBlacklist(h.config, h.ledger, h.processLog, h.now(), listed)
}
+2 -1
View File
@@ -1,6 +1,7 @@
// Package waf is the server the waf command runs. It answers
// rproxy's questions, takes rproxy's reports and answers its own endpoints, and
// writes a log line for each.
// writes a log line for each. It also bans the clients on AbuseIPDB's
// blacklist each time the reputation package fetches it.
package waf
import (
+22 -12
View File
@@ -431,7 +431,8 @@ func startRProxy(
// With no geojsURL, there is no stand-in for GeoJS to look clients up at, and
// SWW_WAF_LOOKUP_SOURCE is off unless env sets it. While it is file, the lookup
// database SWW_WAF_LOOKUP_DB_PATH names is read. Clients are checked with AbuseIPDB
// at abuseIPDBURL while env sets SWW_WAF_ABUSEIPDB_KEY.
// at abuseIPDBURL while env sets SWW_WAF_ABUSEIPDB_KEY, and its blacklist is
// fetched from abuseIPDBBlacklistURL.
// SWW_WAF_CORE_RULE_SET_MODE is off unless env sets it, so that only the tests of the
// Core Rule Set have their requests inspected by it.
func newWAF(
@@ -575,8 +576,9 @@ func addCounters(params *waf.Params) {
}
// addOutsideSources adds to params the lists fetched from URLs, the DNSBL
// zones' verdicts, AbuseIPDB's scores from abuseIPDBCheckURL, routewatch's
// answers while its setting is set, and GeoJS at geojsURL, as sww builds
// zones' verdicts, AbuseIPDB's scores from abuseIPDBCheckURL and its
// blacklist from abuseIPDBBlacklistURL, routewatch's answers while its
// setting is set, and GeoJS at geojsURL, as sww builds
// them from params' settings and the parts addCounters added. The metrics
// read the lists, the zones' verdicts, and AbuseIPDB's while its key is set.
func addOutsideSources(
@@ -605,15 +607,23 @@ func addOutsideSources(
Alerts: alertQueue,
})
params.AbuseIPDB = reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
URL: abuseIPDBCheckURL,
Key: cfg.AbuseIPDBKey,
MinScore: cfg.AbuseIPDBMinScore,
DailyBudget: cfg.AbuseIPDBDailyBudget,
CacheTTL: cfg.IPReputationCacheTTL,
Timeout: cfg.DNSBLAbuseIPDBTimeout,
Now: params.Now,
ProcessLog: processLog,
Alerts: alertQueue,
URL: abuseIPDBCheckURL,
Key: cfg.AbuseIPDBKey,
MinScore: cfg.AbuseIPDBMinScore,
DailyBudget: cfg.AbuseIPDBDailyBudget,
CacheTTL: cfg.IPReputationCacheTTL,
Timeout: cfg.DNSBLAbuseIPDBTimeout,
BlacklistURL: abuseIPDBBlacklistURL,
Threshold: cfg.AbuseIPDBThreshold,
BlacklistRefresh: cfg.AbuseIPDBBlacklistRefresh,
Blacklisted: func(listed []reputation.BlacklistEntry) {
waf.BanAbuseIPDBBlacklist(
cfg, ledger, processLog, params.Now(), listed,
)
},
Now: params.Now,
ProcessLog: processLog,
Alerts: alertQueue,
})
if cfg.RouteWatchURL != "" {
+25 -8
View File
@@ -822,22 +822,39 @@ const abuseipdb = reputation.AbuseIPDBSource
// through which AbuseIPDB is asked.
const abuseIPDBURL = "abuseipdb://stand-in/api/v2/check"
// abuseIPDBBlacklistURL is where newWAF has AbuseIPDB's blacklist fetched
// from: at abuseIPDBStandIn too.
const abuseIPDBBlacklistURL = "abuseipdb://stand-in/api/v2/blacklist"
// abuseIPDBStandIn is a stand-in for AbuseIPDB that gives every client the
// score 100, at once and without the network.
// score 100, at once and without the network. Its blacklist lists client
// alone, with the score 100, to a fetch with accountKey that asks for the
// clients scored 90 or more, the default threshold; it refuses any other
// fetch of it with 400.
type abuseIPDBStandIn struct{}
// RoundTrip answers req with the score 100.
// RoundTrip answers req with the score 100, or with the blacklist.
func (abuseIPDBStandIn) RoundTrip(
req *http.Request,
) (*http.Response, error) {
status, body := http.StatusOK, `{"data":{"abuseConfidenceScore":100}}`
if req.URL.Path == "/api/v2/blacklist" {
body = `{"data":[{"ipAddress":"` + client +
`","abuseConfidenceScore":100}]}`
if req.Header.Get("Key") != accountKey ||
req.URL.RawQuery != "confidenceMinimum=90" {
status, body = http.StatusBadRequest, `{"errors":[]}`
}
}
return &http.Response{
StatusCode: http.StatusOK,
Status: "200 OK",
StatusCode: status,
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
Header: http.Header{},
Body: io.NopCloser(
strings.NewReader(`{"data":{"abuseConfidenceScore":100}}`),
),
Request: req,
Body: io.NopCloser(strings.NewReader(body)),
Request: req,
}, nil
}