waf: ban the clients on AbuseIPDB's blacklist (closes #411)
check / check (push) Successful in 8m16s
check / check (push) Successful in 8m16s
While SWW_WAF_ABUSEIPDB_KEY is set, waf fetches AbuseIPDB's blacklist at the start and every SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS (60), cut at SWW_WAF_ABUSEIPDB_THRESHOLD (90, from 25 to 100). Each listed client's netblock gets a ban of SWW_WAF_ABUSEIPDB_BAN_DAYS (30) with the new cause abuseipdb. The blacklist itself is not kept. These bans do not count toward SWW_WAF_MAX_BANS, so a 10,000-address blacklist cannot push out the bans for limits and attacks. An ended abuseipdb ban is forgotten at each fetch and each read of bans.json. A netblock with an active ban is left alone, and so is one whose abuseipdb ban an admin lifted, until that ban would have ended. Observe mode makes no ban. Model: opus-5-5
This commit was merged in pull request #416.
This commit is contained in:
1 parent
01e6b5d47e
commit
809d2664d7
22 files changed
+1338
-222
No files matched your search
@@ -0,0 +1,46 @@
|
||||
package waf
|
||||
|
||||
// The bans for AbuseIPDB's blacklist. They read SWW_WAF_ABUSEIPDB_BAN_DAYS and
|
||||
// SWW_WAF_MODE; the blacklist is fetched by the reputation package.
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"net/netip"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
)
|
||||
|
||||
// BanAbuseIPDBBlacklist bans in ledger the netblock of each client on
|
||||
// AbuseIPDB's blacklist, listed, for SWW_WAF_ABUSEIPDB_BAN_DAYS from now, as
|
||||
// the ledger's BanForAbuseIPDB describes. It logs to processLog how many
|
||||
// clients were listed and how many bans were made. Clients in one netblock
|
||||
// make one ban, with the highest of their scores. In observe mode it makes
|
||||
// no ban.
|
||||
func BanAbuseIPDBBlacklist(
|
||||
cfg *config.WAF, ledger *bans.Ledger, processLog *slog.Logger,
|
||||
now time.Time, listed []reputation.BlacklistEntry,
|
||||
) {
|
||||
scores := make(map[netip.Prefix]int64, len(listed))
|
||||
|
||||
for _, client := range listed {
|
||||
netblock := banNetblock(cfg, client.Addr)
|
||||
scores[netblock] = max(scores[netblock], client.Score)
|
||||
}
|
||||
|
||||
made := 0
|
||||
|
||||
if !cfg.Observe {
|
||||
made = ledger.BanForAbuseIPDB(
|
||||
now, now.Add(cfg.AbuseIPDBBanDuration), scores,
|
||||
)
|
||||
}
|
||||
|
||||
processLog.Info(
|
||||
"fetched AbuseIPDB's blacklist",
|
||||
"clients", len(listed),
|
||||
"bans_made", made,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package waf_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// abuseIPDBBanDays is the setting of how long a ban for AbuseIPDB's
|
||||
// blacklist lasts, in days.
|
||||
const abuseIPDBBanDays = "SWW_WAF_ABUSEIPDB_BAN_DAYS"
|
||||
|
||||
func TestClientOnTheFetchedAbuseIPDBBlacklistIsRefused(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, clk, server := startWithClock(t, "", map[string]string{
|
||||
abuseIPDBKey: accountKey,
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
stopped := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
server.AbuseIPDB.RunBlacklist(ctx)
|
||||
close(stopped)
|
||||
}()
|
||||
|
||||
t.Cleanup(func() {
|
||||
cancel()
|
||||
<-stopped
|
||||
})
|
||||
|
||||
waitUntil(func() bool {
|
||||
active, _ := server.Ledger.Count(clk.Now())
|
||||
|
||||
return active == 1
|
||||
})
|
||||
|
||||
s.get(client, http.StatusForbidden, requestlog.ActionTemporarilyBanned)
|
||||
s.get(otherClient, http.StatusOK, requestlog.ActionForward)
|
||||
}
|
||||
|
||||
func TestClientsOnTheAbuseIPDBBlacklistAreBannedForTheBanDays(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, clk, server := startWithClock(t, "", map[string]string{
|
||||
abuseIPDBBanDays: "2", banScopeV4Prefix: "24",
|
||||
})
|
||||
|
||||
// Two clients in one netblock make one ban, with the higher score.
|
||||
server.BanAbuseIPDBBlacklist([]reputation.BlacklistEntry{
|
||||
{Addr: netip.MustParseAddr(client), Score: 95},
|
||||
{Addr: netip.MustParseAddr(otherClient), Score: 100},
|
||||
{Addr: netip.MustParseAddr("2001:db8:5::1"), Score: 90},
|
||||
})
|
||||
|
||||
held := server.Ledger.Bans(netip.MustParsePrefix(clientsNet))
|
||||
if len(held) != 1 || held[0].Cause != bans.CauseAbuseIPDB ||
|
||||
*held[0].Notes.Reputation[0].Score != 100 {
|
||||
t.Errorf("bans on %s %+v, want one for the blacklist, with 100",
|
||||
clientsNet, held)
|
||||
}
|
||||
|
||||
// Every client of a banned netblock is refused, the IPv6 address's /64
|
||||
// included.
|
||||
for _, from := range []string{"203.0.113.200", "2001:db8:5::ffff:1"} {
|
||||
s.get(from, http.StatusForbidden, requestlog.ActionTemporarilyBanned)
|
||||
}
|
||||
|
||||
s.get("2001:db8:5:1::1", http.StatusOK, requestlog.ActionForward)
|
||||
|
||||
clk.advance(2 * 24 * time.Hour)
|
||||
s.get(client, http.StatusOK, requestlog.ActionForward)
|
||||
}
|
||||
|
||||
func TestAbuseIPDBBlacklistMakesNoBanInObserveMode(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, clk, server := startWithClock(
|
||||
t,
|
||||
"",
|
||||
map[string]string{mode: "observe"},
|
||||
)
|
||||
|
||||
server.BanAbuseIPDBBlacklist([]reputation.BlacklistEntry{
|
||||
{Addr: netip.MustParseAddr(client), Score: 100},
|
||||
})
|
||||
|
||||
if active, _ := server.Ledger.Count(clk.Now()); active != 0 {
|
||||
t.Errorf("%d bans active, want none", active)
|
||||
}
|
||||
|
||||
s.get(client, http.StatusOK, requestlog.ActionForward)
|
||||
}
|
||||
@@ -1,4 +1,13 @@
|
||||
package waf
|
||||
|
||||
import "sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
|
||||
// BesideLookupTimeLimit is besideLookupTimeLimit.
|
||||
const BesideLookupTimeLimit = besideLookupTimeLimit
|
||||
|
||||
// BanAbuseIPDBBlacklist does on s what a fetch of AbuseIPDB's blacklist with
|
||||
// the clients on listed does.
|
||||
func (s *Server) BanAbuseIPDBBlacklist(listed []reputation.BlacklistEntry) {
|
||||
h := s.Handler.(*handler) //nolint:forcetypeassert // New makes it one
|
||||
BanAbuseIPDBBlacklist(h.config, h.ledger, h.processLog, h.now(), listed)
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
// Package waf is the server the waf command runs. It answers
|
||||
// rproxy's questions, takes rproxy's reports and answers its own endpoints, and
|
||||
// writes a log line for each.
|
||||
// writes a log line for each. It also bans the clients on AbuseIPDB's
|
||||
// blacklist each time the reputation package fetches it.
|
||||
package waf
|
||||
|
||||
import (
|
||||
|
||||
+22
-12
@@ -431,7 +431,8 @@ func startRProxy(
|
||||
// With no geojsURL, there is no stand-in for GeoJS to look clients up at, and
|
||||
// SWW_WAF_LOOKUP_SOURCE is off unless env sets it. While it is file, the lookup
|
||||
// database SWW_WAF_LOOKUP_DB_PATH names is read. Clients are checked with AbuseIPDB
|
||||
// at abuseIPDBURL while env sets SWW_WAF_ABUSEIPDB_KEY.
|
||||
// at abuseIPDBURL while env sets SWW_WAF_ABUSEIPDB_KEY, and its blacklist is
|
||||
// fetched from abuseIPDBBlacklistURL.
|
||||
// SWW_WAF_CORE_RULE_SET_MODE is off unless env sets it, so that only the tests of the
|
||||
// Core Rule Set have their requests inspected by it.
|
||||
func newWAF(
|
||||
@@ -575,8 +576,9 @@ func addCounters(params *waf.Params) {
|
||||
}
|
||||
|
||||
// addOutsideSources adds to params the lists fetched from URLs, the DNSBL
|
||||
// zones' verdicts, AbuseIPDB's scores from abuseIPDBCheckURL, routewatch's
|
||||
// answers while its setting is set, and GeoJS at geojsURL, as sww builds
|
||||
// zones' verdicts, AbuseIPDB's scores from abuseIPDBCheckURL and its
|
||||
// blacklist from abuseIPDBBlacklistURL, routewatch's answers while its
|
||||
// setting is set, and GeoJS at geojsURL, as sww builds
|
||||
// them from params' settings and the parts addCounters added. The metrics
|
||||
// read the lists, the zones' verdicts, and AbuseIPDB's while its key is set.
|
||||
func addOutsideSources(
|
||||
@@ -605,15 +607,23 @@ func addOutsideSources(
|
||||
Alerts: alertQueue,
|
||||
})
|
||||
params.AbuseIPDB = reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
|
||||
URL: abuseIPDBCheckURL,
|
||||
Key: cfg.AbuseIPDBKey,
|
||||
MinScore: cfg.AbuseIPDBMinScore,
|
||||
DailyBudget: cfg.AbuseIPDBDailyBudget,
|
||||
CacheTTL: cfg.IPReputationCacheTTL,
|
||||
Timeout: cfg.DNSBLAbuseIPDBTimeout,
|
||||
Now: params.Now,
|
||||
ProcessLog: processLog,
|
||||
Alerts: alertQueue,
|
||||
URL: abuseIPDBCheckURL,
|
||||
Key: cfg.AbuseIPDBKey,
|
||||
MinScore: cfg.AbuseIPDBMinScore,
|
||||
DailyBudget: cfg.AbuseIPDBDailyBudget,
|
||||
CacheTTL: cfg.IPReputationCacheTTL,
|
||||
Timeout: cfg.DNSBLAbuseIPDBTimeout,
|
||||
BlacklistURL: abuseIPDBBlacklistURL,
|
||||
Threshold: cfg.AbuseIPDBThreshold,
|
||||
BlacklistRefresh: cfg.AbuseIPDBBlacklistRefresh,
|
||||
Blacklisted: func(listed []reputation.BlacklistEntry) {
|
||||
waf.BanAbuseIPDBBlacklist(
|
||||
cfg, ledger, processLog, params.Now(), listed,
|
||||
)
|
||||
},
|
||||
Now: params.Now,
|
||||
ProcessLog: processLog,
|
||||
Alerts: alertQueue,
|
||||
})
|
||||
|
||||
if cfg.RouteWatchURL != "" {
|
||||
|
||||
@@ -822,22 +822,39 @@ const abuseipdb = reputation.AbuseIPDBSource
|
||||
// through which AbuseIPDB is asked.
|
||||
const abuseIPDBURL = "abuseipdb://stand-in/api/v2/check"
|
||||
|
||||
// abuseIPDBBlacklistURL is where newWAF has AbuseIPDB's blacklist fetched
|
||||
// from: at abuseIPDBStandIn too.
|
||||
const abuseIPDBBlacklistURL = "abuseipdb://stand-in/api/v2/blacklist"
|
||||
|
||||
// abuseIPDBStandIn is a stand-in for AbuseIPDB that gives every client the
|
||||
// score 100, at once and without the network.
|
||||
// score 100, at once and without the network. Its blacklist lists client
|
||||
// alone, with the score 100, to a fetch with accountKey that asks for the
|
||||
// clients scored 90 or more, the default threshold; it refuses any other
|
||||
// fetch of it with 400.
|
||||
type abuseIPDBStandIn struct{}
|
||||
|
||||
// RoundTrip answers req with the score 100.
|
||||
// RoundTrip answers req with the score 100, or with the blacklist.
|
||||
func (abuseIPDBStandIn) RoundTrip(
|
||||
req *http.Request,
|
||||
) (*http.Response, error) {
|
||||
status, body := http.StatusOK, `{"data":{"abuseConfidenceScore":100}}`
|
||||
|
||||
if req.URL.Path == "/api/v2/blacklist" {
|
||||
body = `{"data":[{"ipAddress":"` + client +
|
||||
`","abuseConfidenceScore":100}]}`
|
||||
|
||||
if req.Header.Get("Key") != accountKey ||
|
||||
req.URL.RawQuery != "confidenceMinimum=90" {
|
||||
status, body = http.StatusBadRequest, `{"errors":[]}`
|
||||
}
|
||||
}
|
||||
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Status: "200 OK",
|
||||
StatusCode: status,
|
||||
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
|
||||
Header: http.Header{},
|
||||
Body: io.NopCloser(
|
||||
strings.NewReader(`{"data":{"abuseConfidenceScore":100}}`),
|
||||
),
|
||||
Request: req,
|
||||
Body: io.NopCloser(strings.NewReader(body)),
|
||||
Request: req,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user