waf: ban the clients on AbuseIPDB's blacklist (closes #411)
check / check (push) Failing after 8m52s
check / check (push) Failing after 8m52s
While SWW_WAF_ABUSEIPDB_KEY is set, waf fetches AbuseIPDB's blacklist at the start and every SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS (60), cut at SWW_WAF_ABUSEIPDB_THRESHOLD (90, from 25 to 100). Each listed client's netblock gets a ban of SWW_WAF_ABUSEIPDB_BAN_DAYS (30) with the new cause abuseipdb. The blacklist itself is not kept. These bans do not count toward SWW_WAF_MAX_BANS, so a 10,000-address blacklist cannot push out the bans for limits and attacks. An ended abuseipdb ban is forgotten at each fetch and each read of bans.json. A netblock with an active ban is left alone, and so is one whose abuseipdb ban an admin lifted, until that ban would have ended. Observe mode makes no ban. Model: opus-5-5
This commit is contained in:
1 parent
9f42460f70
commit
dfffeb73ec
22 files changed
+1304
-205
No files matched your search
@@ -11,6 +11,7 @@ import (
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -23,6 +24,9 @@ const (
|
||||
// AbuseIPDBURL is where clients are checked: the check endpoint of
|
||||
// AbuseIPDB's API.
|
||||
AbuseIPDBURL = "https://api.abuseipdb.com/api/v2/check"
|
||||
// AbuseIPDBBlacklistURL is where AbuseIPDB's blacklist is fetched from:
|
||||
// the blacklist endpoint of AbuseIPDB's API.
|
||||
AbuseIPDBBlacklistURL = "https://api.abuseipdb.com/api/v2/blacklist"
|
||||
// AbuseIPDBSource is how the request log, the alerts and the metrics
|
||||
// name AbuseIPDB.
|
||||
AbuseIPDBSource = "abuseipdb"
|
||||
@@ -32,6 +36,8 @@ const (
|
||||
|
||||
var (
|
||||
errNoScore = errors.New("the answer gives no abuseConfidenceScore")
|
||||
errNoBlacklist = errors.New("the answer gives no data")
|
||||
errNotAddress = errors.New("is not an address")
|
||||
errBudgetUsedUp = errors.New(
|
||||
"checks spent; none is made until the day ends at 00:00 UTC")
|
||||
)
|
||||
@@ -53,6 +59,13 @@ type ChecksSpent struct {
|
||||
Spent int
|
||||
}
|
||||
|
||||
// BlacklistEntry is a client on AbuseIPDB's blacklist: its address and its
|
||||
// abuse confidence score.
|
||||
type BlacklistEntry struct {
|
||||
Addr netip.Addr
|
||||
Score int64
|
||||
}
|
||||
|
||||
// AbuseIPDBParams are what NewAbuseIPDB needs.
|
||||
type AbuseIPDBParams struct {
|
||||
// URL is where clients are checked, normally AbuseIPDBURL, with Key,
|
||||
@@ -69,10 +82,19 @@ type AbuseIPDBParams struct {
|
||||
// take (SWW_WAF_DNSBL_ABUSEIPDB_TIMEOUT_MS).
|
||||
CacheTTL time.Duration
|
||||
Timeout time.Duration
|
||||
// BlacklistURL is where the blacklist is fetched from, normally
|
||||
// AbuseIPDBBlacklistURL, with the clients scored Threshold or more
|
||||
// (SWW_WAF_ABUSEIPDB_THRESHOLD), every BlacklistRefresh
|
||||
// (SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS). Blacklisted
|
||||
// receives the clients on each blacklist fetched.
|
||||
BlacklistURL string
|
||||
Threshold int64
|
||||
BlacklistRefresh time.Duration
|
||||
Blacklisted func([]BlacklistEntry)
|
||||
// Now tells the time, normally time.Now in UTC.
|
||||
Now func() time.Time
|
||||
// ProcessLog receives each check that fails, and why, and the day's
|
||||
// budget used up.
|
||||
// ProcessLog receives each check or fetch of the blacklist that fails,
|
||||
// and why, and the day's budget used up.
|
||||
ProcessLog *slog.Logger
|
||||
// Alerts receive a source_failure alert for each.
|
||||
Alerts *alerts.Queue
|
||||
@@ -256,6 +278,40 @@ func (a *AbuseIPDB) LoadChecksSpent(checks ChecksSpent) {
|
||||
a.day, a.spent = checks.Day, checks.Spent
|
||||
}
|
||||
|
||||
// RunBlacklist fetches AbuseIPDB's blacklist at once, and again every
|
||||
// BlacklistRefresh, until ctx is done, and gives the clients on each copy
|
||||
// fetched to Blacklisted. While Key is unset it does nothing. A fetch that
|
||||
// fails is logged and raised as a source_failure alert, unless it was cut
|
||||
// off as ctx ended.
|
||||
func (a *AbuseIPDB) RunBlacklist(ctx context.Context) {
|
||||
if a.params.Key == "" {
|
||||
return
|
||||
}
|
||||
|
||||
for {
|
||||
listed, err := a.fetchBlacklist(ctx)
|
||||
|
||||
switch {
|
||||
case ctx.Err() != nil:
|
||||
return
|
||||
case err != nil:
|
||||
a.alert("fetching AbuseIPDB's blacklist failed", err)
|
||||
default:
|
||||
a.params.Blacklisted(listed)
|
||||
}
|
||||
|
||||
timer := time.NewTimer(a.params.BlacklistRefresh)
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
|
||||
return
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
|
||||
// the score as client's, and notes the check as no longer under way. A
|
||||
// check that fails gives no score: it is counted, logged and raised as a
|
||||
@@ -344,6 +400,94 @@ func (a *AbuseIPDB) ask(
|
||||
return *answer.Data.AbuseConfidenceScore, nil
|
||||
}
|
||||
|
||||
// fetchBlacklist fetches the clients on AbuseIPDB's blacklist scored
|
||||
// Threshold or more, sending the key in the header Key. An answer other
|
||||
// than 200, one longer than maxListBytes or without the list, an entry
|
||||
// whose address does not read, and none within fetchTimeout, fail.
|
||||
func (a *AbuseIPDB) fetchBlacklist(
|
||||
ctx context.Context,
|
||||
) ([]BlacklistEntry, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, fetchTimeout)
|
||||
defer cancel()
|
||||
|
||||
query := url.Values{
|
||||
"confidenceMinimum": {strconv.FormatInt(a.params.Threshold, 10)},
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet,
|
||||
a.params.BlacklistURL+"?"+query.Encode(), http.NoBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("make the request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Key", a.params.Key)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
res, err := a.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetch the blacklist: %w", errors.Unwrap(err))
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = res.Body.Close()
|
||||
}()
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("%w %s", errStatus, res.Status)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(res.Body, maxListBytes+1))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read the blacklist: %w", err)
|
||||
}
|
||||
|
||||
if len(body) > maxListBytes {
|
||||
return nil, errTooLong
|
||||
}
|
||||
|
||||
return parseBlacklist(body)
|
||||
}
|
||||
|
||||
// parseBlacklist reads body, AbuseIPDB's answer with its blacklist.
|
||||
func parseBlacklist(body []byte) ([]BlacklistEntry, error) {
|
||||
var answer struct {
|
||||
Data *[]struct {
|
||||
IPAddress string `json:"ipAddress"`
|
||||
AbuseConfidenceScore int64 `json:"abuseConfidenceScore"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
err := json.Unmarshal(body, &answer)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read the blacklist: %w", err)
|
||||
}
|
||||
|
||||
if answer.Data == nil {
|
||||
return nil, errNoBlacklist
|
||||
}
|
||||
|
||||
listed := make([]BlacklistEntry, 0, len(*answer.Data))
|
||||
|
||||
for i, entry := range *answer.Data {
|
||||
addr, err := netip.ParseAddr(entry.IPAddress)
|
||||
if err != nil || addr.Zone() != "" {
|
||||
// The error leaves out what the entry holds, as it does a
|
||||
// client's address.
|
||||
return nil, fmt.Errorf(
|
||||
"entry %d's ipAddress %w",
|
||||
i+1,
|
||||
errNotAddress,
|
||||
)
|
||||
}
|
||||
|
||||
listed = append(listed, BlacklistEntry{
|
||||
Addr: addr.Unmap(), Score: entry.AbuseConfidenceScore,
|
||||
})
|
||||
}
|
||||
|
||||
return listed, nil
|
||||
}
|
||||
|
||||
// alert raises a source_failure alert from AbuseIPDB with reason and err,
|
||||
// and logs them.
|
||||
func (a *AbuseIPDB) alert(reason string, err error) {
|
||||
|
||||
Reference in new issue
Block a user