AbuseIPDB scores for clients that committed an offence, within a daily budget (closes #105)
check / check (push) Waiting to run
check / check (push) Waiting to run
With SWWAF_ABUSEIPDB_KEY set, a client whose history counts an offence (a broken limit, a ban rule's match or a block rule's refusal, counted by kind) is checked in the background, at most SWWAF_ABUSEIPDB_DAILY_BUDGET checks a day, the count kept in reputation.json. A client, an IPv4 address or an IPv6 /64, is checked by the address it sent from, and its score serves all its addresses. A score at or over SWWAF_ABUSEIPDB_MIN_SCORE is a hit for SWWAF_REPUTATION_ACTION, logged as abuseipdb and alerted with its score. A failure or the used-up budget gives no score and raises source_failure. The key goes only in the Key header. Judgement call: the budget's day is UTC; AbuseIPDB documents no reset time. Judgement call: each check sent spends budget; a minute's pause after a failure. Model: opus-5-5
This commit was merged in pull request #111.
This commit is contained in:
+14
-13
@@ -28,18 +28,19 @@ func biasedThresholdsSet(cfg *config.Config) bool {
|
||||
|
||||
// limitPercentages returns the client's limit percentages, for the rate
|
||||
// limits and for the byte limits, by its AS number and country as looked
|
||||
// up, each "" when unknown, and the blocklists and DNSBL zones that list
|
||||
// it. Each is the lowest of those the settings give it, the first of them
|
||||
// in the order below when several are lowest: the percentage
|
||||
// SWWAF_ASN_LIMIT_PERCENT gives its AS number, the one the file
|
||||
// up, each "" when unknown, and the blocklists, DNSBL zones and AbuseIPDB
|
||||
// that list it. Each is the lowest of those the settings give it, the
|
||||
// first of them in the order below when several are lowest: the
|
||||
// percentage SWWAF_ASN_LIMIT_PERCENT gives its AS number, the one the file
|
||||
// SWWAF_ASN_LIMIT_PERCENT_URL names gives it, the one
|
||||
// SWWAF_COUNTRY_LIMIT_PERCENT gives its country, for a client without a
|
||||
// country, SWWAF_UNKNOWN_LIMIT_PERCENT, for a client a blocklist lists,
|
||||
// the percentage of SWWAF_BLOCKLIST_ACTION while it is limit, and for a
|
||||
// client a DNSBL zone's verdict lists, the percentage of
|
||||
// SWWAF_REPUTATION_ACTION while it is limit. For the byte limits,
|
||||
// SWWAF_ASN_BYTES_PERCENT and SWWAF_COUNTRY_BYTES_PERCENT take the place
|
||||
// of the first three for an AS number or a country they list.
|
||||
// client a DNSBL zone's verdict lists, or whose AbuseIPDB score is a hit,
|
||||
// the percentage of SWWAF_REPUTATION_ACTION while it is limit. For the
|
||||
// byte limits, SWWAF_ASN_BYTES_PERCENT and SWWAF_COUNTRY_BYTES_PERCENT
|
||||
// take the place of the first three for an AS number or a country they
|
||||
// list.
|
||||
func (rq *request) limitPercentages() (percentage, percentage) {
|
||||
cfg := rq.h.config
|
||||
asn, country := rq.line.ASN, rq.line.Country
|
||||
@@ -59,9 +60,9 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
||||
blocklisted = percentage{cfg.BlocklistLimitPercent, "SWWAF_BLOCKLIST_ACTION"}
|
||||
}
|
||||
|
||||
dnsblListed := percentage{percent: whole}
|
||||
if rq.dnsblListed && cfg.ReputationAction == "limit" {
|
||||
dnsblListed = percentage{cfg.ReputationLimitPercent, "SWWAF_REPUTATION_ACTION"}
|
||||
reputationListed := percentage{percent: whole}
|
||||
if (rq.dnsblListed || rq.abuseIPDBHit) && cfg.ReputationAction == "limit" {
|
||||
reputationListed = percentage{cfg.ReputationLimitPercent, "SWWAF_REPUTATION_ACTION"}
|
||||
}
|
||||
|
||||
asnRequests := lowest(given(cfg.ASNLimitPercent, asn, "SWWAF_ASN_LIMIT_PERCENT"),
|
||||
@@ -78,8 +79,8 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
||||
countryBytes = given(cfg.CountryBytesPercent, country, "SWWAF_COUNTRY_BYTES_PERCENT")
|
||||
}
|
||||
|
||||
return lowest(asnRequests, countryRequests, unknown, blocklisted, dnsblListed),
|
||||
lowest(asnBytes, countryBytes, unknown, blocklisted, dnsblListed)
|
||||
return lowest(asnRequests, countryRequests, unknown, blocklisted, reputationListed),
|
||||
lowest(asnBytes, countryBytes, unknown, blocklisted, reputationListed)
|
||||
}
|
||||
|
||||
// given returns the percentage percents, the setting named setting, gives
|
||||
|
||||
@@ -349,6 +349,7 @@ const unansweredGeoJSURL = "unanswered://geojs/v1/ip/geo.json"
|
||||
func TestMain(m *testing.M) {
|
||||
transport, _ := http.DefaultTransport.(*http.Transport)
|
||||
transport.RegisterProtocol("unanswered", unansweredGeoJS{})
|
||||
transport.RegisterProtocol("abuseipdb", abuseIPDBStandIn{})
|
||||
|
||||
m.Run()
|
||||
}
|
||||
|
||||
+33
-10
@@ -59,6 +59,9 @@ type Params struct {
|
||||
// GeoJSURL is where clients' AS numbers and countries are looked up
|
||||
// while SWWAF_LOOKUP_SOURCE is geojs, normally lookup.URL.
|
||||
GeoJSURL string
|
||||
// AbuseIPDBURL is where clients are checked with AbuseIPDB while
|
||||
// SWWAF_ABUSEIPDB_KEY is set, normally reputation.AbuseIPDBURL.
|
||||
AbuseIPDBURL string
|
||||
// LookupFile is the lookup database they are looked up in while
|
||||
// SWWAF_LOOKUP_SOURCE is file, and nil otherwise.
|
||||
LookupFile *lookup.File
|
||||
@@ -71,15 +74,17 @@ type Params struct {
|
||||
Rules *rules.Files
|
||||
// Alerts receive the alert for each ban the proxy makes or makes
|
||||
// permanent, for each count over an anomaly threshold, for each request
|
||||
// whose client a blocklist or a DNSBL zone lists, and for GeoJS failing,
|
||||
// a fetch of a list failing or a query to a DNSBL zone failing.
|
||||
// whose client a blocklist, a DNSBL zone or AbuseIPDB lists, and for
|
||||
// GeoJS failing, a fetch of a list failing, a query to a DNSBL zone or
|
||||
// a check with AbuseIPDB failing, or the day's AbuseIPDB checks used up.
|
||||
Alerts *alerts.Queue
|
||||
}
|
||||
|
||||
// Server is the server smallwebwaf runs, with the parts of the proxy
|
||||
// whose state the state files keep, the lookup database, nil unless
|
||||
// SWWAF_LOOKUP_SOURCE is file, the lists fetched from URLs, which its Run
|
||||
// fetches, the DNSBL zones' verdicts, and the metrics.
|
||||
// fetches, the DNSBL zones' verdicts, AbuseIPDB's scores and checks
|
||||
// spent, and the metrics.
|
||||
type Server struct {
|
||||
*http.Server
|
||||
|
||||
@@ -90,6 +95,7 @@ type Server struct {
|
||||
LookupFile *lookup.File
|
||||
Lists *reputation.Lists
|
||||
DNSBL *reputation.DNSBL
|
||||
AbuseIPDB *reputation.AbuseIPDB
|
||||
Metrics *metrics.Metrics
|
||||
}
|
||||
|
||||
@@ -102,7 +108,7 @@ type Server struct {
|
||||
func New(params Params) *Server {
|
||||
errorLog := slog.NewLogLogger(params.ProcessLog.Handler(), slog.LevelWarn)
|
||||
m := metrics.New(params.Config.MetricsTopN, params.Config.InstanceName)
|
||||
lists, dnsbl := newReputation(params)
|
||||
lists, dnsbl, abuseIPDB := newReputation(params, m)
|
||||
h := &handler{
|
||||
config: params.Config,
|
||||
requestLog: params.RequestLog,
|
||||
@@ -140,6 +146,7 @@ func New(params Params) *Server {
|
||||
lookupFile: params.LookupFile,
|
||||
lists: lists,
|
||||
dnsbl: dnsbl,
|
||||
abuseIPDB: abuseIPDB,
|
||||
rules: params.Rules,
|
||||
alerts: params.Alerts,
|
||||
}
|
||||
@@ -159,7 +166,6 @@ func New(params Params) *Server {
|
||||
})
|
||||
m.AddBansAndClients(h.ledger, h.limiter, params.Now)
|
||||
m.AddRules(params.Rules)
|
||||
m.AddReputation(h.lists, h.dnsbl)
|
||||
|
||||
return &Server{
|
||||
Server: &http.Server{
|
||||
@@ -181,14 +187,18 @@ func New(params Params) *Server {
|
||||
LookupFile: h.lookupFile,
|
||||
Lists: h.lists,
|
||||
DNSBL: h.dnsbl,
|
||||
AbuseIPDB: h.abuseIPDB,
|
||||
Metrics: m,
|
||||
}
|
||||
}
|
||||
|
||||
// newReputation returns the lists fetched from URLs and the DNSBL zones'
|
||||
// verdicts, as the settings in params name them, with none fetched or
|
||||
// asked for yet.
|
||||
func newReputation(params Params) (*reputation.Lists, *reputation.DNSBL) {
|
||||
// newReputation returns the lists fetched from URLs, the DNSBL zones'
|
||||
// verdicts and AbuseIPDB's scores, as the settings in params name them,
|
||||
// with none fetched, asked for or checked yet, and adds their metrics to
|
||||
// m, AbuseIPDB's while SWWAF_ABUSEIPDB_KEY is set.
|
||||
func newReputation(
|
||||
params Params, m *metrics.Metrics,
|
||||
) (*reputation.Lists, *reputation.DNSBL, *reputation.AbuseIPDB) {
|
||||
cfg := params.Config
|
||||
lists := reputation.New(reputation.Params{
|
||||
BlocklistURLs: cfg.BlocklistURLs, Refresh: cfg.BlocklistRefresh,
|
||||
@@ -200,8 +210,20 @@ func newReputation(params Params) (*reputation.Lists, *reputation.DNSBL) {
|
||||
Timeout: cfg.ReputationTimeout, Now: params.Now, ProcessLog: params.ProcessLog,
|
||||
Alerts: params.Alerts,
|
||||
})
|
||||
abuseIPDB := reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
|
||||
URL: params.AbuseIPDBURL, Key: cfg.AbuseIPDBKey, MinScore: cfg.AbuseIPDBMinScore,
|
||||
DailyBudget: cfg.AbuseIPDBDailyBudget, CacheTTL: cfg.ReputationCacheTTL,
|
||||
Timeout: cfg.ReputationTimeout, Now: params.Now, ProcessLog: params.ProcessLog,
|
||||
Alerts: params.Alerts,
|
||||
})
|
||||
|
||||
return lists, dnsbl
|
||||
m.AddReputation(lists, dnsbl)
|
||||
|
||||
if cfg.AbuseIPDBKey != "" {
|
||||
m.AddAbuseIPDB(abuseIPDB)
|
||||
}
|
||||
|
||||
return lists, dnsbl, abuseIPDB
|
||||
}
|
||||
|
||||
// handler is the proxy. It holds what every request shares; what belongs
|
||||
@@ -221,6 +243,7 @@ type handler struct {
|
||||
lookupFile *lookup.File
|
||||
lists *reputation.Lists
|
||||
dnsbl *reputation.DNSBL
|
||||
abuseIPDB *reputation.AbuseIPDB
|
||||
rules *rules.Files
|
||||
alerts *alerts.Queue
|
||||
}
|
||||
|
||||
@@ -268,7 +268,8 @@ func startProxyWithAlerts(
|
||||
// queue: they wait in it, for the test to look at. With no geojsURL, there
|
||||
// is no stand-in for GeoJS to look clients up at, and SWWAF_LOOKUP_SOURCE
|
||||
// is off unless env sets it. While it is file, the lookup database
|
||||
// SWWAF_LOOKUP_DB_PATH names is read.
|
||||
// SWWAF_LOOKUP_DB_PATH names is read. Clients are checked with AbuseIPDB
|
||||
// at abuseIPDBURL while env sets SWWAF_ABUSEIPDB_KEY.
|
||||
func newProxy(
|
||||
t *testing.T, appURL, geojsURL string, now func() time.Time,
|
||||
env map[string]string,
|
||||
@@ -325,14 +326,15 @@ func newProxy(
|
||||
}
|
||||
|
||||
server := proxy.New(proxy.Params{
|
||||
Config: cfg,
|
||||
RequestLog: out,
|
||||
ProcessLog: processLog,
|
||||
GeoJSURL: geojsURL,
|
||||
LookupFile: lookupFile,
|
||||
Now: now,
|
||||
Rules: ruleFiles,
|
||||
Alerts: alertQueue,
|
||||
Config: cfg,
|
||||
RequestLog: out,
|
||||
ProcessLog: processLog,
|
||||
GeoJSURL: geojsURL,
|
||||
AbuseIPDBURL: abuseIPDBURL,
|
||||
LookupFile: lookupFile,
|
||||
Now: now,
|
||||
Rules: ruleFiles,
|
||||
Alerts: alertQueue,
|
||||
})
|
||||
|
||||
return server, out, alertQueue
|
||||
|
||||
@@ -4,8 +4,14 @@ import (
|
||||
"context"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
)
|
||||
|
||||
// deny is the SWWAF_BLOCKLIST_ACTION and the SWWAF_REPUTATION_ACTION that
|
||||
// refuses the requests of a client a source lists.
|
||||
const deny = "deny"
|
||||
|
||||
// blocklistDenied notes the blocklists that list the client, as
|
||||
// noteListed does, and reports whether SWWAF_BLOCKLIST_ACTION, being deny,
|
||||
// refuses the request. Being limit, it lowers the client's limits instead
|
||||
@@ -15,7 +21,7 @@ func (rq *request) blocklistDenied() bool {
|
||||
rq.blocklisted = len(listedBy) > 0
|
||||
rq.noteListed(listedBy, "listed by a blocklist")
|
||||
|
||||
return rq.blocklisted && rq.h.config.BlocklistAction == "deny"
|
||||
return rq.blocklisted && rq.h.config.BlocklistAction == deny
|
||||
}
|
||||
|
||||
// dnsblDenied notes the DNSBL zones whose verdict lists the client, as
|
||||
@@ -30,27 +36,63 @@ func (rq *request) dnsblDenied(ctx context.Context) bool {
|
||||
rq.dnsblListed = len(listedBy) > 0
|
||||
rq.noteListed(listedBy, "listed by a DNSBL zone")
|
||||
|
||||
return rq.dnsblListed && rq.h.config.ReputationAction == "deny"
|
||||
return rq.dnsblListed && rq.h.config.ReputationAction == deny
|
||||
}
|
||||
|
||||
// noteListed adds sources, the URLs of the blocklists or the DNSBL zones,
|
||||
// their keys masked, that list the client, to the log line's reputation,
|
||||
// counts each of them in the metrics, and raises a reputation_hit alert,
|
||||
// with reason, for each.
|
||||
func (rq *request) noteListed(sources []string, reason string) {
|
||||
rq.line.Reputation = append(rq.line.Reputation, sources...)
|
||||
// abuseIPDBDenied notes AbuseIPDB, as noteHit does, with the score, when
|
||||
// its score of the client is a hit, and reports whether
|
||||
// SWWAF_REPUTATION_ACTION, being deny, refuses the request, as dnsblDenied
|
||||
// does for a zone. While SWWAF_ABUSEIPDB_KEY is unset it does nothing. A
|
||||
// client without a score is checked in the background, by the request's
|
||||
// address, if its history counts an offence, and the request does not
|
||||
// wait for the answer. The score is then used for each address of the
|
||||
// client. ctx is the request's own context.
|
||||
func (rq *request) abuseIPDBDenied(ctx context.Context) bool {
|
||||
if rq.h.config.AbuseIPDBKey == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
client := clientGroup(rq.client)
|
||||
held, _ := rq.h.limiter.Client(client)
|
||||
offender := held.History.Offences != ratelimit.Offences{}
|
||||
|
||||
score, hit := rq.h.abuseIPDB.Hit(ctx, client, rq.client, offender)
|
||||
if !hit {
|
||||
return false
|
||||
}
|
||||
|
||||
rq.abuseIPDBHit = true
|
||||
rq.noteHit(reputation.AbuseIPDBSource, "scored by AbuseIPDB at or over "+
|
||||
"SWWAF_ABUSEIPDB_MIN_SCORE", map[string]any{
|
||||
"source": reputation.AbuseIPDBSource, "score": score,
|
||||
})
|
||||
|
||||
return rq.h.config.ReputationAction == deny
|
||||
}
|
||||
|
||||
// noteListed notes each of sources, the URLs of the blocklists or the
|
||||
// DNSBL zones, their keys masked, that list the client, as noteHit does,
|
||||
// with reason, and the source in the alert's detail.
|
||||
func (rq *request) noteListed(sources []string, reason string) {
|
||||
for _, source := range sources {
|
||||
rq.h.metrics.ReputationHit(source)
|
||||
rq.h.alerts.Raise(alerts.Alert{
|
||||
Event: alerts.EventReputationHit,
|
||||
Client: rq.client,
|
||||
Netblock: clientGroup(rq.client),
|
||||
ASN: rq.line.ASN,
|
||||
ASName: rq.line.ASName,
|
||||
Country: rq.line.Country,
|
||||
Reason: reason,
|
||||
Detail: map[string]any{"source": source},
|
||||
})
|
||||
rq.noteHit(source, reason, map[string]any{"source": source})
|
||||
}
|
||||
}
|
||||
|
||||
// noteHit adds source, which lists the client, to the log line's
|
||||
// reputation, counts it in the metrics, and raises a reputation_hit alert
|
||||
// with reason and detail.
|
||||
func (rq *request) noteHit(source, reason string, detail map[string]any) {
|
||||
rq.line.Reputation = append(rq.line.Reputation, source)
|
||||
rq.h.metrics.ReputationHit(source)
|
||||
rq.h.alerts.Raise(alerts.Alert{
|
||||
Event: alerts.EventReputationHit,
|
||||
Client: rq.client,
|
||||
Netblock: clientGroup(rq.client),
|
||||
ASN: rq.line.ASN,
|
||||
ASName: rq.line.ASName,
|
||||
Country: rq.line.Country,
|
||||
Reason: reason,
|
||||
Detail: detail,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,16 +1,20 @@
|
||||
package proxy_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/proxy"
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
@@ -599,6 +603,277 @@ func TestRequestFromAClientWithoutAVerdictHasTheZoneAskedAboutIt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The AbuseIPDB settings, and accountKey, the key the tests set.
|
||||
const (
|
||||
abuseIPDBKey = "SWWAF_ABUSEIPDB_KEY"
|
||||
accountKey = "abuseipdb-key-0123456789abcdef"
|
||||
)
|
||||
|
||||
// abuseipdb is how the request log, the alerts and the metrics name
|
||||
// AbuseIPDB.
|
||||
const abuseipdb = reputation.AbuseIPDBSource
|
||||
|
||||
// abuseIPDBURL is where newProxy has clients checked with AbuseIPDB: at
|
||||
// abuseIPDBStandIn, which TestMain registers with Go's default transport,
|
||||
// through which AbuseIPDB is asked.
|
||||
const abuseIPDBURL = "abuseipdb://stand-in/api/v2/check"
|
||||
|
||||
// abuseIPDBStandIn is a stand-in for AbuseIPDB that gives every client the
|
||||
// score 100, at once and without the network.
|
||||
type abuseIPDBStandIn struct{}
|
||||
|
||||
// RoundTrip answers req with the score 100.
|
||||
func (abuseIPDBStandIn) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Status: "200 OK",
|
||||
Header: http.Header{},
|
||||
Body: io.NopCloser(strings.NewReader(`{"data":{"abuseConfidenceScore":100}}`)),
|
||||
Request: req,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func TestOnlyAClientThatHasCommittedAnOffenceIsCheckedWithAbuseIPDB(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
forward := requestlog.ActionForward
|
||||
|
||||
s, clk, server, _ := startWithLookupsAndClock(t, map[string]string{
|
||||
abuseIPDBKey: accountKey, rateLimitPerMinute: "2", reputationAction: actionLog,
|
||||
})
|
||||
|
||||
// Neither fromDE, until it breaks a rate limit, nor fromKP, which never
|
||||
// does, is checked, nor fromDE under the ban that makes.
|
||||
s.get(fromDE, http.StatusOK, forward)
|
||||
s.get(fromDE, http.StatusOK, forward)
|
||||
s.get(fromKP, http.StatusOK, forward)
|
||||
s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited)
|
||||
s.get(fromDE, http.StatusForbidden, requestlog.ActionBanned)
|
||||
wantAbuseIPDBChecks(t, server, 0)
|
||||
|
||||
// Once the ban has ended, fromDE's first request has it checked in the
|
||||
// background, and goes on without its score, which its next request
|
||||
// finds.
|
||||
clk.advance(time.Hour)
|
||||
wantReputation(t, s.get(fromDE, http.StatusOK, forward))
|
||||
wantAbuseIPDBChecks(t, server, 1)
|
||||
waitUntil(func() bool { return len(server.AbuseIPDB.Snapshot().Scores) == 1 })
|
||||
wantReputation(t, s.get(fromDE, http.StatusOK, forward), abuseipdb)
|
||||
|
||||
s.get(fromKP, http.StatusOK, forward)
|
||||
wantAbuseIPDBChecks(t, server, 1)
|
||||
}
|
||||
|
||||
func TestIPv6ClientCostsOneAbuseIPDBCheckWhicheverOfItsAddressesSends(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
forward := requestlog.ActionForward
|
||||
|
||||
// 15 addresses of 2001:db8:1:2::/64, one client, each in a part of it
|
||||
// of its own.
|
||||
var addresses []string
|
||||
for i := 1; i < 16; i++ {
|
||||
addresses = append(addresses, fmt.Sprintf("2001:db8:1:2:%x::9", i<<12))
|
||||
}
|
||||
|
||||
s, clk, server := startWithClock(t, "", map[string]string{
|
||||
abuseIPDBKey: accountKey, reputationAction: actionLog,
|
||||
rateLimitPerMinute: strconv.Itoa(len(addresses)),
|
||||
})
|
||||
|
||||
// The client breaks the rate limit from its first address, which bans
|
||||
// it for an hour.
|
||||
for range addresses {
|
||||
s.get(addresses[0], http.StatusOK, forward)
|
||||
}
|
||||
|
||||
s.get(addresses[0], http.StatusForbidden, requestlog.ActionRateLimited)
|
||||
clk.advance(time.Hour)
|
||||
|
||||
// Once the ban has ended, which set its counters back to zero, a
|
||||
// request from each of its addresses has it checked once.
|
||||
for _, address := range addresses {
|
||||
s.get(address, http.StatusOK, forward)
|
||||
}
|
||||
|
||||
wantAbuseIPDBChecks(t, server, 1)
|
||||
}
|
||||
|
||||
// probePath is the path the ban rule of testRules, probe, matches.
|
||||
const probePath = "/.env"
|
||||
|
||||
func TestClientARuleRefusedIsCheckedWithAbuseIPDBAtItsNextRequest(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
// path is what the client asks for, status and action what that
|
||||
// request is answered and logged with, and want the offences its
|
||||
// history then counts.
|
||||
path string
|
||||
status int
|
||||
action string
|
||||
want ratelimit.Offences
|
||||
}{
|
||||
{
|
||||
"a block rule", "/blocked", http.StatusForbidden, requestlog.ActionRuleBlocked,
|
||||
ratelimit.Offences{RuleBlocked: 1},
|
||||
},
|
||||
{
|
||||
"a ban rule", probePath, http.StatusForbidden, requestlog.ActionBanned,
|
||||
ratelimit.Offences{Attack: 1},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, clk, server := startWithClock(t, "", map[string]string{
|
||||
abuseIPDBKey: accountKey, reputationAction: actionLog,
|
||||
rulesDir: writeRules(t, testRules), attackBanDuration: "1h",
|
||||
})
|
||||
|
||||
s.request(client, tc.path, tc.status, tc.action)
|
||||
wantAbuseIPDBChecks(t, server, 0)
|
||||
|
||||
if got := historyOf(t, server, client).Offences; got != tc.want {
|
||||
t.Errorf("history counts the offences %+v, want %+v", got, tc.want)
|
||||
}
|
||||
|
||||
// Its next request, once a ban rule's ban has ended, has it
|
||||
// checked.
|
||||
clk.advance(time.Hour)
|
||||
s.get(client, http.StatusOK, requestlog.ActionForward)
|
||||
wantAbuseIPDBChecks(t, server, 1)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEachReputationActionForAClientAbuseIPDBScoresAtOrOverTheMinimum(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
forward, denied := requestlog.ActionForward, requestlog.ActionDenied
|
||||
|
||||
for _, tc := range []struct {
|
||||
action string
|
||||
// statuses and actions are those of fromDE's three requests, and
|
||||
// percent their limit_percent, as percentText gives it.
|
||||
statuses []int
|
||||
actions []string
|
||||
percent string
|
||||
}{
|
||||
{
|
||||
actionDeny, []int{http.StatusForbidden, http.StatusForbidden, http.StatusForbidden},
|
||||
[]string{denied, denied, denied}, none,
|
||||
},
|
||||
{
|
||||
// Half of 4 requests a minute: the third breaks the limit.
|
||||
limitHalf, []int{http.StatusOK, http.StatusOK, http.StatusForbidden},
|
||||
[]string{forward, forward, requestlog.ActionRateLimited},
|
||||
"50 from " + reputationAction,
|
||||
},
|
||||
{
|
||||
actionLog, []int{http.StatusOK, http.StatusOK, http.StatusOK},
|
||||
[]string{forward, forward, forward}, none,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.action, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s, server, _ := startWithLookups(t, map[string]string{
|
||||
rateLimitPerMinute: fourAMinute, abuseIPDBKey: accountKey,
|
||||
reputationAction: tc.action,
|
||||
})
|
||||
// At SWWAF_ABUSEIPDB_MIN_SCORE, 75 by default, and just under it.
|
||||
loadScores(server, map[string]int64{fromDE: 75, fromKP: 74})
|
||||
|
||||
for i := range 3 {
|
||||
line := s.get(fromDE, tc.statuses[i], tc.actions[i])
|
||||
wantReputation(t, line, abuseipdb)
|
||||
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
|
||||
tc.percent)
|
||||
|
||||
// A request refused for the score is not counted.
|
||||
counted := line.fields["counts"] != nil
|
||||
if counted != (tc.actions[i] != denied) {
|
||||
t.Errorf("request counted %t, logged %s", counted, tc.actions[i])
|
||||
}
|
||||
}
|
||||
|
||||
// fromKP's score is no hit, and it has the whole limit.
|
||||
for range 3 {
|
||||
line := s.get(fromKP, http.StatusOK, forward)
|
||||
wantReputation(t, line)
|
||||
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
|
||||
none)
|
||||
}
|
||||
|
||||
// A refusal for the score makes no ban.
|
||||
if held := server.Ledger.Snapshot(); tc.action == actionDeny && len(held) != 0 {
|
||||
t.Errorf("bans %+v, want none", held)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAbuseIPDBHitRaisesAnAlertWithTheScoreOncePerCooldownAndIsCounted(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
s, server, queue := startWithLookups(t, map[string]string{
|
||||
abuseIPDBKey: accountKey, reputationAction: actionLog, metricsToken: token,
|
||||
})
|
||||
loadScores(server, map[string]int64{fromDE: 90})
|
||||
|
||||
// The second request's alert is a repeat, which the cooldown holds back.
|
||||
for range 2 {
|
||||
wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward),
|
||||
abuseipdb)
|
||||
}
|
||||
|
||||
// The alert is made as a DNSBL zone's is, with the score besides.
|
||||
waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
|
||||
if len(waiting) != 1 || waiting[0].Event != alerts.EventReputationHit ||
|
||||
waiting[0].Reason != "scored by AbuseIPDB at or over SWWAF_ABUSEIPDB_MIN_SCORE" ||
|
||||
waiting[0].Detail["source"] != abuseipdb || waiting[0].Detail["score"] != int64(90) ||
|
||||
queue.Suppressed() != 1 {
|
||||
t.Errorf("alerts waiting %+v, %d held back, want AbuseIPDB's reputation_hit "+
|
||||
"with the score 90, and 1", waiting, queue.Suppressed())
|
||||
}
|
||||
|
||||
// The hits, and the checks, none, since no client committed an
|
||||
// offence, so that the whole budget is left.
|
||||
metrics := s.scrape(unplaced)
|
||||
labels := `{instance="` + alertInstance + `",source="` + abuseipdb + `"}`
|
||||
|
||||
wantMetric(t, metrics, "smallwebwaf_reputation_hits_total"+labels, 2)
|
||||
wantMetric(t, metrics, "smallwebwaf_reputation_queries_total"+labels, 0)
|
||||
wantMetric(t, metrics, "smallwebwaf_reputation_failures_total"+labels, 0)
|
||||
wantMetric(t, metrics, "smallwebwaf_reputation_daily_budget_remaining"+labels, 900)
|
||||
}
|
||||
|
||||
func TestWithoutAnAbuseIPDBKeyNoClientIsCheckedNorAScoreUsed(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
forward := requestlog.ActionForward
|
||||
|
||||
s, clk, server, _ := startWithLookupsAndClock(t, map[string]string{
|
||||
rateLimitPerMinute: "1", metricsToken: token,
|
||||
})
|
||||
loadScores(server, map[string]int64{fromDE: 100})
|
||||
|
||||
// fromDE's score is not used, and once it has committed an offence it
|
||||
// is not checked either.
|
||||
wantReputation(t, s.get(fromDE, http.StatusOK, forward))
|
||||
s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited)
|
||||
clk.advance(time.Hour)
|
||||
wantReputation(t, s.get(fromDE, http.StatusOK, forward))
|
||||
wantAbuseIPDBChecks(t, server, 0)
|
||||
|
||||
wantNoSeries(t, s.scrape(unplaced), `smallwebwaf_reputation_daily_budget_remaining{`+
|
||||
`instance="`+alertInstance+`",source="`+abuseipdb+`"}`)
|
||||
}
|
||||
|
||||
// listsFetched is when loadLists has the copies fetched.
|
||||
func listsFetched() time.Time {
|
||||
return time.Date(2026, 10, 5, 0, 0, 0, 0, time.UTC)
|
||||
@@ -631,6 +906,31 @@ func loadVerdicts(server *proxy.Server, listedBy map[string][]string) {
|
||||
server.DNSBL.Load(verdicts)
|
||||
}
|
||||
|
||||
// loadScores puts into server's AbuseIPDB the score scores gives each
|
||||
// client, an IPv4 address, fetched at verdictsFetched, as reputation.json
|
||||
// would at start.
|
||||
func loadScores(server *proxy.Server, scores map[string]int64) {
|
||||
kept := make([]reputation.Score, 0, len(scores))
|
||||
for client, score := range scores {
|
||||
kept = append(kept, reputation.Score{
|
||||
Client: netip.MustParsePrefix(client + "/32"), Score: score,
|
||||
Fetched: verdictsFetched(),
|
||||
})
|
||||
}
|
||||
|
||||
server.AbuseIPDB.Load(reputation.Checks{Scores: kept})
|
||||
}
|
||||
|
||||
// wantAbuseIPDBChecks checks how many clients server has checked with
|
||||
// AbuseIPDB.
|
||||
func wantAbuseIPDBChecks(t *testing.T, server *proxy.Server, want int) {
|
||||
t.Helper()
|
||||
|
||||
if got := server.AbuseIPDB.Checked(); got != want {
|
||||
t.Errorf("%d clients checked with AbuseIPDB, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// loadLists puts copies of lists into server's lists, by URL, each with
|
||||
// its lines, fetched at listsFetched, as reputation.json would at start.
|
||||
func loadLists(t *testing.T, server *proxy.Server, copies map[string][]string) {
|
||||
|
||||
+19
-11
@@ -63,10 +63,15 @@ type request struct {
|
||||
// limits and for the byte limits.
|
||||
counted bool
|
||||
limitPercent, bytesPercent percentage
|
||||
// attack is true for a request that matched a ban rule, and
|
||||
// ruleBlocked for one a block rule refused, each an offence its
|
||||
// client's history counts.
|
||||
attack, ruleBlocked bool
|
||||
// blocklisted is true once a blocklist is found to list the client,
|
||||
// and dnsblListed once a DNSBL zone's verdict is.
|
||||
blocklisted, dnsblListed bool
|
||||
start time.Time
|
||||
// dnsblListed once a DNSBL zone's verdict is, and abuseIPDBHit once
|
||||
// AbuseIPDB's score of it is a hit.
|
||||
blocklisted, dnsblListed, abuseIPDBHit bool
|
||||
start time.Time
|
||||
// checked is when the checks were done, and upstreamStart when the
|
||||
// request was handed to the app.
|
||||
checked time.Time
|
||||
@@ -217,13 +222,14 @@ func (rq *request) check(ctx context.Context) *refusal {
|
||||
// other client, SWWAF_DENY_NETS comes first, then a ban on its netblock,
|
||||
// so that a client either refuses is not looked up, then the lookup of
|
||||
// its AS number and country, then the country lists, then the blocklists,
|
||||
// and then the DNSBL zones' verdicts; a request any of them refuses is not
|
||||
// counted for the rate limits. Then come the rate limits, unless the
|
||||
// client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the request's path is
|
||||
// exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that every other request
|
||||
// is counted, each of them by the client's limit percentages, and last the
|
||||
// rule files. A request exempt from the rate limits is exempt from the
|
||||
// byte limits too. ctx is the request's own context.
|
||||
// then the DNSBL zones' verdicts, and then AbuseIPDB's score; a request
|
||||
// any of them refuses is not counted for the rate limits. Then come the
|
||||
// rate limits, unless the client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the
|
||||
// request's path is exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that
|
||||
// every other request is counted, each of them by the client's limit
|
||||
// percentages, and last the rule files. A request exempt from the rate
|
||||
// limits is exempt from the byte limits too. ctx is the request's own
|
||||
// context.
|
||||
func (rq *request) checkClient(ctx context.Context) string {
|
||||
cfg := rq.h.config
|
||||
if isInside(rq.client, cfg.AllowNets) {
|
||||
@@ -250,7 +256,7 @@ func (rq *request) checkClient(ctx context.Context) string {
|
||||
return requestlog.ActionDenied
|
||||
}
|
||||
|
||||
if rq.dnsblDenied(ctx) {
|
||||
if rq.dnsblDenied(ctx) || rq.abuseIPDBDenied(ctx) {
|
||||
return requestlog.ActionDenied
|
||||
}
|
||||
|
||||
@@ -540,6 +546,8 @@ func (rq *request) addToHistory() {
|
||||
RequestBytes: rq.requestBytes(),
|
||||
ResponseBytes: rq.out.bytes,
|
||||
BrokeLimit: rq.line.Offence == requestlog.OffenceLimit,
|
||||
Attack: rq.attack,
|
||||
RuleBlocked: rq.ruleBlocked,
|
||||
})
|
||||
|
||||
answer, found := rq.answerAtTheEnd()
|
||||
|
||||
@@ -12,7 +12,8 @@ import (
|
||||
// action of the rule that refuses it, ActionRuleBlocked for a block rule
|
||||
// and ActionBanned for a ban rule, or "" when none does. A ban rule bans
|
||||
// the client's netblock for a clear sign of attack, or in observe mode
|
||||
// raises the alert for the ban it would have made.
|
||||
// raises the alert for the ban it would have made. Either rule's match
|
||||
// is noted as an offence, for the client's history.
|
||||
func (rq *request) checkRules(now time.Time) string {
|
||||
matched := rq.h.rules.Match(rq.in)
|
||||
|
||||
@@ -28,8 +29,11 @@ func (rq *request) checkRules(now time.Time) string {
|
||||
// Only the last rule matched can refuse the request.
|
||||
switch last := matched[len(matched)-1]; last.Action {
|
||||
case rules.ActionBlock:
|
||||
rq.ruleBlocked = true
|
||||
|
||||
return requestlog.ActionRuleBlocked
|
||||
case rules.ActionBan:
|
||||
rq.attack = true
|
||||
rq.banForAttack(now, last)
|
||||
|
||||
return requestlog.ActionBanned
|
||||
|
||||
Reference in New Issue
Block a user