AbuseIPDB scores for clients that broke a limit, within a daily budget (closes #105)
check / check (push) Waiting to run

With SWWAF_ABUSEIPDB_KEY set, a client whose history counts an offence
is checked in the background, at most SWWAF_ABUSEIPDB_DAILY_BUDGET checks
a day, the count kept in reputation.json. A score at or over
SWWAF_ABUSEIPDB_MIN_SCORE is a hit for SWWAF_REPUTATION_ACTION, logged as
abuseipdb and alerted with its score. A failure or the used-up budget
gives no score and raises source_failure. The key goes only in the Key
header.

Judgement call: the budget's day is UTC; AbuseIPDB documents no reset time.
Judgement call: each check sent spends budget; a minute's pause after a failure.
Judgement call: offences are those the history counts, so far broken limits only.

Model: opus-5-5
This commit is contained in:
2026-10-07 19:53:17 +00:00
parent 0b0f207423
commit 4bc8d9edb4
21 changed files with 1862 additions and 304 deletions
+3 -2
View File
@@ -47,10 +47,11 @@ const (
// EventWAFBlock comes with the Core Rule Set; nothing raises it yet.
EventWAFBlock = "waf_block"
// EventReputationHit is a request whose client a blocklist or a DNSBL
// zone lists.
// zone lists, or whose AbuseIPDB score is a hit.
EventReputationHit = "reputation_hit"
// EventSourceFailure is GeoJS failing or refusing smallwebwaf, a fetch
// of a list failing, or a query to a DNSBL zone failing or refused.
// of a list failing, a query to a DNSBL zone or a check with AbuseIPDB
// failing or refused, or the day's AbuseIPDB checks used up.
EventSourceFailure = "source_failure"
// EventFileError is a rule file or state file edited while smallwebwaf
// runs that does not parse, a replacement of the lookup database that
+21 -10
View File
@@ -155,14 +155,22 @@ type Config struct {
// DNSBLZones are the DNSBL zones clients are asked about
// (SWWAF_DNSBL_ZONES), through DNSBLResolver (SWWAF_DNSBL_RESOLVER), or
// the host's resolver while that is the zero AddrPort.
// ReputationAction is what is done with a client a zone's verdict lists
// (SWWAF_REPUTATION_ACTION): deny, limit or log; for limit,
// ReputationLimitPercent is the percentage of every limit it gets. A
// verdict is used for ReputationCacheTTL after it was fetched
// (SWWAF_REPUTATION_CACHE_TTL), and a query may take ReputationTimeout
// (SWWAF_REPUTATION_TIMEOUT). Neither can be off.
// AbuseIPDBKey is the key of the AbuseIPDB account clients are checked
// with (SWWAF_ABUSEIPDB_KEY), "" while it is unset and none is. A score
// of AbuseIPDBMinScore or more is a hit (SWWAF_ABUSEIPDB_MIN_SCORE), and
// at most AbuseIPDBDailyBudget checks are made a day
// (SWWAF_ABUSEIPDB_DAILY_BUDGET).
// ReputationAction is what is done with a client a zone's verdict lists,
// or whose score is a hit (SWWAF_REPUTATION_ACTION): deny, limit or log;
// for limit, ReputationLimitPercent is the percentage of every limit it
// gets. A verdict or a score is used for ReputationCacheTTL after it was
// fetched (SWWAF_REPUTATION_CACHE_TTL), and a query or a check may take
// ReputationTimeout (SWWAF_REPUTATION_TIMEOUT). Neither can be off.
DNSBLZones []string
DNSBLResolver netip.AddrPort
AbuseIPDBKey string
AbuseIPDBMinScore int64
AbuseIPDBDailyBudget int
ReputationAction string
ReputationLimitPercent int64
ReputationCacheTTL time.Duration
@@ -440,6 +448,9 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
BlocklistRefresh: env.refresh("SWWAF_BLOCKLIST_REFRESH", "24h"),
DNSBLZones: env.zones("SWWAF_DNSBL_ZONES"),
DNSBLResolver: env.resolver("SWWAF_DNSBL_RESOLVER"),
AbuseIPDBKey: env.secret("SWWAF_ABUSEIPDB_KEY"),
AbuseIPDBMinScore: env.percent("SWWAF_ABUSEIPDB_MIN_SCORE", "75"),
AbuseIPDBDailyBudget: env.numberNotOff("SWWAF_ABUSEIPDB_DAILY_BUDGET", "900"),
ReputationCacheTTL: env.durationNotOff("SWWAF_REPUTATION_CACHE_TTL", "24h"),
ReputationTimeout: env.durationNotOff("SWWAF_REPUTATION_TIMEOUT", "2s"),
BanResponse: env.banResponse("SWWAF_BAN_RESPONSE", "403"),
@@ -1110,10 +1121,10 @@ func (e *environment) webhookHeaders(name string) http.Header {
}
// secret reads a setting that is a secret another service gave, such as
// an ntfy token, "" while it is unset. It is sent in a header, which
// cannot hold a control character, so one in it is an error. The log
// shows ******** in place of a value that is not empty, and an error
// shows none of it.
// an ntfy token or an AbuseIPDB key, "" while it is unset. It is sent in
// a header, which cannot hold a control character, so one in it is an
// error. The log shows ******** in place of a value that is not empty, and
// an error shows none of it.
func (e *environment) secret(name string) string {
value, _ := e.lookup(name)
+76
View File
@@ -63,6 +63,9 @@ const (
blocklistAction = "SWWAF_BLOCKLIST_ACTION"
dnsblZones = "SWWAF_DNSBL_ZONES"
dnsblResolver = "SWWAF_DNSBL_RESOLVER"
abuseIPDBKey = "SWWAF_ABUSEIPDB_KEY"
abuseIPDBMinScore = "SWWAF_ABUSEIPDB_MIN_SCORE"
abuseIPDBDailyBudget = "SWWAF_ABUSEIPDB_DAILY_BUDGET"
reputationAction = "SWWAF_REPUTATION_ACTION"
reputationCacheTTL = "SWWAF_REPUTATION_CACHE_TTL"
reputationTimeout = "SWWAF_REPUTATION_TIMEOUT"
@@ -1526,6 +1529,76 @@ func TestDNSBLZoneKeyIsLoggedMaskedAndNeverShown(t *testing.T) {
}
}
func TestAbuseIPDBSettingsAsSet(t *testing.T) {
t.Parallel()
cfg := fromEnvironment(t, environment{})
if cfg.AbuseIPDBKey != "" || cfg.AbuseIPDBMinScore != 75 ||
cfg.AbuseIPDBDailyBudget != 900 {
t.Errorf("by default, the key %q, the minimum score %d and the daily budget %d, "+
"want none, 75 and 900", cfg.AbuseIPDBKey, cfg.AbuseIPDBMinScore,
cfg.AbuseIPDBDailyBudget)
}
cfg = fromEnvironment(t, environment{
abuseIPDBKey: token, abuseIPDBMinScore: "0", abuseIPDBDailyBudget: "1",
})
if cfg.AbuseIPDBKey != token || cfg.AbuseIPDBMinScore != 0 ||
cfg.AbuseIPDBDailyBudget != 1 {
t.Errorf("set, the key %q, the minimum score %d and the daily budget %d, "+
"want %s, 0 and 1", cfg.AbuseIPDBKey, cfg.AbuseIPDBMinScore,
cfg.AbuseIPDBDailyBudget, token)
}
}
func TestInvalidAbuseIPDBSettingStopsTheStartSayingWhatIsWrong(t *testing.T) {
t.Parallel()
const (
notScore = " is not a percentage, a whole number from 0 to 100"
notBudget = " is not a whole number above zero, such as 5000"
)
for _, tc := range []struct{ name, value, want string }{
{abuseIPDBMinScore, "101", `"101"` + notScore},
{abuseIPDBMinScore, off, `"off"` + notScore},
{abuseIPDBDailyBudget, "0", `"0"` + notBudget},
{abuseIPDBDailyBudget, off, `"off"` + notBudget},
// The key itself is never shown.
{
abuseIPDBKey, token + "\r",
"holds a control character, such as the carriage return of a Windows line end",
},
} {
t.Run(tc.name+"="+tc.value, func(t *testing.T) {
t.Parallel()
_, err := config.FromEnvironment(environment{tc.name: tc.value}.lookupEnv)
want := tc.name + ": " + tc.want
if err == nil || err.Error() != want {
t.Errorf("error %v, want %s", err, want)
}
})
}
}
func TestAbuseIPDBKeyIsLoggedMasked(t *testing.T) {
t.Parallel()
cfg := fromEnvironment(t, environment{abuseIPDBKey: token})
var out bytes.Buffer
slog.New(slog.NewJSONHandler(&out, nil)).Info("starting", "settings", cfg)
logged := out.String()
if strings.Contains(logged, token) ||
!strings.Contains(logged, `"`+abuseIPDBKey+`":"********"`) {
t.Errorf("the key is not logged masked: %s", logged)
}
}
func TestSizesAndOff(t *testing.T) {
t.Parallel()
@@ -1956,6 +2029,9 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
blocklistAction: actionDeny,
dnsblZones: "",
dnsblResolver: "",
abuseIPDBKey: "",
abuseIPDBMinScore: "75",
abuseIPDBDailyBudget: "900",
reputationAction: "limit:25",
reputationCacheTTL: defaultReputationCacheTTL,
reputationTimeout: "2s",
+62 -36
View File
@@ -257,57 +257,37 @@ func (m *Metrics) AddLookupFile(lastRead func() time.Time, readFailures func() i
)
}
// sourceLabel is the label of the reputation metrics: a list's URL, a
// DNSBL zone, its key masked, or abuseipdb.
const sourceLabel = "source"
// AddReputation adds the metrics of the lists fetched from URLs and of the
// DNSBL zones, by source, each list's URL or each zone, its key masked as
// config.MaskZoneKey masks it: the requests whose client a blocklist or a
// zone's verdict lists, which ReputationHit counts, and, read from lists
// and dnsbl as the metrics are asked for, for a list, the fetches that
// failed and when the copy in use was fetched, and for a zone, the queries
// made and those that failed. It is called once, before ReputationHit.
// config.MaskZoneKey masks it: the requests whose client a blocklist, a
// zone's verdict or AbuseIPDB's score lists, which ReputationHit counts,
// and, read from lists and dnsbl as the metrics are asked for, for a list,
// the fetches that failed and when the copy in use was fetched, and for a
// zone, the queries made and those that failed. It is called once, before
// ReputationHit.
func (m *Metrics) AddReputation(lists *reputation.Lists, dnsbl *reputation.DNSBL) {
const (
sourceLabel = "source"
failuresHelp = "Fetches of the list, or queries to the DNSBL zone, that failed."
)
m.reputationHits = counterVec("smallwebwaf_reputation_hits_total",
"Requests whose client a blocklist or a DNSBL zone lists, by the "+
"blocklist's URL or the zone.",
"Requests whose client a blocklist, a DNSBL zone or AbuseIPDB lists, by "+
"the blocklist's URL, the zone, or abuseipdb.",
[]string{sourceLabel})
m.registry.MustRegister(m.reputationHits)
for _, zone := range dnsbl.Zones() {
source := prometheus.Labels{sourceLabel: config.MaskZoneKey(zone)}
m.registry.MustRegister(
prometheus.NewCounterFunc(prometheus.CounterOpts{
Name: "smallwebwaf_reputation_queries_total",
Help: "Queries to the DNSBL zone.",
ConstLabels: source,
}, func() float64 {
return float64(dnsbl.Queries(zone))
}),
prometheus.NewCounterFunc(prometheus.CounterOpts{
Name: "smallwebwaf_reputation_failures_total",
Help: failuresHelp,
ConstLabels: source,
}, func() float64 {
return float64(dnsbl.Failures(zone))
}),
)
m.addReputationQueries(source, func() int { return dnsbl.Queries(zone) })
m.addReputationFailures(source, func() int { return dnsbl.Failures(zone) })
}
for _, listURL := range lists.URLs() {
source := prometheus.Labels{sourceLabel: listURL}
m.addReputationFailures(source, func() int { return lists.Failures(listURL) })
m.registry.MustRegister(
prometheus.NewCounterFunc(prometheus.CounterOpts{
Name: "smallwebwaf_reputation_failures_total",
Help: failuresHelp,
ConstLabels: source,
}, func() float64 {
return float64(lists.Failures(listURL))
}),
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
Name: "smallwebwaf_reputation_last_fetch_timestamp_seconds",
Help: "When the copy of the list in use was fetched, in seconds since " +
@@ -325,8 +305,28 @@ func (m *Metrics) AddReputation(lists *reputation.Lists, dnsbl *reputation.DNSBL
}
}
// AddAbuseIPDB adds the metrics of AbuseIPDB, with the source abuseipdb,
// read from abuseIPDB as the metrics are asked for: the checks made, those
// that failed, and how many checks the day's budget has left. It is
// called once, after AddReputation, while SWWAF_ABUSEIPDB_KEY is set.
func (m *Metrics) AddAbuseIPDB(abuseIPDB *reputation.AbuseIPDB) {
source := prometheus.Labels{sourceLabel: reputation.AbuseIPDBSource}
m.addReputationQueries(source, abuseIPDB.Checked)
m.addReputationFailures(source, abuseIPDB.Failures)
m.registry.MustRegister(
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
Name: "smallwebwaf_reputation_daily_budget_remaining",
Help: "Checks of the day's SWWAF_ABUSEIPDB_DAILY_BUDGET not yet spent.",
ConstLabels: source,
}, func() float64 {
return float64(abuseIPDB.BudgetLeft())
}),
)
}
// ReputationHit counts a request whose client source lists: a blocklist,
// by its URL, or a DNSBL zone, its key masked.
// by its URL, a DNSBL zone, its key masked, or AbuseIPDB, abuseipdb.
func (m *Metrics) ReputationHit(source string) {
m.reputationHits.WithLabelValues(source).Inc()
}
@@ -470,6 +470,32 @@ func (m *Metrics) StateFileEditSetAside(name string) {
m.stateFileEditsSetAside.WithLabelValues(name).Inc()
}
// addReputationQueries adds the counter of the queries to source, a DNSBL
// zone, or of the checks of clients with AbuseIPDB, which count tells.
func (m *Metrics) addReputationQueries(source prometheus.Labels, count func() int) {
m.registry.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{
Name: "smallwebwaf_reputation_queries_total",
Help: "Queries to the DNSBL zone, or checks of clients with AbuseIPDB.",
ConstLabels: source,
}, func() float64 {
return float64(count())
}))
}
// addReputationFailures adds the counter of the fetches of source, a
// list, the queries to it, a DNSBL zone, or the checks with it, AbuseIPDB,
// that failed, which count tells.
func (m *Metrics) addReputationFailures(source prometheus.Labels, count func() int) {
m.registry.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{
Name: "smallwebwaf_reputation_failures_total",
Help: "Fetches of the list, queries to the DNSBL zone, or checks with " +
"AbuseIPDB, that failed.",
ConstLabels: source,
}, func() float64 {
return float64(count())
}))
}
// statusClass returns the class of status, such as 2xx, or none when no
// status was sent.
func statusClass(status int) string {
+14 -13
View File
@@ -28,18 +28,19 @@ func biasedThresholdsSet(cfg *config.Config) bool {
// limitPercentages returns the client's limit percentages, for the rate
// limits and for the byte limits, by its AS number and country as looked
// up, each "" when unknown, and the blocklists and DNSBL zones that list
// it. Each is the lowest of those the settings give it, the first of them
// in the order below when several are lowest: the percentage
// SWWAF_ASN_LIMIT_PERCENT gives its AS number, the one the file
// up, each "" when unknown, and the blocklists, DNSBL zones and AbuseIPDB
// that list it. Each is the lowest of those the settings give it, the
// first of them in the order below when several are lowest: the
// percentage SWWAF_ASN_LIMIT_PERCENT gives its AS number, the one the file
// SWWAF_ASN_LIMIT_PERCENT_URL names gives it, the one
// SWWAF_COUNTRY_LIMIT_PERCENT gives its country, for a client without a
// country, SWWAF_UNKNOWN_LIMIT_PERCENT, for a client a blocklist lists,
// the percentage of SWWAF_BLOCKLIST_ACTION while it is limit, and for a
// client a DNSBL zone's verdict lists, the percentage of
// SWWAF_REPUTATION_ACTION while it is limit. For the byte limits,
// SWWAF_ASN_BYTES_PERCENT and SWWAF_COUNTRY_BYTES_PERCENT take the place
// of the first three for an AS number or a country they list.
// client a DNSBL zone's verdict lists, or whose AbuseIPDB score is a hit,
// the percentage of SWWAF_REPUTATION_ACTION while it is limit. For the
// byte limits, SWWAF_ASN_BYTES_PERCENT and SWWAF_COUNTRY_BYTES_PERCENT
// take the place of the first three for an AS number or a country they
// list.
func (rq *request) limitPercentages() (percentage, percentage) {
cfg := rq.h.config
asn, country := rq.line.ASN, rq.line.Country
@@ -59,9 +60,9 @@ func (rq *request) limitPercentages() (percentage, percentage) {
blocklisted = percentage{cfg.BlocklistLimitPercent, "SWWAF_BLOCKLIST_ACTION"}
}
dnsblListed := percentage{percent: whole}
if rq.dnsblListed && cfg.ReputationAction == "limit" {
dnsblListed = percentage{cfg.ReputationLimitPercent, "SWWAF_REPUTATION_ACTION"}
reputationListed := percentage{percent: whole}
if (rq.dnsblListed || rq.abuseIPDBHit) && cfg.ReputationAction == "limit" {
reputationListed = percentage{cfg.ReputationLimitPercent, "SWWAF_REPUTATION_ACTION"}
}
asnRequests := lowest(given(cfg.ASNLimitPercent, asn, "SWWAF_ASN_LIMIT_PERCENT"),
@@ -78,8 +79,8 @@ func (rq *request) limitPercentages() (percentage, percentage) {
countryBytes = given(cfg.CountryBytesPercent, country, "SWWAF_COUNTRY_BYTES_PERCENT")
}
return lowest(asnRequests, countryRequests, unknown, blocklisted, dnsblListed),
lowest(asnBytes, countryBytes, unknown, blocklisted, dnsblListed)
return lowest(asnRequests, countryRequests, unknown, blocklisted, reputationListed),
lowest(asnBytes, countryBytes, unknown, blocklisted, reputationListed)
}
// given returns the percentage percents, the setting named setting, gives
+1
View File
@@ -349,6 +349,7 @@ const unansweredGeoJSURL = "unanswered://geojs/v1/ip/geo.json"
func TestMain(m *testing.M) {
transport, _ := http.DefaultTransport.(*http.Transport)
transport.RegisterProtocol("unanswered", unansweredGeoJS{})
transport.RegisterProtocol("abuseipdb", abuseIPDBStandIn{})
m.Run()
}
+33 -10
View File
@@ -59,6 +59,9 @@ type Params struct {
// GeoJSURL is where clients' AS numbers and countries are looked up
// while SWWAF_LOOKUP_SOURCE is geojs, normally lookup.URL.
GeoJSURL string
// AbuseIPDBURL is where clients are checked with AbuseIPDB while
// SWWAF_ABUSEIPDB_KEY is set, normally reputation.AbuseIPDBURL.
AbuseIPDBURL string
// LookupFile is the lookup database they are looked up in while
// SWWAF_LOOKUP_SOURCE is file, and nil otherwise.
LookupFile *lookup.File
@@ -71,15 +74,17 @@ type Params struct {
Rules *rules.Files
// Alerts receive the alert for each ban the proxy makes or makes
// permanent, for each count over an anomaly threshold, for each request
// whose client a blocklist or a DNSBL zone lists, and for GeoJS failing,
// a fetch of a list failing or a query to a DNSBL zone failing.
// whose client a blocklist, a DNSBL zone or AbuseIPDB lists, and for
// GeoJS failing, a fetch of a list failing, a query to a DNSBL zone or
// a check with AbuseIPDB failing, or the day's AbuseIPDB checks used up.
Alerts *alerts.Queue
}
// Server is the server smallwebwaf runs, with the parts of the proxy
// whose state the state files keep, the lookup database, nil unless
// SWWAF_LOOKUP_SOURCE is file, the lists fetched from URLs, which its Run
// fetches, the DNSBL zones' verdicts, and the metrics.
// fetches, the DNSBL zones' verdicts, AbuseIPDB's scores and checks
// spent, and the metrics.
type Server struct {
*http.Server
@@ -90,6 +95,7 @@ type Server struct {
LookupFile *lookup.File
Lists *reputation.Lists
DNSBL *reputation.DNSBL
AbuseIPDB *reputation.AbuseIPDB
Metrics *metrics.Metrics
}
@@ -102,7 +108,7 @@ type Server struct {
func New(params Params) *Server {
errorLog := slog.NewLogLogger(params.ProcessLog.Handler(), slog.LevelWarn)
m := metrics.New(params.Config.MetricsTopN, params.Config.InstanceName)
lists, dnsbl := newReputation(params)
lists, dnsbl, abuseIPDB := newReputation(params, m)
h := &handler{
config: params.Config,
requestLog: params.RequestLog,
@@ -140,6 +146,7 @@ func New(params Params) *Server {
lookupFile: params.LookupFile,
lists: lists,
dnsbl: dnsbl,
abuseIPDB: abuseIPDB,
rules: params.Rules,
alerts: params.Alerts,
}
@@ -159,7 +166,6 @@ func New(params Params) *Server {
})
m.AddBansAndClients(h.ledger, h.limiter, params.Now)
m.AddRules(params.Rules)
m.AddReputation(h.lists, h.dnsbl)
return &Server{
Server: &http.Server{
@@ -181,14 +187,18 @@ func New(params Params) *Server {
LookupFile: h.lookupFile,
Lists: h.lists,
DNSBL: h.dnsbl,
AbuseIPDB: h.abuseIPDB,
Metrics: m,
}
}
// newReputation returns the lists fetched from URLs and the DNSBL zones'
// verdicts, as the settings in params name them, with none fetched or
// asked for yet.
func newReputation(params Params) (*reputation.Lists, *reputation.DNSBL) {
// newReputation returns the lists fetched from URLs, the DNSBL zones'
// verdicts and AbuseIPDB's scores, as the settings in params name them,
// with none fetched, asked for or checked yet, and adds their metrics to
// m, AbuseIPDB's while SWWAF_ABUSEIPDB_KEY is set.
func newReputation(
params Params, m *metrics.Metrics,
) (*reputation.Lists, *reputation.DNSBL, *reputation.AbuseIPDB) {
cfg := params.Config
lists := reputation.New(reputation.Params{
BlocklistURLs: cfg.BlocklistURLs, Refresh: cfg.BlocklistRefresh,
@@ -200,8 +210,20 @@ func newReputation(params Params) (*reputation.Lists, *reputation.DNSBL) {
Timeout: cfg.ReputationTimeout, Now: params.Now, ProcessLog: params.ProcessLog,
Alerts: params.Alerts,
})
abuseIPDB := reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
URL: params.AbuseIPDBURL, Key: cfg.AbuseIPDBKey, MinScore: cfg.AbuseIPDBMinScore,
DailyBudget: cfg.AbuseIPDBDailyBudget, CacheTTL: cfg.ReputationCacheTTL,
Timeout: cfg.ReputationTimeout, Now: params.Now, ProcessLog: params.ProcessLog,
Alerts: params.Alerts,
})
return lists, dnsbl
m.AddReputation(lists, dnsbl)
if cfg.AbuseIPDBKey != "" {
m.AddAbuseIPDB(abuseIPDB)
}
return lists, dnsbl, abuseIPDB
}
// handler is the proxy. It holds what every request shares; what belongs
@@ -221,6 +243,7 @@ type handler struct {
lookupFile *lookup.File
lists *reputation.Lists
dnsbl *reputation.DNSBL
abuseIPDB *reputation.AbuseIPDB
rules *rules.Files
alerts *alerts.Queue
}
+11 -9
View File
@@ -268,7 +268,8 @@ func startProxyWithAlerts(
// queue: they wait in it, for the test to look at. With no geojsURL, there
// is no stand-in for GeoJS to look clients up at, and SWWAF_LOOKUP_SOURCE
// is off unless env sets it. While it is file, the lookup database
// SWWAF_LOOKUP_DB_PATH names is read.
// SWWAF_LOOKUP_DB_PATH names is read. Clients are checked with AbuseIPDB
// at abuseIPDBURL while env sets SWWAF_ABUSEIPDB_KEY.
func newProxy(
t *testing.T, appURL, geojsURL string, now func() time.Time,
env map[string]string,
@@ -325,14 +326,15 @@ func newProxy(
}
server := proxy.New(proxy.Params{
Config: cfg,
RequestLog: out,
ProcessLog: processLog,
GeoJSURL: geojsURL,
LookupFile: lookupFile,
Now: now,
Rules: ruleFiles,
Alerts: alertQueue,
Config: cfg,
RequestLog: out,
ProcessLog: processLog,
GeoJSURL: geojsURL,
AbuseIPDBURL: abuseIPDBURL,
LookupFile: lookupFile,
Now: now,
Rules: ruleFiles,
Alerts: alertQueue,
})
return server, out, alertQueue
+59 -19
View File
@@ -4,8 +4,14 @@ import (
"context"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/reputation"
)
// deny is the SWWAF_BLOCKLIST_ACTION and the SWWAF_REPUTATION_ACTION that
// refuses the requests of a client a source lists.
const deny = "deny"
// blocklistDenied notes the blocklists that list the client, as
// noteListed does, and reports whether SWWAF_BLOCKLIST_ACTION, being deny,
// refuses the request. Being limit, it lowers the client's limits instead
@@ -15,7 +21,7 @@ func (rq *request) blocklistDenied() bool {
rq.blocklisted = len(listedBy) > 0
rq.noteListed(listedBy, "listed by a blocklist")
return rq.blocklisted && rq.h.config.BlocklistAction == "deny"
return rq.blocklisted && rq.h.config.BlocklistAction == deny
}
// dnsblDenied notes the DNSBL zones whose verdict lists the client, as
@@ -30,27 +36,61 @@ func (rq *request) dnsblDenied(ctx context.Context) bool {
rq.dnsblListed = len(listedBy) > 0
rq.noteListed(listedBy, "listed by a DNSBL zone")
return rq.dnsblListed && rq.h.config.ReputationAction == "deny"
return rq.dnsblListed && rq.h.config.ReputationAction == deny
}
// noteListed adds sources, the URLs of the blocklists or the DNSBL zones,
// their keys masked, that list the client, to the log line's reputation,
// counts each of them in the metrics, and raises a reputation_hit alert,
// with reason, for each.
func (rq *request) noteListed(sources []string, reason string) {
rq.line.Reputation = append(rq.line.Reputation, sources...)
// abuseIPDBDenied notes AbuseIPDB, as noteHit does, with the score, when
// its score of the client is a hit, and reports whether
// SWWAF_REPUTATION_ACTION, being deny, refuses the request, as dnsblDenied
// does for a zone. While SWWAF_ABUSEIPDB_KEY is unset it does nothing. A
// client without a score is checked in the background if its history
// counts an offence, and the request does not wait for the answer. ctx is
// the request's own context.
func (rq *request) abuseIPDBDenied(ctx context.Context) bool {
if rq.h.config.AbuseIPDBKey == "" {
return false
}
held, _ := rq.h.limiter.Client(clientGroup(rq.client))
offender := held.History.Offences != ratelimit.Offences{}
score, hit := rq.h.abuseIPDB.Hit(ctx, rq.client, offender)
if !hit {
return false
}
rq.abuseIPDBHit = true
rq.noteHit(reputation.AbuseIPDBSource, "scored by AbuseIPDB at or over "+
"SWWAF_ABUSEIPDB_MIN_SCORE", map[string]any{
"source": reputation.AbuseIPDBSource, "score": score,
})
return rq.h.config.ReputationAction == deny
}
// noteListed notes each of sources, the URLs of the blocklists or the
// DNSBL zones, their keys masked, that list the client, as noteHit does,
// with reason, and the source in the alert's detail.
func (rq *request) noteListed(sources []string, reason string) {
for _, source := range sources {
rq.h.metrics.ReputationHit(source)
rq.h.alerts.Raise(alerts.Alert{
Event: alerts.EventReputationHit,
Client: rq.client,
Netblock: clientGroup(rq.client),
ASN: rq.line.ASN,
ASName: rq.line.ASName,
Country: rq.line.Country,
Reason: reason,
Detail: map[string]any{"source": source},
})
rq.noteHit(source, reason, map[string]any{"source": source})
}
}
// noteHit adds source, which lists the client, to the log line's
// reputation, counts it in the metrics, and raises a reputation_hit alert
// with reason and detail.
func (rq *request) noteHit(source, reason string, detail map[string]any) {
rq.line.Reputation = append(rq.line.Reputation, source)
rq.h.metrics.ReputationHit(source)
rq.h.alerts.Raise(alerts.Alert{
Event: alerts.EventReputationHit,
Client: rq.client,
Netblock: clientGroup(rq.client),
ASN: rq.line.ASN,
ASName: rq.line.ASName,
Country: rq.line.Country,
Reason: reason,
Detail: detail,
})
}
+211
View File
@@ -1,6 +1,7 @@
package proxy_test
import (
"io"
"maps"
"net/http"
"net/netip"
@@ -599,6 +600,193 @@ func TestRequestFromAClientWithoutAVerdictHasTheZoneAskedAboutIt(t *testing.T) {
}
}
// The AbuseIPDB settings, and accountKey, the key the tests set.
const (
abuseIPDBKey = "SWWAF_ABUSEIPDB_KEY"
accountKey = "abuseipdb-key-0123456789abcdef"
)
// abuseipdb is how the request log, the alerts and the metrics name
// AbuseIPDB.
const abuseipdb = reputation.AbuseIPDBSource
// abuseIPDBURL is where newProxy has clients checked with AbuseIPDB: at
// abuseIPDBStandIn, which TestMain registers with Go's default transport,
// through which AbuseIPDB is asked.
const abuseIPDBURL = "abuseipdb://stand-in/api/v2/check"
// abuseIPDBStandIn is a stand-in for AbuseIPDB that gives every client the
// score 100, at once and without the network.
type abuseIPDBStandIn struct{}
// RoundTrip answers req with the score 100.
func (abuseIPDBStandIn) RoundTrip(req *http.Request) (*http.Response, error) {
return &http.Response{
StatusCode: http.StatusOK,
Status: "200 OK",
Header: http.Header{},
Body: io.NopCloser(strings.NewReader(`{"data":{"abuseConfidenceScore":100}}`)),
Request: req,
}, nil
}
func TestOnlyAClientThatHasCommittedAnOffenceIsCheckedWithAbuseIPDB(t *testing.T) {
t.Parallel()
forward := requestlog.ActionForward
s, clk, server, _ := startWithLookupsAndClock(t, map[string]string{
abuseIPDBKey: accountKey, rateLimitPerMinute: "2", reputationAction: actionLog,
})
// Neither fromDE, until it breaks a rate limit, nor fromKP, which never
// does, is checked, nor fromDE under the ban that makes.
s.get(fromDE, http.StatusOK, forward)
s.get(fromDE, http.StatusOK, forward)
s.get(fromKP, http.StatusOK, forward)
s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited)
s.get(fromDE, http.StatusForbidden, requestlog.ActionBanned)
wantAbuseIPDBChecks(t, server, 0)
// Once the ban has ended, fromDE's first request has it checked in the
// background, and goes on without its score, which its next request
// finds.
clk.advance(time.Hour)
wantReputation(t, s.get(fromDE, http.StatusOK, forward))
wantAbuseIPDBChecks(t, server, 1)
waitUntil(func() bool { return len(server.AbuseIPDB.Snapshot().Scores) == 1 })
wantReputation(t, s.get(fromDE, http.StatusOK, forward), abuseipdb)
s.get(fromKP, http.StatusOK, forward)
wantAbuseIPDBChecks(t, server, 1)
}
func TestEachReputationActionForAClientAbuseIPDBScoresAtOrOverTheMinimum(t *testing.T) {
t.Parallel()
forward, denied := requestlog.ActionForward, requestlog.ActionDenied
for _, tc := range []struct {
action string
// statuses and actions are those of fromDE's three requests, and
// percent their limit_percent, as percentText gives it.
statuses []int
actions []string
percent string
}{
{
actionDeny, []int{http.StatusForbidden, http.StatusForbidden, http.StatusForbidden},
[]string{denied, denied, denied}, none,
},
{
// Half of 4 requests a minute: the third breaks the limit.
limitHalf, []int{http.StatusOK, http.StatusOK, http.StatusForbidden},
[]string{forward, forward, requestlog.ActionRateLimited},
"50 from " + reputationAction,
},
{
actionLog, []int{http.StatusOK, http.StatusOK, http.StatusOK},
[]string{forward, forward, forward}, none,
},
} {
t.Run(tc.action, func(t *testing.T) {
t.Parallel()
s, server, _ := startWithLookups(t, map[string]string{
rateLimitPerMinute: fourAMinute, abuseIPDBKey: accountKey,
reputationAction: tc.action,
})
// At SWWAF_ABUSEIPDB_MIN_SCORE, 75 by default, and just under it.
loadScores(server, map[string]int64{fromDE: 75, fromKP: 74})
for i := range 3 {
line := s.get(fromDE, tc.statuses[i], tc.actions[i])
wantReputation(t, line, abuseipdb)
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
tc.percent)
// A request refused for the score is not counted.
counted := line.fields["counts"] != nil
if counted != (tc.actions[i] != denied) {
t.Errorf("request counted %t, logged %s", counted, tc.actions[i])
}
}
// fromKP's score is no hit, and it has the whole limit.
for range 3 {
line := s.get(fromKP, http.StatusOK, forward)
wantReputation(t, line)
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
none)
}
// A refusal for the score makes no ban.
if held := server.Ledger.Snapshot(); tc.action == actionDeny && len(held) != 0 {
t.Errorf("bans %+v, want none", held)
}
})
}
}
func TestAbuseIPDBHitRaisesAnAlertWithTheScoreOncePerCooldownAndIsCounted(
t *testing.T,
) {
t.Parallel()
s, server, queue := startWithLookups(t, map[string]string{
abuseIPDBKey: accountKey, reputationAction: actionLog, metricsToken: token,
})
loadScores(server, map[string]int64{fromDE: 90})
// The second request's alert is a repeat, which the cooldown holds back.
for range 2 {
wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward),
abuseipdb)
}
// The alert is made as a DNSBL zone's is, with the score besides.
waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
if len(waiting) != 1 || waiting[0].Event != alerts.EventReputationHit ||
waiting[0].Reason != "scored by AbuseIPDB at or over SWWAF_ABUSEIPDB_MIN_SCORE" ||
waiting[0].Detail["source"] != abuseipdb || waiting[0].Detail["score"] != int64(90) ||
queue.Suppressed() != 1 {
t.Errorf("alerts waiting %+v, %d held back, want AbuseIPDB's reputation_hit "+
"with the score 90, and 1", waiting, queue.Suppressed())
}
// The hits, and the checks, none, since no client committed an
// offence, so that the whole budget is left.
metrics := s.scrape(unplaced)
labels := `{instance="` + alertInstance + `",source="` + abuseipdb + `"}`
wantMetric(t, metrics, "smallwebwaf_reputation_hits_total"+labels, 2)
wantMetric(t, metrics, "smallwebwaf_reputation_queries_total"+labels, 0)
wantMetric(t, metrics, "smallwebwaf_reputation_failures_total"+labels, 0)
wantMetric(t, metrics, "smallwebwaf_reputation_daily_budget_remaining"+labels, 900)
}
func TestWithoutAnAbuseIPDBKeyNoClientIsCheckedNorAScoreUsed(t *testing.T) {
t.Parallel()
forward := requestlog.ActionForward
s, clk, server, _ := startWithLookupsAndClock(t, map[string]string{
rateLimitPerMinute: "1", metricsToken: token,
})
loadScores(server, map[string]int64{fromDE: 100})
// fromDE's score is not used, and once it has committed an offence it
// is not checked either.
wantReputation(t, s.get(fromDE, http.StatusOK, forward))
s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited)
clk.advance(time.Hour)
wantReputation(t, s.get(fromDE, http.StatusOK, forward))
wantAbuseIPDBChecks(t, server, 0)
wantNoSeries(t, s.scrape(unplaced), `smallwebwaf_reputation_daily_budget_remaining{`+
`instance="`+alertInstance+`",source="`+abuseipdb+`"}`)
}
// listsFetched is when loadLists has the copies fetched.
func listsFetched() time.Time {
return time.Date(2026, 10, 5, 0, 0, 0, 0, time.UTC)
@@ -631,6 +819,29 @@ func loadVerdicts(server *proxy.Server, listedBy map[string][]string) {
server.DNSBL.Load(verdicts)
}
// loadScores puts into server's AbuseIPDB the score scores gives each
// client, fetched at verdictsFetched, as reputation.json would at start.
func loadScores(server *proxy.Server, scores map[string]int64) {
kept := make([]reputation.Score, 0, len(scores))
for client, score := range scores {
kept = append(kept, reputation.Score{
Client: netip.MustParseAddr(client), Score: score, Fetched: verdictsFetched(),
})
}
server.AbuseIPDB.Load(reputation.Checks{Scores: kept})
}
// wantAbuseIPDBChecks checks how many clients server has checked with
// AbuseIPDB.
func wantAbuseIPDBChecks(t *testing.T, server *proxy.Server, want int) {
t.Helper()
if got := server.AbuseIPDB.Checked(); got != want {
t.Errorf("%d clients checked with AbuseIPDB, want %d", got, want)
}
}
// loadLists puts copies of lists into server's lists, by URL, each with
// its lines, fetched at listsFetched, as reputation.json would at start.
func loadLists(t *testing.T, server *proxy.Server, copies map[string][]string) {
+13 -11
View File
@@ -64,9 +64,10 @@ type request struct {
counted bool
limitPercent, bytesPercent percentage
// blocklisted is true once a blocklist is found to list the client,
// and dnsblListed once a DNSBL zone's verdict is.
blocklisted, dnsblListed bool
start time.Time
// dnsblListed once a DNSBL zone's verdict is, and abuseIPDBHit once
// AbuseIPDB's score of it is a hit.
blocklisted, dnsblListed, abuseIPDBHit bool
start time.Time
// checked is when the checks were done, and upstreamStart when the
// request was handed to the app.
checked time.Time
@@ -217,13 +218,14 @@ func (rq *request) check(ctx context.Context) *refusal {
// other client, SWWAF_DENY_NETS comes first, then a ban on its netblock,
// so that a client either refuses is not looked up, then the lookup of
// its AS number and country, then the country lists, then the blocklists,
// and then the DNSBL zones' verdicts; a request any of them refuses is not
// counted for the rate limits. Then come the rate limits, unless the
// client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the request's path is
// exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that every other request
// is counted, each of them by the client's limit percentages, and last the
// rule files. A request exempt from the rate limits is exempt from the
// byte limits too. ctx is the request's own context.
// then the DNSBL zones' verdicts, and then AbuseIPDB's score; a request
// any of them refuses is not counted for the rate limits. Then come the
// rate limits, unless the client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the
// request's path is exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that
// every other request is counted, each of them by the client's limit
// percentages, and last the rule files. A request exempt from the rate
// limits is exempt from the byte limits too. ctx is the request's own
// context.
func (rq *request) checkClient(ctx context.Context) string {
cfg := rq.h.config
if isInside(rq.client, cfg.AllowNets) {
@@ -250,7 +252,7 @@ func (rq *request) checkClient(ctx context.Context) string {
return requestlog.ActionDenied
}
if rq.dnsblDenied(ctx) {
if rq.dnsblDenied(ctx) || rq.abuseIPDBDenied(ctx) {
return requestlog.ActionDenied
}
+326
View File
@@ -0,0 +1,326 @@
package reputation
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/netip"
"net/url"
"slices"
"sync"
"time"
"github.com/hashicorp/golang-lru/v2/simplelru"
"sneak.berlin/go/smallwebwaf/internal/alerts"
)
const (
// AbuseIPDBURL is where clients are checked: the check endpoint of
// AbuseIPDB's API.
AbuseIPDBURL = "https://api.abuseipdb.com/api/v2/check"
// AbuseIPDBSource is how the request log, the alerts and the metrics
// name AbuseIPDB.
AbuseIPDBSource = "abuseipdb"
// maxAnswerBytes is the most of an answer of AbuseIPDB that is read.
maxAnswerBytes = 64 << 10
// day is the length of the day the checks are counted in, in UTC.
day = 24 * time.Hour
)
var (
errNoScore = errors.New("the answer gives no abuseConfidenceScore")
errBudgetUsedUp = errors.New(
"checks spent; none is made until the day ends at 00:00 UTC")
)
// Score is what AbuseIPDB said about a client, as reputation.json holds
// it: the client's address, its abuse confidence score, from 0 to 100,
// and when AbuseIPDB answered.
type Score struct {
Client netip.Addr `json:"client"`
Score int64 `json:"score"`
Fetched time.Time `json:"fetched"`
}
// Checks are what reputation.json keeps of the checks of clients with
// AbuseIPDB: the day, in UTC, of the checks Spent counts, zero before the
// first, and the scores still in use.
type Checks struct {
Day time.Time `json:"day,omitzero"`
Spent int `json:"spent"`
Scores []Score `json:"scores"`
}
// AbuseIPDBParams are what NewAbuseIPDB needs.
type AbuseIPDBParams struct {
// URL is where clients are checked, normally AbuseIPDBURL, with Key,
// the account's key (SWWAF_ABUSEIPDB_KEY).
URL string
Key string
// MinScore is the least score that is a hit (SWWAF_ABUSEIPDB_MIN_SCORE),
// and DailyBudget the most checks made in a day, in UTC
// (SWWAF_ABUSEIPDB_DAILY_BUDGET).
MinScore int64
DailyBudget int
// CacheTTL is how long a score is used after it was fetched
// (SWWAF_REPUTATION_CACHE_TTL), and Timeout how long a check may take
// (SWWAF_REPUTATION_TIMEOUT).
CacheTTL time.Duration
Timeout time.Duration
// Now tells the time, normally time.Now in UTC.
Now func() time.Time
// ProcessLog receives each check that fails, and why, and the day's
// budget used up.
ProcessLog *slog.Logger
// Alerts receive a source_failure alert for each.
Alerts *alerts.Queue
}
// AbuseIPDB checks clients with AbuseIPDB, in the background, and keeps
// their scores. It is safe for concurrent use.
type AbuseIPDB struct {
params AbuseIPDBParams
httpClient *http.Client
mu sync.Mutex
// scores are by client. Each is added as it is fetched and never moved
// up, so that the one fetched longest ago is the first dropped.
scores *simplelru.LRU[netip.Addr, Score]
// checking are the clients whose check is under way.
checking map[netip.Addr]bool
// day is the day, in UTC, of the checks spent counts.
day time.Time
spent int
// checks and failures count the checks made and those that failed,
// and retryAt is when a client may be checked again after the last
// check failed.
checks int
failures int
retryAt time.Time
}
// NewAbuseIPDB returns an AbuseIPDB with no score yet, and no check spent.
func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
scores, err := simplelru.NewLRU[netip.Addr, Score](maxVerdicts, nil)
if err != nil {
panic(err) // NewLRU fails only for a size below one
}
return &AbuseIPDB{
params: params,
httpClient: &http.Client{},
scores: scores,
checking: map[netip.Addr]bool{},
}
}
// Hit returns AbuseIPDB's score of addr, a client's address, and whether
// it is a hit: MinScore or more. A score is used until CacheTTL has passed
// since it was fetched. A client without one is checked in the
// background if offender, if it has committed an offence, unless its
// check is under way, a check failed less than failureDelay ago, or the
// day's checks have used up DailyBudget; Hit never waits for a check. The
// check that uses the budget up is logged and raised as a source_failure
// alert. ctx is the context of the client's request, and a check goes on
// after the request ends.
func (a *AbuseIPDB) Hit(
ctx context.Context, addr netip.Addr, offender bool,
) (int64, bool) {
a.mu.Lock()
now := a.params.Now()
kept, found := a.scores.Peek(addr)
if found && now.Sub(kept.Fetched) < a.params.CacheTTL {
a.mu.Unlock()
return kept.Score, kept.Score >= a.params.MinScore
}
if today := now.Truncate(day); !a.day.Equal(today) {
a.day, a.spent = today, 0
}
check := offender && !a.checking[addr] && !now.Before(a.retryAt) &&
a.spent < a.params.DailyBudget
if check {
a.checking[addr] = true
a.checks++
a.spent++
go a.check(context.WithoutCancel(ctx), addr)
}
usedUp := check && a.spent == a.params.DailyBudget
a.mu.Unlock()
if usedUp {
a.alert("the daily budget of AbuseIPDB checks is used up",
fmt.Errorf("%d %w", a.params.DailyBudget, errBudgetUsedUp))
}
return 0, false
}
// Checked returns how many checks were made.
func (a *AbuseIPDB) Checked() int {
a.mu.Lock()
defer a.mu.Unlock()
return a.checks
}
// Failures returns how many checks failed.
func (a *AbuseIPDB) Failures() int {
a.mu.Lock()
defer a.mu.Unlock()
return a.failures
}
// BudgetLeft returns how many checks the day's budget has left.
func (a *AbuseIPDB) BudgetLeft() int {
a.mu.Lock()
defer a.mu.Unlock()
if !a.day.Equal(a.params.Now().Truncate(day)) {
return a.params.DailyBudget
}
return max(a.params.DailyBudget-a.spent, 0)
}
// Snapshot returns the checks spent and every score still in use, sorted
// by client, as reputation.json keeps them.
func (a *AbuseIPDB) Snapshot() Checks {
a.mu.Lock()
now := a.params.Now()
checks := Checks{Day: a.day, Spent: a.spent, Scores: make([]Score, 0, a.scores.Len())}
for _, kept := range a.scores.Values() {
if now.Sub(kept.Fetched) < a.params.CacheTTL {
checks.Scores = append(checks.Scores, kept)
}
}
a.mu.Unlock()
slices.SortFunc(checks.Scores, func(x, y Score) int {
return x.Client.Compare(y.Client)
})
return checks
}
// Load keeps checks, read from reputation.json, in place of those it
// keeps, but for the scores past maxVerdicts, those fetched longest ago.
// One fetched CacheTTL ago or more is neither used nor written, as for any
// score.
func (a *AbuseIPDB) Load(checks Checks) {
scores := slices.Clone(checks.Scores)
slices.SortStableFunc(scores, func(x, y Score) int {
return x.Fetched.Compare(y.Fetched)
})
a.mu.Lock()
defer a.mu.Unlock()
a.day, a.spent = checks.Day, checks.Spent
a.scores.Purge()
for _, kept := range scores {
a.scores.Add(kept.Client, kept)
}
}
// check checks addr with AbuseIPDB, keeps the score, and notes the check
// as no longer under way. A check that fails gives no score: it is
// counted, logged and raised as a source_failure alert, and no client is
// checked for failureDelay.
func (a *AbuseIPDB) check(ctx context.Context, addr netip.Addr) {
score, err := a.ask(ctx, addr)
now := a.params.Now()
a.mu.Lock()
delete(a.checking, addr)
if err == nil {
a.scores.Add(addr, Score{Client: addr, Score: score, Fetched: now})
} else {
a.failures++
a.retryAt = now.Add(failureDelay)
}
a.mu.Unlock()
if err != nil {
a.alert("checking a client with AbuseIPDB failed", err)
}
}
// ask asks AbuseIPDB for addr's abuse confidence score, sending the key
// in the header Key. An answer other than 200, one that gives no score,
// and none within Timeout, fail.
func (a *AbuseIPDB) ask(ctx context.Context, addr netip.Addr) (int64, error) {
ctx, cancel := context.WithTimeout(ctx, a.params.Timeout)
defer cancel()
query := url.Values{"ipAddress": {addr.String()}}
req, err := http.NewRequestWithContext(ctx, http.MethodGet,
a.params.URL+"?"+query.Encode(), http.NoBody)
if err != nil {
return 0, fmt.Errorf("make the request: %w", err)
}
req.Header.Set("Key", a.params.Key)
req.Header.Set("Accept", "application/json")
res, err := a.httpClient.Do(req)
if err != nil {
// Do's error names the URL, which holds the client's address, which
// is not to be logged: only what went wrong is kept.
return 0, fmt.Errorf("check the client: %w", errors.Unwrap(err))
}
defer func() {
_ = res.Body.Close()
}()
if res.StatusCode != http.StatusOK {
return 0, fmt.Errorf("%w %s", errStatus, res.Status)
}
var answer struct {
Data struct {
AbuseConfidenceScore *int64 `json:"abuseConfidenceScore"`
} `json:"data"`
}
err = json.NewDecoder(io.LimitReader(res.Body, maxAnswerBytes)).Decode(&answer)
if err != nil {
return 0, fmt.Errorf("read the answer: %w", err)
}
if answer.Data.AbuseConfidenceScore == nil {
return 0, errNoScore
}
return *answer.Data.AbuseConfidenceScore, nil
}
// alert raises a source_failure alert from AbuseIPDB with reason and err,
// and logs them.
func (a *AbuseIPDB) alert(reason string, err error) {
// Raised before it is logged, so that the alert is there once the log
// line is.
raiseFailure(a.params.Alerts, reason, AbuseIPDBSource, err)
a.params.ProcessLog.Warn(reason, "source", AbuseIPDBSource, "error", err.Error())
}
+610
View File
@@ -0,0 +1,610 @@
package reputation_test
import (
"bytes"
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"net/netip"
"reflect"
"slices"
"strings"
"sync"
"testing"
"testing/synctest"
"time"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/reputation"
)
// The tests of AbuseIPDB run in synctest bubbles, as those of the lists
// do, and AbuseIPDB is a stand-in reached without the network, for the
// same reason. A bubble's clock starts at midnight UTC, as a day the
// checks are counted in starts.
const (
// key is the account's key the tests give, the only one the stand-in
// takes.
key = "abuseipdb-key-0123456789abcdef"
// suspect and other are clients that have committed an offence.
suspect = "203.0.113.9"
other = "2001:db8::9"
)
func TestOnlyAnOffenderWithoutAScoreIsChecked(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
abuseIPDB := &abuseIPDBStandIn{scores: map[string]int64{suspect: 100}}
checker := newAbuseIPDB(abuseIPDB, abuseIPDBParams())
// A client that has committed no offence is not checked.
wantScore(t, checker, suspect, false, 0, false)
synctest.Wait()
wantChecked(t, abuseIPDB)
// An offender is, and from then on its score is used, whether or not
// it is an offender.
wantScore(t, checker, suspect, true, 0, false)
synctest.Wait()
wantScore(t, checker, suspect, true, 100, true)
wantScore(t, checker, suspect, false, 100, true)
synctest.Wait()
wantChecked(t, abuseIPDB, suspect)
})
}
func TestScoreAtOrOverTheMinimumIsAHit(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
scores := map[string]int64{"192.0.2.74": 74, "192.0.2.75": 75, "192.0.2.100": 100}
p := abuseIPDBParams()
p.MinScore = 75
checker := newAbuseIPDB(&abuseIPDBStandIn{scores: scores}, p)
for client := range scores {
checker.Hit(t.Context(), netip.MustParseAddr(client), true)
}
synctest.Wait()
for client, score := range scores {
wantScore(t, checker, client, true, score, score >= 75)
}
})
}
func TestScoreUsedUntilTheCacheTTLHasPassedSinceItWasFetched(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
abuseIPDB := &abuseIPDBStandIn{scores: map[string]int64{suspect: 100}}
checker := newAbuseIPDB(abuseIPDB, abuseIPDBParams())
checker.Hit(t.Context(), netip.MustParseAddr(suspect), true)
synctest.Wait()
// AbuseIPDB gives another score from now on, but the one kept is
// used, and the client is not checked again, until the TTL has
// passed.
abuseIPDB.setScore(suspect, 80)
time.Sleep(cacheTTL - time.Nanosecond)
wantScore(t, checker, suspect, true, 100, true)
synctest.Wait()
wantChecked(t, abuseIPDB, suspect)
// Then it is not used, and the client is checked again.
time.Sleep(time.Nanosecond)
wantScore(t, checker, suspect, true, 0, false)
synctest.Wait()
wantScore(t, checker, suspect, true, 80, true)
wantChecked(t, abuseIPDB, suspect, suspect)
})
}
func TestDailyBudgetKeptAcrossARestartAndWholeAgainAsTheDayEnds(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
var log bytes.Buffer
queue := newQueue()
p := abuseIPDBParams()
p.DailyBudget = 3
p.Alerts = queue
p.ProcessLog = slog.New(slog.NewJSONHandler(&log, nil))
abuseIPDB := &abuseIPDBStandIn{scores: map[string]int64{suspect: 100}}
checker := newAbuseIPDB(abuseIPDB, p)
// At noon, the first three offenders spend the budget, and the
// fourth, unchecked, is not.
time.Sleep(12 * time.Hour)
const unchecked = "192.0.2.4"
clients := []string{suspect, "192.0.2.2", "192.0.2.3", unchecked}
for _, client := range clients {
checker.Hit(t.Context(), netip.MustParseAddr(client), true)
}
synctest.Wait()
wantChecked(t, abuseIPDB, clients[:3]...)
wantBudgetLeft(t, checker, 0)
// The check that used the budget up raised the alert, and logged it.
const usedUp = "the daily budget of AbuseIPDB checks is used up"
wantFailureAlert(t, queue, time.Now(), usedUp,
"3 checks spent; none is made until the day ends at 00:00 UTC", 0)
if !strings.Contains(log.String(), `"msg":"`+usedUp+`"`) {
t.Errorf("logged\n%s\nwant the budget used up", log.String())
}
// Restarted with what reputation.json keeps, it uses the scores, and
// checks no client until the day ends.
restarted := &abuseIPDBStandIn{}
again := newAbuseIPDB(restarted, p)
again.Load(checker.Snapshot())
wantScore(t, again, suspect, true, 100, true)
wantBudgetLeft(t, again, 0)
time.Sleep(12*time.Hour - time.Nanosecond)
wantScore(t, again, unchecked, true, 0, false)
synctest.Wait()
wantChecked(t, restarted)
// At midnight the budget is whole again.
time.Sleep(time.Nanosecond)
wantBudgetLeft(t, again, 3)
wantScore(t, again, unchecked, true, 0, false)
synctest.Wait()
wantChecked(t, restarted, unchecked)
wantBudgetLeft(t, again, 2)
})
}
func TestFailedCheckGivesNoScoreAndNoClientIsCheckedForAMinute(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
name string
// key is the key sent, status and body what AbuseIPDB answers with,
// and error the failure.
key, body string
status int
error string
}{
{
"a refusal, past AbuseIPDB's own limit", key,
`{"errors":[{"detail":"Daily rate limit of 1000 requests exceeded"}]}`,
http.StatusTooManyRequests, "the server answered 429 Too Many Requests",
},
{
"a refusal of a wrong key", "wrong-key-0123456789abcdef", "", 0,
"the server answered 401 Unauthorized",
},
{
"a server failure", key, "", http.StatusInternalServerError,
"the server answered 500 Internal Server Error",
},
{
"an answer without a score", key, `{"data":{"ipAddress":"` + suspect + `"}}`,
http.StatusOK, "the answer gives no abuseConfidenceScore",
},
{
"an answer that is not JSON", key, "<html>", http.StatusOK,
"read the answer: invalid character '<' looking for beginning of value",
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
var log bytes.Buffer
queue := newQueue()
p := abuseIPDBParams()
p.Key = tc.key
p.Alerts = queue
p.ProcessLog = slog.New(slog.NewJSONHandler(&log, nil))
abuseIPDB := &abuseIPDBStandIn{status: tc.status, body: tc.body}
checker := newAbuseIPDB(abuseIPDB, p)
// The failure gives no score, and no client is checked within a
// minute of it.
wantScore(t, checker, suspect, true, 0, false)
synctest.Wait()
time.Sleep(time.Minute - time.Nanosecond)
wantScore(t, checker, other, true, 0, false)
synctest.Wait()
wantChecked(t, abuseIPDB, suspect)
wantFailures(t, checker, 1)
time.Sleep(time.Nanosecond)
wantScore(t, checker, other, true, 0, false)
synctest.Wait()
wantChecked(t, abuseIPDB, suspect, other)
wantFailures(t, checker, 2)
if scores := checker.Snapshot().Scores; len(scores) != 0 {
t.Errorf("scores %+v, want none", scores)
}
// One alert for the first failure; the cooldown holds back the
// second.
wantFailureAlert(t, queue, time.Now().Add(-time.Minute),
"checking a client with AbuseIPDB failed", tc.error, 1)
if !strings.Contains(log.String(), `"msg":"checking a client with `+
`AbuseIPDB failed","source":"abuseipdb","error":"`+tc.error) {
t.Errorf("logged\n%s\nwant the failures", log.String())
}
})
})
}
}
func TestCheckNotAnsweredWithinTheTimeoutFailsAndHitNeverWaits(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
queue := newQueue()
p := abuseIPDBParams()
p.Alerts = queue
abuseIPDB := &abuseIPDBStandIn{hanging: true}
checker := newAbuseIPDB(abuseIPDB, p)
began := time.Now()
// The second, while the first's check is under way, starts none.
wantScore(t, checker, suspect, true, 0, false)
wantScore(t, checker, suspect, true, 0, false)
if waited := time.Since(began); waited != 0 {
t.Errorf("waited %s for the check, want no wait", waited)
}
time.Sleep(timeout - time.Nanosecond)
synctest.Wait()
wantChecked(t, abuseIPDB, suspect)
wantFailures(t, checker, 0)
time.Sleep(time.Nanosecond)
synctest.Wait()
wantFailures(t, checker, 1)
wantFailureAlert(t, queue, time.Now(), "checking a client with AbuseIPDB failed",
"check the client: context deadline exceeded", 0)
})
}
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
var log bytes.Buffer
queue := newQueue()
p := abuseIPDBParams()
p.Alerts = queue
p.ProcessLog = slog.New(slog.NewJSONHandler(&log, nil))
abuseIPDB := &abuseIPDBStandIn{scores: map[string]int64{suspect: 100}}
checker := newAbuseIPDB(abuseIPDB, p)
m := metrics.New(1, "app")
m.AddReputation(reputation.New(params()), reputation.NewDNSBL(dnsblParams()))
m.AddAbuseIPDB(checker)
// One check that AbuseIPDB answers, and one it refuses with an answer
// that names the key.
wantScore(t, checker, suspect, true, 0, false)
synctest.Wait()
wantScore(t, checker, suspect, true, 100, true)
abuseIPDB.answerWith(http.StatusUnauthorized, `{"errors":[{"detail":"`+key+`"}]}`)
wantScore(t, checker, other, true, 0, false)
synctest.Wait()
wantFailures(t, checker, 1)
abuseIPDB.mu.Lock()
sent := slices.Clone(abuseIPDB.keys)
abuseIPDB.mu.Unlock()
if !slices.Equal(sent, []string{key, key}) {
t.Errorf("checks sent the keys %v, want %s twice", sent, key)
}
alerted, err := json.Marshal(waiting(queue))
if err != nil {
t.Fatalf("encode the alerts: %v", err)
}
kept, err := json.Marshal(checker.Snapshot())
if err != nil {
t.Fatalf("encode the checks: %v", err)
}
for name, shown := range map[string]string{
"the log": log.String(), "the alerts": string(alerted),
"the metrics": scrapeMetrics(t, m), "reputation.json": string(kept),
} {
if strings.Contains(shown, key) {
t.Errorf("%s shows the key:\n%s", name, shown)
}
}
})
}
func TestMetricsCountTheChecksTheFailuresAndTheBudgetLeft(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
abuseIPDB := &abuseIPDBStandIn{}
p := abuseIPDBParams()
p.DailyBudget = 5
checker := newAbuseIPDB(abuseIPDB, p)
m := metrics.New(1, "app")
m.AddReputation(reputation.New(params()), reputation.NewDNSBL(dnsblParams()))
m.AddAbuseIPDB(checker)
// One check that AbuseIPDB answers, and one that fails.
checker.Hit(t.Context(), netip.MustParseAddr(suspect), true)
synctest.Wait()
abuseIPDB.answerWith(http.StatusInternalServerError, "")
checker.Hit(t.Context(), netip.MustParseAddr(other), true)
synctest.Wait()
scraped := scrapeMetrics(t, m)
for series, want := range map[string]string{
"queries_total": "2",
"failures_total": "1",
"daily_budget_remaining": "3",
} {
line := "\nsmallwebwaf_reputation_" + series +
`{instance="app",source="abuseipdb"} ` + want + "\n"
if !strings.Contains(scraped, line) {
t.Errorf("metrics\n%s\nwant%s", scraped, line)
}
}
})
}
func TestScoreFetchedATTLAgoIsNeitherUsedNorKept(t *testing.T) {
t.Parallel()
now := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC)
p := abuseIPDBParams()
p.Now = func() time.Time { return now }
checker := reputation.NewAbuseIPDB(p)
// The last score still in use, and one fetched a TTL ago.
inUse := reputation.Score{
Client: netip.MustParseAddr(suspect), Score: 100,
Fetched: now.Add(-cacheTTL + time.Nanosecond),
}
stale := reputation.Score{
Client: netip.MustParseAddr(other), Score: 100, Fetched: now.Add(-cacheTTL),
}
checker.Load(reputation.Checks{Scores: []reputation.Score{stale, inUse}})
wantScore(t, checker, suspect, false, 100, true)
wantScore(t, checker, other, false, 0, false)
got := checker.Snapshot().Scores
if !reflect.DeepEqual(got, []reputation.Score{inUse}) {
t.Errorf("scores %+v, want only %+v", got, inUse)
}
}
func TestAtMost100000ScoresKeptTheOneFetchedLongestAgoDroppedFirst(t *testing.T) {
t.Parallel()
now := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC)
p := abuseIPDBParams()
p.Now = func() time.Time { return now }
checker := reputation.NewAbuseIPDB(p)
// 100,001 scores, listed by client, as reputation.json lists them, each
// fetched a millisecond before the one before it: the last is one too
// many.
const count = 100001
scores := make([]reputation.Score, 0, count)
client := netip.MustParseAddr("198.18.0.0")
for i := range count {
scores = append(scores, reputation.Score{
Client: client, Fetched: now.Add(-time.Duration(i) * time.Millisecond),
})
client = client.Next()
}
checker.Load(reputation.Checks{Scores: scores})
got := checker.Snapshot().Scores
if len(got) != count-1 || !slices.Contains(got, scores[0]) ||
slices.Contains(got, scores[count-1]) {
t.Errorf("%d scores kept, want all but the one fetched longest ago", len(got))
}
}
// abuseIPDBStandIn is a stand-in for AbuseIPDB. It answers a check sent
// with key by the client's score, as scores gives it, 0 for a client it
// does not give; a check sent with another key with 401; and, while
// status is not 0, every check with status and body; and while hanging,
// none at all. It notes each client checked, and the key sent.
type abuseIPDBStandIn struct {
mu sync.Mutex
scores map[string]int64
status int
body string
hanging bool
checked []string
keys []string
}
// RoundTrip has the stand-in answer req, in place of the network. A check
// abandoned before the stand-in answers fails, as over the network.
func (s *abuseIPDBStandIn) RoundTrip(req *http.Request) (*http.Response, error) {
client := req.URL.Query().Get("ipAddress")
sent := req.Header.Get("Key")
s.mu.Lock()
s.checked = append(s.checked, client)
s.keys = append(s.keys, sent)
score := s.scores[client]
status, body, hanging := s.status, s.body, s.hanging
s.mu.Unlock()
switch {
case hanging:
<-req.Context().Done()
return nil, req.Context().Err()
case sent != key:
status = http.StatusUnauthorized
case status == 0:
status = http.StatusOK
body = fmt.Sprintf(`{"data":{"ipAddress":%q,"abuseConfidenceScore":%d}}`, client,
score)
}
return &http.Response{
StatusCode: status,
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
Header: http.Header{},
Body: io.NopCloser(strings.NewReader(body)),
Request: req,
}, nil
}
// setScore has the stand-in give client score.
func (s *abuseIPDBStandIn) setScore(client string, score int64) {
s.mu.Lock()
defer s.mu.Unlock()
s.scores[client] = score
}
// answerWith has the stand-in answer every check with status and body.
func (s *abuseIPDBStandIn) answerWith(status int, body string) {
s.mu.Lock()
defer s.mu.Unlock()
s.status, s.body = status, body
}
// abuseIPDBParams returns the AbuseIPDBParams of the tests: key, a minimum
// score of 75, a daily budget of 900, and the cache TTL and timeout of the
// DNSBL tests, by the bubble's clock, with alerts to a queue that sends
// none.
func abuseIPDBParams() reputation.AbuseIPDBParams {
return reputation.AbuseIPDBParams{
URL: "https://abuseipdb.example/api/v2/check",
Key: key,
MinScore: 75,
DailyBudget: 900,
CacheTTL: cacheTTL,
Timeout: timeout,
Now: time.Now,
ProcessLog: slog.New(slog.DiscardHandler),
Alerts: newQueue(),
}
}
// newAbuseIPDB returns the AbuseIPDB of p, checking clients with
// abuseIPDB.
func newAbuseIPDB(
abuseIPDB *abuseIPDBStandIn, p reputation.AbuseIPDBParams,
) *reputation.AbuseIPDB {
checker := reputation.NewAbuseIPDB(p)
checker.SetTransport(abuseIPDB)
return checker
}
// wantScore checks the score checker gives client, and whether it is a
// hit, as a request from client finds them, offender or not.
func wantScore(
t *testing.T, checker *reputation.AbuseIPDB, client string, offender bool,
score int64, hit bool,
) {
t.Helper()
gotScore, gotHit := checker.Hit(t.Context(), netip.MustParseAddr(client), offender)
if gotScore != score || gotHit != hit {
t.Errorf("%s has the score %d, a hit %t, want %d, %t", client, gotScore, gotHit,
score, hit)
}
}
// wantChecked checks the clients the stand-in was asked about, in any
// order.
func wantChecked(t *testing.T, abuseIPDB *abuseIPDBStandIn, want ...string) {
t.Helper()
abuseIPDB.mu.Lock()
got := slices.Sorted(slices.Values(abuseIPDB.checked))
abuseIPDB.mu.Unlock()
want = slices.Sorted(slices.Values(want))
if !slices.Equal(got, want) {
t.Errorf("checked %v, want %v", got, want)
}
}
// wantFailures checks how many checks failed.
func wantFailures(t *testing.T, checker *reputation.AbuseIPDB, want int) {
t.Helper()
if got := checker.Failures(); got != want {
t.Errorf("%d checks failed, want %d", got, want)
}
}
// wantFailureAlert checks that the one alert waiting in queue is a
// source_failure alert from AbuseIPDB, raised at raised, with reason and
// the error failure, and that the cooldown has held back held repeats of
// it.
func wantFailureAlert(
t *testing.T, queue *alerts.Queue, raised time.Time, reason, failure string,
held int64,
) {
t.Helper()
got := waiting(queue)
if len(got) != 1 || !got[0].Time.Equal(raised) ||
got[0].Event != alerts.EventSourceFailure || got[0].Reason != reason ||
got[0].Detail["source"] != reputation.AbuseIPDBSource ||
got[0].Detail["error"] != failure || queue.Suppressed() != held {
t.Errorf("alerts waiting %+v, %d held back, want only AbuseIPDB's %q with %q, "+
"and %d", got, queue.Suppressed(), reason, failure, held)
}
}
// wantBudgetLeft checks how many checks the day's budget has left.
func wantBudgetLeft(t *testing.T, checker *reputation.AbuseIPDB, want int) {
t.Helper()
if got := checker.BudgetLeft(); got != want {
t.Errorf("%d checks left, want %d", got, want)
}
}
// scrapeMetrics returns the metrics m serves.
func scrapeMetrics(t *testing.T, m *metrics.Metrics) string {
t.Helper()
scraped := httptest.NewRecorder()
m.ServeHTTP(scraped, httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/",
http.NoBody))
return scraped.Body.String()
}
+6 -9
View File
@@ -20,16 +20,17 @@ import (
)
const (
// maxVerdicts is how many verdicts are kept. Past it, the one fetched
// longest ago is dropped.
// maxVerdicts is how many verdicts of the DNSBL zones are kept, and how
// many scores of AbuseIPDB. Past it, the one fetched longest ago is
// dropped.
maxVerdicts = 100000
// maxQueries is how many queries may be under way at once. Past it, a
// zone is not asked about a client until the client's next request, so
// that a swarm of new addresses cannot fill the memory.
maxQueries = 1000
// failureDelay is how long a zone is not asked again after a query to
// it fails, so that a zone refusing queries is not asked on every
// request.
// it fails, and no client is checked with AbuseIPDB after a check
// fails, so that a source refusing them is not asked on every request.
failureDelay = time.Minute
)
@@ -261,11 +262,7 @@ func (d *DNSBL) ask(ctx context.Context, q query) {
// Raised before it is logged, so that the alert is there once the
// log line is.
d.params.Alerts.Raise(alerts.Alert{
Event: alerts.EventSourceFailure,
Reason: failed,
Detail: map[string]any{"source": shown, "error": err.Error()},
})
raiseFailure(d.params.Alerts, failed, shown, err)
d.params.ProcessLog.Warn(failed, "zone", shown, "error", err.Error())
}
}
+6
View File
@@ -13,6 +13,12 @@ func (l *Lists) SetTransport(transport http.RoundTripper) {
l.httpClient.Transport = transport
}
// SetTransport has a's checks go through transport instead of the
// network.
func (a *AbuseIPDB) SetTransport(transport http.RoundTripper) {
a.httpClient.Transport = transport
}
// SetDial has d's queries go through dial instead of the network.
func (d *DNSBL) SetDial(
dial func(ctx context.Context, network, address string) (net.Conn, error),
+16 -8
View File
@@ -3,9 +3,10 @@
// SWWAF_ASN_LIMIT_PERCENT_URL names. It keeps the last good copy of each,
// whole, comment lines included, which is used while a fetch fails, and
// when each was last tried. It also asks the DNSBL zones of
// SWWAF_DNSBL_ZONES about clients, and keeps their verdicts. The state
// package writes all of these to reputation.json and reads them from it,
// so that a restart keeps them too.
// SWWAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks
// clients with AbuseIPDB, and keeps their scores and the checks spent
// today. The state package writes all of these to reputation.json and
// reads them from it, so that a restart keeps them too.
package reputation
import (
@@ -329,11 +330,7 @@ func (l *Lists) fetch(ctx context.Context, listURL string) {
// Raised before it is logged, so that the alert is there once the
// log line is.
l.params.Alerts.Raise(alerts.Alert{
Event: alerts.EventSourceFailure,
Reason: failed,
Detail: map[string]any{"source": listURL, "error": err.Error()},
})
raiseFailure(l.params.Alerts, failed, listURL, err)
l.params.ProcessLog.Warn(failed, "url", listURL, "error", err.Error())
return
@@ -342,6 +339,17 @@ func (l *Lists) fetch(ctx context.Context, listURL string) {
l.params.ProcessLog.Info("fetched a list", "url", listURL, "lines", len(lines))
}
// raiseFailure raises a source_failure alert into queue, with reason, and
// in its detail the source that failed, a list's URL, a zone with its key
// masked or abuseipdb, and err.
func raiseFailure(queue *alerts.Queue, reason, source string, err error) {
queue.Raise(alerts.Alert{
Event: alerts.EventSourceFailure,
Reason: reason,
Detail: map[string]any{"source": source, "error": err.Error()},
})
}
// get fetches the list at listURL, and returns its lines. An answer other
// than 200, or a list longer than maxListBytes, is a failure.
func (l *Lists) get(ctx context.Context, listURL string) ([]string, error) {
+4 -2
View File
@@ -36,7 +36,8 @@ const (
ActionRuleBlocked = "rule_blocked"
// ActionDenied is a request refused because its client is in
// SWWAF_DENY_NETS, in a blocklist while SWWAF_BLOCKLIST_ACTION is deny,
// or listed by a DNSBL zone while SWWAF_REPUTATION_ACTION is deny.
// or listed by a DNSBL zone, or scored a hit by AbuseIPDB, while
// SWWAF_REPUTATION_ACTION is deny.
ActionDenied = "denied"
// ActionCountryDenied is a request refused for its client's country.
ActionCountryDenied = "country_denied"
@@ -139,7 +140,8 @@ type Line struct {
// minute_bytes, hour_bytes or day_bytes for a byte limit.
LimitHit string `json:"limit_hit,omitempty"`
// Reputation are the URLs of the blocklists that list the client, then
// the DNSBL zones whose verdict lists it, their keys masked.
// the DNSBL zones whose verdict lists it, their keys masked, then
// abuseipdb when its score is a hit.
Reputation []string `json:"reputation,omitempty"`
// Offence is the offence the request was held as, OffenceLimit.
Offence string `json:"offence,omitempty"`
+11 -8
View File
@@ -21,6 +21,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/proxy"
"sneak.berlin/go/smallwebwaf/internal/remotelog"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rules"
"sneak.berlin/go/smallwebwaf/internal/state"
@@ -169,14 +170,15 @@ func newServer(
}
server := proxy.New(proxy.Params{
Config: cfg,
RequestLog: stdout,
ProcessLog: processLog,
GeoJSURL: lookup.URL,
LookupFile: lookupFile,
Now: now,
Rules: ruleFiles,
Alerts: alertQueue,
Config: cfg,
RequestLog: stdout,
ProcessLog: processLog,
GeoJSURL: lookup.URL,
AbuseIPDBURL: reputation.AbuseIPDBURL,
LookupFile: lookupFile,
Now: now,
Rules: ruleFiles,
Alerts: alertQueue,
})
server.Metrics.AddAlerts(alertQueue)
@@ -202,6 +204,7 @@ func loadStateFiles(
GeoJS: server.GeoJS,
Lists: server.Lists,
DNSBL: server.DNSBL,
AbuseIPDB: server.AbuseIPDB,
Alerts: alertQueue,
Anomalies: server.Anomalies,
Now: now,
+47 -12
View File
@@ -2,8 +2,8 @@
// SWWAF_STATE_DIR, as the "Persistent state" section of SPEC.md describes:
// bans.json holds the bans, clients.json each client's counters and
// history, lookups.json GeoJS's answers, reputation.json the last try and
// last good copy of each list fetched from a URL and the DNSBL zones'
// verdicts, and alerts.json the
// last good copy of each list fetched from a URL, the DNSBL zones'
// verdicts, and AbuseIPDB's scores and checks spent, and alerts.json the
// cooldowns, the hour under way, the alerts waiting for each destination
// and the anomaly counters. Load
// reads them at start, Watch takes in an admin's edit of one while
@@ -77,14 +77,15 @@ type Params struct {
// is (SWWAF_STATE_COUNTER_INTERVAL).
WriteDelay time.Duration
CounterInterval time.Duration
// Ledger, Limiter, GeoJS, Lists, DNSBL, Alerts and Anomalies hold the
// state. Alerts also receive a file_error alert for an edit set aside,
// and for a write that fails while smallwebwaf runs.
// Ledger, Limiter, GeoJS, Lists, DNSBL, AbuseIPDB, Alerts and Anomalies
// hold the state. Alerts also receive a file_error alert for an edit set
// aside, and for a write that fails while smallwebwaf runs.
Ledger *bans.Ledger
Limiter *ratelimit.Limiter
GeoJS *lookup.GeoJS
Lists *reputation.Lists
DNSBL *reputation.DNSBL
AbuseIPDB *reputation.AbuseIPDB
Alerts *alerts.Queue
Anomalies *anomaly.Counters
// Now tells the time by which the counters' buckets run out, normally
@@ -147,9 +148,10 @@ type lookupsFile struct {
// reputationFile is reputation.json, indented for an admin to read and
// edit, so that each line of a list's copy is on a line of its own.
type reputationFile struct {
Version int `json:"version"`
Lists []reputation.List `json:"lists"`
Verdicts []reputation.Verdict `json:"verdicts"`
Version int `json:"version"`
Lists []reputation.List `json:"lists"`
Verdicts []reputation.Verdict `json:"verdicts"`
AbuseIPDB reputation.Checks `json:"abuseipdb"`
}
// alertsFile is alerts.json, indented for an admin to read and edit.
@@ -436,6 +438,7 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) {
}
f.params.DNSBL.Load(file.Verdicts)
f.params.AbuseIPDB.Load(file.AbuseIPDB)
entries = len(file.Lists)
case alertsJSON:
waiting, err := f.takeInAlerts(path, data)
@@ -571,7 +574,7 @@ func (f *Files) encode(name string) ([]byte, error) {
case reputationJSON:
return encodeIndented(reputationFile{
Version: version, Lists: f.params.Lists.Snapshot(),
Verdicts: f.params.DNSBL.Snapshot(),
Verdicts: f.params.DNSBL.Snapshot(), AbuseIPDB: f.params.AbuseIPDB.Snapshot(),
})
default: // alerts.json
held := f.params.Alerts.Snapshot()
@@ -736,9 +739,11 @@ func (f *lookupsFile) check(data []byte) error {
// of it without the time it was fetched, or without its lines, which hold
// the list. It refuses a verdict without its zone or its client, which
// would be about no one, whether the zone lists the client, or the time
// it was fetched, which would drop it. A verdict's listed is false for a
// client the zone does not list, which Verdicts cannot tell from a
// missing one, so each listed is read again as written.
// it was fetched, which would drop it, and so an AbuseIPDB score without
// its client, the score, or the time it was fetched. A verdict's listed is
// false for a client the zone does not list, and a score can be 0, which
// the structs cannot tell from a missing one, so each is read again as
// written.
func (f *reputationFile) check(data []byte) error {
for i, kept := range f.Lists {
switch {
@@ -777,6 +782,36 @@ func (f *reputationFile) check(data []byte) error {
}
}
return checkScores(f.AbuseIPDB.Scores, data)
}
// checkScores refuses an AbuseIPDB score, of scores, read from data, as
// reputationFile's check describes.
func checkScores(scores []reputation.Score, data []byte) error {
var written struct {
AbuseIPDB struct {
Scores []struct {
Score *int64 `json:"score"`
} `json:"scores"`
} `json:"abuseipdb"`
}
err := json.Unmarshal(data, &written)
if err != nil {
return err
}
for i, kept := range scores {
switch {
case !kept.Client.IsValid():
return fmt.Errorf("abuseipdb scores %w", missing(i, "client"))
case written.AbuseIPDB.Scores[i].Score == nil:
return fmt.Errorf("abuseipdb scores %w", missing(i, "score"))
case kept.Fetched.IsZero():
return fmt.Errorf("abuseipdb scores %w", missing(i, "fetched"))
}
}
return nil
}
+93 -8
View File
@@ -212,8 +212,9 @@ const filledAlertsJSON = `{
`
// filledReputationJSON is reputation.json holding the blocklists' last
// tries and the copy of one, with its comment line, and two verdicts of a
// DNSBL zone, as fill puts them in.
// tries and the copy of one, with its comment line, two verdicts of a
// DNSBL zone, and the AbuseIPDB checks spent today with two scores, as
// fill puts them in.
const filledReputationJSON = `{
"version": 1,
"lists": [
@@ -245,7 +246,23 @@ const filledReputationJSON = `{
"listed": false,
"fetched": "2026-10-05T22:00:00Z"
}
]
],
"abuseipdb": {
"day": "2026-10-06T00:00:00Z",
"spent": 3,
"scores": [
{
"client": "203.0.113.9",
"score": 100,
"fetched": "2026-10-05T23:00:00Z"
},
{
"client": "2001:db8::1",
"score": 0,
"fetched": "2026-10-05T22:00:00Z"
}
]
}
}
`
@@ -282,6 +299,11 @@ func TestFilesWrittenAndReadBack(t *testing.T) {
wantEqual(t, reputationJSON, after.DNSBL.Snapshot(), before.DNSBL.Snapshot())
checks, wantChecks := after.AbuseIPDB.Snapshot(), before.AbuseIPDB.Snapshot()
if !reflect.DeepEqual(checks, wantChecks) {
t.Errorf("%s read back\n%+v\nwant\n%+v", reputationJSON, checks, wantChecks)
}
if got, want := after.Alerts.Snapshot(), before.Alerts.Snapshot(); !reflect.DeepEqual(
got, want) {
t.Errorf("%s read back\n%+v\nwant\n%+v", alertsJSON, got, want)
@@ -439,10 +461,13 @@ func TestMissingFilesAreEmptyState(t *testing.T) {
load(t, params)
held := params.Alerts.Snapshot()
checks := params.AbuseIPDB.Snapshot()
if len(params.Ledger.Snapshot()) != 0 || len(params.Limiter.Snapshot()) != 0 ||
len(params.GeoJS.Snapshot()) != 0 || len(params.Lists.Snapshot()) != 0 ||
len(params.DNSBL.Snapshot()) != 0 || len(held.Cooldowns) != 0 ||
len(held.Waiting[alerts.DestinationWebhook]) != 0 || held.Hour.Sent != 0 {
len(params.DNSBL.Snapshot()) != 0 || len(checks.Scores) != 0 || checks.Spent != 0 ||
len(held.Cooldowns) != 0 || len(held.Waiting[alerts.DestinationWebhook]) != 0 ||
held.Hour.Sent != 0 {
t.Error("state from no files")
}
}
@@ -678,6 +703,47 @@ func TestReputationJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
}
}
func TestReputationJSONScoreWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
t.Parallel()
// scores opens the list of AbuseIPDB scores, and ends closes it; client,
// score and fetched make a score.
const (
scores = `{"version": 1, "abuseipdb": {"scores": [`
client = `"client": "198.51.100.7", `
score = `"score": 0, `
fetched = `"fetched": "2026-10-06T00:00:00Z"`
ends = `}]}}`
)
for _, tc := range []struct {
name, content string
// want is what the error says after the file's path.
want string
}{
{
"without its client", scores + `{` + score + fetched + ends,
`: abuseipdb scores entry 1 has no "client"`,
},
{
// A score of 0 is not having none.
"without the score",
scores + `{` + client + score + fetched + `}, {` + client + fetched + ends,
`: abuseipdb scores entry 2 has no "score"`,
},
{
"without the time it was fetched", scores + `{` + client + `"score": 100` + ends,
`: abuseipdb scores entry 1 has no "fetched"`,
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
wantRefused(t, reputationJSON, tc.content, tc.want)
})
}
}
func TestAlertsJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
t.Parallel()
@@ -1182,7 +1248,9 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
edit(t, dir, reputationJSON, `{"version": 1, "lists": [{"url": "`+blocklistURL+`", `+
`"tried": "2026-10-06T00:00:00Z", "fetched": "2026-10-06T00:00:00Z", `+
`"lines": ["198.51.100.7"]}], "verdicts": [{"zone": "`+dnsblZone+`", `+
`"client": "198.51.100.7", "listed": true, "fetched": "2026-10-06T00:00:00Z"}]}`)
`"client": "198.51.100.7", "listed": true, "fetched": "2026-10-06T00:00:00Z"}], `+
`"abuseipdb": {"day": "2026-10-06T00:00:00Z", "spent": 9, "scores": [`+
`{"client": "198.51.100.7", "score": 80, "fetched": "2026-10-06T00:00:00Z"}]}}`)
wantTakenIn(t, lines, dir, reputationJSON)
listedBy := params.Lists.ListedBy(client.Addr())
@@ -1195,6 +1263,14 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
Zone: dnsblZone, Client: client.Addr(), Listed: true, Fetched: midnight(),
}})
checks := reputation.Checks{
Day: midnight(), Spent: 9,
Scores: []reputation.Score{{Client: client.Addr(), Score: 80, Fetched: midnight()}},
}
if got := params.AbuseIPDB.Snapshot(); !reflect.DeepEqual(got, checks) {
t.Errorf("%s taken in as\n%+v\nwant\n%+v", reputationJSON, got, checks)
}
// A netblock with bits past its length is read as the netblock it is
// in.
edit(t, dir, alertsJSON, `{"version": 1, "cooldowns": [{"event": "ban", `+
@@ -1615,6 +1691,10 @@ func newParams(dir string) state.Params {
Zones: []string{dnsblZone}, CacheTTL: 24 * time.Hour, Timeout: time.Second,
Now: midnight, ProcessLog: discard, Alerts: queue,
}),
AbuseIPDB: reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
MinScore: 75, DailyBudget: 900, CacheTTL: 24 * time.Hour, Timeout: time.Second,
Now: midnight, ProcessLog: discard, Alerts: queue,
}),
Alerts: queue,
Anomalies: anomaly.New(anomaly.Params{
Net: anomaly.Thresholds{RequestsPerMinute: 1000},
@@ -1639,8 +1719,9 @@ func office() netip.Prefix {
// fill puts a permanent ban an admin made, a ban for a broken limit and
// one for a clear sign of attack, clients with counts and histories,
// GeoJS answers, the blocklists' last tries and the copy of one, and two
// verdicts of a DNSBL zone, as filledReputationJSON holds them, and alerts
// GeoJS answers, the blocklists' last tries and the copy of one, two
// verdicts of a DNSBL zone, and the AbuseIPDB checks spent today with two
// scores, as filledReputationJSON holds them, and alerts
// and anomaly counters, as filledAlertsJSON holds them, into the parts of
// params.
func fill(params state.Params) {
@@ -1696,6 +1777,10 @@ func fill(params state.Params) {
},
{Zone: dnsblZone, Client: client.Addr(), Listed: true, Fetched: now.Add(-time.Hour)},
})
params.AbuseIPDB.Load(reputation.Checks{Day: now, Spent: 3, Scores: []reputation.Score{
{Client: netip.MustParseAddr("2001:db8::1"), Fetched: now.Add(-2 * time.Hour)},
{Client: client.Addr(), Score: 100, Fetched: now.Add(-time.Hour)},
}})
// An alert waiting, a repeat of it the cooldown holds back, another
// alert waiting, and one past the two an hour, for the hour's summary.