clawbot
271aaafccc
check / check (push) Waiting to run
SWWAF_BLOCKLIST_URLS names lists of addresses and netblocks, fetched every SWWAF_BLOCKLIST_REFRESH (24h, never under 1h); an IPv4-mapped line stands for its IPv4 address or netblock. reputation.json keeps each list's last try, failed or not, even one cut off by a stop, which a restart waits on as a running instance does, and its last good copy, whole, used while a fetch fails. SWWAF_BLOCKLIST_ACTION denies, limits or only logs a listed client; the log line names the lists, each raises reputation_hit, and a failed fetch raises source_failure. SWWAF_ASN_LIMIT_PERCENT_URL is fetched the same way and counts as SWWAF_ASN_LIMIT_PERCENT does, the lower winning. Judgement call: a failed fetch is retried after the refresh, not sooner. Not done: ban notes do not name the lists yet. Model: opus-5-5
334 lines
10 KiB
Go
334 lines
10 KiB
Go
package proxy_test
|
|
|
|
import (
|
|
"maps"
|
|
"net/http"
|
|
"net/netip"
|
|
"slices"
|
|
"testing"
|
|
"time"
|
|
|
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
|
"sneak.berlin/go/smallwebwaf/internal/proxy"
|
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
)
|
|
|
|
// The reputation settings.
|
|
const (
|
|
blocklistURLs = "SWWAF_BLOCKLIST_URLS"
|
|
blocklistAction = "SWWAF_BLOCKLIST_ACTION"
|
|
asnLimitPercentURL = "SWWAF_ASN_LIMIT_PERCENT_URL"
|
|
)
|
|
|
|
// The actions of SWWAF_BLOCKLIST_ACTION but limit, which has a
|
|
// percentage.
|
|
const (
|
|
actionDeny = "deny"
|
|
actionLog = "log"
|
|
)
|
|
|
|
// The lists these tests name, which are never fetched: each test puts in
|
|
// the copies it needs, as reputation.json would at start.
|
|
const (
|
|
dropURL = "https://lists.example/drop.txt"
|
|
torURL = "https://lists.example/tor.txt"
|
|
asnURL = "https://lists.example/asn.txt"
|
|
)
|
|
|
|
func TestEachBlocklistActionForAListedAddressAndAListedNetblock(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
forward, denied := requestlog.ActionForward, requestlog.ActionDenied
|
|
|
|
for _, tc := range []struct {
|
|
action string
|
|
// statuses and actions are those of a listed client's three
|
|
// requests, and percent their limit_percent, as percentText gives it.
|
|
statuses []int
|
|
actions []string
|
|
percent string
|
|
}{
|
|
{
|
|
actionDeny, []int{http.StatusForbidden, http.StatusForbidden, http.StatusForbidden},
|
|
[]string{denied, denied, denied}, none,
|
|
},
|
|
{
|
|
// Half of 4 requests a minute: the third breaks the limit.
|
|
"limit:50", []int{http.StatusOK, http.StatusOK, http.StatusForbidden},
|
|
[]string{forward, forward, requestlog.ActionRateLimited},
|
|
"50 from " + blocklistAction,
|
|
},
|
|
{
|
|
actionLog, []int{http.StatusOK, http.StatusOK, http.StatusOK},
|
|
[]string{forward, forward, forward}, none,
|
|
},
|
|
} {
|
|
t.Run(tc.action, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
s, server, _ := startWithLookups(t, map[string]string{
|
|
rateLimitPerMinute: fourAMinute, blocklistURLs: dropURL,
|
|
blocklistAction: tc.action,
|
|
})
|
|
// fromDE is listed as an address, and fromKP in a netblock.
|
|
loadLists(t, server, map[string][]string{
|
|
dropURL: {"; DROP", fromDE, "198.51.100.0/24 ; SBL1"},
|
|
})
|
|
|
|
for _, from := range []string{fromDE, fromKP} {
|
|
for i := range 3 {
|
|
line := s.get(from, tc.statuses[i], tc.actions[i])
|
|
wantReputation(t, line, dropURL)
|
|
wantPercent(t, "limit_percent", line.LimitPercent,
|
|
line.LimitPercentSetting, tc.percent)
|
|
|
|
// A request refused for the list is not counted.
|
|
counted := line.fields["counts"] != nil
|
|
if counted != (tc.actions[i] != denied) {
|
|
t.Errorf("request from %s counted %t, logged %s", from, counted,
|
|
tc.actions[i])
|
|
}
|
|
}
|
|
}
|
|
|
|
// A client no list lists has the whole limit.
|
|
for range 3 {
|
|
line := s.get(unplaced, http.StatusOK, forward)
|
|
wantReputation(t, line)
|
|
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
|
|
none)
|
|
}
|
|
|
|
// A refusal for the list makes no ban.
|
|
if held := server.Ledger.Snapshot(); tc.action == actionDeny && len(held) != 0 {
|
|
t.Errorf("bans %+v, want none", held)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestBlocklistsComeAfterTheCountryListsAndSkipAllowNets(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
s, server, queue := startWithLookups(t, map[string]string{
|
|
blocklistURLs: dropURL, deniedCountries: "kp", allowNets: fromDE,
|
|
})
|
|
loadLists(t, server, map[string][]string{dropURL: {fromDE, fromKP}})
|
|
|
|
// fromKP's country refuses it before the list is looked at, and fromDE,
|
|
// in SWWAF_ALLOW_NETS, is not checked at all: neither is noted, nor
|
|
// alerted.
|
|
wantReputation(t, s.get(fromKP, http.StatusForbidden, requestlog.ActionCountryDenied))
|
|
wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward))
|
|
wantAlerts(t, queue)
|
|
}
|
|
|
|
func TestObserveModeForwardsAClientABlocklistDeniesAndAlertsIt(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
s, server, queue := startWithLookups(t, map[string]string{
|
|
blocklistURLs: dropURL, mode: observe,
|
|
})
|
|
loadLists(t, server, map[string][]string{dropURL: {fromDE}})
|
|
|
|
line := s.get(fromDE, http.StatusOK, requestlog.ActionForward)
|
|
wantWouldAction(t, line, requestlog.ActionDenied)
|
|
wantReputation(t, line, dropURL)
|
|
|
|
waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
|
|
if len(waiting) != 1 || waiting[0].Event != alerts.EventReputationHit {
|
|
t.Errorf("alerts waiting %+v, want a reputation_hit alert", waiting)
|
|
}
|
|
}
|
|
|
|
func TestBlocklistLimitTakesPartInTheLowestPercentageOfEveryLimit(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
for _, tc := range []struct {
|
|
action, asnPercent string
|
|
// want is the upload's limit_percent and bytes_percent, as
|
|
// percentText gives them, and limitHit its limit_hit.
|
|
want, limitHit string
|
|
}{
|
|
{"limit:50", asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes},
|
|
{"limit:25", asnDEHalf, "25 from " + blocklistAction, minuteBytes},
|
|
// The AS number's, the first of two alike.
|
|
{"limit:25", asnDEQuarter, "25 from " + asnLimitPercent, minuteBytes},
|
|
{actionLog, asnDE + ":100", none, ""},
|
|
} {
|
|
t.Run(tc.action+" "+tc.asnPercent, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
s, server, _ := startWithLookups(t, map[string]string{
|
|
bytesLimitPerMinute: twoUploads, blocklistURLs: dropURL,
|
|
blocklistAction: tc.action, asnLimitPercent: tc.asnPercent,
|
|
})
|
|
loadLists(t, server, map[string][]string{dropURL: {fromDE}})
|
|
|
|
// The upload's 100 bytes are over 49, a quarter of 199, and 99,
|
|
// half of it, and within 199.
|
|
line := s.uploadFrom(fromDE)
|
|
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
|
|
tc.want)
|
|
wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting,
|
|
tc.want)
|
|
|
|
if line.LimitHit != tc.limitHit {
|
|
t.Errorf("log line has limit_hit %q, want %q", line.LimitHit, tc.limitHit)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestASNLimitPercentFileCountsAsTheSettingDoesTheLowerWinning(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
const (
|
|
fromURL = "25 from " + asnLimitPercentURL
|
|
fromSetting = "25 from " + asnLimitPercent
|
|
)
|
|
|
|
for _, tc := range []struct {
|
|
name string
|
|
env map[string]string
|
|
file string
|
|
// limitPercent and bytesPercent are the upload's, as percentText
|
|
// gives them.
|
|
limitPercent, bytesPercent string
|
|
}{
|
|
{"the file's alone", nil, asnDEQuarter, fromURL, fromURL},
|
|
{
|
|
"the file's, lower than the setting's",
|
|
map[string]string{asnLimitPercent: asnDEHalf}, asnDEQuarter, fromURL, fromURL,
|
|
},
|
|
{
|
|
"the setting's, lower than the file's",
|
|
map[string]string{asnLimitPercent: asnDEQuarter}, asnDEHalf,
|
|
fromSetting, fromSetting,
|
|
},
|
|
{
|
|
"the setting's, the first of two alike",
|
|
map[string]string{asnLimitPercent: asnDEQuarter}, asnDEQuarter,
|
|
fromSetting, fromSetting,
|
|
},
|
|
{"none, for an AS number the file does not list", nil, asnKP + ":25", none, none},
|
|
{
|
|
"SWWAF_ASN_BYTES_PERCENT's in place of the file's for the byte limits",
|
|
map[string]string{asnBytesPercent: asnDE + ":100"}, asnDEQuarter, fromURL, none,
|
|
},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
env := map[string]string{asnLimitPercentURL: asnURL}
|
|
maps.Copy(env, tc.env)
|
|
|
|
s, server, _ := startWithLookups(t, env)
|
|
loadLists(t, server, map[string][]string{asnURL: {"# by AS number", tc.file}})
|
|
|
|
line := s.uploadFrom(fromDE)
|
|
wantPercent(t, "limit_percent", line.LimitPercent, line.LimitPercentSetting,
|
|
tc.limitPercent)
|
|
wantPercent(t, "bytes_percent", line.BytesPercent, line.BytesPercentSetting,
|
|
tc.bytesPercent)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestEachBlocklistThatListsAClientRaisesAnAlertOncePerCooldownAndIsCounted(
|
|
t *testing.T,
|
|
) {
|
|
t.Parallel()
|
|
|
|
const emptyURL = "https://lists.example/empty.txt"
|
|
|
|
s, clk, server, queue := startWithLookupsAndClock(t, map[string]string{
|
|
blocklistURLs: dropURL + "," + torURL + "," + emptyURL,
|
|
blocklistAction: actionLog,
|
|
metricsToken: token,
|
|
})
|
|
loadLists(t, server, map[string][]string{dropURL: {fromDE}, torURL: {fromDE}})
|
|
|
|
// The second request's alerts are repeats, which the cooldown holds
|
|
// back.
|
|
for range 2 {
|
|
wantReputation(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward),
|
|
dropURL, torURL)
|
|
}
|
|
|
|
hit := func(source string) alerts.Alert {
|
|
return alerts.Alert{
|
|
Instance: alertInstance,
|
|
Time: clk.Now(),
|
|
Event: alerts.EventReputationHit,
|
|
Client: netip.MustParseAddr(fromDE),
|
|
Netblock: netip.MustParsePrefix(fromDE + "/32"),
|
|
ASN: asnDE,
|
|
ASName: asNameDE,
|
|
Country: "DE",
|
|
Reason: "listed by a blocklist",
|
|
Detail: map[string]any{"source": source},
|
|
}
|
|
}
|
|
wantAlerts(t, queue, hit(dropURL), hit(torURL))
|
|
|
|
if queue.Suppressed() != 2 {
|
|
t.Errorf("%d alerts held back, want the second request's 2", queue.Suppressed())
|
|
}
|
|
|
|
// Each list's hits, none of its fetches failed, and when its copy was
|
|
// fetched, 0 for the one without.
|
|
metrics := s.scrape(unplaced)
|
|
fetched := float64(listsFetched().Unix())
|
|
|
|
for listURL, want := range map[string]struct{ hits, fetched float64 }{
|
|
dropURL: {2, fetched}, torURL: {2, fetched}, emptyURL: {0, 0},
|
|
} {
|
|
labels := `{instance="` + alertInstance + `",source="` + listURL + `"}`
|
|
|
|
if want.hits == 0 {
|
|
wantNoSeries(t, metrics, "smallwebwaf_reputation_hits_total"+labels)
|
|
} else {
|
|
wantMetric(t, metrics, "smallwebwaf_reputation_hits_total"+labels, want.hits)
|
|
}
|
|
|
|
wantMetric(t, metrics, "smallwebwaf_reputation_failures_total"+labels, 0)
|
|
wantMetric(t, metrics, "smallwebwaf_reputation_last_fetch_timestamp_seconds"+labels,
|
|
want.fetched)
|
|
}
|
|
}
|
|
|
|
// listsFetched is when loadLists has the copies fetched.
|
|
func listsFetched() time.Time {
|
|
return time.Date(2026, 10, 5, 0, 0, 0, 0, time.UTC)
|
|
}
|
|
|
|
// loadLists puts copies of lists into server's lists, by URL, each with
|
|
// its lines, fetched at listsFetched, as reputation.json would at start.
|
|
func loadLists(t *testing.T, server *proxy.Server, copies map[string][]string) {
|
|
t.Helper()
|
|
|
|
lists := make([]reputation.List, 0, len(copies))
|
|
for listURL, lines := range copies {
|
|
lists = append(lists, reputation.List{
|
|
URL: listURL, Fetched: listsFetched(), Lines: lines,
|
|
})
|
|
}
|
|
|
|
err := server.Lists.Load(lists)
|
|
if err != nil {
|
|
t.Fatalf("load the lists: %v", err)
|
|
}
|
|
}
|
|
|
|
// wantReputation checks the URLs of the blocklists the log line names in
|
|
// its reputation.
|
|
func wantReputation(t *testing.T, line logLine, want ...string) {
|
|
t.Helper()
|
|
|
|
if !slices.Equal(line.Reputation, want) {
|
|
t.Errorf("log line has reputation %v, want %v", line.Reputation, want)
|
|
}
|
|
}
|