Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ac46289c62 |
No files matched your search
@@ -22,42 +22,31 @@ import (
|
||||
// an offence, and the request does not wait for the answer. The score is then
|
||||
// used for each address of the client. Its figures are the score and
|
||||
// SWW_WAF_ABUSEIPDB_MIN_SCORE.
|
||||
type abuseIPDB struct {
|
||||
abuseIPDB *reputation.AbuseIPDB
|
||||
// limiter is the table of clients the rate limits count, whose histories
|
||||
// say which clients are offenders.
|
||||
limiter *ratelimit.Limiter
|
||||
keySet bool // SWW_WAF_ABUSEIPDB_KEY is set
|
||||
minScore int64 // SWW_WAF_ABUSEIPDB_MIN_SCORE
|
||||
action string // SWW_WAF_DNSBL_ABUSEIPDB_ACTION
|
||||
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
||||
hitNoter reputationHitNoter
|
||||
}
|
||||
type abuseIPDB struct{}
|
||||
|
||||
func (abuseIPDB) name() string { return "abuseipdb" }
|
||||
|
||||
func (a abuseIPDB) check(
|
||||
func (abuseIPDB) check(
|
||||
ctx context.Context,
|
||||
rq *request,
|
||||
_ time.Time,
|
||||
) *verdict {
|
||||
if !a.keySet {
|
||||
if rq.h.config.AbuseIPDBKey == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
client := clientGroup(a.ipv6GroupPrefix, rq.client)
|
||||
held, _ := a.limiter.Client(client)
|
||||
client := clientGroup(rq.h.config, rq.client)
|
||||
held, _ := rq.h.limiter.Client(client)
|
||||
offender := held.History.Offences != ratelimit.Offences{}
|
||||
|
||||
score, hit := a.abuseIPDB.Hit(ctx, client, rq.client, offender)
|
||||
score, hit := rq.h.abuseIPDB.Hit(ctx, client, rq.client, offender)
|
||||
if !hit {
|
||||
return nil
|
||||
}
|
||||
|
||||
rq.abuseIPDBHit = true
|
||||
rq.addTag(config.AbuseIPDBTag)
|
||||
a.hitNoter.noteHit(
|
||||
rq,
|
||||
rq.noteHit(
|
||||
bans.ReputationHit{
|
||||
Source: reputation.AbuseIPDBSource,
|
||||
Score: &score,
|
||||
@@ -65,14 +54,14 @@ func (a abuseIPDB) check(
|
||||
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
|
||||
)
|
||||
|
||||
if a.action != deny {
|
||||
if rq.h.config.DNSBLAbuseIPDBAction != deny {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &verdict{
|
||||
action: requestlog.ActionClientDenied,
|
||||
figures: map[string]any{
|
||||
figureScore: score, "min_score": a.minScore,
|
||||
figureScore: score, "min_score": rq.h.config.AbuseIPDBMinScore,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -26,9 +26,7 @@ func BanAbuseIPDBBlacklist(
|
||||
scores := make(map[netip.Prefix]int64, len(listed))
|
||||
|
||||
for _, client := range listed {
|
||||
netblock := banNetblock(
|
||||
cfg.BanScopeV4Prefix, cfg.IPv6GroupPrefix, client.Addr,
|
||||
)
|
||||
netblock := banNetblock(cfg, client.Addr)
|
||||
scores[netblock] = max(scores[netblock], client.Score)
|
||||
}
|
||||
|
||||
|
||||
@@ -54,7 +54,7 @@ func TestClientsOnTheAbuseIPDBBlacklistAreBannedForTheBanDays(t *testing.T) {
|
||||
})
|
||||
|
||||
// Two clients in one netblock make one ban, with the higher score.
|
||||
server.BanAbuseIPDBBlacklist([]reputation.BlacklistEntry{
|
||||
server.banAbuseIPDBBlacklist([]reputation.BlacklistEntry{
|
||||
{Addr: netip.MustParseAddr(client), Score: 95},
|
||||
{Addr: netip.MustParseAddr(otherClient), Score: 100},
|
||||
{Addr: netip.MustParseAddr("2001:db8:5::1"), Score: 90},
|
||||
@@ -88,7 +88,7 @@ func TestAbuseIPDBBlacklistMakesNoBanInObserveMode(t *testing.T) {
|
||||
map[string]string{mode: "observe"},
|
||||
)
|
||||
|
||||
server.BanAbuseIPDBBlacklist([]reputation.BlacklistEntry{
|
||||
server.banAbuseIPDBBlacklist([]reputation.BlacklistEntry{
|
||||
{Addr: netip.MustParseAddr(client), Score: 100},
|
||||
})
|
||||
|
||||
|
||||
@@ -224,11 +224,7 @@ func (h *handler) parseBanNetblock(value string) (netip.Prefix, error) {
|
||||
return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errZone)
|
||||
}
|
||||
|
||||
return banNetblock(
|
||||
h.config.BanScopeV4Prefix,
|
||||
h.config.IPv6GroupPrefix,
|
||||
addr,
|
||||
), nil
|
||||
return banNetblock(h.config, addr), nil
|
||||
}
|
||||
|
||||
// expiry returns when a ban made at now for duration ends: duration
|
||||
@@ -299,9 +295,7 @@ func (rq *request) showClient() {
|
||||
Bans: state.BanEntries(rq.h.ledger.Covering(addr)),
|
||||
}
|
||||
|
||||
client, seen := rq.h.limiter.Client(
|
||||
clientGroup(rq.h.config.IPv6GroupPrefix, addr),
|
||||
)
|
||||
client, seen := rq.h.limiter.Client(clientGroup(rq.h.config, addr))
|
||||
if seen {
|
||||
answer.Client = &client
|
||||
}
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -252,7 +251,7 @@ func TestObserveModeWorksOutABanOnlyWhenItsAlertWouldBeSent(t *testing.T) {
|
||||
// never sent them, and returns the queue they wait in as well.
|
||||
func startWithAlerts(
|
||||
t *testing.T, env map[string]string,
|
||||
) (*sender, *clock, *waf.Server, *alerts.Queue) {
|
||||
) (*sender, *clock, *wafAndParts, *alerts.Queue) {
|
||||
t.Helper()
|
||||
|
||||
return startAppWithAlerts(
|
||||
@@ -265,7 +264,7 @@ func startWithAlerts(
|
||||
// startAppWithAlerts is startWithAlerts in front of the app handler.
|
||||
func startAppWithAlerts(
|
||||
t *testing.T, handler http.HandlerFunc, env map[string]string,
|
||||
) (*sender, *clock, *waf.Server, *alerts.Queue) {
|
||||
) (*sender, *clock, *wafAndParts, *alerts.Queue) {
|
||||
t.Helper()
|
||||
|
||||
app := startApp(t, handler)
|
||||
|
||||
@@ -15,7 +15,6 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
)
|
||||
|
||||
// The anomaly thresholds: the prefix of a scope followed by the end of a
|
||||
@@ -332,7 +331,7 @@ func TestRequestCountsForTheASNumberGeoJSGivesBeforeItEnds(t *testing.T) {
|
||||
// kept.
|
||||
geojsURL, _, release := startHeldGeoJS(t)
|
||||
|
||||
var server atomic.Pointer[waf.Server]
|
||||
var server atomic.Pointer[wafAndParts]
|
||||
|
||||
app := startApp(t, func(http.ResponseWriter, *http.Request) {
|
||||
release()
|
||||
|
||||
@@ -411,7 +411,7 @@ func TestTheHeaderLimitsComeBeforeEveryCheckThatRefuses(t *testing.T) {
|
||||
env map[string]string
|
||||
path string
|
||||
// prepare gets the server ready, at now, before the question.
|
||||
prepare func(t *testing.T, server *waf.Server, now time.Time)
|
||||
prepare func(t *testing.T, server *wafAndParts, now time.Time)
|
||||
}{
|
||||
{"SWW_WAF_DENY_NETS", map[string]string{denyNets: client}, "/", leaveAsItIs},
|
||||
{"a ban", nil, "/", banClient},
|
||||
@@ -426,21 +426,21 @@ func TestTheHeaderLimitsComeBeforeEveryCheckThatRefuses(t *testing.T) {
|
||||
"a DNS blocklist", map[string]string{
|
||||
dnsblZones: dnsblZone, dnsblResolver: noResolver, dnsblAbuseIPDBAction: actionDeny,
|
||||
},
|
||||
"/", func(_ *testing.T, server *waf.Server, _ time.Time) {
|
||||
"/", func(_ *testing.T, server *wafAndParts, _ time.Time) {
|
||||
loadVerdicts(server, map[string][]string{client: {dnsblZone}})
|
||||
},
|
||||
},
|
||||
{
|
||||
"AbuseIPDB",
|
||||
map[string]string{abuseIPDBKey: accountKey, dnsblAbuseIPDBAction: actionDeny},
|
||||
"/", func(_ *testing.T, server *waf.Server, _ time.Time) {
|
||||
"/", func(_ *testing.T, server *wafAndParts, _ time.Time) {
|
||||
// Over SWW_WAF_ABUSEIPDB_MIN_SCORE, 75 by default.
|
||||
loadScores(server, map[string]int64{client: 80})
|
||||
},
|
||||
},
|
||||
{
|
||||
"SWW_WAF_TAG_DENY", map[string]string{tagLists: "vpn=" + vpnList, tagDeny: "vpn"},
|
||||
"/", func(t *testing.T, server *waf.Server, _ time.Time) {
|
||||
"/", func(t *testing.T, server *wafAndParts, _ time.Time) {
|
||||
t.Helper()
|
||||
loadLists(t, server, map[string][]string{vpnList: {client}})
|
||||
},
|
||||
@@ -647,7 +647,7 @@ func TestObserveModeAsksGeoJSNothingForARequestOverTheHeaderLimits(
|
||||
// for env. No request to it may use the network.
|
||||
func newWAFInBubble(
|
||||
t *testing.T, env map[string]string,
|
||||
) (*waf.Server, *output) {
|
||||
) (*wafAndParts, *output) {
|
||||
t.Helper()
|
||||
|
||||
settings := map[string]string{tagDeny: "country=KP"}
|
||||
@@ -661,7 +661,7 @@ func newWAFInBubble(
|
||||
// askDirectly hands a question about a request from client straight to server,
|
||||
// as change leaves it, and returns the answer's status and how long it took.
|
||||
func askDirectly(
|
||||
t *testing.T, server *waf.Server, change func(*http.Request),
|
||||
t *testing.T, server *wafAndParts, change func(*http.Request),
|
||||
) (int, time.Duration) {
|
||||
t.Helper()
|
||||
|
||||
@@ -685,7 +685,7 @@ func askDirectly(
|
||||
}
|
||||
|
||||
// banClient bans client at now.
|
||||
func banClient(_ *testing.T, server *waf.Server, now time.Time) {
|
||||
func banClient(_ *testing.T, server *wafAndParts, now time.Time) {
|
||||
server.Ledger.BanForLimit(
|
||||
netip.MustParsePrefix(client+"/32"),
|
||||
now,
|
||||
@@ -694,7 +694,7 @@ func banClient(_ *testing.T, server *waf.Server, now time.Time) {
|
||||
}
|
||||
|
||||
// countClient counts one request from client at now for the rate limits.
|
||||
func countClient(_ *testing.T, server *waf.Server, now time.Time) {
|
||||
func countClient(_ *testing.T, server *wafAndParts, now time.Time) {
|
||||
server.Limiter.Count(
|
||||
netip.MustParsePrefix(client+"/32"),
|
||||
now,
|
||||
@@ -703,7 +703,7 @@ func countClient(_ *testing.T, server *waf.Server, now time.Time) {
|
||||
}
|
||||
|
||||
// listClient puts client on the blocklist at dropURL.
|
||||
func listClient(t *testing.T, server *waf.Server, now time.Time) {
|
||||
func listClient(t *testing.T, server *wafAndParts, now time.Time) {
|
||||
t.Helper()
|
||||
|
||||
err := server.Lists.Load([]reputation.List{{
|
||||
@@ -715,7 +715,7 @@ func listClient(t *testing.T, server *waf.Server, now time.Time) {
|
||||
}
|
||||
|
||||
// leaveAsItIs prepares nothing.
|
||||
func leaveAsItIs(*testing.T, *waf.Server, time.Time) {}
|
||||
func leaveAsItIs(*testing.T, *wafAndParts, time.Time) {}
|
||||
|
||||
func TestARefusedRequestDoesNotWaitForASlowGeoJS(t *testing.T) {
|
||||
t.Parallel()
|
||||
@@ -726,7 +726,7 @@ func TestARefusedRequestDoesNotWaitForASlowGeoJS(t *testing.T) {
|
||||
name string
|
||||
env map[string]string
|
||||
// prepare gets the server ready, at now, before the question.
|
||||
prepare func(t *testing.T, server *waf.Server, now time.Time)
|
||||
prepare func(t *testing.T, server *wafAndParts, now time.Time)
|
||||
// cookie is the request's Cookie header.
|
||||
cookie string
|
||||
status int
|
||||
@@ -811,7 +811,7 @@ func TestAQuestionWaitsForASlowGeoJSOnlyAsLongAsTheDefaultSays(
|
||||
name string
|
||||
env map[string]string
|
||||
// prepare gets the server ready, at now, before the question.
|
||||
prepare func(t *testing.T, server *waf.Server, now time.Time)
|
||||
prepare func(t *testing.T, server *wafAndParts, now time.Time)
|
||||
status int
|
||||
action string
|
||||
}{
|
||||
@@ -877,7 +877,7 @@ func TestWAFAnswersWithinTheTimeLimitWhileEveryOutsideSourceHangs(
|
||||
held, _, _ := startHeldGeoJS(t)
|
||||
resolver, releaseResolver := startHeldResolver(t)
|
||||
|
||||
server, out, _ := newWAFWith(t, waf.New,
|
||||
server, out, _ := newWAFWith(t, func(*waf.Params) {},
|
||||
silentGeoJSURL, silentAbuseIPDBURL, time.Now, map[string]string{
|
||||
tagDeny: "country=KP", coreRuleSetMode: block,
|
||||
dnsblZones: dnsblZone, dnsblResolver: resolver,
|
||||
|
||||
+61
-91
@@ -1,7 +1,7 @@
|
||||
package waf
|
||||
|
||||
// The heuristic active_ban, and banMaker, which makes the bans of several
|
||||
// heuristics. active_ban reads SWW_WAF_MODE; the bans are the ban ledger's.
|
||||
// The heuristic active_ban, and the code that makes bans, which several
|
||||
// heuristics share. active_ban reads SWW_WAF_MODE; the bans are the ban ledger's.
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
@@ -25,22 +25,18 @@ func (rq *request) banResponse(action string) *refusal {
|
||||
// in, in force at now. It notes for the log line when that ban ends. A request
|
||||
// that makes the ban permanent, or in observe mode would have, raises the alert
|
||||
// for it. Its figures are the ban's netblock and when it ends.
|
||||
type activeBan struct {
|
||||
banMaker banMaker
|
||||
}
|
||||
type activeBan struct{}
|
||||
|
||||
func (activeBan) name() string { return "active_ban" }
|
||||
|
||||
func (a activeBan) check(
|
||||
func (activeBan) check(
|
||||
_ context.Context,
|
||||
rq *request,
|
||||
now time.Time,
|
||||
) *verdict {
|
||||
ledger := a.banMaker.ledger
|
||||
|
||||
check := ledger.Check
|
||||
if a.banMaker.observe {
|
||||
check = ledger.Find // the ban refuses nothing, and stays as it is
|
||||
check := rq.h.ledger.Check
|
||||
if rq.h.config.Observe {
|
||||
check = rq.h.ledger.Find // the ban refuses nothing, and stays as it is
|
||||
}
|
||||
|
||||
ban, banned, madePermanent := check(rq.client, now)
|
||||
@@ -50,7 +46,7 @@ func (a activeBan) check(
|
||||
|
||||
if madePermanent {
|
||||
ban.Expires = time.Time{} // the ban made permanent, which Find leaves as it is
|
||||
a.banMaker.alertBan(rq, ban)
|
||||
rq.alertBan(ban)
|
||||
}
|
||||
|
||||
if !banned {
|
||||
@@ -85,12 +81,11 @@ func limitFigures(hit ratelimit.Hit) map[string]any {
|
||||
|
||||
// countedBytes returns the request's bytes, once it has ended, as the
|
||||
// byte limits and the anomaly thresholds count them: the response's body
|
||||
// bytes, the request's, or both, as bytesCounted, SWW_WAF_BYTES_COUNTED,
|
||||
// says.
|
||||
func (rq *request) countedBytes(bytesCounted string) int64 {
|
||||
// bytes, the request's, or both, as SWW_WAF_BYTES_COUNTED says.
|
||||
func (rq *request) countedBytes() int64 {
|
||||
response, request := rq.out.bytes, rq.requestBytes
|
||||
|
||||
switch bytesCounted {
|
||||
switch rq.h.config.BytesCounted {
|
||||
case "response":
|
||||
return response
|
||||
case "request":
|
||||
@@ -100,29 +95,7 @@ func (rq *request) countedBytes(bytesCounted string) int64 {
|
||||
}
|
||||
}
|
||||
|
||||
// banMaker makes the bans of the heuristics that ban a client, in the ban
|
||||
// ledger, and raises the alert for each. In observe mode it makes no ban, and
|
||||
// raises the alert for the ban it would have made.
|
||||
type banMaker struct {
|
||||
ledger *bans.Ledger
|
||||
// limiter is the table of clients the rate limits count. A ban for a limit
|
||||
// sets the client's counters back to zero, and a ban's notes give how many
|
||||
// requests its netblock has sent.
|
||||
limiter *ratelimit.Limiter
|
||||
metrics *metrics.Metrics
|
||||
alerts *alerts.Queue
|
||||
// observe is true in observe mode (SWW_WAF_MODE).
|
||||
observe bool
|
||||
// banResponse is SWW_WAF_BAN_RESPONSE, the status the notes of a ban for a
|
||||
// clear sign of attack give.
|
||||
banResponse int
|
||||
// banScopeV4Prefix is SWW_WAF_BAN_SCOPE_V4_PREFIX and ipv6GroupPrefix
|
||||
// SWW_WAF_IPV6_GROUP_PREFIX, which set the netblock a ban covers.
|
||||
banScopeV4Prefix int
|
||||
ipv6GroupPrefix int
|
||||
}
|
||||
|
||||
// banForLimit bans the netblock of rq's client at now for a broken limit, the
|
||||
// banForLimit bans the client's netblock at now for a broken limit, the
|
||||
// one hit names, notes the offence for the log line and counts the hit in
|
||||
// the metrics. status is what the client was sent, or is sent:
|
||||
// SWW_WAF_BAN_RESPONSE for a request over a rate limit, what the report says for
|
||||
@@ -132,8 +105,7 @@ type banMaker struct {
|
||||
// the client's counters back to zero. In observe mode it makes no ban and
|
||||
// sets nothing back, and raises the alert for the ban it would have made,
|
||||
// if that alert would be sent.
|
||||
func (b banMaker) banForLimit(
|
||||
rq *request,
|
||||
func (rq *request) banForLimit(
|
||||
now time.Time,
|
||||
hit ratelimit.Hit,
|
||||
status int,
|
||||
@@ -148,11 +120,11 @@ func (b banMaker) banForLimit(
|
||||
}
|
||||
|
||||
rq.line.Offence = requestlog.OffenceLimit
|
||||
rq.h.metrics.LimitHit(hit)
|
||||
|
||||
b.metrics.LimitHit(hit)
|
||||
|
||||
netblock := banNetblock(b.banScopeV4Prefix, b.ipv6GroupPrefix, rq.client)
|
||||
if b.observe && !b.wouldAlertBan(netblock, now, bans.CauseLimit) {
|
||||
netblock := banNetblock(rq.h.config, rq.client)
|
||||
if rq.h.config.Observe &&
|
||||
!rq.wouldAlertBan(netblock, now, bans.CauseLimit) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -166,7 +138,7 @@ func (b banMaker) banForLimit(
|
||||
Count: hit.Count,
|
||||
Reputation: rq.reputation,
|
||||
Request: rq.noted(now, status),
|
||||
Requests: b.netblockRequests(netblock),
|
||||
Requests: rq.netblockRequests(netblock),
|
||||
}
|
||||
|
||||
switch hit.Kind {
|
||||
@@ -176,36 +148,35 @@ func (b banMaker) banForLimit(
|
||||
notes.LimitPercent, notes.LimitPercentSetting = rq.bytesLimitPercent.logged()
|
||||
}
|
||||
|
||||
if b.observe {
|
||||
ban, wouldBan := b.ledger.WouldBanForLimit(netblock, now, notes)
|
||||
if rq.h.config.Observe {
|
||||
ban, wouldBan := rq.h.ledger.WouldBanForLimit(netblock, now, notes)
|
||||
if wouldBan {
|
||||
b.alertBan(rq, ban)
|
||||
rq.alertBan(ban)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
ban, made := b.ledger.BanForLimit(netblock, now, notes)
|
||||
b.limiter.Reset(clientGroup(b.ipv6GroupPrefix, rq.client))
|
||||
ban, made := rq.h.ledger.BanForLimit(netblock, now, notes)
|
||||
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client))
|
||||
rq.line.BanExpires = banExpires(ban)
|
||||
|
||||
if made {
|
||||
b.alertBan(rq, ban)
|
||||
rq.alertBan(ban)
|
||||
}
|
||||
}
|
||||
|
||||
// banForAttack bans the netblock of rq's client at now for a clear sign of
|
||||
// banForAttack bans the client's netblock at now for a clear sign of
|
||||
// attack, which notes name: the ban rule that matched, or the trap path
|
||||
// asked for. It fills in the rest of the notes. In observe mode it makes
|
||||
// no ban, and raises the alert for the ban it would have made, if that
|
||||
// alert would be sent. It returns the action for that ban.
|
||||
func (b banMaker) banForAttack(
|
||||
rq *request, now time.Time, notes bans.Notes,
|
||||
) string {
|
||||
netblock := banNetblock(b.banScopeV4Prefix, b.ipv6GroupPrefix, rq.client)
|
||||
if b.observe && !b.wouldAlertBan(netblock, now, bans.CauseAttack) {
|
||||
func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
|
||||
netblock := banNetblock(rq.h.config, rq.client)
|
||||
if rq.h.config.Observe &&
|
||||
!rq.wouldAlertBan(netblock, now, bans.CauseAttack) {
|
||||
return banAction(
|
||||
b.ledger.WouldBePermanent(netblock, now, bans.CauseAttack),
|
||||
rq.h.ledger.WouldBePermanent(netblock, now, bans.CauseAttack),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -213,23 +184,23 @@ func (b banMaker) banForAttack(
|
||||
notes.ASName = rq.line.ASName
|
||||
notes.Country = rq.line.Country
|
||||
notes.Reputation = rq.reputation
|
||||
notes.Request = rq.noted(now, b.banResponse)
|
||||
notes.Requests = b.netblockRequests(netblock)
|
||||
notes.Request = rq.noted(now, rq.h.config.BanResponse)
|
||||
notes.Requests = rq.netblockRequests(netblock)
|
||||
|
||||
if b.observe {
|
||||
ban, wouldBan := b.ledger.WouldBanForAttack(netblock, now, notes)
|
||||
if rq.h.config.Observe {
|
||||
ban, wouldBan := rq.h.ledger.WouldBanForAttack(netblock, now, notes)
|
||||
if wouldBan {
|
||||
b.alertBan(rq, ban)
|
||||
rq.alertBan(ban)
|
||||
}
|
||||
|
||||
return banAction(ban.Permanent())
|
||||
}
|
||||
|
||||
ban, made := b.ledger.BanForAttack(netblock, now, notes)
|
||||
ban, made := rq.h.ledger.BanForAttack(netblock, now, notes)
|
||||
rq.line.BanExpires = banExpires(ban)
|
||||
|
||||
if made {
|
||||
b.alertBan(rq, ban)
|
||||
rq.alertBan(ban)
|
||||
}
|
||||
|
||||
return banAction(ban.Permanent())
|
||||
@@ -240,22 +211,23 @@ func (b banMaker) banForAttack(
|
||||
// have made is worked out only then, at most once per
|
||||
// SWW_WAF_ALERT_COOLDOWN and never with no webhook set: its notes count the
|
||||
// netblock's requests, which can mean going through every client.
|
||||
func (b banMaker) wouldAlertBan(
|
||||
func (rq *request) wouldAlertBan(
|
||||
netblock netip.Prefix, now time.Time, cause string,
|
||||
) bool {
|
||||
event := alerts.EventBan
|
||||
if b.ledger.WouldBePermanent(netblock, now, cause) {
|
||||
if rq.h.ledger.WouldBePermanent(netblock, now, cause) {
|
||||
event = alerts.EventPermanentBan
|
||||
}
|
||||
|
||||
return b.alerts.WouldSend(event, netblock)
|
||||
return rq.h.alerts.WouldSend(event, netblock)
|
||||
}
|
||||
|
||||
// alertBan raises the alert for ban, which rq made, or made permanent:
|
||||
// permanent_ban for a permanent ban, ban for another. Its detail gives the
|
||||
// ban's cause, when it ends, and its notes, and in observe mode, where ban is
|
||||
// the ban that would have been made, or made permanent, mode, observe.
|
||||
func (b banMaker) alertBan(rq *request, ban bans.Ban) {
|
||||
// alertBan raises the alert for ban, which the request made, or made
|
||||
// permanent: permanent_ban for a permanent ban, ban for another. Its
|
||||
// detail gives the ban's cause, when it ends, and its notes, and in
|
||||
// observe mode, where ban is the ban that would have been made, or made
|
||||
// permanent, mode, observe.
|
||||
func (rq *request) alertBan(ban bans.Ban) {
|
||||
event := alerts.EventBan
|
||||
if ban.Permanent() {
|
||||
event = alerts.EventPermanentBan
|
||||
@@ -264,11 +236,11 @@ func (b banMaker) alertBan(rq *request, ban bans.Ban) {
|
||||
detail := map[string]any{
|
||||
"cause": ban.Cause, "ban_expires": banExpires(ban), "notes": ban.Notes,
|
||||
}
|
||||
if b.observe {
|
||||
if rq.h.config.Observe {
|
||||
detail["mode"] = "observe"
|
||||
}
|
||||
|
||||
b.alerts.Raise(alerts.Alert{
|
||||
rq.h.alerts.Raise(alerts.Alert{
|
||||
Event: event,
|
||||
Host: rq.site,
|
||||
Client: rq.client,
|
||||
@@ -281,13 +253,6 @@ func (b banMaker) alertBan(rq *request, ban bans.Ban) {
|
||||
})
|
||||
}
|
||||
|
||||
// netblockRequests is how many requests netblock has sent since it was
|
||||
// first seen, the request being banned for included: the histories count it
|
||||
// only once it has ended.
|
||||
func (b banMaker) netblockRequests(netblock netip.Prefix) int64 {
|
||||
return b.limiter.Requests(netblock) + 1
|
||||
}
|
||||
|
||||
// noted is the request, at now, with status, what the client was sent, or
|
||||
// in observe mode would have been, as the notes of the ban it makes keep
|
||||
// it.
|
||||
@@ -302,18 +267,23 @@ func (rq *request) noted(now time.Time, status int) bans.Request {
|
||||
}
|
||||
}
|
||||
|
||||
// netblockRequests is how many requests netblock has sent since it was
|
||||
// first seen, this one included: the histories count it only once it has
|
||||
// ended.
|
||||
func (rq *request) netblockRequests(netblock netip.Prefix) int64 {
|
||||
return rq.h.limiter.Requests(netblock) + 1
|
||||
}
|
||||
|
||||
// banNetblock is the netblock a ban on client covers: its IPv4 address,
|
||||
// widened to banScopeV4Prefix, SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group
|
||||
// clientGroup counts it in, of length ipv6GroupPrefix.
|
||||
func banNetblock(
|
||||
banScopeV4Prefix, ipv6GroupPrefix int, client netip.Addr,
|
||||
) netip.Prefix {
|
||||
// widened to SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group clientGroup
|
||||
// counts it in.
|
||||
func banNetblock(cfg *config.WAF, client netip.Addr) netip.Prefix {
|
||||
addr := client.Unmap()
|
||||
if addr.Is4() {
|
||||
return netip.PrefixFrom(addr, banScopeV4Prefix).Masked()
|
||||
return netip.PrefixFrom(addr, cfg.BanScopeV4Prefix).Masked()
|
||||
}
|
||||
|
||||
return clientGroup(ipv6GroupPrefix, addr)
|
||||
return clientGroup(cfg, addr)
|
||||
}
|
||||
|
||||
// banExpires is when ban ends, as the log line gives it: a time, or
|
||||
|
||||
@@ -16,7 +16,6 @@ import (
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -466,7 +465,7 @@ func (c *clock) advance(d time.Duration) {
|
||||
// window.
|
||||
func startWithClock(
|
||||
t *testing.T, geojsURL string, env map[string]string,
|
||||
) (*sender, *clock, *waf.Server) {
|
||||
) (*sender, *clock, *wafAndParts) {
|
||||
t.Helper()
|
||||
|
||||
app := startApp(t, func(http.ResponseWriter, *http.Request) {})
|
||||
|
||||
@@ -54,7 +54,7 @@ func (rq *request) runBesideLookup(
|
||||
) *verdict {
|
||||
var running atomic.Pointer[string]
|
||||
|
||||
running.Store(new(rq.h.trapPaths.name()))
|
||||
running.Store(new(trapPaths{}.name()))
|
||||
|
||||
found := make(chan *besideLookupFindings, 1)
|
||||
|
||||
@@ -92,7 +92,7 @@ func (rq *request) runBesideLookup(
|
||||
return v
|
||||
}
|
||||
|
||||
return rq.runHeuristics(ctx, rq.h.afterBesideLookup, rq.h.now())
|
||||
return rq.runHeuristics(ctx, afterBesideLookup(), rq.h.now())
|
||||
}
|
||||
|
||||
// findBesideLookup does the work of the heuristics of besideLookup on the
|
||||
@@ -105,39 +105,39 @@ func (rq *request) findBesideLookup(
|
||||
) *besideLookupFindings {
|
||||
findings := &besideLookupFindings{}
|
||||
|
||||
findings.trapPath = rq.h.trapPaths.trapPath(rq)
|
||||
findings.trapPath = rq.trapPath()
|
||||
if findings.trapPath != "" {
|
||||
findings.refuses = true
|
||||
|
||||
return findings
|
||||
}
|
||||
|
||||
running.Store(new(rq.h.ruleFiles.name()))
|
||||
running.Store(new(ruleFiles{}.name()))
|
||||
|
||||
findings.rules = rq.h.ruleFiles.rules.Match(rq.in)
|
||||
findings.rules = rq.h.rules.Match(rq.in)
|
||||
if refusedByRule(findings.rules) {
|
||||
findings.refuses = true
|
||||
|
||||
return findings
|
||||
}
|
||||
|
||||
coreRuleSet := rq.h.coreRuleSet
|
||||
running.Store(new(coreRuleSet.name()))
|
||||
running.Store(new(coreRuleSet{}.name()))
|
||||
|
||||
if coreRuleSet.mode == config.CoreRuleSetModeOff ||
|
||||
pathExempt(rq.in, coreRuleSet.exemptPaths) {
|
||||
cfg := rq.h.config
|
||||
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff ||
|
||||
pathExempt(rq.in, cfg.CoreRuleSetExemptPaths) {
|
||||
return findings
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
// A question's body has been read into memory, so that reading it here
|
||||
// cannot fail.
|
||||
findings.inspection, findings.read, _ = coreRuleSet.inspector.Inspect(
|
||||
findings.inspection, findings.read, _ = rq.h.coreRuleSet.Inspect(
|
||||
rq.in, rq.client, rq.in.Body,
|
||||
)
|
||||
findings.inspectionTime = time.Since(start)
|
||||
findings.inspected = true
|
||||
findings.refuses = coreRuleSet.refusedByCoreRuleSet(findings.inspection)
|
||||
findings.refuses = rq.refusedByCoreRuleSet(findings.inspection)
|
||||
|
||||
return findings
|
||||
}
|
||||
@@ -151,7 +151,7 @@ func (rq *request) applyFindings(
|
||||
) *verdict {
|
||||
rq.findings = findings
|
||||
|
||||
return rq.runHeuristics(ctx, rq.h.besideLookup, rq.h.now())
|
||||
return rq.runHeuristics(ctx, besideLookup(), rq.h.now())
|
||||
}
|
||||
|
||||
// timedOut refuses the request for the heuristics of besideLookup not having
|
||||
|
||||
@@ -62,7 +62,7 @@ func coreRuleSetMatch() coreruleset.Result {
|
||||
// at its default but for env. No request to it may use the network.
|
||||
func newWAFHoldingTheCoreRuleSet(
|
||||
t *testing.T, crs *heldCoreRuleSet, env map[string]string,
|
||||
) (*waf.Server, *output, *alerts.Queue) {
|
||||
) (*wafAndParts, *output, *alerts.Queue) {
|
||||
t.Helper()
|
||||
|
||||
settings := map[string]string{
|
||||
@@ -71,11 +71,9 @@ func newWAFHoldingTheCoreRuleSet(
|
||||
}
|
||||
maps.Copy(settings, env)
|
||||
|
||||
server, out, queue := newWAFWith(t, func(params waf.Params) *waf.Server {
|
||||
server, out, queue := newWAFWith(t, func(params *waf.Params) {
|
||||
params.CoreRuleSet = crs
|
||||
params.BesideLookupTimeLimit = waf.BesideLookupTimeLimit
|
||||
|
||||
return waf.New(params)
|
||||
}, silentGeoJSURL, abuseIPDBURL, time.Now, settings)
|
||||
|
||||
return server, out, queue
|
||||
@@ -105,7 +103,7 @@ func wantAnswer(
|
||||
// left no trace: line has none of its fields, and server has counted no match
|
||||
// of it and raised no alert for one.
|
||||
func wantNoCoreRuleSetTrace(
|
||||
t *testing.T, server *waf.Server, queue *alerts.Queue, line logLine,
|
||||
t *testing.T, server *wafAndParts, queue *alerts.Queue, line logLine,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
|
||||
+15
-36
@@ -2,7 +2,6 @@ package waf
|
||||
|
||||
import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
)
|
||||
|
||||
// whole is the percentage of each limit a client gets when no biased
|
||||
@@ -30,25 +29,6 @@ func biasedThresholdsSet(cfg *config.WAF) bool {
|
||||
cfg.ASNLimitPercentURL != ""
|
||||
}
|
||||
|
||||
// biasedThresholds are the settings and the lists by which limitPercentages
|
||||
// lowers a client's limits.
|
||||
type biasedThresholds struct {
|
||||
// lists hold the file SWW_WAF_ASN_LIMIT_PERCENT_URL names.
|
||||
lists *reputation.Lists
|
||||
tagLimitPercent map[string]int64 // SWW_WAF_TAG_LIMIT_PERCENT
|
||||
asnBytesLimitPercent map[string]int64 // SWW_WAF_ASN_BYTES_LIMIT_PERCENT
|
||||
countryBytesLimitPercent map[string]int64 // SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT
|
||||
unknownCountryLimitPercent int64 // SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT
|
||||
// blocklistAction and blocklistLimitPercent are SWW_WAF_BLOCKLIST_ACTION
|
||||
// and its percentage.
|
||||
blocklistAction string
|
||||
blocklistLimitPercent int64
|
||||
// dnsblAbuseIPDBAction and dnsblAbuseIPDBLimitPercent are
|
||||
// SWW_WAF_DNSBL_ABUSEIPDB_ACTION and its percentage.
|
||||
dnsblAbuseIPDBAction string
|
||||
dnsblAbuseIPDBLimitPercent int64
|
||||
}
|
||||
|
||||
// limitPercentages returns the client's limit percentages, for the rate
|
||||
// limits and for the byte limits, by its tags, its AS number and country
|
||||
// as looked up, each "" when unknown, and the blocklists, DNSBL zones and
|
||||
@@ -65,37 +45,36 @@ type biasedThresholds struct {
|
||||
// the byte limits, SWW_WAF_ASN_BYTES_LIMIT_PERCENT and
|
||||
// SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT take the place of the first three for
|
||||
// an AS number or a country they list.
|
||||
func (b biasedThresholds) limitPercentages(
|
||||
rq *request,
|
||||
) (percentage, percentage) {
|
||||
func (rq *request) limitPercentages() (percentage, percentage) {
|
||||
cfg := rq.h.config
|
||||
asn, country := rq.line.ASN, rq.line.Country
|
||||
|
||||
unknown := percentage{percent: whole}
|
||||
if country == "" && !rq.skippedLookupWait {
|
||||
unknown = percentage{
|
||||
b.unknownCountryLimitPercent,
|
||||
cfg.UnknownCountryLimitPercent,
|
||||
"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
|
||||
}
|
||||
}
|
||||
|
||||
fetched := percentage{percent: whole}
|
||||
if percent, listed := b.lists.ASNLimitPercent(asn); listed {
|
||||
if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed {
|
||||
fetched = percentage{percent, "SWW_WAF_ASN_LIMIT_PERCENT_URL"}
|
||||
}
|
||||
|
||||
blocklisted := percentage{percent: whole}
|
||||
if rq.blocklisted && b.blocklistAction == "limit" {
|
||||
if rq.blocklisted && cfg.BlocklistAction == "limit" {
|
||||
blocklisted = percentage{
|
||||
b.blocklistLimitPercent,
|
||||
cfg.BlocklistLimitPercent,
|
||||
"SWW_WAF_BLOCKLIST_ACTION",
|
||||
}
|
||||
}
|
||||
|
||||
dnsblAbuseIPDBListed := percentage{percent: whole}
|
||||
if (rq.dnsblListed || rq.abuseIPDBHit) &&
|
||||
b.dnsblAbuseIPDBAction == "limit" {
|
||||
cfg.DNSBLAbuseIPDBAction == "limit" {
|
||||
dnsblAbuseIPDBListed = percentage{
|
||||
b.dnsblAbuseIPDBLimitPercent,
|
||||
cfg.DNSBLAbuseIPDBLimitPercent,
|
||||
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
|
||||
}
|
||||
}
|
||||
@@ -103,11 +82,11 @@ func (b biasedThresholds) limitPercentages(
|
||||
const tagSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
|
||||
|
||||
asnRequests := lowest(
|
||||
given(b.tagLimitPercent, asnTag(asn), tagSetting),
|
||||
given(cfg.TagLimitPercent, asnTag(asn), tagSetting),
|
||||
fetched,
|
||||
)
|
||||
countryRequests := given(
|
||||
b.tagLimitPercent,
|
||||
cfg.TagLimitPercent,
|
||||
countryTag(country),
|
||||
tagSetting,
|
||||
)
|
||||
@@ -118,23 +97,23 @@ func (b biasedThresholds) limitPercentages(
|
||||
if tag != asnTag(asn) && tag != countryTag(country) {
|
||||
otherTags = lowest(
|
||||
otherTags,
|
||||
given(b.tagLimitPercent, tag, tagSetting),
|
||||
given(cfg.TagLimitPercent, tag, tagSetting),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
asnBytes, countryBytes := asnRequests, countryRequests
|
||||
if _, listed := b.asnBytesLimitPercent[asn]; listed {
|
||||
if _, listed := cfg.ASNBytesLimitPercent[asn]; listed {
|
||||
asnBytes = given(
|
||||
b.asnBytesLimitPercent,
|
||||
cfg.ASNBytesLimitPercent,
|
||||
asn,
|
||||
"SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
|
||||
)
|
||||
}
|
||||
|
||||
if _, listed := b.countryBytesLimitPercent[country]; listed {
|
||||
if _, listed := cfg.CountryBytesLimitPercent[country]; listed {
|
||||
countryBytes = given(
|
||||
b.countryBytesLimitPercent,
|
||||
cfg.CountryBytesLimitPercent,
|
||||
country,
|
||||
"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
|
||||
)
|
||||
|
||||
@@ -529,7 +529,7 @@ func TestBanForALoweredLimitGivesThePercentageInItsNotesAndItsAlert(
|
||||
// clock.
|
||||
func startWithLookups(
|
||||
t *testing.T, env map[string]string,
|
||||
) (*sender, *waf.Server, *alerts.Queue) {
|
||||
) (*sender, *wafAndParts, *alerts.Queue) {
|
||||
t.Helper()
|
||||
|
||||
s, _, server, queue := startWithLookupsAndClock(t, env)
|
||||
@@ -544,7 +544,7 @@ func startWithLookups(
|
||||
// country, and no other address.
|
||||
func startWithLookupsAndClock(
|
||||
t *testing.T, env map[string]string,
|
||||
) (*sender, *clock, *waf.Server, *alerts.Queue) {
|
||||
) (*sender, *clock, *wafAndParts, *alerts.Queue) {
|
||||
t.Helper()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
|
||||
@@ -584,7 +584,7 @@ func (s *sender) uploadFrom(from string) logLine {
|
||||
// straight to server's handler, without the network, and returns once it is
|
||||
// answered.
|
||||
func serveFromDE(
|
||||
t *testing.T, server *waf.Server, method string, body io.Reader,
|
||||
t *testing.T, server *wafAndParts, method string, body io.Reader,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
@@ -15,24 +14,20 @@ import (
|
||||
// and refuses the request when SWW_WAF_BLOCKLIST_ACTION is deny. Being limit, it
|
||||
// lowers the client's limits instead (see limitPercentages), and being log, it
|
||||
// does nothing more. Its figure is the blocklists that list the client.
|
||||
type blocklists struct {
|
||||
lists *reputation.Lists
|
||||
action string // SWW_WAF_BLOCKLIST_ACTION
|
||||
hitNoter reputationHitNoter
|
||||
}
|
||||
type blocklists struct{}
|
||||
|
||||
func (blocklists) name() string { return "blocklists" }
|
||||
|
||||
func (b blocklists) check(
|
||||
func (blocklists) check(
|
||||
_ context.Context,
|
||||
rq *request,
|
||||
_ time.Time,
|
||||
) *verdict {
|
||||
listedBy := b.lists.ListedBy(rq.client)
|
||||
listedBy := rq.h.lists.ListedBy(rq.client)
|
||||
rq.blocklisted = len(listedBy) > 0
|
||||
b.hitNoter.noteListed(rq, listedBy, "listed by a blocklist")
|
||||
rq.noteListed(listedBy, "listed by a blocklist")
|
||||
|
||||
if !b.refusedByBlocklists(listedBy) {
|
||||
if !rq.refusedByBlocklists(listedBy) {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -45,6 +40,6 @@ func (b blocklists) check(
|
||||
// refusedByBlocklists reports whether the heuristic blocklists refuses a
|
||||
// client, given listedBy, the blocklists that list it: it does when one lists
|
||||
// it and SWW_WAF_BLOCKLIST_ACTION is deny.
|
||||
func (b blocklists) refusedByBlocklists(listedBy []string) bool {
|
||||
return len(listedBy) > 0 && b.action == deny
|
||||
func (rq *request) refusedByBlocklists(listedBy []string) bool {
|
||||
return len(listedBy) > 0 && rq.h.config.BlocklistAction == deny
|
||||
}
|
||||
@@ -6,8 +6,6 @@ package waf
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
)
|
||||
|
||||
// byteLimits counts the request's bytes, as countedBytes gives them, for the
|
||||
@@ -18,16 +16,11 @@ import (
|
||||
// the client over a byte limit, as its limit percentage for the byte limits
|
||||
// lowers it, break it, which bans the client; the response was passed on whole.
|
||||
// Its figures are the window, the count and the limit.
|
||||
type byteLimits struct {
|
||||
limiter *ratelimit.Limiter
|
||||
bytesCounted string // SWW_WAF_BYTES_COUNTED
|
||||
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
||||
banMaker banMaker
|
||||
}
|
||||
type byteLimits struct{}
|
||||
|
||||
func (byteLimits) name() string { return "byte_limits" }
|
||||
|
||||
func (b byteLimits) check(
|
||||
func (byteLimits) check(
|
||||
_ context.Context,
|
||||
rq *request,
|
||||
now time.Time,
|
||||
@@ -36,10 +29,10 @@ func (b byteLimits) check(
|
||||
return nil
|
||||
}
|
||||
|
||||
counts, hit, over := b.limiter.CountBytes(
|
||||
clientGroup(b.ipv6GroupPrefix, rq.client),
|
||||
counts, hit, over := rq.h.limiter.CountBytes(
|
||||
clientGroup(rq.h.config, rq.client),
|
||||
now,
|
||||
rq.countedBytes(b.bytesCounted),
|
||||
rq.countedBytes(),
|
||||
rq.bytesLimitPercent.percent,
|
||||
)
|
||||
rq.line.Counts.MinuteBytes = counts.MinuteBytes
|
||||
@@ -50,7 +43,7 @@ func (b byteLimits) check(
|
||||
return nil
|
||||
}
|
||||
|
||||
b.banMaker.banForLimit(rq, now, hit, rq.out.status)
|
||||
rq.banForLimit(now, hit, rq.out.status)
|
||||
|
||||
return &verdict{figures: limitFigures(hit)}
|
||||
}
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"slices"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
)
|
||||
|
||||
// peerAddress is the address of the request's TCP peer.
|
||||
@@ -43,13 +45,12 @@ func scheme(r *http.Request, isQuestion bool) string {
|
||||
|
||||
// clientGroup is the client a request is counted toward: its IPv4
|
||||
// address, or its IPv6 group, the netblock its IPv6 address is in of the
|
||||
// length ipv6GroupPrefix, SWW_WAF_IPV6_GROUP_PREFIX, a /64 by default, since
|
||||
// one abuser usually holds a whole /64. An IPv4 address in IPv6 form counts as
|
||||
// IPv4.
|
||||
func clientGroup(ipv6GroupPrefix int, addr netip.Addr) netip.Prefix {
|
||||
// length SWW_WAF_IPV6_GROUP_PREFIX sets, a /64 by default, since one abuser
|
||||
// usually holds a whole /64. An IPv4 address in IPv6 form counts as IPv4.
|
||||
func clientGroup(cfg *config.WAF, addr netip.Addr) netip.Prefix {
|
||||
addr = addr.Unmap()
|
||||
if addr.Is6() {
|
||||
return netip.PrefixFrom(addr, ipv6GroupPrefix).Masked()
|
||||
return netip.PrefixFrom(addr, cfg.IPv6GroupPrefix).Masked()
|
||||
}
|
||||
|
||||
return netip.PrefixFrom(addr, addr.BitLen())
|
||||
|
||||
+18
-30
@@ -10,7 +10,6 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
@@ -23,22 +22,11 @@ import (
|
||||
// exploitsignature, raises the core_rule_set_match alert, and in block mode
|
||||
// refuses the request with 403, which is an offence its client's history
|
||||
// counts. Its figures are the score and the threshold.
|
||||
type coreRuleSet struct {
|
||||
// inspector is the Core Rule Set, nil while SWW_WAF_CORE_RULE_SET_MODE is
|
||||
// off.
|
||||
inspector CoreRuleSetInspector
|
||||
metrics *metrics.Metrics
|
||||
alerts *alerts.Queue
|
||||
mode string // SWW_WAF_CORE_RULE_SET_MODE
|
||||
exemptPaths []string // SWW_WAF_CORE_RULE_SET_EXEMPT_PATHS
|
||||
scoreThreshold int // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD
|
||||
observe bool // SWW_WAF_MODE is observe
|
||||
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
||||
}
|
||||
type coreRuleSet struct{}
|
||||
|
||||
func (coreRuleSet) name() string { return "core_rule_set" }
|
||||
|
||||
func (c coreRuleSet) check(
|
||||
func (coreRuleSet) check(
|
||||
_ context.Context,
|
||||
rq *request,
|
||||
_ time.Time,
|
||||
@@ -48,6 +36,7 @@ func (c coreRuleSet) check(
|
||||
return nil
|
||||
}
|
||||
|
||||
cfg := rq.h.config
|
||||
result := findings.inspection
|
||||
rq.requestBytes = int64(len(findings.read))
|
||||
|
||||
@@ -58,18 +47,18 @@ func (c coreRuleSet) check(
|
||||
rq.line.CoreRuleSetScore = &result.Score
|
||||
|
||||
for _, id := range result.RuleIDs {
|
||||
c.metrics.CoreRuleSetMatched(c.mode, id)
|
||||
rq.h.metrics.CoreRuleSetMatched(cfg.CoreRuleSetMode, id)
|
||||
}
|
||||
|
||||
threshold := c.scoreThreshold
|
||||
threshold := cfg.CoreRuleSetScoreThreshold
|
||||
if threshold == 0 || result.Score < threshold {
|
||||
return nil
|
||||
}
|
||||
|
||||
rq.addTag(config.ExploitSignatureTag)
|
||||
c.alertCoreRuleSetMatch(rq, result)
|
||||
rq.alertCoreRuleSetMatch(result)
|
||||
|
||||
if !c.refusedByCoreRuleSet(result) {
|
||||
if !rq.refusedByCoreRuleSet(result) {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -87,21 +76,20 @@ func (c coreRuleSet) check(
|
||||
// refusedByCoreRuleSet reports whether coreRuleSet refuses a request the Core
|
||||
// Rule Set found result of: whether the score is at or over
|
||||
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD in block mode.
|
||||
func (c coreRuleSet) refusedByCoreRuleSet(result coreruleset.Result) bool {
|
||||
threshold := c.scoreThreshold
|
||||
func (rq *request) refusedByCoreRuleSet(result coreruleset.Result) bool {
|
||||
cfg := rq.h.config
|
||||
threshold := cfg.CoreRuleSetScoreThreshold
|
||||
|
||||
return c.mode == config.CoreRuleSetModeBlock &&
|
||||
return cfg.CoreRuleSetMode == config.CoreRuleSetModeBlock &&
|
||||
threshold != 0 && result.Score >= threshold
|
||||
}
|
||||
|
||||
// alertCoreRuleSetMatch raises the core_rule_set_match alert for rq, which the
|
||||
// Core Rule Set scored at result, at or over
|
||||
// alertCoreRuleSetMatch raises the core_rule_set_match alert for the request,
|
||||
// which the Core Rule Set scored at result, at or over
|
||||
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD. Its detail gives the rule ids, the
|
||||
// score, the method and the path with the query, and, for a request that is
|
||||
// not refused for it, the mode: detect, or observe in observe mode.
|
||||
func (c coreRuleSet) alertCoreRuleSetMatch(
|
||||
rq *request, result coreruleset.Result,
|
||||
) {
|
||||
func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
|
||||
detail := map[string]any{
|
||||
"rule_ids": result.RuleIDs,
|
||||
"score": result.Score,
|
||||
@@ -110,17 +98,17 @@ func (c coreRuleSet) alertCoreRuleSetMatch(
|
||||
}
|
||||
|
||||
switch {
|
||||
case c.mode == config.CoreRuleSetModeDetect:
|
||||
case rq.h.config.CoreRuleSetMode == config.CoreRuleSetModeDetect:
|
||||
detail["mode"] = config.CoreRuleSetModeDetect
|
||||
case c.observe:
|
||||
case rq.h.config.Observe:
|
||||
detail["mode"] = "observe"
|
||||
}
|
||||
|
||||
c.alerts.Raise(alerts.Alert{
|
||||
rq.h.alerts.Raise(alerts.Alert{
|
||||
Event: alerts.EventCoreRuleSetMatch,
|
||||
Host: rq.site,
|
||||
Client: rq.client,
|
||||
Netblock: clientGroup(c.ipv6GroupPrefix, rq.client),
|
||||
Netblock: clientGroup(rq.h.config, rq.client),
|
||||
ASN: rq.line.ASN,
|
||||
ASName: rq.line.ASName,
|
||||
Country: rq.line.Country,
|
||||
|
||||
@@ -14,18 +14,16 @@ import (
|
||||
// does not hold the client's country as it was looked up. A client without a
|
||||
// country, or whose country cannot be found, is refused. Its figures are the
|
||||
// country and the setting that refused it.
|
||||
type countries struct {
|
||||
allowed []string // SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES
|
||||
}
|
||||
type countries struct{}
|
||||
|
||||
func (countries) name() string { return "countries" }
|
||||
|
||||
func (c countries) check(
|
||||
func (countries) check(
|
||||
_ context.Context,
|
||||
rq *request,
|
||||
_ time.Time,
|
||||
) *verdict {
|
||||
allowed := c.allowed
|
||||
allowed := rq.h.config.ExclusivelyAllowedCountries
|
||||
country := rq.line.Country
|
||||
|
||||
if len(allowed) == 0 || slices.Contains(allowed, country) {
|
||||
|
||||
@@ -4,7 +4,6 @@ package waf
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/netip"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
@@ -12,18 +11,12 @@ import (
|
||||
|
||||
// denyNets refuses a client in SWW_WAF_DENY_NETS. Its figure is the netblock of
|
||||
// the setting the client is in.
|
||||
type denyNets struct {
|
||||
netblocks []netip.Prefix // SWW_WAF_DENY_NETS
|
||||
}
|
||||
type denyNets struct{}
|
||||
|
||||
func (denyNets) name() string { return "deny_nets" }
|
||||
|
||||
func (d denyNets) check(
|
||||
_ context.Context,
|
||||
rq *request,
|
||||
_ time.Time,
|
||||
) *verdict {
|
||||
for _, netblock := range d.netblocks {
|
||||
func (denyNets) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
||||
for _, netblock := range rq.h.config.DenyNets {
|
||||
if netblock.Contains(rq.client) {
|
||||
return &verdict{
|
||||
action: requestlog.ActionClientDenied,
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
package waf
|
||||
|
||||
import (
|
||||
"net/netip"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
func TestDenyNetsRefusesOnlyAClientInsideItsNetblocks(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
heuristic := denyNets{
|
||||
netblocks: []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")},
|
||||
}
|
||||
|
||||
// The request has no handler: the heuristic reads only what it holds.
|
||||
inside := &request{
|
||||
decision: decision{client: netip.MustParseAddr("198.51.100.7")},
|
||||
}
|
||||
|
||||
v := heuristic.check(t.Context(), inside, time.Time{})
|
||||
if v == nil {
|
||||
t.Fatal("a client inside the netblock was let through")
|
||||
}
|
||||
|
||||
if v.action != requestlog.ActionClientDenied {
|
||||
t.Errorf(
|
||||
"action = %q, want %q",
|
||||
v.action,
|
||||
requestlog.ActionClientDenied,
|
||||
)
|
||||
}
|
||||
|
||||
if got := v.figures[figureNetblock]; got != "198.51.100.0/24" {
|
||||
t.Errorf("netblock figure = %v, want 198.51.100.0/24", got)
|
||||
}
|
||||
|
||||
outside := &request{
|
||||
decision: decision{client: netip.MustParseAddr("203.0.113.7")},
|
||||
}
|
||||
|
||||
if v := heuristic.check(t.Context(), outside, time.Time{}); v != nil {
|
||||
t.Errorf("a client outside the netblock was refused: %+v", v)
|
||||
}
|
||||
}
|
||||
+5
-14
@@ -9,7 +9,6 @@ import (
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
@@ -20,28 +19,20 @@ import (
|
||||
// being log, it does nothing more. A zone without a verdict on the client is
|
||||
// asked about it in the background, and the request does not wait for the
|
||||
// answer. Its figure is the zones that list the client, their keys masked.
|
||||
type dnsbl struct {
|
||||
dnsbl *reputation.DNSBL
|
||||
action string // SWW_WAF_DNSBL_ABUSEIPDB_ACTION
|
||||
hitNoter reputationHitNoter
|
||||
}
|
||||
type dnsbl struct{}
|
||||
|
||||
func (dnsbl) name() string { return "dnsbl" }
|
||||
|
||||
func (d dnsbl) check(
|
||||
ctx context.Context,
|
||||
rq *request,
|
||||
_ time.Time,
|
||||
) *verdict {
|
||||
listedBy := d.dnsbl.ListedBy(ctx, rq.client)
|
||||
func (dnsbl) check(ctx context.Context, rq *request, _ time.Time) *verdict {
|
||||
listedBy := rq.h.dnsbl.ListedBy(ctx, rq.client)
|
||||
rq.dnsblListed = len(listedBy) > 0
|
||||
d.hitNoter.noteListed(rq, listedBy, "listed by a DNSBL zone")
|
||||
rq.noteListed(listedBy, "listed by a DNSBL zone")
|
||||
|
||||
for _, zone := range listedBy {
|
||||
rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone))
|
||||
}
|
||||
|
||||
if !rq.dnsblListed || d.action != deny {
|
||||
if !rq.dnsblListed || rq.h.config.DNSBLAbuseIPDBAction != deny {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,13 +1,4 @@
|
||||
package waf
|
||||
|
||||
import "sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
|
||||
// BesideLookupTimeLimit is besideLookupTimeLimit.
|
||||
const BesideLookupTimeLimit = besideLookupTimeLimit
|
||||
|
||||
// BanAbuseIPDBBlacklist does on s what a fetch of AbuseIPDB's blacklist with
|
||||
// the clients on listed does.
|
||||
func (s *Server) BanAbuseIPDBBlacklist(listed []reputation.BlacklistEntry) {
|
||||
h := s.Handler.(*handler) //nolint:forcetypeassert // New makes it one
|
||||
BanAbuseIPDBBlacklist(h.config, h.ledger, h.processLog, h.now(), listed)
|
||||
}
|
||||
@@ -5,10 +5,9 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// To add a heuristic, write one file with a struct that implements heuristic
|
||||
// and holds the settings and parts it reads. Add one line for it to one of the
|
||||
// lists that newHandler builds, which hands it those settings and parts. Its
|
||||
// settings go in config.WAF, and the comment at the top of its file names them.
|
||||
// To add a heuristic, write one file with a type that implements heuristic,
|
||||
// and add one line for it to one of the lists below. Its settings go in
|
||||
// config.WAF, and the comment at the top of its file names them.
|
||||
|
||||
// heuristic is one of the checks that decide what waf does with a
|
||||
// request.
|
||||
@@ -37,10 +36,78 @@ const (
|
||||
figureNetblock = "netblock"
|
||||
)
|
||||
|
||||
// The lists of heuristics, each in the order its heuristics run. A client in
|
||||
// SWW_WAF_ALLOW_NETS skips them all.
|
||||
|
||||
// beforeLookup are the heuristics that run before the client's AS number and
|
||||
// country are looked up, so that a client they refuse is not looked up.
|
||||
// header_limits comes first: it only measures and counts, and a request it
|
||||
// refuses causes no other work.
|
||||
func beforeLookup() []heuristic {
|
||||
return []heuristic{
|
||||
headerLimits{},
|
||||
tagLists{},
|
||||
routeWatchTags{},
|
||||
denyNets{},
|
||||
activeBan{},
|
||||
}
|
||||
}
|
||||
|
||||
// afterLookup are the heuristics that run once the lookup has been made.
|
||||
func afterLookup() []heuristic {
|
||||
return []heuristic{
|
||||
countries{},
|
||||
blocklists{},
|
||||
dnsbl{},
|
||||
abuseIPDB{},
|
||||
tagDeny{},
|
||||
rateLimits{},
|
||||
}
|
||||
}
|
||||
|
||||
// besideLookup are the heuristics that only read the request. Their work runs
|
||||
// beside the lookup and afterLookup, in findBesideLookup, which changes nothing
|
||||
// of the request. Their check then applies what that work found, and runs only
|
||||
// once waf takes it.
|
||||
func besideLookup() []heuristic {
|
||||
return []heuristic{
|
||||
trapPaths{},
|
||||
ruleFiles{},
|
||||
coreRuleSet{},
|
||||
}
|
||||
}
|
||||
|
||||
// afterBesideLookup are the heuristics that run once afterLookup and
|
||||
// besideLookup have both let the request through. tag_deny runs a second time,
|
||||
// on the tag rule_files and core_rule_set can add.
|
||||
func afterBesideLookup() []heuristic {
|
||||
return []heuristic{
|
||||
tagDeny{},
|
||||
}
|
||||
}
|
||||
|
||||
// afterAnswer are the heuristics that run on a request once waf has
|
||||
// answered it, before its decision line is written. They ban the client, and do
|
||||
// not change what the request got: their verdict's action is not used.
|
||||
func afterAnswer() []heuristic {
|
||||
return []heuristic{
|
||||
refusalLimit{},
|
||||
}
|
||||
}
|
||||
|
||||
// onReport are the heuristics that run on an allowed request when rproxy's
|
||||
// report of it comes, before its report line is written. Like afterAnswer, they
|
||||
// only ban.
|
||||
func onReport() []heuristic {
|
||||
return []heuristic{
|
||||
byteLimits{},
|
||||
}
|
||||
}
|
||||
|
||||
// runAfterAnswer runs the heuristics of afterAnswer on the request, at the time
|
||||
// it is called.
|
||||
func (rq *request) runAfterAnswer(ctx context.Context) {
|
||||
rq.runBanningHeuristics(ctx, rq.h.afterAnswer)
|
||||
rq.runBanningHeuristics(ctx, afterAnswer())
|
||||
}
|
||||
|
||||
// runHeuristics runs list, one of those checkClient runs, on the request at now,
|
||||
|
||||
@@ -150,7 +150,7 @@ func TestPathUnderTheOwnPrefixIsNeitherForwardedNorRefused(t *testing.T) {
|
||||
|
||||
// historyOf returns the history of the client at addr.
|
||||
func historyOf(
|
||||
t *testing.T, server *waf.Server, addr string,
|
||||
t *testing.T, server *wafAndParts, addr string,
|
||||
) ratelimit.History {
|
||||
t.Helper()
|
||||
|
||||
|
||||
@@ -55,7 +55,7 @@ func (rq *request) startLookUp(
|
||||
return result
|
||||
}
|
||||
|
||||
client := clientGroup(rq.h.config.IPv6GroupPrefix, rq.client)
|
||||
client := clientGroup(rq.h.config, rq.client)
|
||||
|
||||
if rq.h.config.LookupSource == "file" {
|
||||
result <- lookupResult{
|
||||
@@ -142,10 +142,8 @@ func AddLookup(
|
||||
) {
|
||||
limiter.AddLookup(answer.Client, answer.Answered,
|
||||
answer.ASN, answer.ASName, answer.Country)
|
||||
netblock := banNetblock(
|
||||
cfg.BanScopeV4Prefix, cfg.IPv6GroupPrefix, answer.Client.Addr(),
|
||||
)
|
||||
ledger.AddLookup(netblock, answer.ASN, answer.ASName, answer.Country)
|
||||
ledger.AddLookup(banNetblock(cfg, answer.Client.Addr()),
|
||||
answer.ASN, answer.ASName, answer.Country)
|
||||
}
|
||||
|
||||
// GeoJSFailed logs a failure of GeoJS to processLog, raises a
|
||||
|
||||
+32
-181
@@ -116,37 +116,21 @@ type CoreRuleSetInspector interface {
|
||||
) (coreruleset.Result, []byte, error)
|
||||
}
|
||||
|
||||
// Server is the server waf runs, with the parts of its Params. The tests
|
||||
// read them.
|
||||
// Server is the server waf runs.
|
||||
type Server struct {
|
||||
*http.Server
|
||||
|
||||
Ledger *bans.Ledger
|
||||
Limiter *ratelimit.Limiter
|
||||
GeoJS *geojs.GeoJS
|
||||
Anomalies *anomaly.Counters
|
||||
LookupFile *lookup.File
|
||||
Lists *reputation.Lists
|
||||
DNSBL *reputation.DNSBL
|
||||
AbuseIPDB *reputation.AbuseIPDB
|
||||
RouteWatch *routewatch.Client
|
||||
Metrics *metrics.Metrics
|
||||
}
|
||||
|
||||
// newHandler returns the handler with what every request shares, the parts
|
||||
// in params among it. It builds the heuristics, each with the settings and
|
||||
// parts it reads, and the lists they run in.
|
||||
//
|
||||
//nolint:funlen // a few lines for each heuristic, a list that grows with them
|
||||
// in params among it.
|
||||
func newHandler(params Params) *handler {
|
||||
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
|
||||
if err != nil {
|
||||
panic(err) // New fails only for a size below one
|
||||
}
|
||||
|
||||
cfg := params.Config
|
||||
h := &handler{
|
||||
config: cfg,
|
||||
return &handler{
|
||||
config: params.Config,
|
||||
requestLog: params.RequestLog,
|
||||
processLog: params.ProcessLog,
|
||||
now: params.Now,
|
||||
@@ -157,127 +141,25 @@ func newHandler(params Params) *handler {
|
||||
geojs: params.GeoJS,
|
||||
anomalies: params.Anomalies,
|
||||
lookupFile: params.LookupFile,
|
||||
lists: params.Lists,
|
||||
dnsbl: params.DNSBL,
|
||||
abuseIPDB: params.AbuseIPDB,
|
||||
routeWatch: params.RouteWatch,
|
||||
rules: params.Rules,
|
||||
coreRuleSet: params.CoreRuleSet,
|
||||
alerts: params.Alerts,
|
||||
// The country list, the headers, the tag settings that name an asn=
|
||||
// or country= tag and the biased thresholds act on the answer before
|
||||
// the request goes on.
|
||||
waitForGeoJS: len(cfg.ExclusivelyAllowedCountries) > 0 ||
|
||||
cfg.AddLookupHeaders ||
|
||||
cfg.TagsNeedLookup() ||
|
||||
biasedThresholdsSet(cfg),
|
||||
waitForGeoJS: len(params.Config.ExclusivelyAllowedCountries) > 0 ||
|
||||
params.Config.AddLookupHeaders ||
|
||||
params.Config.TagsNeedLookup() ||
|
||||
biasedThresholdsSet(params.Config),
|
||||
besideLookupLimit: cmp.Or(
|
||||
params.BesideLookupTimeLimit,
|
||||
besideLookupTimeLimit,
|
||||
),
|
||||
}
|
||||
|
||||
sharedBanMaker := banMaker{
|
||||
ledger: params.Ledger,
|
||||
limiter: params.Limiter,
|
||||
metrics: params.Metrics,
|
||||
alerts: params.Alerts,
|
||||
observe: cfg.Observe,
|
||||
banResponse: cfg.BanResponse,
|
||||
banScopeV4Prefix: cfg.BanScopeV4Prefix,
|
||||
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||
}
|
||||
hitNoter := reputationHitNoter{
|
||||
metrics: params.Metrics,
|
||||
alerts: params.Alerts,
|
||||
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||
}
|
||||
|
||||
h.blocklists = blocklists{
|
||||
lists: params.Lists,
|
||||
action: cfg.BlocklistAction,
|
||||
hitNoter: hitNoter,
|
||||
}
|
||||
h.rateLimits = rateLimits{
|
||||
limiter: params.Limiter,
|
||||
exemptNets: cfg.RateLimitExemptNets,
|
||||
exemptPaths: cfg.RateLimitExemptPaths,
|
||||
banResponse: cfg.BanResponse,
|
||||
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||
biasedThresholds: biasedThresholds{
|
||||
lists: params.Lists,
|
||||
tagLimitPercent: cfg.TagLimitPercent,
|
||||
asnBytesLimitPercent: cfg.ASNBytesLimitPercent,
|
||||
countryBytesLimitPercent: cfg.CountryBytesLimitPercent,
|
||||
unknownCountryLimitPercent: cfg.UnknownCountryLimitPercent,
|
||||
blocklistAction: cfg.BlocklistAction,
|
||||
blocklistLimitPercent: cfg.BlocklistLimitPercent,
|
||||
dnsblAbuseIPDBAction: cfg.DNSBLAbuseIPDBAction,
|
||||
dnsblAbuseIPDBLimitPercent: cfg.DNSBLAbuseIPDBLimitPercent,
|
||||
},
|
||||
banMaker: sharedBanMaker,
|
||||
}
|
||||
h.trapPaths = trapPaths{paths: cfg.TrapPaths, banMaker: sharedBanMaker}
|
||||
h.ruleFiles = ruleFiles{
|
||||
rules: params.Rules,
|
||||
metrics: params.Metrics,
|
||||
banMaker: sharedBanMaker,
|
||||
}
|
||||
h.coreRuleSet = coreRuleSet{
|
||||
inspector: params.CoreRuleSet,
|
||||
metrics: params.Metrics,
|
||||
alerts: params.Alerts,
|
||||
mode: cfg.CoreRuleSetMode,
|
||||
exemptPaths: cfg.CoreRuleSetExemptPaths,
|
||||
scoreThreshold: cfg.CoreRuleSetScoreThreshold,
|
||||
observe: cfg.Observe,
|
||||
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||
}
|
||||
|
||||
h.beforeLookup = []heuristic{
|
||||
headerLimits{},
|
||||
tagLists{lists: params.Lists},
|
||||
routeWatchTags{
|
||||
routeWatch: params.RouteWatch,
|
||||
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||
},
|
||||
denyNets{netblocks: cfg.DenyNets},
|
||||
activeBan{banMaker: sharedBanMaker},
|
||||
}
|
||||
h.afterLookup = []heuristic{
|
||||
countries{allowed: cfg.ExclusivelyAllowedCountries},
|
||||
h.blocklists,
|
||||
dnsbl{
|
||||
dnsbl: params.DNSBL,
|
||||
action: cfg.DNSBLAbuseIPDBAction,
|
||||
hitNoter: hitNoter,
|
||||
},
|
||||
abuseIPDB{
|
||||
abuseIPDB: params.AbuseIPDB,
|
||||
limiter: params.Limiter,
|
||||
keySet: cfg.AbuseIPDBKey != "",
|
||||
minScore: cfg.AbuseIPDBMinScore,
|
||||
action: cfg.DNSBLAbuseIPDBAction,
|
||||
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||
hitNoter: hitNoter,
|
||||
},
|
||||
tagDeny{tags: cfg.TagDeny},
|
||||
h.rateLimits,
|
||||
}
|
||||
h.besideLookup = []heuristic{h.trapPaths, h.ruleFiles, h.coreRuleSet}
|
||||
h.afterBesideLookup = []heuristic{tagDeny{tags: cfg.TagDeny}}
|
||||
h.afterAnswer = []heuristic{
|
||||
refusalLimit{
|
||||
limiter: params.Limiter,
|
||||
perMinute: cfg.RefusalsLimitPerMinute,
|
||||
banResponse: cfg.BanResponse,
|
||||
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||
banMaker: sharedBanMaker,
|
||||
},
|
||||
}
|
||||
h.onReport = []heuristic{
|
||||
byteLimits{
|
||||
limiter: params.Limiter,
|
||||
bytesCounted: cfg.BytesCounted,
|
||||
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||
banMaker: sharedBanMaker,
|
||||
},
|
||||
}
|
||||
|
||||
return h
|
||||
}
|
||||
|
||||
// NewCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
|
||||
@@ -317,16 +199,23 @@ func NewCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
|
||||
// handler is waf's handler. It holds what every request shares; what
|
||||
// belongs to one request is in a request.
|
||||
type handler struct {
|
||||
config *config.WAF
|
||||
requestLog io.Writer
|
||||
processLog *slog.Logger
|
||||
now func() time.Time
|
||||
metrics *metrics.Metrics
|
||||
limiter *ratelimit.Limiter
|
||||
ledger *bans.Ledger
|
||||
geojs *geojs.GeoJS
|
||||
anomalies *anomaly.Counters
|
||||
lookupFile *lookup.File
|
||||
config *config.WAF
|
||||
requestLog io.Writer
|
||||
processLog *slog.Logger
|
||||
now func() time.Time
|
||||
metrics *metrics.Metrics
|
||||
limiter *ratelimit.Limiter
|
||||
ledger *bans.Ledger
|
||||
geojs *geojs.GeoJS
|
||||
anomalies *anomaly.Counters
|
||||
lookupFile *lookup.File
|
||||
lists *reputation.Lists
|
||||
dnsbl *reputation.DNSBL
|
||||
abuseIPDB *reputation.AbuseIPDB
|
||||
routeWatch *routewatch.Client
|
||||
rules *rules.Files
|
||||
coreRuleSet CoreRuleSetInspector
|
||||
alerts *alerts.Queue
|
||||
// waitForGeoJS is true when a setting needs each request's answer from
|
||||
// GeoJS before the request goes on.
|
||||
waitForGeoJS bool
|
||||
@@ -336,42 +225,4 @@ type handler struct {
|
||||
// awaitingReport holds the decision on each answered question by its id
|
||||
// until its report comes, and nil for a question still being handled.
|
||||
awaitingReport *lru.Cache[string, *decision]
|
||||
|
||||
// The lists of heuristics, each in the order its heuristics run. A client
|
||||
// in SWW_WAF_ALLOW_NETS skips them all.
|
||||
|
||||
// beforeLookup are the heuristics that run before the client's AS number
|
||||
// and country are looked up, so that a client they refuse is not looked
|
||||
// up. header_limits comes first: it only measures and counts, and a
|
||||
// request it refuses causes no other work.
|
||||
beforeLookup []heuristic
|
||||
// afterLookup are the heuristics that run once the lookup has been made.
|
||||
afterLookup []heuristic
|
||||
// besideLookup are the heuristics that only read the request. Their work
|
||||
// runs beside the lookup and afterLookup, in findBesideLookup, which
|
||||
// changes nothing of the request. Their check then applies what that work
|
||||
// found, and runs only once waf takes it.
|
||||
besideLookup []heuristic
|
||||
// afterBesideLookup are the heuristics that run once afterLookup and
|
||||
// besideLookup have both let the request through. tag_deny runs a second
|
||||
// time, on the tag rule_files and core_rule_set can add.
|
||||
afterBesideLookup []heuristic
|
||||
// afterAnswer are the heuristics that run on a request once waf has
|
||||
// answered it, before its decision line is written. They ban the client,
|
||||
// and do not change what the request got: their verdict's action is not
|
||||
// used.
|
||||
afterAnswer []heuristic
|
||||
// onReport are the heuristics that run on an allowed request when
|
||||
// rproxy's report of it comes, before its report line is written. Like
|
||||
// afterAnswer, they only ban.
|
||||
onReport []heuristic
|
||||
|
||||
// The heuristics of the lists that other steps of a request use too:
|
||||
// refusedWhateverTheAnswer asks blocklists and rateLimits, and
|
||||
// findBesideLookup does the work of trapPaths, ruleFiles and coreRuleSet.
|
||||
blocklists blocklists
|
||||
rateLimits rateLimits
|
||||
trapPaths trapPaths
|
||||
ruleFiles ruleFiles
|
||||
coreRuleSet coreRuleSet
|
||||
}
|
||||
+36
-12
@@ -318,7 +318,7 @@ func startProxyWithGeoJS(
|
||||
func startProxyWithClock(
|
||||
t *testing.T, appURL, geojsURL string, now func() time.Time,
|
||||
env map[string]string,
|
||||
) (string, *output, *waf.Server) {
|
||||
) (string, *output, *wafAndParts) {
|
||||
t.Helper()
|
||||
|
||||
addr, out, server, _ := startProxyWithAlerts(
|
||||
@@ -337,7 +337,7 @@ func startProxyWithClock(
|
||||
func startProxyWithAlerts(
|
||||
t *testing.T, appURL, geojsURL string, now func() time.Time,
|
||||
env map[string]string,
|
||||
) (string, *output, *waf.Server, *alerts.Queue) {
|
||||
) (string, *output, *wafAndParts, *alerts.Queue) {
|
||||
t.Helper()
|
||||
|
||||
server, out, alertQueue := newWAF(t, geojsURL, now, env)
|
||||
@@ -348,7 +348,7 @@ func startProxyWithAlerts(
|
||||
|
||||
// serve serves server on a port of its own until the test ends, and sets its
|
||||
// Addr to where it listens.
|
||||
func serve(t *testing.T, server *waf.Server) {
|
||||
func serve(t *testing.T, server *wafAndParts) {
|
||||
t.Helper()
|
||||
|
||||
listener, err := (&net.ListenConfig{}).Listen(
|
||||
@@ -424,8 +424,29 @@ func startRProxy(
|
||||
return listener.Addr().String()
|
||||
}
|
||||
|
||||
// newWAF makes waf's server, without starting it, and
|
||||
// returns it, what it writes, and the queue of the alerts it raises, as the
|
||||
// wafAndParts is waf's server with the Params it was built from, so that a
|
||||
// test can read the parts the server uses.
|
||||
type wafAndParts struct {
|
||||
*waf.Server
|
||||
waf.Params
|
||||
}
|
||||
|
||||
// banAbuseIPDBBlacklist does on w what a fetch of AbuseIPDB's blacklist with
|
||||
// the clients on listed does.
|
||||
func (w *wafAndParts) banAbuseIPDBBlacklist(
|
||||
listed []reputation.BlacklistEntry,
|
||||
) {
|
||||
waf.BanAbuseIPDBBlacklist(
|
||||
w.Config,
|
||||
w.Ledger,
|
||||
w.ProcessLog,
|
||||
w.Now(),
|
||||
listed,
|
||||
)
|
||||
}
|
||||
|
||||
// newWAF makes waf's server, without starting it, and returns it with its
|
||||
// parts, what it writes, and the queue of the alerts it raises, as the
|
||||
// settings in env make it. No alert is sent from the queue: they wait in it,
|
||||
// for the test to look at.
|
||||
// With no geojsURL, there is no stand-in for GeoJS to look clients up at, and
|
||||
@@ -440,19 +461,21 @@ func newWAF(
|
||||
geojsURL string,
|
||||
now func() time.Time,
|
||||
env map[string]string,
|
||||
) (*waf.Server, *output, *alerts.Queue) {
|
||||
) (*wafAndParts, *output, *alerts.Queue) {
|
||||
t.Helper()
|
||||
|
||||
return newWAFWith(t, waf.New, geojsURL, abuseIPDBURL, now, env)
|
||||
return newWAFWith(
|
||||
t, func(*waf.Params) {}, geojsURL, abuseIPDBURL, now, env,
|
||||
)
|
||||
}
|
||||
|
||||
// newWAFWith is newWAF for the server build makes, with clients checked with
|
||||
// AbuseIPDB at abuseIPDBCheckURL.
|
||||
// newWAFWith is newWAF with change applied to the Params before the server is
|
||||
// built from them, and clients checked with AbuseIPDB at abuseIPDBCheckURL.
|
||||
func newWAFWith(
|
||||
t *testing.T, build func(waf.Params) *waf.Server,
|
||||
t *testing.T, change func(*waf.Params),
|
||||
geojsURL, abuseIPDBCheckURL string, now func() time.Time,
|
||||
env map[string]string,
|
||||
) (*waf.Server, *output, *alerts.Queue) {
|
||||
) (*wafAndParts, *output, *alerts.Queue) {
|
||||
t.Helper()
|
||||
|
||||
settings := map[string]string{
|
||||
@@ -524,8 +547,9 @@ func newWAFWith(
|
||||
}
|
||||
addCounters(¶ms)
|
||||
addOutsideSources(¶ms, geojsURL, abuseIPDBCheckURL)
|
||||
change(¶ms)
|
||||
|
||||
server := build(params)
|
||||
server := &wafAndParts{Server: waf.New(params), Params: params}
|
||||
|
||||
return server, out, alertQueue
|
||||
}
|
||||
|
||||
+22
-34
@@ -6,10 +6,8 @@ package waf
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/netip"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
@@ -20,37 +18,28 @@ import (
|
||||
// refused, and bans the client. A request exempt from the rate limits is exempt
|
||||
// from the byte limits too. Its figures are the window, the count and the
|
||||
// limit.
|
||||
type rateLimits struct {
|
||||
limiter *ratelimit.Limiter
|
||||
exemptNets []netip.Prefix // SWW_WAF_RATE_LIMIT_EXEMPT_NETS
|
||||
exemptPaths []string // SWW_WAF_RATE_LIMIT_EXEMPT_PATHS
|
||||
banResponse int // SWW_WAF_BAN_RESPONSE
|
||||
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
||||
biasedThresholds biasedThresholds
|
||||
banMaker banMaker
|
||||
}
|
||||
type rateLimits struct{}
|
||||
|
||||
func (rateLimits) name() string { return "rate_limits" }
|
||||
|
||||
func (r rateLimits) check(
|
||||
func (rateLimits) check(
|
||||
_ context.Context,
|
||||
rq *request,
|
||||
now time.Time,
|
||||
) *verdict {
|
||||
rq.counted = r.countedByRateLimits(rq)
|
||||
rq.counted = rq.countedByRateLimits()
|
||||
if !rq.counted {
|
||||
return nil
|
||||
}
|
||||
|
||||
rq.rateLimitPercent, rq.bytesLimitPercent =
|
||||
r.biasedThresholds.limitPercentages(rq)
|
||||
rq.rateLimitPercent, rq.bytesLimitPercent = rq.limitPercentages()
|
||||
rq.line.RateLimitPercent, rq.line.RateLimitPercentSetting =
|
||||
rq.rateLimitPercent.logged()
|
||||
rq.line.BytesLimitPercent, rq.line.BytesLimitPercentSetting =
|
||||
rq.bytesLimitPercent.logged()
|
||||
|
||||
counts, hit, over := r.limiter.Count(
|
||||
clientGroup(r.ipv6GroupPrefix, rq.client),
|
||||
counts, hit, over := rq.h.limiter.Count(
|
||||
clientGroup(rq.h.config, rq.client),
|
||||
now,
|
||||
rq.rateLimitPercent.percent,
|
||||
)
|
||||
@@ -60,29 +49,28 @@ func (r rateLimits) check(
|
||||
return nil
|
||||
}
|
||||
|
||||
r.banMaker.banForLimit(rq, now, hit, r.banResponse)
|
||||
rq.banForLimit(now, hit, rq.h.config.BanResponse)
|
||||
|
||||
return &verdict{
|
||||
action: requestlog.ActionClientIsRateLimited, figures: limitFigures(hit),
|
||||
}
|
||||
}
|
||||
|
||||
// overAWholeRateLimit reports whether the heuristic rate_limits will refuse rq
|
||||
// at now, whatever its client's limit percentages: the rate limits count it,
|
||||
// and one more request from its client takes it over a whole rate limit. It
|
||||
// counts nothing.
|
||||
func (r rateLimits) overAWholeRateLimit(rq *request, now time.Time) bool {
|
||||
return r.countedByRateLimits(rq) &&
|
||||
r.limiter.OverARateLimit(
|
||||
clientGroup(r.ipv6GroupPrefix, rq.client),
|
||||
now,
|
||||
)
|
||||
// overAWholeRateLimit reports whether the heuristic rate_limits will refuse the
|
||||
// request at now, whatever its client's limit percentages: the rate limits
|
||||
// count it, and one more request from its client takes it over a whole rate
|
||||
// limit. It counts nothing.
|
||||
func (rq *request) overAWholeRateLimit(now time.Time) bool {
|
||||
return rq.countedByRateLimits() &&
|
||||
rq.h.limiter.OverARateLimit(clientGroup(rq.h.config, rq.client), now)
|
||||
}
|
||||
|
||||
// countedByRateLimits reports whether the rate limits count rq: its client is
|
||||
// not in SWW_WAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt under
|
||||
// SWW_WAF_RATE_LIMIT_EXEMPT_PATHS.
|
||||
func (r rateLimits) countedByRateLimits(rq *request) bool {
|
||||
return !isInside(rq.client, r.exemptNets) &&
|
||||
!pathExempt(rq.in, r.exemptPaths)
|
||||
// countedByRateLimits reports whether the rate limits count the request: its
|
||||
// client is not in SWW_WAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt
|
||||
// under SWW_WAF_RATE_LIMIT_EXEMPT_PATHS.
|
||||
func (rq *request) countedByRateLimits() bool {
|
||||
cfg := rq.h.config
|
||||
|
||||
return !isInside(rq.client, cfg.RateLimitExemptNets) &&
|
||||
!pathExempt(rq.in, cfg.RateLimitExemptPaths)
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
@@ -19,22 +18,17 @@ import (
|
||||
// a limit, which bans the client. A client in SWW_WAF_ALLOW_NETS, which the
|
||||
// heuristics skip, is not counted, and nothing is while the limit is off.
|
||||
// Its figures are the window, the count and the limit.
|
||||
type refusalLimit struct {
|
||||
limiter *ratelimit.Limiter
|
||||
perMinute int64 // SWW_WAF_REFUSALS_LIMIT_PER_MINUTE
|
||||
banResponse int // SWW_WAF_BAN_RESPONSE
|
||||
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
||||
banMaker banMaker
|
||||
}
|
||||
type refusalLimit struct{}
|
||||
|
||||
func (refusalLimit) name() string { return "refusal_limit" }
|
||||
|
||||
func (r refusalLimit) check(
|
||||
func (refusalLimit) check(
|
||||
_ context.Context,
|
||||
rq *request,
|
||||
now time.Time,
|
||||
) *verdict {
|
||||
if r.perMinute == 0 {
|
||||
cfg := rq.h.config
|
||||
if cfg.RefusalsLimitPerMinute == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -42,10 +36,10 @@ func (r refusalLimit) check(
|
||||
return nil
|
||||
}
|
||||
|
||||
hit, over := r.limiter.CountRefusal(
|
||||
clientGroup(r.ipv6GroupPrefix, rq.client),
|
||||
hit, over := rq.h.limiter.CountRefusal(
|
||||
clientGroup(rq.h.config, rq.client),
|
||||
now,
|
||||
r.perMinute,
|
||||
cfg.RefusalsLimitPerMinute,
|
||||
)
|
||||
if !over {
|
||||
return nil
|
||||
@@ -60,10 +54,10 @@ func (r refusalLimit) check(
|
||||
status = http.StatusForbidden
|
||||
case requestlog.ActionTemporarilyBanned,
|
||||
requestlog.ActionPermanentlyBanned:
|
||||
status = r.banResponse
|
||||
status = cfg.BanResponse
|
||||
}
|
||||
|
||||
r.banMaker.banForLimit(rq, now, hit, status)
|
||||
rq.banForLimit(now, hit, status)
|
||||
|
||||
return &verdict{figures: limitFigures(hit)}
|
||||
}
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
)
|
||||
|
||||
const refusalsLimitPerMinute = "SWW_WAF_REFUSALS_LIMIT_PER_MINUTE"
|
||||
@@ -52,7 +51,7 @@ func (r refused) send(s *sender, from string) logLine {
|
||||
// SWW_WAF_REFUSALS_LIMIT_PER_MINUTE at limit, and the settings in env.
|
||||
func startForRefusalLimit(
|
||||
t *testing.T, limit string, env map[string]string,
|
||||
) (*sender, *clock, *waf.Server) {
|
||||
) (*sender, *clock, *wafAndParts) {
|
||||
t.Helper()
|
||||
|
||||
settings := map[string]string{
|
||||
|
||||
@@ -5,39 +5,26 @@ package waf
|
||||
import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||
)
|
||||
|
||||
// deny is the SWW_WAF_BLOCKLIST_ACTION and the SWW_WAF_DNSBL_ABUSEIPDB_ACTION that
|
||||
// refuses the requests of a client a source lists.
|
||||
const deny = "deny"
|
||||
|
||||
// reputationHitNoter notes the reputation sources that list a request's
|
||||
// client, for the reputation heuristics.
|
||||
type reputationHitNoter struct {
|
||||
metrics *metrics.Metrics
|
||||
alerts *alerts.Queue
|
||||
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
||||
}
|
||||
|
||||
// noteListed notes each of sources, the URLs of the blocklists or the
|
||||
// DNSBL zones, their keys masked, that list rq's client, as noteHit does,
|
||||
// DNSBL zones, their keys masked, that list the client, as noteHit does,
|
||||
// with reason.
|
||||
func (n reputationHitNoter) noteListed(
|
||||
rq *request, sources []string, reason string,
|
||||
) {
|
||||
func (rq *request) noteListed(sources []string, reason string) {
|
||||
for _, source := range sources {
|
||||
n.noteHit(rq, bans.ReputationHit{Source: source}, reason)
|
||||
rq.noteHit(bans.ReputationHit{Source: source}, reason)
|
||||
}
|
||||
}
|
||||
|
||||
// noteHit adds hit's source, which lists rq's client, to the log line's
|
||||
// noteHit adds hit's source, which lists the client, to the log line's
|
||||
// reputation, and hit to the notes of a ban the request makes, counts the
|
||||
// source in the metrics, and raises a reputation_hit alert with reason,
|
||||
// whose detail gives hit's source and score.
|
||||
func (n reputationHitNoter) noteHit(
|
||||
rq *request, hit bans.ReputationHit, reason string,
|
||||
) {
|
||||
func (rq *request) noteHit(hit bans.ReputationHit, reason string) {
|
||||
detail := map[string]any{"source": hit.Source}
|
||||
if hit.Score != nil {
|
||||
detail["score"] = *hit.Score
|
||||
@@ -45,12 +32,12 @@ func (n reputationHitNoter) noteHit(
|
||||
|
||||
rq.line.Reputation = append(rq.line.Reputation, hit.Source)
|
||||
rq.reputation = append(rq.reputation, hit)
|
||||
n.metrics.ReputationHit(hit.Source)
|
||||
n.alerts.Raise(alerts.Alert{
|
||||
rq.h.metrics.ReputationHit(hit.Source)
|
||||
rq.h.alerts.Raise(alerts.Alert{
|
||||
Event: alerts.EventReputationHit,
|
||||
Host: rq.site,
|
||||
Client: rq.client,
|
||||
Netblock: clientGroup(n.ipv6GroupPrefix, rq.client),
|
||||
Netblock: clientGroup(rq.h.config, rq.client),
|
||||
ASN: rq.line.ASN,
|
||||
ASName: rq.line.ASName,
|
||||
Country: rq.line.Country,
|
||||
|
||||
@@ -18,7 +18,6 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
)
|
||||
|
||||
// The reputation settings.
|
||||
@@ -139,13 +138,13 @@ func TestReputationRefusalNamesItsHeuristicAndWhatListedTheClient(
|
||||
for _, tc := range []struct {
|
||||
heuristic string
|
||||
env map[string]string
|
||||
load func(*testing.T, *waf.Server)
|
||||
load func(*testing.T, *wafAndParts)
|
||||
figures string
|
||||
}{
|
||||
{
|
||||
"blocklists",
|
||||
map[string]string{blocklistURLs: dropURL, blocklistAction: actionDeny},
|
||||
func(t *testing.T, server *waf.Server) {
|
||||
func(t *testing.T, server *wafAndParts) {
|
||||
t.Helper()
|
||||
loadLists(t, server, map[string][]string{dropURL: {client}})
|
||||
},
|
||||
@@ -157,7 +156,7 @@ func TestReputationRefusalNamesItsHeuristicAndWhatListedTheClient(
|
||||
dnsblZones: dnsblZone, dnsblResolver: noResolver,
|
||||
dnsblAbuseIPDBAction: actionDeny,
|
||||
},
|
||||
func(_ *testing.T, server *waf.Server) {
|
||||
func(_ *testing.T, server *wafAndParts) {
|
||||
loadVerdicts(server, map[string][]string{client: {dnsblZone}})
|
||||
},
|
||||
`{"zones": ["` + dnsblZone + `"]}`,
|
||||
@@ -165,7 +164,7 @@ func TestReputationRefusalNamesItsHeuristicAndWhatListedTheClient(
|
||||
{
|
||||
"abuseipdb",
|
||||
map[string]string{abuseIPDBKey: accountKey, dnsblAbuseIPDBAction: actionDeny},
|
||||
func(_ *testing.T, server *waf.Server) {
|
||||
func(_ *testing.T, server *wafAndParts) {
|
||||
loadScores(server, map[string]int64{client: 80})
|
||||
},
|
||||
`{"score": 80, "min_score": 75}`,
|
||||
@@ -560,7 +559,7 @@ func TestReputationSourcesTagTheRequestWhateverTheirAction(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
env map[string]string
|
||||
load func(*waf.Server)
|
||||
load func(*wafAndParts)
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
@@ -568,7 +567,7 @@ func TestReputationSourcesTagTheRequestWhateverTheirAction(t *testing.T) {
|
||||
map[string]string{
|
||||
dnsblZones: dnsblZone, dnsblResolver: noResolver, dnsblAbuseIPDBAction: actionLog,
|
||||
},
|
||||
func(server *waf.Server) {
|
||||
func(server *wafAndParts) {
|
||||
loadVerdicts(server, map[string][]string{fromDE: {dnsblZone}})
|
||||
},
|
||||
[]string{"asn=64496", "country=DE", "dnsbl=" + dnsblZone},
|
||||
@@ -576,7 +575,7 @@ func TestReputationSourcesTagTheRequestWhateverTheirAction(t *testing.T) {
|
||||
{
|
||||
abuseipdb,
|
||||
map[string]string{abuseIPDBKey: accountKey, dnsblAbuseIPDBAction: actionLog},
|
||||
func(server *waf.Server) {
|
||||
func(server *wafAndParts) {
|
||||
loadScores(server, map[string]int64{fromDE: 80})
|
||||
},
|
||||
[]string{abuseipdb, "asn=64496", "country=DE"},
|
||||
@@ -1256,7 +1255,7 @@ func verdictsFetched() time.Time {
|
||||
// verdictsFetched, as dnsbl.json would at start: one that lists the
|
||||
// client from each zone listedBy gives for it, and one that does not from
|
||||
// each other zone.
|
||||
func loadVerdicts(server *waf.Server, listedBy map[string][]string) {
|
||||
func loadVerdicts(server *wafAndParts, listedBy map[string][]string) {
|
||||
verdicts := make(
|
||||
[]reputation.Verdict,
|
||||
0,
|
||||
@@ -1281,7 +1280,7 @@ func loadVerdicts(server *waf.Server, listedBy map[string][]string) {
|
||||
// loadScores puts into server's AbuseIPDB the score scores gives each
|
||||
// client, an IPv4 address, fetched at verdictsFetched, as
|
||||
// abuseipdb-scores.json would at start.
|
||||
func loadScores(server *waf.Server, scores map[string]int64) {
|
||||
func loadScores(server *wafAndParts, scores map[string]int64) {
|
||||
kept := make([]reputation.Score, 0, len(scores))
|
||||
for client, score := range scores {
|
||||
kept = append(kept, reputation.Score{
|
||||
@@ -1295,7 +1294,7 @@ func loadScores(server *waf.Server, scores map[string]int64) {
|
||||
|
||||
// wantAbuseIPDBChecks checks how many clients server has checked with
|
||||
// AbuseIPDB.
|
||||
func wantAbuseIPDBChecks(t *testing.T, server *waf.Server, want int) {
|
||||
func wantAbuseIPDBChecks(t *testing.T, server *wafAndParts, want int) {
|
||||
t.Helper()
|
||||
|
||||
if got := server.AbuseIPDB.Checked(); got != want {
|
||||
@@ -1307,7 +1306,7 @@ func wantAbuseIPDBChecks(t *testing.T, server *waf.Server, want int) {
|
||||
// its lines, fetched at listsFetched, as lists.json would at start.
|
||||
func loadLists(
|
||||
t *testing.T,
|
||||
server *waf.Server,
|
||||
server *wafAndParts,
|
||||
copies map[string][]string,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
+8
-13
@@ -122,11 +122,8 @@ func (h *handler) startRequest(
|
||||
RequestID: requestID(r, isQuestion),
|
||||
PeerIP: peer.String(),
|
||||
ForwardedFor: strings.Join(forwardedFor, ", "),
|
||||
ClientGroup: clientGroup(
|
||||
h.config.IPv6GroupPrefix,
|
||||
client,
|
||||
).String(),
|
||||
ContentType: r.Header.Get("Content-Type"),
|
||||
ClientGroup: clientGroup(h.config, client).String(),
|
||||
ContentType: r.Header.Get("Content-Type"),
|
||||
RequestHeaders: requestHeaders(
|
||||
r,
|
||||
h.config.LogRequestHeaders,
|
||||
@@ -236,7 +233,7 @@ func (rq *request) checkClient(ctx context.Context) *verdict {
|
||||
|
||||
now := rq.h.now()
|
||||
|
||||
v := rq.runHeuristics(ctx, rq.h.beforeLookup, now)
|
||||
v := rq.runHeuristics(ctx, beforeLookup(), now)
|
||||
if v != nil {
|
||||
return v
|
||||
}
|
||||
@@ -268,7 +265,7 @@ func (rq *request) runAfterLookup(
|
||||
// again: a request counted at the time it arrived could be counted
|
||||
// after a later one of its client's, which the rate limits would take
|
||||
// for the clock set back.
|
||||
return rq.runHeuristics(ctx, rq.h.afterLookup, rq.h.now())
|
||||
return rq.runHeuristics(ctx, afterLookup(), rq.h.now())
|
||||
}
|
||||
|
||||
// pathExempt reports whether r is exempt under prefixes,
|
||||
@@ -368,7 +365,7 @@ func (rq *request) addToHistory() {
|
||||
}
|
||||
|
||||
rq.h.limiter.AddToHistory(
|
||||
clientGroup(rq.h.config.IPv6GroupPrefix, rq.client),
|
||||
clientGroup(rq.h.config, rq.client),
|
||||
rq.h.now(),
|
||||
request,
|
||||
)
|
||||
@@ -397,11 +394,11 @@ func (rq *request) countAnomalies() {
|
||||
rq.h.anomalies.Count(rq.h.now(), anomaly.Request{
|
||||
Host: rq.site,
|
||||
Client: rq.client,
|
||||
ClientGroup: clientGroup(rq.h.config.IPv6GroupPrefix, rq.client),
|
||||
ClientGroup: clientGroup(rq.h.config, rq.client),
|
||||
ASN: answer.ASN,
|
||||
ASName: answer.ASName,
|
||||
Country: answer.Country,
|
||||
Bytes: rq.countedBytes(rq.h.config.BytesCounted),
|
||||
Bytes: rq.countedBytes(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -428,9 +425,7 @@ func (rq *request) answerAtTheEnd() (geojs.Answer, bool) {
|
||||
return rq.lookupAnswer, true
|
||||
}
|
||||
|
||||
return rq.h.geojs.Kept(
|
||||
clientGroup(rq.h.config.IPv6GroupPrefix, rq.client),
|
||||
)
|
||||
return rq.h.geojs.Kept(clientGroup(rq.h.config, rq.client))
|
||||
}
|
||||
|
||||
// responseWriter is waf's answer to a request. It notes the status and
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
||||
)
|
||||
|
||||
// routeWatchTags adds to the request the tags asn=<number> and prefix=<netblock>
|
||||
@@ -18,26 +17,22 @@ import (
|
||||
// SWW_WAF_ROUTEWATCH_URL is unset it does nothing. Answers are kept by client and
|
||||
// asked for by the request's address, and a route without an origin AS gives only
|
||||
// the prefix= tag.
|
||||
type routeWatchTags struct {
|
||||
// routeWatch is nil while SWW_WAF_ROUTEWATCH_URL is unset.
|
||||
routeWatch *routewatch.Client
|
||||
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
||||
}
|
||||
type routeWatchTags struct{}
|
||||
|
||||
func (routeWatchTags) name() string { return "routewatch_tags" }
|
||||
|
||||
func (r routeWatchTags) check(
|
||||
func (routeWatchTags) check(
|
||||
ctx context.Context,
|
||||
rq *request,
|
||||
_ time.Time,
|
||||
) *verdict {
|
||||
if r.routeWatch == nil {
|
||||
if rq.h.routeWatch == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
route, missing := r.routeWatch.Route(
|
||||
route, missing := rq.h.routeWatch.Route(
|
||||
ctx,
|
||||
clientGroup(r.ipv6GroupPrefix, rq.client),
|
||||
clientGroup(rq.h.config, rq.client),
|
||||
rq.client,
|
||||
)
|
||||
if missing != "" {
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
)
|
||||
|
||||
// routeWatchURL is SWW_WAF_ROUTEWATCH_URL, and tagsMissing the log line's field
|
||||
@@ -58,7 +57,7 @@ func startRouteWatch(t *testing.T, routes map[string]standInRoute) string {
|
||||
// waitForRouteWatch waits until routewatch's answer about from, in the client
|
||||
// group of the given length, is kept, so that the next request gets its tags.
|
||||
func waitForRouteWatch(
|
||||
t *testing.T, server *waf.Server, from string, bits int,
|
||||
t *testing.T, server *wafAndParts, from string, bits int,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||
)
|
||||
@@ -23,15 +22,11 @@ import (
|
||||
// mode raises the alert for the ban
|
||||
// it would have made. Either rule's match is noted as an offence, for the
|
||||
// client's history. Its figures are the rule's id and action.
|
||||
type ruleFiles struct {
|
||||
rules *rules.Files
|
||||
metrics *metrics.Metrics
|
||||
banMaker banMaker
|
||||
}
|
||||
type ruleFiles struct{}
|
||||
|
||||
func (ruleFiles) name() string { return "rule_files" }
|
||||
|
||||
func (r ruleFiles) check(
|
||||
func (ruleFiles) check(
|
||||
_ context.Context,
|
||||
rq *request,
|
||||
now time.Time,
|
||||
@@ -40,7 +35,7 @@ func (r ruleFiles) check(
|
||||
|
||||
for _, rule := range matched {
|
||||
rq.line.RuleIDs = append(rq.line.RuleIDs, rule.ID)
|
||||
r.metrics.RuleMatched(rule.ID, rule.Action)
|
||||
rq.h.metrics.RuleMatched(rule.ID, rule.Action)
|
||||
}
|
||||
|
||||
if len(matched) == 0 {
|
||||
@@ -65,8 +60,7 @@ func (r ruleFiles) check(
|
||||
rq.attack = true
|
||||
|
||||
return &verdict{
|
||||
action: r.banMaker.banForAttack(
|
||||
rq,
|
||||
action: rq.banForAttack(
|
||||
now,
|
||||
bans.Notes{RuleID: last.ID, Target: last.Target},
|
||||
),
|
||||
|
||||
@@ -12,19 +12,13 @@ import (
|
||||
|
||||
// tagDeny refuses a request that carries a tag in SWW_WAF_TAG_DENY. Its figure is
|
||||
// the first such tag, in sorted order.
|
||||
type tagDeny struct {
|
||||
tags []string // SWW_WAF_TAG_DENY
|
||||
}
|
||||
type tagDeny struct{}
|
||||
|
||||
func (tagDeny) name() string { return "tag_deny" }
|
||||
|
||||
func (t tagDeny) check(
|
||||
_ context.Context,
|
||||
rq *request,
|
||||
_ time.Time,
|
||||
) *verdict {
|
||||
func (tagDeny) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
||||
for _, tag := range rq.line.Tags {
|
||||
if slices.Contains(t.tags, tag) {
|
||||
if slices.Contains(rq.h.config.TagDeny, tag) {
|
||||
return &verdict{
|
||||
action: requestlog.ActionClientDenied,
|
||||
figures: map[string]any{"tag": tag},
|
||||
|
||||
@@ -6,25 +6,17 @@ package waf
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
)
|
||||
|
||||
// tagLists adds to the request one tag for each list of SWW_WAF_TAG_LISTS that
|
||||
// holds the client, the tag being the list's name. It refuses nothing, and runs
|
||||
// first so that every refusal's log line carries these tags.
|
||||
type tagLists struct {
|
||||
lists *reputation.Lists
|
||||
}
|
||||
type tagLists struct{}
|
||||
|
||||
func (tagLists) name() string { return "tag_lists" }
|
||||
|
||||
func (t tagLists) check(
|
||||
_ context.Context,
|
||||
rq *request,
|
||||
_ time.Time,
|
||||
) *verdict {
|
||||
for _, name := range t.lists.TagsOf(rq.client) {
|
||||
func (tagLists) check(_ context.Context, rq *request, _ time.Time) *verdict {
|
||||
for _, name := range rq.h.lists.TagsOf(rq.client) {
|
||||
rq.addTag(name)
|
||||
}
|
||||
|
||||
|
||||
@@ -16,14 +16,11 @@ import (
|
||||
// it bans the client's netblock, or in observe mode raises the alert for the
|
||||
// ban it would have made. The ban's notes give the path as sent, and so does
|
||||
// its figure.
|
||||
type trapPaths struct {
|
||||
paths []string // SWW_WAF_TRAP_PATHS
|
||||
banMaker banMaker
|
||||
}
|
||||
type trapPaths struct{}
|
||||
|
||||
func (trapPaths) name() string { return "trap_paths" }
|
||||
|
||||
func (t trapPaths) check(
|
||||
func (trapPaths) check(
|
||||
_ context.Context,
|
||||
rq *request,
|
||||
now time.Time,
|
||||
@@ -36,22 +33,18 @@ func (t trapPaths) check(
|
||||
rq.attack = true
|
||||
|
||||
return &verdict{
|
||||
action: t.banMaker.banForAttack(
|
||||
rq,
|
||||
now,
|
||||
bans.Notes{TrapPath: path},
|
||||
),
|
||||
action: rq.banForAttack(now, bans.Notes{TrapPath: path}),
|
||||
figures: map[string]any{"path": path},
|
||||
}
|
||||
}
|
||||
|
||||
// trapPath returns rq's path as sent, when its path as a path rule sees it,
|
||||
// without the query, as sent or once percent-decoded, is in
|
||||
// trapPath returns the request's path as sent, when its path as a path rule
|
||||
// sees it, without the query, as sent or once percent-decoded, is in
|
||||
// SWW_WAF_TRAP_PATHS, and "" otherwise.
|
||||
func (t trapPaths) trapPath(rq *request) string {
|
||||
func (rq *request) trapPath() string {
|
||||
path := rules.Path(rq.in)
|
||||
if !slices.Contains(t.paths, path) &&
|
||||
!slices.Contains(t.paths, rules.DecodeOnce(path)) {
|
||||
if !slices.Contains(rq.h.config.TrapPaths, path) &&
|
||||
!slices.Contains(rq.h.config.TrapPaths, rules.DecodeOnce(path)) {
|
||||
return ""
|
||||
}
|
||||
|
||||
|
||||
+4
-18
@@ -104,12 +104,10 @@ type Report struct {
|
||||
// is far above rproxy's, and a question refused for its size would have rproxy
|
||||
// let the request through.
|
||||
func New(params Params) *Server {
|
||||
h := newHandler(params)
|
||||
|
||||
return &Server{
|
||||
Server: &http.Server{
|
||||
Addr: params.Config.ListenAddr,
|
||||
Handler: h,
|
||||
Handler: newHandler(params),
|
||||
ErrorLog: slog.NewLogLogger(
|
||||
params.ProcessLog.Handler(),
|
||||
slog.LevelWarn,
|
||||
@@ -118,16 +116,6 @@ func New(params Params) *Server {
|
||||
WriteTimeout: wafTimeout,
|
||||
IdleTimeout: wafIdleTimeout,
|
||||
},
|
||||
Ledger: h.ledger,
|
||||
Limiter: h.limiter,
|
||||
GeoJS: h.geojs,
|
||||
Anomalies: h.anomalies,
|
||||
LookupFile: h.lookupFile,
|
||||
Lists: params.Lists,
|
||||
DNSBL: params.DNSBL,
|
||||
AbuseIPDB: params.AbuseIPDB,
|
||||
RouteWatch: params.RouteWatch,
|
||||
Metrics: h.metrics,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -154,10 +142,8 @@ func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
listedBy := rq.h.blocklists.lists.ListedBy(rq.client)
|
||||
|
||||
return rq.h.blocklists.refusedByBlocklists(listedBy) ||
|
||||
rq.h.rateLimits.overAWholeRateLimit(rq, now)
|
||||
return rq.refusedByBlocklists(rq.h.lists.ListedBy(rq.client)) ||
|
||||
rq.overAWholeRateLimit(now)
|
||||
}
|
||||
|
||||
// answerQuestion runs the checks on the visitor's request a question
|
||||
@@ -493,7 +479,7 @@ func (h *handler) finishWith(
|
||||
if rq.allowed {
|
||||
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
|
||||
|
||||
rq.runBanningHeuristics(ctx, rq.h.onReport)
|
||||
rq.runBanningHeuristics(ctx, onReport())
|
||||
rq.countAnomalies()
|
||||
rq.addToHistory()
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ type WAF struct {
|
||||
t *testing.T
|
||||
addr string
|
||||
out *output
|
||||
server *waf.Server
|
||||
server *wafAndParts
|
||||
clock *clock
|
||||
queue *alerts.Queue
|
||||
}
|
||||
@@ -69,14 +69,14 @@ func startWAF(
|
||||
) *WAF {
|
||||
t.Helper()
|
||||
|
||||
return startWAFWith(t, geojsURL, env, func(*waf.Server) {})
|
||||
return startWAFWith(t, geojsURL, env, func(*wafAndParts) {})
|
||||
}
|
||||
|
||||
// startWAFWith is startWAF with change applied to the server before
|
||||
// it starts.
|
||||
func startWAFWith(
|
||||
t *testing.T, geojsURL string, env map[string]string,
|
||||
change func(*waf.Server),
|
||||
change func(*wafAndParts),
|
||||
) *WAF {
|
||||
t.Helper()
|
||||
|
||||
@@ -663,7 +663,7 @@ func TestQuestionWhoseBodyStopsArrivingGetsNoAnswer(t *testing.T) {
|
||||
t,
|
||||
"",
|
||||
map[string]string{metricsToken: token},
|
||||
func(server *waf.Server) { server.ReadTimeout = readTimeout },
|
||||
func(server *wafAndParts) { server.ReadTimeout = readTimeout },
|
||||
)
|
||||
|
||||
conn := dial(t, s.addr)
|
||||
|
||||
Reference in new issue
Block a user