sww: wire the daemons with fx (closes #414)
check / check (push) Failing after 3m12s

go.uber.org/fx builds the parts of waf and rproxy in internal/sww, one
provider per part. Each background loop is a lifecycle hook added where
its part is built. waf.Params takes every part built, and waf.New builds
none. GeoJS calls back into waf, so waf exports AddLookup and
GeoJSFailed, and clientGroup and banNetblock take the settings.

newServer, loadStateFiles, the inBackground list and serveRProxy are
gone. serve is the one hook that serves both daemons, and reports a
failure to run on a channel of its own; fx's signal handling is not
used. Once fx has stopped, run takes each daemon's last step: waf writes
the state files, and rproxy waits for its reports.

Model: opus-5-5
This commit is contained in:
sneak committed 2026-11-19 09:59:52 +00:00
1 parent 9f42460f70
commit 5a73af61b0
24 files changed
+1008 -649

No files matched your search

+1 -1
View File
@@ -35,7 +35,7 @@ func (abuseIPDB) check(
return nil
}
client := rq.h.clientGroup(rq.client)
client := clientGroup(rq.h.config, rq.client)
held, _ := rq.h.limiter.Client(client)
offender := held.History.Offences != ratelimit.Offences{}
+5 -5
View File
@@ -139,7 +139,7 @@ func (rq *request) addBan() {
)
if err == nil {
netblock, err = rq.h.banNetblock(toAdd.Netblock)
netblock, err = rq.h.parseBanNetblock(toAdd.Netblock)
}
if err == nil {
@@ -190,12 +190,12 @@ func (rq *request) readBanToAdd() (banToAdd, error) {
return toAdd, nil
}
// banNetblock reads value, a netblock such as 203.0.113.0/24, or a
// parseBanNetblock reads value, a netblock such as 203.0.113.0/24, or a
// client's address, which stands for the netblock a ban on that client
// covers. An IPv4-mapped netblock, such as ::ffff:203.0.113.0/120, is
// refused, since a client's address is looked up as IPv4 and a ban on it
// would refuse nothing, and so is a value with a zone.
func (h *handler) banNetblock(value string) (netip.Prefix, error) {
func (h *handler) parseBanNetblock(value string) (netip.Prefix, error) {
netblock, err := netip.ParsePrefix(value)
if err == nil {
if netblock.Addr().Is4In6() {
@@ -224,7 +224,7 @@ func (h *handler) banNetblock(value string) (netip.Prefix, error) {
return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errZone)
}
return h.netblock(addr), nil
return banNetblock(h.config, addr), nil
}
// expiry returns when a ban made at now for duration ends: duration
@@ -295,7 +295,7 @@ func (rq *request) showClient() {
Bans: state.BanEntries(rq.h.ledger.Covering(addr)),
}
client, seen := rq.h.limiter.Client(rq.h.clientGroup(addr))
client, seen := rq.h.limiter.Client(clientGroup(rq.h.config, addr))
if seen {
answer.Client = &client
}
+7 -12
View File
@@ -877,18 +877,13 @@ func TestWAFAnswersWithinTheTimeLimitWhileEveryOutsideSourceHangs(
held, _, _ := startHeldGeoJS(t)
resolver, releaseResolver := startHeldResolver(t)
server, out, _ := newWAFWith(t, func(
params waf.Params,
) *waf.Server {
params.AbuseIPDBURL = silentAbuseIPDBURL
return waf.New(params)
}, silentGeoJSURL, time.Now, map[string]string{
tagDeny: "country=KP", coreRuleSetMode: block,
dnsblZones: dnsblZone, dnsblResolver: resolver,
abuseIPDBKey: accountKey,
blocklistURLs: held + "/drop.txt", asnLimitPercentURL: held + "/asn.txt",
})
server, out, _ := newWAFWith(t, waf.New,
silentGeoJSURL, silentAbuseIPDBURL, time.Now, map[string]string{
tagDeny: "country=KP", coreRuleSetMode: block,
dnsblZones: dnsblZone, dnsblResolver: resolver,
abuseIPDBKey: accountKey,
blocklistURLs: held + "/drop.txt", asnLimitPercentURL: held + "/asn.txt",
})
// The lists are fetched outside the synctest bubble below: a wait on the
// network inside it would keep its clock from moving on.
+8 -7
View File
@@ -10,6 +10,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
@@ -121,7 +122,7 @@ func (rq *request) banForLimit(
rq.line.Offence = requestlog.OffenceLimit
rq.h.metrics.LimitHit(hit)
netblock := rq.h.netblock(rq.client)
netblock := banNetblock(rq.h.config, rq.client)
if rq.h.config.Observe &&
!rq.wouldAlertBan(netblock, now, bans.CauseLimit) {
return
@@ -157,7 +158,7 @@ func (rq *request) banForLimit(
}
ban, made := rq.h.ledger.BanForLimit(netblock, now, notes)
rq.h.limiter.Reset(rq.h.clientGroup(rq.client))
rq.h.limiter.Reset(clientGroup(rq.h.config, rq.client))
rq.line.BanExpires = banExpires(ban)
if made {
@@ -171,7 +172,7 @@ func (rq *request) banForLimit(
// no ban, and raises the alert for the ban it would have made, if that
// alert would be sent. It returns the action for that ban.
func (rq *request) banForAttack(now time.Time, notes bans.Notes) string {
netblock := rq.h.netblock(rq.client)
netblock := banNetblock(rq.h.config, rq.client)
if rq.h.config.Observe &&
!rq.wouldAlertBan(netblock, now, bans.CauseAttack) {
return banAction(
@@ -273,16 +274,16 @@ func (rq *request) netblockRequests(netblock netip.Prefix) int64 {
return rq.h.limiter.Requests(netblock) + 1
}
// netblock is the netblock a ban on client covers: its IPv4 address,
// banNetblock is the netblock a ban on client covers: its IPv4 address,
// widened to SWW_WAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group clientGroup
// counts it in.
func (h *handler) netblock(client netip.Addr) netip.Prefix {
func banNetblock(cfg *config.WAF, client netip.Addr) netip.Prefix {
addr := client.Unmap()
if addr.Is4() {
return netip.PrefixFrom(addr, h.config.BanScopeV4Prefix).Masked()
return netip.PrefixFrom(addr, cfg.BanScopeV4Prefix).Masked()
}
return h.clientGroup(addr)
return clientGroup(cfg, addr)
}
// banExpires is when ban ends, as the log line gives it: a time, or
+1 -1
View File
@@ -76,7 +76,7 @@ func newWAFHoldingTheCoreRuleSet(
params.BesideLookupTimeLimit = waf.BesideLookupTimeLimit
return waf.New(params)
}, silentGeoJSURL, time.Now, settings)
}, silentGeoJSURL, abuseIPDBURL, time.Now, settings)
return server, out, queue
}
+1 -1
View File
@@ -30,7 +30,7 @@ func (byteLimits) check(
}
counts, hit, over := rq.h.limiter.CountBytes(
rq.h.clientGroup(rq.client),
clientGroup(rq.h.config, rq.client),
now,
rq.countedBytes(),
rq.bytesLimitPercent.percent,
+4 -2
View File
@@ -5,6 +5,8 @@ import (
"net/http"
"net/netip"
"slices"
"sneak.berlin/go/smallwebwaf/internal/config"
)
// peerAddress is the address of the request's TCP peer.
@@ -45,10 +47,10 @@ func scheme(r *http.Request, isQuestion bool) string {
// address, or its IPv6 group, the netblock its IPv6 address is in of the
// length SWW_WAF_IPV6_GROUP_PREFIX sets, a /64 by default, since one abuser
// usually holds a whole /64. An IPv4 address in IPv6 form counts as IPv4.
func (h *handler) clientGroup(addr netip.Addr) netip.Prefix {
func clientGroup(cfg *config.WAF, addr netip.Addr) netip.Prefix {
addr = addr.Unmap()
if addr.Is6() {
return netip.PrefixFrom(addr, h.config.IPv6GroupPrefix).Masked()
return netip.PrefixFrom(addr, cfg.IPv6GroupPrefix).Masked()
}
return netip.PrefixFrom(addr, addr.BitLen())
+1 -1
View File
@@ -108,7 +108,7 @@ func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
Event: alerts.EventCoreRuleSetMatch,
Host: rq.site,
Client: rq.client,
Netblock: rq.h.clientGroup(rq.client),
Netblock: clientGroup(rq.h.config, rq.client),
ASN: rq.line.ASN,
ASName: rq.line.ASName,
Country: rq.line.Country,
+25 -13
View File
@@ -2,14 +2,18 @@ package waf
import (
"context"
"log/slog"
"net/http"
"net/netip"
"strings"
"time"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
)
// The headers in which an answer of allow gives rproxy the client's AS number
@@ -51,7 +55,7 @@ func (rq *request) startLookUp(
return result
}
client := rq.h.clientGroup(rq.client)
client := clientGroup(rq.h.config, rq.client)
if rq.h.config.LookupSource == "file" {
result <- lookupResult{
@@ -128,22 +132,30 @@ func countryTag(country string) string {
return config.CountryTagPrefix + country
}
// addLookup adds answer, an answer about a client from the lookup
// database or GeoJS, to the client's history, and to the notes of the bans
// on its netblock that have no AS number, AS name or country yet.
func (h *handler) addLookup(answer geojs.Answer) {
h.limiter.AddLookup(answer.Client, answer.Answered,
// AddLookup adds answer, an answer about a client from the lookup database
// or GeoJS, to the client's history in limiter, and to the notes of the bans
// in ledger on its netblock that have no AS number, AS name or country yet.
// cfg sets the netblock's size.
func AddLookup(
cfg *config.WAF, limiter *ratelimit.Limiter, ledger *bans.Ledger,
answer geojs.Answer,
) {
limiter.AddLookup(answer.Client, answer.Answered,
answer.ASN, answer.ASName, answer.Country)
h.ledger.AddLookup(h.netblock(answer.Client.Addr()),
ledger.AddLookup(banNetblock(cfg, answer.Client.Addr()),
answer.ASN, answer.ASName, answer.Country)
}
// geojsFailed logs a failure of GeoJS, raises a source_failure alert for it
// and counts it. askingAgainIn is how long GeoJS is left alone after it.
func (h *handler) geojsFailed(err error, askingAgainIn time.Duration) {
h.processLog.Warn("asking GeoJS failed",
// GeoJSFailed logs a failure of GeoJS to processLog, raises a
// source_failure alert for it on alertQueue and counts it in m.
// askingAgainIn is how long GeoJS is left alone after it.
func GeoJSFailed(
processLog *slog.Logger, alertQueue *alerts.Queue, m *metrics.Metrics,
err error, askingAgainIn time.Duration,
) {
processLog.Warn("asking GeoJS failed",
"error", err.Error(), "asking_again_in", askingAgainIn.String())
h.alerts.Raise(alerts.Alert{
alertQueue.Raise(alerts.Alert{
Event: alerts.EventSourceFailure,
Reason: "asking GeoJS failed",
Detail: map[string]any{
@@ -151,7 +163,7 @@ func (h *handler) geojsFailed(err error, askingAgainIn time.Duration) {
"asking_again_in": askingAgainIn.String(),
},
})
h.metrics.GeoJSFailures.Inc()
m.GeoJSFailures.Inc()
}
// setLookupHeaders sets the headers that give the client's AS number and
+47 -118
View File
@@ -57,12 +57,28 @@ type Params struct {
RequestLog io.Writer
// ProcessLog receives the process's own messages.
ProcessLog *slog.Logger
// GeoJSURL is where clients' AS numbers and countries are looked up
// while SWW_WAF_LOOKUP_SOURCE is geojs, normally geojs.URL.
GeoJSURL string
// AbuseIPDBURL is where clients are checked with AbuseIPDB while
// SWW_WAF_ABUSEIPDB_KEY is set, normally reputation.AbuseIPDBURL.
AbuseIPDBURL string
// Metrics count what waf does. Whoever builds them adds the metrics
// read from the other parts.
Metrics *metrics.Metrics
// Limiter is the table of clients the rate limits count, with each
// client's history.
Limiter *ratelimit.Limiter
// Ledger holds the bans.
Ledger *bans.Ledger
// Anomalies are the counters of the anomaly thresholds.
Anomalies *anomaly.Counters
// Lists are the lists fetched from URLs, DNSBL the DNSBL zones'
// verdicts, and AbuseIPDB AbuseIPDB's scores and the checks spent.
Lists *reputation.Lists
DNSBL *reputation.DNSBL
AbuseIPDB *reputation.AbuseIPDB
// RouteWatch asks routewatch about clients, and is nil while
// SWW_WAF_ROUTEWATCH_URL is unset.
RouteWatch *routewatch.Client
// GeoJS looks clients' AS numbers and countries up while
// SWW_WAF_LOOKUP_SOURCE is geojs. It is given AddLookup as its Answered
// and GeoJSFailed as its Failed.
GeoJS *geojs.GeoJS
// LookupFile is the lookup database they are looked up in while
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
LookupFile *lookup.File
@@ -99,11 +115,8 @@ type CoreRuleSetInspector interface {
) (coreruleset.Result, []byte, error)
}
// Server is the server waf runs, with the parts whose state the state
// files keep, the lookup database, nil unless SWW_WAF_LOOKUP_SOURCE is file, the
// lists fetched from URLs, which its Run fetches, the DNSBL zones' verdicts,
// AbuseIPDB's scores and checks spent, routewatch's answers, and the metrics.
// RouteWatch is nil while SWW_WAF_ROUTEWATCH_URL is unset.
// Server is the server waf runs, with the parts of its Params. The tests
// read them.
type Server struct {
*http.Server
@@ -119,129 +132,45 @@ type Server struct {
Metrics *metrics.Metrics
}
// newHandler returns the handler with what every request shares, as the
// settings in params make it.
// newHandler returns the handler with what every request shares, the parts
// in params among it.
func newHandler(params Params) *handler {
m := metrics.New(
params.Config.MetricsBusiestCountriesAndASNs,
params.Config.InstanceName,
)
lists, dnsbl, abuseIPDB := newReputation(params, m)
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
if err != nil {
panic(err) // New fails only for a size below one
}
h := &handler{
return &handler{
config: params.Config,
requestLog: params.RequestLog,
processLog: params.ProcessLog,
now: params.Now,
metrics: m,
metrics: params.Metrics,
awaitingReport: awaitingReport,
limiter: ratelimit.New(ratelimit.Limits{
PerMinute: params.Config.RateLimitPerMinute,
PerHour: params.Config.RateLimitPerHour,
PerDay: params.Config.RateLimitPerDay,
BytesPerMinute: params.Config.BytesLimitPerMinute,
BytesPerHour: params.Config.BytesLimitPerHour,
BytesPerDay: params.Config.BytesLimitPerDay,
}, params.Config.MaxTrackedClients),
ledger: bans.New(bans.Params{
LimitBanDuration: params.Config.LimitBanDuration,
LimitBanRepeatWindow: params.Config.LimitBanRepeatWindow,
LimitBanPermanentAbove: params.Config.LimitBanPermanentAbove,
AttackBanDuration: params.Config.AttackBanDuration,
MaxBans: params.Config.MaxBans,
}),
anomalies: anomaly.New(anomaly.Params{
Client: params.Config.AnomalyClient,
Net: params.Config.AnomalyNet,
ASN: params.Config.AnomalyASN,
Total: params.Config.AnomalyTotal,
WatchedNet: params.Config.AnomalyWatchedNet,
NetV4Prefix: params.Config.AnomalyNetV4Prefix,
NetV6Prefix: params.Config.AnomalyNetV6Prefix,
NamedNetblocks: params.Config.AnomalyWatchedNets,
Alerts: params.Alerts,
}),
lookupFile: params.LookupFile,
lists: lists,
dnsbl: dnsbl,
abuseIPDB: abuseIPDB,
rules: params.Rules,
coreRuleSet: params.CoreRuleSet,
alerts: params.Alerts,
limiter: params.Limiter,
ledger: params.Ledger,
geojs: params.GeoJS,
anomalies: params.Anomalies,
lookupFile: params.LookupFile,
lists: params.Lists,
dnsbl: params.DNSBL,
abuseIPDB: params.AbuseIPDB,
routeWatch: params.RouteWatch,
rules: params.Rules,
coreRuleSet: params.CoreRuleSet,
alerts: params.Alerts,
// The country list, the headers, the tag settings that name an asn=
// or country= tag and the biased thresholds act on the answer before
// the request goes on.
waitForGeoJS: len(params.Config.ExclusivelyAllowedCountries) > 0 ||
params.Config.AddLookupHeaders ||
params.Config.TagsNeedLookup() ||
biasedThresholdsSet(params.Config),
besideLookupLimit: cmp.Or(
params.BesideLookupTimeLimit,
besideLookupTimeLimit,
),
}
if params.Config.RouteWatchURL != "" {
h.routeWatch = routewatch.New(routewatch.Params{
URL: params.Config.RouteWatchURL, Now: params.Now, ProcessLog: params.ProcessLog,
})
}
// The country list, the headers, the tag settings that name an asn= or
// country= tag and the biased thresholds act on the answer before the
// request goes on.
h.waitForGeoJS = len(params.Config.ExclusivelyAllowedCountries) > 0 ||
params.Config.AddLookupHeaders || params.Config.TagsNeedLookup() ||
biasedThresholdsSet(params.Config)
h.geojs = geojs.New(geojs.Params{
URL: params.GeoJSURL,
Answered: h.addLookup,
Now: params.Now,
Asked: m.GeoJSRequests.Inc,
Failed: h.geojsFailed,
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
})
m.AddBansAndClients(h.ledger, h.limiter, params.Now)
m.AddRules(params.Rules)
return h
}
// newReputation returns the lists fetched from URLs, the DNSBL zones'
// verdicts and AbuseIPDB's scores, as the settings in params name them,
// with none fetched, asked for or checked yet, and adds their metrics to
// m, AbuseIPDB's while SWW_WAF_ABUSEIPDB_KEY is set.
func newReputation(
params Params, m *metrics.Metrics,
) (*reputation.Lists, *reputation.DNSBL, *reputation.AbuseIPDB) {
cfg := params.Config
lists := reputation.New(reputation.Params{
BlocklistURLs: cfg.BlocklistURLs,
Refresh: cfg.BlocklistRefresh,
ASNLimitPercentURL: cfg.ASNLimitPercentURL,
TagLists: cfg.TagLists,
Now: params.Now,
ProcessLog: params.ProcessLog,
Alerts: params.Alerts,
})
dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{
Zones: cfg.DNSBLZones, Resolver: cfg.DNSBLResolver,
CacheTTL: cfg.IPReputationCacheTTL,
Timeout: cfg.DNSBLAbuseIPDBTimeout, Now: params.Now, ProcessLog: params.ProcessLog,
Alerts: params.Alerts,
})
abuseIPDB := reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
URL: params.AbuseIPDBURL, Key: cfg.AbuseIPDBKey, MinScore: cfg.AbuseIPDBMinScore,
DailyBudget: cfg.AbuseIPDBDailyBudget, CacheTTL: cfg.IPReputationCacheTTL,
Timeout: cfg.DNSBLAbuseIPDBTimeout, Now: params.Now, ProcessLog: params.ProcessLog,
Alerts: params.Alerts,
})
m.AddReputation(lists, dnsbl)
if cfg.AbuseIPDBKey != "" {
m.AddAbuseIPDB(abuseIPDB)
}
return lists, dnsbl, abuseIPDB
}
// NewCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
+141 -15
View File
@@ -16,10 +16,17 @@ import (
"time"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/anomaly"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/routewatch"
"sneak.berlin/go/smallwebwaf/internal/rproxy"
"sneak.berlin/go/smallwebwaf/internal/rules"
"sneak.berlin/go/smallwebwaf/internal/waf"
@@ -435,13 +442,15 @@ func newWAF(
) (*waf.Server, *output, *alerts.Queue) {
t.Helper()
return newWAFWith(t, waf.New, geojsURL, now, env)
return newWAFWith(t, waf.New, geojsURL, abuseIPDBURL, now, env)
}
// newWAFWith is newWAF for the server build makes.
// newWAFWith is newWAF for the server build makes, with clients checked with
// AbuseIPDB at abuseIPDBCheckURL.
func newWAFWith(
t *testing.T, build func(waf.Params) *waf.Server,
geojsURL string, now func() time.Time, env map[string]string,
geojsURL, abuseIPDBCheckURL string, now func() time.Time,
env map[string]string,
) (*waf.Server, *output, *alerts.Queue) {
t.Helper()
@@ -499,25 +508,142 @@ func newWAFWith(
t.Fatalf("Core Rule Set: %v", err)
}
server := build(waf.Params{
Config: cfg,
RequestLog: out,
ProcessLog: processLog,
GeoJSURL: geojsURL,
AbuseIPDBURL: abuseIPDBURL,
LookupFile: lookupFile,
CoreRuleSet: coreRuleSet,
Now: now,
Rules: ruleFiles,
Alerts: alertQueue,
params := waf.Params{
Config: cfg,
RequestLog: out,
ProcessLog: processLog,
LookupFile: lookupFile,
CoreRuleSet: coreRuleSet,
Now: now,
Rules: ruleFiles,
Alerts: alertQueue,
// The tests of the limit set it back, and hold the Core Rule Set
// with a stand-in.
BesideLookupTimeLimit: time.Hour,
})
}
addCounters(&params)
addOutsideSources(&params, geojsURL, abuseIPDBCheckURL)
server := build(params)
return server, out, alertQueue
}
// addCounters adds to params the metrics, the table of clients, the ledger
// and the anomaly counters, as sww builds them from params' settings. The
// metrics read the ledger, the table of clients and the rules.
func addCounters(params *waf.Params) {
cfg := params.Config
params.Metrics = metrics.New(
cfg.MetricsBusiestCountriesAndASNs,
cfg.InstanceName,
)
params.Limiter = ratelimit.New(ratelimit.Limits{
PerMinute: cfg.RateLimitPerMinute,
PerHour: cfg.RateLimitPerHour,
PerDay: cfg.RateLimitPerDay,
BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerHour: cfg.BytesLimitPerHour,
BytesPerDay: cfg.BytesLimitPerDay,
}, cfg.MaxTrackedClients)
params.Ledger = bans.New(bans.Params{
LimitBanDuration: cfg.LimitBanDuration,
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
LimitBanPermanentAbove: cfg.LimitBanPermanentAbove,
AttackBanDuration: cfg.AttackBanDuration,
MaxBans: cfg.MaxBans,
})
params.Anomalies = anomaly.New(anomaly.Params{
Client: cfg.AnomalyClient,
Net: cfg.AnomalyNet,
ASN: cfg.AnomalyASN,
Total: cfg.AnomalyTotal,
WatchedNet: cfg.AnomalyWatchedNet,
NetV4Prefix: cfg.AnomalyNetV4Prefix,
NetV6Prefix: cfg.AnomalyNetV6Prefix,
NamedNetblocks: cfg.AnomalyWatchedNets,
Alerts: params.Alerts,
})
params.Metrics.AddBansAndClients(
params.Ledger,
params.Limiter,
params.Now,
)
params.Metrics.AddRules(params.Rules)
}
// addOutsideSources adds to params the lists fetched from URLs, the DNSBL
// zones' verdicts, AbuseIPDB's scores from abuseIPDBCheckURL, routewatch's
// answers while its setting is set, and GeoJS at geojsURL, as sww builds
// them from params' settings and the parts addCounters added. The metrics
// read the lists, the zones' verdicts, and AbuseIPDB's while its key is set.
func addOutsideSources(
params *waf.Params, geojsURL, abuseIPDBCheckURL string,
) {
cfg := params.Config
m, limiter, ledger := params.Metrics, params.Limiter, params.Ledger
processLog, alertQueue := params.ProcessLog, params.Alerts
params.Lists = reputation.New(reputation.Params{
BlocklistURLs: cfg.BlocklistURLs,
Refresh: cfg.BlocklistRefresh,
ASNLimitPercentURL: cfg.ASNLimitPercentURL,
TagLists: cfg.TagLists,
Now: params.Now,
ProcessLog: processLog,
Alerts: alertQueue,
})
params.DNSBL = reputation.NewDNSBL(reputation.DNSBLParams{
Zones: cfg.DNSBLZones,
Resolver: cfg.DNSBLResolver,
CacheTTL: cfg.IPReputationCacheTTL,
Timeout: cfg.DNSBLAbuseIPDBTimeout,
Now: params.Now,
ProcessLog: processLog,
Alerts: alertQueue,
})
params.AbuseIPDB = reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
URL: abuseIPDBCheckURL,
Key: cfg.AbuseIPDBKey,
MinScore: cfg.AbuseIPDBMinScore,
DailyBudget: cfg.AbuseIPDBDailyBudget,
CacheTTL: cfg.IPReputationCacheTTL,
Timeout: cfg.DNSBLAbuseIPDBTimeout,
Now: params.Now,
ProcessLog: processLog,
Alerts: alertQueue,
})
if cfg.RouteWatchURL != "" {
params.RouteWatch = routewatch.New(routewatch.Params{
URL: cfg.RouteWatchURL,
Now: params.Now,
ProcessLog: processLog,
})
}
params.GeoJS = geojs.New(geojs.Params{
URL: geojsURL,
Answered: func(answer geojs.Answer) {
waf.AddLookup(cfg, limiter, ledger, answer)
},
Now: params.Now,
Asked: m.GeoJSRequests.Inc,
Failed: func(err error, askingAgainIn time.Duration) {
waf.GeoJSFailed(processLog, alertQueue, m, err, askingAgainIn)
},
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
})
m.AddReputation(params.Lists, params.DNSBL)
if cfg.AbuseIPDBKey != "" {
m.AddAbuseIPDB(params.AbuseIPDB)
}
}
// newClient returns an HTTP client that sends requests as they are made,
// with no compression of its own.
func newClient(t *testing.T) *http.Client {
+6 -3
View File
@@ -38,8 +38,11 @@ func (rateLimits) check(
rq.line.BytesLimitPercent, rq.line.BytesLimitPercentSetting =
rq.bytesLimitPercent.logged()
counts, hit, over := rq.h.limiter.Count(rq.h.clientGroup(rq.client), now,
rq.rateLimitPercent.percent)
counts, hit, over := rq.h.limiter.Count(
clientGroup(rq.h.config, rq.client),
now,
rq.rateLimitPercent.percent,
)
rq.line.Counts = counts
if !over {
@@ -59,7 +62,7 @@ func (rateLimits) check(
// limit. It counts nothing.
func (rq *request) overAWholeRateLimit(now time.Time) bool {
return rq.countedByRateLimits() &&
rq.h.limiter.OverARateLimit(rq.h.clientGroup(rq.client), now)
rq.h.limiter.OverARateLimit(clientGroup(rq.h.config, rq.client), now)
}
// countedByRateLimits reports whether the rate limits count the request: its
+5 -2
View File
@@ -36,8 +36,11 @@ func (refusalLimit) check(
return nil
}
hit, over := rq.h.limiter.CountRefusal(rq.h.clientGroup(rq.client), now,
cfg.RefusalsLimitPerMinute)
hit, over := rq.h.limiter.CountRefusal(
clientGroup(rq.h.config, rq.client),
now,
cfg.RefusalsLimitPerMinute,
)
if !over {
return nil
}
+1 -1
View File
@@ -37,7 +37,7 @@ func (rq *request) noteHit(hit bans.ReputationHit, reason string) {
Event: alerts.EventReputationHit,
Host: rq.site,
Client: rq.client,
Netblock: rq.h.clientGroup(rq.client),
Netblock: clientGroup(rq.h.config, rq.client),
ASN: rq.line.ASN,
ASName: rq.line.ASName,
Country: rq.line.Country,
+5 -5
View File
@@ -122,7 +122,7 @@ func (h *handler) startRequest(
RequestID: requestID(r, isQuestion),
PeerIP: peer.String(),
ForwardedFor: strings.Join(forwardedFor, ", "),
ClientGroup: h.clientGroup(client).String(),
ClientGroup: clientGroup(h.config, client).String(),
ContentType: r.Header.Get("Content-Type"),
RequestHeaders: requestHeaders(
r,
@@ -365,7 +365,7 @@ func (rq *request) addToHistory() {
}
rq.h.limiter.AddToHistory(
rq.h.clientGroup(rq.client),
clientGroup(rq.h.config, rq.client),
rq.h.now(),
request,
)
@@ -373,7 +373,7 @@ func (rq *request) addToHistory() {
answer, found := rq.answerAtTheEnd()
if found {
rq.h.addLookup(answer)
AddLookup(rq.h.config, rq.h.limiter, rq.h.ledger, answer)
}
}
@@ -394,7 +394,7 @@ func (rq *request) countAnomalies() {
rq.h.anomalies.Count(rq.h.now(), anomaly.Request{
Host: rq.site,
Client: rq.client,
ClientGroup: rq.h.clientGroup(rq.client),
ClientGroup: clientGroup(rq.h.config, rq.client),
ASN: answer.ASN,
ASName: answer.ASName,
Country: answer.Country,
@@ -425,7 +425,7 @@ func (rq *request) answerAtTheEnd() (geojs.Answer, bool) {
return rq.lookupAnswer, true
}
return rq.h.geojs.Kept(rq.h.clientGroup(rq.client))
return rq.h.geojs.Kept(clientGroup(rq.h.config, rq.client))
}
// responseWriter is waf's answer to a request. It notes the status and
+1 -1
View File
@@ -32,7 +32,7 @@ func (routeWatchTags) check(
route, missing := rq.h.routeWatch.Route(
ctx,
rq.h.clientGroup(rq.client),
clientGroup(rq.h.config, rq.client),
rq.client,
)
if missing != "" {