Compare commits

..
Author SHA1 Message Date
clawbot b7e7557047 waf: test the request_bytes of a ban added through the endpoint
check / check (push) Waiting to run
No test read the request_bytes of an own endpoint's log line, so
removing the counting in logOwnEndpoint went unnoticed. The test checks
that a ban to add is logged with its body's bytes, and that the health
endpoint, which reads no body, logs 0.

Model: opus-5-5
2026-11-19 17:07:16 +00:00
clawbot e84156ef04 waf: list its own endpoints on a chi router in routes.go (closes #426)
check / check (push) Waiting to run
routes.go lists every own endpoint. One middleware checks a token and
another writes the log line. answerOwnEndpoint, answerAdmin and
endpoint() are gone; the handlers return http.HandlerFunc. The router
matches the decoded path, as the old code did.

Model: opus-5-5
2026-11-19 16:40:27 +00:00
44 changed files with 744 additions and 1097 deletions

No files matched your search

+3 -5
View File
@@ -906,11 +906,9 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
`internal/alerts`: one check or source each.
- `internal/state`: the state and cache files.
- `internal/requestlog`: the request log.
- `internal/processlog`: the process's own messages.
- `internal/remotelog`: the syslog sender.
- `internal/metrics`: the metrics.
- `internal/state`, `internal/requestlog`, `internal/remotelog` and
`internal/metrics`: the state and cache files, the log lines, the syslog
sender and the metrics.
## Rationale
+6 -7
View File
@@ -74,9 +74,9 @@ never as a time.
and stops their background work.
- `internal/config`: reads the settings, the one place they are read.
- `internal/waf`: `waf`'s server. It runs the checks in `check`, and holds
`waf`'s question, answer and report. It answers `waf`'s own endpoints in
`answerOwnEndpoint`. It counts bytes, refusals and anomalies, and writes
the log lines.
`waf`'s question, answer and report. Its `routes.go` lists `waf`'s own
endpoints. It counts bytes, refusals and anomalies, and writes the log
lines.
- `internal/rproxy`: the reverse proxy in front of one app that asks `waf`
about each request. It passes a request to the app with the standard
library's `httputil.ReverseProxy`, within the time and size limits.
@@ -104,10 +104,8 @@ never as a time.
its own.
- `internal/state`: reads the state and cache files at start, takes in edits
while running, and writes the files when due and at the stop.
- `internal/requestlog`: the request log, as JSON lines on stdout.
- `internal/processlog`: the process lines on stdout, which hold the
process's own messages.
- `internal/remotelog`: sends the lines on stdout to the syslog server.
- `internal/requestlog` and `internal/remotelog`: the lines on stdout, and
sending them to the syslog server.
- `internal/metrics`: the metrics, in the Prometheus text format.
- `Dockerfile`: the lint and test phases, then the image.
`deploy/example-app` is an app built on the image, with `rproxy` in its
@@ -121,6 +119,7 @@ never as a time.
default `SWW_WAF_CORE_RULE_SET_DISABLED_RULES`, are built for that
version. They are checked again before the version changes, since rule
ids, and what each rule matches, change between versions;
- `github.com/go-chi/chi/v5` to route the requests to `waf`'s own endpoints;
- `github.com/hashicorp/golang-lru/v2` for the bounded tables;
- `github.com/oschwald/maxminddb-golang/v2` to read the lookup database;
- `github.com/fsnotify/fsnotify` to notice edited files;
+1
View File
@@ -7,6 +7,7 @@ require (
github.com/corazawaf/coraza-coreruleset/v4 v4.25.0
github.com/corazawaf/coraza/v3 v3.8.2-0.20261022102553-d028be399c68
github.com/fsnotify/fsnotify v1.10.1
github.com/go-chi/chi/v5 v5.5.0
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/maxmind/mmdbwriter v1.2.0
github.com/oschwald/maxminddb-golang/v2 v2.7.0
+2
View File
@@ -20,6 +20,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
github.com/go-chi/chi/v5 v5.5.0 h1:xdKQ1dJmVJsmgahrXSKhUx8NyVPzpwdlyrrQFpMPJgg=
github.com/go-chi/chi/v5 v5.5.0/go.mod h1:d1IG7GtKPSjPyBP365PH8sAGVHk3qfB/IhjFNRH1wWM=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
+32 -93
View File
@@ -75,7 +75,7 @@ type WAF struct {
BytesLimitPerMinute int64
BytesLimitPerHour int64
BytesLimitPerDay int64
BytesCounted BytesCounted
BytesCounted string
// LookupSource is where each client's AS number and country are looked
// up (SWW_WAF_LOOKUP_SOURCE): geojs, file, or off for nowhere.
// LookupDBPath is the lookup database, the IPinfo Lite file looked up in
@@ -86,7 +86,7 @@ type WAF struct {
// AddLookupHeaders is true when the app is passed the client's AS number
// and country in headers (SWW_WAF_ADD_LOOKUP_HEADERS), by default unless
// LookupSource is off.
LookupSource LookupSource
LookupSource string
LookupDBPath string
ASNAndCountryLookupTimeout time.Duration
AddLookupHeaders bool
@@ -131,7 +131,7 @@ type WAF struct {
// BlocklistLimitPercent is the percentage of every limit it gets.
BlocklistURLs []string
BlocklistRefresh time.Duration
BlocklistAction ReputationAction
BlocklistAction string
BlocklistLimitPercent int64
// DNSBLZones are the DNSBL zones clients are asked about
// (SWW_WAF_DNSBL_ZONES), through DNSBLResolver (SWW_WAF_DNSBL_RESOLVER), or
@@ -154,7 +154,7 @@ type WAF struct {
AbuseIPDBKey string
AbuseIPDBMinScore int64
AbuseIPDBDailyBudget int
DNSBLAbuseIPDBAction ReputationAction
DNSBLAbuseIPDBAction string
DNSBLAbuseIPDBLimitPercent int64
IPReputationCacheTTL time.Duration
DNSBLAbuseIPDBTimeout time.Duration
@@ -237,7 +237,7 @@ type WAF struct {
// reads none.
// CoreRuleSetExclusionsFile is the path of the file of its exclusions
// (SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE), "" while there is none.
CoreRuleSetMode CoreRuleSetMode
CoreRuleSetMode string
CoreRuleSetParanoiaLevel int
CoreRuleSetScoreThreshold int
CoreRuleSetDisabledRules []int
@@ -316,66 +316,20 @@ type WAF struct {
settings []slog.Attr
}
// BytesCounted is which body bytes count toward the byte limits and the
// byte anomaly thresholds, a value of SWW_WAF_BYTES_COUNTED.
type BytesCounted string
// The values of SWW_WAF_BYTES_COUNTED.
const (
// BytesCountedResponse counts the response's body bytes.
BytesCountedResponse BytesCounted = "response"
// BytesCountedRequest counts the request's body bytes.
BytesCountedRequest BytesCounted = "request"
// BytesCountedBoth counts the body bytes of both.
BytesCountedBoth BytesCounted = "both"
)
// LookupSource is where each client's AS number and country are looked up,
// a value of SWW_WAF_LOOKUP_SOURCE.
type LookupSource string
// The values of SWW_WAF_LOOKUP_SOURCE.
const (
// LookupSourceGeoJS asks GeoJS.
LookupSourceGeoJS LookupSource = "geojs"
// LookupSourceFile looks clients up in the lookup database, the file
// SWW_WAF_LOOKUP_DB_PATH names.
LookupSourceFile LookupSource = "file"
// LookupSourceOff looks no client up.
LookupSourceOff LookupSource = "off"
)
// ReputationAction is what is done with a client a reputation source
// lists, a value of SWW_WAF_BLOCKLIST_ACTION and of
// SWW_WAF_DNSBL_ABUSEIPDB_ACTION.
type ReputationAction string
// The values of SWW_WAF_BLOCKLIST_ACTION and SWW_WAF_DNSBL_ABUSEIPDB_ACTION.
const (
// ReputationActionDeny refuses the client's requests.
ReputationActionDeny ReputationAction = "deny"
// ReputationActionLimit gives the client a percentage of every limit,
// written limit:<percent> in the setting.
ReputationActionLimit ReputationAction = "limit"
// ReputationActionLog only notes the hit in the log line and raises an
// alert.
ReputationActionLog ReputationAction = "log"
)
// CoreRuleSetMode is what the Core Rule Set does, a value of
// SWW_WAF_CORE_RULE_SET_MODE.
type CoreRuleSetMode string
// The values of SWW_WAF_CORE_RULE_SET_MODE.
const (
// CoreRuleSetModeOff runs no request through the Core Rule Set.
CoreRuleSetModeOff CoreRuleSetMode = "off"
CoreRuleSetModeOff = off
// CoreRuleSetModeDetect logs and alerts a match, and refuses nothing.
CoreRuleSetModeDetect CoreRuleSetMode = "detect"
CoreRuleSetModeDetect = "detect"
// CoreRuleSetModeBlock refuses a match with 403.
CoreRuleSetModeBlock CoreRuleSetMode = "block"
CoreRuleSetModeBlock = "block"
)
// fileSource is the SWW_WAF_LOOKUP_SOURCE that looks clients up in the
// lookup database, the file SWW_WAF_LOOKUP_DB_PATH names.
const fileSource = "file"
const (
day = 24 * time.Hour
ipv4Bits = 32
@@ -604,13 +558,10 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
),
BytesLimitPerHour: env.size("SWW_WAF_BYTES_LIMIT_PER_HOUR", "50G"),
BytesLimitPerDay: env.size("SWW_WAF_BYTES_LIMIT_PER_DAY", "100G"),
BytesCounted: env.bytesCounted(
"SWW_WAF_BYTES_COUNTED",
BytesCountedBoth,
),
BytesCounted: env.bytesCounted("SWW_WAF_BYTES_COUNTED", "both"),
LookupSource: env.lookupSource(
"SWW_WAF_LOOKUP_SOURCE",
LookupSourceGeoJS,
"geojs",
),
LookupDBPath: env.value("SWW_WAF_LOOKUP_DB_PATH", ""),
ASNAndCountryLookupTimeout: env.milliseconds(
@@ -807,9 +758,9 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
cfg.InstanceName, cfg.LogRemoteURL != nil)
// The app gets the headers by default, unless no client is looked up.
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
strconv.FormatBool(cfg.LookupSource != off))
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
"SWW_WAF_BLOCKLIST_ACTION", string(ReputationActionDeny))
"SWW_WAF_BLOCKLIST_ACTION", "deny")
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "limit:25")
@@ -907,12 +858,9 @@ func (e *environment) count(name, defaultValue string) int64 {
// bytesCounted reads the setting that is which body bytes count toward the
// byte limits and the byte anomaly thresholds: response, request or both.
func (e *environment) bytesCounted(
name string, defaultValue BytesCounted,
) BytesCounted {
value := BytesCounted(e.value(name, string(defaultValue)))
if value != BytesCountedResponse && value != BytesCountedRequest &&
value != BytesCountedBoth {
func (e *environment) bytesCounted(name, defaultValue string) string {
value := e.value(name, defaultValue)
if value != "response" && value != "request" && value != "both" {
e.check(name, fmt.Errorf("%q %w", value, errNotBytesCounted))
}
@@ -957,10 +905,8 @@ func (e *environment) trapPaths(name string) []string {
// coreRuleSetMode reads the setting that is what the Core Rule Set does: off,
// detect or block.
func (e *environment) coreRuleSetMode(
name string, defaultValue CoreRuleSetMode,
) CoreRuleSetMode {
mode := CoreRuleSetMode(e.value(name, string(defaultValue)))
func (e *environment) coreRuleSetMode(name, defaultValue string) string {
mode := e.value(name, defaultValue)
if mode != CoreRuleSetModeOff && mode != CoreRuleSetModeDetect &&
mode != CoreRuleSetModeBlock {
e.check(name, fmt.Errorf("%q %w", mode, errNotCoreRuleSetMode))
@@ -1110,24 +1056,20 @@ func (e *environment) refresh(name, defaultValue string) time.Duration {
// action reads a setting that is what is done with a client a blocklist
// or a DNSBL zone lists: deny, log, or limit:<percent>, which it returns
// as limit and the percentage.
func (e *environment) action(
name, defaultValue string,
) (ReputationAction, int64) {
value := ReputationAction(e.value(name, defaultValue))
if value == ReputationActionDeny || value == ReputationActionLog {
func (e *environment) action(name, defaultValue string) (string, int64) {
value := e.value(name, defaultValue)
if value == "deny" || value == "log" {
return value, 0
}
percentText, isLimit := strings.CutPrefix(
string(value), string(ReputationActionLimit)+":",
)
percentText, isLimit := strings.CutPrefix(value, "limit:")
percent, err := ParsePercent(percentText)
if !isLimit || err != nil {
e.check(name, fmt.Errorf("%q %w", value, errNotAction))
}
return ReputationActionLimit, percent
return "limit", percent
}
// zones reads the setting that is the list of DNSBL zones. It is empty by
@@ -1178,12 +1120,9 @@ func (e *environment) routeWatchURL(name string) string {
// lookupSource reads the setting that is where clients are looked up:
// geojs, file, or off.
func (e *environment) lookupSource(
name string, defaultValue LookupSource,
) LookupSource {
source := LookupSource(e.value(name, string(defaultValue)))
if source != LookupSourceGeoJS && source != LookupSourceFile &&
source != LookupSourceOff {
func (e *environment) lookupSource(name, defaultValue string) string {
source := e.value(name, defaultValue)
if source != "geojs" && source != fileSource && source != off {
e.check(name, fmt.Errorf("%q %w", source, errNotLookupSource))
}
@@ -1195,12 +1134,12 @@ func (e *environment) lookupSource(
// one source at a time.
func (e *environment) checkLookupDBPath(cfg *WAF) {
switch {
case cfg.LookupSource == LookupSourceFile && cfg.LookupDBPath == "":
case cfg.LookupSource == fileSource && cfg.LookupDBPath == "":
e.check("SWW_WAF_LOOKUP_SOURCE", fmt.Errorf(
"is file while SWW_WAF_LOOKUP_DB_PATH is unset; %w",
errNeedsDBPath,
))
case cfg.LookupSource != LookupSourceFile && cfg.LookupDBPath != "":
case cfg.LookupSource != fileSource && cfg.LookupDBPath != "":
e.check("SWW_WAF_LOOKUP_DB_PATH", fmt.Errorf(
"is set while SWW_WAF_LOOKUP_SOURCE is %s; %w",
cfg.LookupSource,
@@ -1225,7 +1164,7 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
}
}
if cfg.LookupSource != LookupSourceOff {
if cfg.LookupSource != off {
return
}
+5 -6
View File
@@ -614,7 +614,7 @@ func TestCoreRuleSetSettings(t *testing.T) {
},
{
environment{
coreRuleSetMode: string(config.CoreRuleSetModeDetect),
coreRuleSetMode: config.CoreRuleSetModeDetect,
coreRuleSetParanoiaLevel: "4",
coreRuleSetScoreThreshold: "10",
coreRuleSetDisabledRules: "942100, 920350",
@@ -2033,9 +2033,8 @@ func TestReputationSettingsAsSet(t *testing.T) {
}
for _, tc := range []struct {
value string
action config.ReputationAction
percent int64
value, action string
percent int64
}{
{actionDeny, actionDeny, 0},
{actionLog, actionLog, 0},
@@ -2168,7 +2167,7 @@ func TestDNSBLSettingsAsSet(t *testing.T) {
// want are the zones, resolver, action and percent config.WAF gives.
want []string
wantResolver string
wantAction config.ReputationAction
wantAction string
wantPercent int64
}{
{
@@ -3050,7 +3049,7 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
logLevel: "info",
ruleFilesDir: "/etc/sww/rules.d",
ruleFilesEnabled: "true",
coreRuleSetMode: string(config.CoreRuleSetModeBlock),
coreRuleSetMode: config.CoreRuleSetModeBlock,
coreRuleSetParanoiaLevel: "1",
coreRuleSetScoreThreshold: "5",
coreRuleSetDisabledRules: defaultCoreRuleSetDisabledRules,
+37 -60
View File
@@ -84,7 +84,12 @@ type Params struct {
// one request to GeoJS is under way at a time, and it asks about every
// client waiting, up to maxPerRequest. It is safe for concurrent use.
type GeoJS struct {
params Params
url string
answered func(Answer)
now func() time.Time
asked func()
failed func(err error, askingAgainIn time.Duration)
wentWithoutAnswer func()
// httpClient follows no redirect, so that visitors' addresses go to
// GeoJS alone: a redirect is a failure.
httpClient *http.Client
@@ -94,13 +99,8 @@ type GeoJS struct {
// waiting are the clients without an answer: those to ask GeoJS about,
// and those it is being asked about.
waiting map[netip.Prefix]*wait
// asking is true while a request to GeoJS is under way, and endAsking
// ends it. stopped is true once StopAsking has been called, and
// askGoroutines waits for the goroutine that makes the requests.
asking bool
endAsking context.CancelFunc
stopped bool
askGoroutines sync.WaitGroup
// asking is true while a request to GeoJS is under way.
asking bool
// retryDelay is how long GeoJS is left alone after its last failure,
// zero after an answer; retryAt is when it may be asked again.
retryDelay time.Duration
@@ -140,7 +140,12 @@ func New(params Params) *GeoJS {
}
return &GeoJS{
params: params,
url: params.URL,
answered: params.Answered,
now: params.Now,
asked: params.Asked,
failed: params.Failed,
wentWithoutAnswer: params.WentWithoutAnswer,
httpClient: &http.Client{
CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
@@ -183,7 +188,7 @@ func (g *GeoJS) LookUp(
answer, found := g.kept(client)
if !found {
g.params.WentWithoutAnswer()
g.wentWithoutAnswer()
}
w, waiting := g.waiting[client]
@@ -205,23 +210,6 @@ func (g *GeoJS) LookUpWithoutWaiting(
return answer
}
// StopAsking ends the request to GeoJS under way, and returns once the
// goroutine that made it has. A request it ends is no failure: Failed is not
// called, and GeoJS is not left alone after it. After StopAsking, GeoJS is
// asked nothing.
func (g *GeoJS) StopAsking() {
g.mu.Lock()
g.stopped = true
if g.asking {
g.endAsking()
}
g.mu.Unlock()
g.askGoroutines.Wait()
}
// Kept returns client's answer, if one is kept, without asking GeoJS.
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
g.mu.Lock()
@@ -263,7 +251,7 @@ func (g *GeoJS) Load(answers []Answer) {
g.answers.Purge()
now := g.params.Now()
now := g.now()
for _, answer := range answers {
if now.Sub(answer.Answered) < keepFor {
@@ -300,19 +288,18 @@ func (g *GeoJS) answerOrWait(
}
if w == nil {
g.params.WentWithoutAnswer()
g.wentWithoutAnswer()
return Answer{}, nil // too many clients wait already
}
if !g.asking {
// GeoJS is left alone after a failure, or asked nothing after
// StopAsking, so no answer can come.
// GeoJS is left alone after a failure, so no answer can come.
w.late = true
}
if w.late {
g.params.WentWithoutAnswer()
g.wentWithoutAnswer()
return Answer{}, nil
}
@@ -323,7 +310,7 @@ func (g *GeoJS) answerOrWait(
// kept returns client's answer, if GeoJS gave it less than keepFor ago,
// and notes that it was used.
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
now := g.params.Now()
now := g.now()
kept, found := g.answers.Get(client)
if !found || now.Sub(kept.Answered) >= keepFor {
@@ -336,28 +323,22 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
}
// ask starts asking GeoJS about the waiting clients, unless a request to
// it is under way, it is left alone after a failure, or StopAsking has
// been called. The requests to GeoJS are for every client waiting, so they
// go on when the caller whose ctx is given stops waiting, until StopAsking.
// it is under way or it is left alone after a failure. The requests to
// GeoJS are for every client waiting, so they go on when the caller whose
// ctx is given stops waiting.
func (g *GeoJS) ask(ctx context.Context) {
if g.asking || g.stopped || g.params.Now().Before(g.retryAt) {
if g.asking || g.now().Before(g.retryAt) {
return
}
askCtx, end := context.WithCancel(context.WithoutCancel(ctx))
g.asking, g.endAsking = true, end
g.asking = true
g.askGoroutines.Go(func() {
defer end()
g.askAboutWaiting(askCtx)
})
go g.askAboutWaiting(context.WithoutCancel(ctx))
}
// askAboutWaiting asks GeoJS about the waiting clients, one request at a
// time, until none is left, GeoJS fails, or StopAsking ends ctx. Each
// answer kept is given to Answered, outside the lock, since Answered takes
// locks of its own.
// time, until none is left or GeoJS fails. Each answer kept is given to
// Answered, outside the lock, since Answered takes locks of its own.
func (g *GeoJS) askAboutWaiting(ctx context.Context) {
for {
clients := g.nextClients()
@@ -366,15 +347,11 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
}
given, err := g.request(ctx, clients)
if err != nil && ctx.Err() != nil {
continue // ended by StopAsking, which is no failure
}
kept, answered := g.keep(clients, given, err)
if g.params.Answered != nil {
if g.answered != nil {
for _, answer := range kept {
g.params.Answered(answer)
g.answered(answer)
}
}
@@ -385,13 +362,13 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
}
// nextClients returns up to maxPerRequest of the waiting clients. When
// none is waiting, or StopAsking has been called, it returns none and
// notes that no request to GeoJS is under way.
// none is waiting, it returns none and notes that no request to GeoJS is
// under way.
func (g *GeoJS) nextClients() []netip.Prefix {
g.mu.Lock()
defer g.mu.Unlock()
if len(g.waiting) == 0 || g.stopped {
if len(g.waiting) == 0 {
g.asking = false
return nil
@@ -423,7 +400,7 @@ func (g *GeoJS) keep(
g.mu.Lock()
defer g.mu.Unlock()
now := g.params.Now()
now := g.now()
kept := make([]Answer, 0, len(clients))
leftOut := 0
@@ -467,7 +444,7 @@ func (g *GeoJS) keep(
w.late = true
}
g.params.Failed(err, g.retryDelay)
g.failed(err, g.retryDelay)
return kept, false
}
@@ -497,7 +474,7 @@ func (g *GeoJS) request(
req, err := http.NewRequestWithContext(
ctx,
http.MethodGet,
g.params.URL,
g.url,
http.NoBody,
)
if err != nil {
@@ -506,7 +483,7 @@ func (g *GeoJS) request(
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
g.params.Asked()
g.asked()
res, err := g.httpClient.Do(req)
if err != nil {
-38
View File
@@ -318,44 +318,6 @@ func TestCallerWhoseContextEndsEarlyDoesNotStopTheRequest(t *testing.T) {
})
}
func TestStopAskingEndsTheRequestUnderWayAsNoFailureAndAsksNoMore(
t *testing.T,
) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
service, _, g, reports := startWithReports()
clients := newClients()
service.set(hanging)
g.LookUpWithoutWaiting(t.Context(), clients())
waitForRequests(t, service, 1)
// The request is ended, not waited out, and is no failure.
began := time.Now()
g.StopAsking()
if waited := time.Since(began); waited != 0 {
t.Errorf("StopAsking returned after %s, want at once", waited)
}
reports.wantFailures(t)
// A new client is then not asked about, and its lookup does not
// wait.
service.set(answering)
wantCountry(t, g, clients(), "")
if waited := time.Since(began); waited != 0 {
t.Errorf("the lookup returned after %s, want at once", waited)
}
waitForRequests(t, service, 1)
reports.wantFailures(t)
})
}
func TestRequestOver10SecondsIsAFailure(t *testing.T) {
t.Parallel()
-44
View File
@@ -1,44 +0,0 @@
// Package processlog builds the logger for the process's own messages,
// which it writes as JSON lines marked "type":"process". The request log
// is package requestlog's.
package processlog
import (
"io"
"log/slog"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
// Params are what New needs.
type Params struct {
// Output is where the lines are written.
Output io.Writer
// InstanceName, SWW_WAF_INSTANCE_NAME, is each line's instance.
InstanceName string
// Level, SWW_WAF_LOG_LEVEL, is the least severe level written.
Level slog.Level
}
// New returns the logger for the process's own messages: JSON lines on
// params.Output, marked "type":"process", with the time in the same form
// as a request line's. It writes only the messages at params.Level or more
// severe; the request lines requestlog.Write writes are never held back.
func New(params Params) *slog.Logger {
handler := slog.NewJSONHandler(params.Output, &slog.HandlerOptions{
Level: params.Level,
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
if attr.Key == slog.TimeKey && len(groups) == 0 {
return slog.String(
slog.TimeKey,
requestlog.FormatTime(attr.Value.Time()),
)
}
return attr
},
})
return slog.New(handler).
With("type", "process", "instance", params.InstanceName)
}
-105
View File
@@ -1,105 +0,0 @@
package processlog_test
import (
"bytes"
"encoding/json"
"log/slog"
"slices"
"strings"
"testing"
"time"
"sneak.berlin/go/smallwebwaf/internal/processlog"
)
// instanceName is the instance the tests' loggers are built with.
const instanceName = "host1/blog"
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processlog.New(processlog.Params{
Output: &out,
InstanceName: instanceName,
Level: slog.LevelInfo,
}).
Info("starting",
"version", "v1")
var fields map[string]any
err := json.Unmarshal(out.Bytes(), &fields)
if err != nil {
t.Fatalf("decode %q: %v", out.String(), err)
}
if fields["type"] != "process" ||
fields["instance"] != "host1/blog" ||
fields["msg"] != "starting" ||
fields["level"] != "INFO" ||
fields["version"] != "v1" {
t.Errorf("process line %v", fields)
}
timeText, _ := fields["time"].(string)
logged, err := time.Parse(time.RFC3339, timeText)
if err != nil || !strings.HasSuffix(timeText, "Z") ||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
time.Since(logged) > time.Minute {
t.Errorf(
"process line time %q, want now in UTC with milliseconds",
timeText,
)
}
}
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
t.Parallel()
levels := []slog.Level{
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
}
for i, level := range levels {
t.Run(level.String(), func(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processLog := processlog.New(processlog.Params{
Output: &out,
InstanceName: instanceName,
Level: level,
})
for _, at := range levels {
processLog.Log(t.Context(), at, "message")
}
var got, want []string
for line := range strings.Lines(out.String()) {
var fields struct {
Level string `json:"level"`
}
err := json.Unmarshal([]byte(line), &fields)
if err != nil {
t.Fatalf("decode %q: %v", line, err)
}
got = append(got, fields.Level)
}
for _, written := range levels[i:] {
want = append(want, written.String())
}
if !slices.Equal(got, want) {
t.Errorf("lines at %v, want %v", got, want)
}
})
}
}
+4 -7
View File
@@ -11,7 +11,7 @@ import (
func TestHistoryKeepsEveryRequest(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -57,7 +57,7 @@ func TestHistoryKeepsEveryRequest(t *testing.T) {
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
other := netip.MustParsePrefix("198.51.100.7/32")
start := midnight()
@@ -100,10 +100,7 @@ func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
func TestResetKeepsTheHistory(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit},
MaxClients: tableSize,
})
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -126,7 +123,7 @@ func TestResetKeepsTheHistory(t *testing.T) {
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
for client, requests := range map[string]int{
"198.51.100.9/32": 2,
+6 -18
View File
@@ -156,28 +156,16 @@ type Request struct {
BlockedByCoreRuleSet bool
}
// Params are what New needs.
type Params struct {
// Limits are what each client's requests and bytes are counted against.
Limits Limits
// MaxClients is the most clients the table holds,
// SWW_WAF_MAX_TRACKED_CLIENTS. Past it, the least recently seen client
// is dropped, with its history, and starts afresh if it comes back.
MaxClients int
}
// New returns a Limiter with no client counted yet.
func New(params Params) *Limiter {
clients, err := simplelru.NewLRU[netip.Prefix, *Client](
params.MaxClients,
nil,
)
// New returns a Limiter for limits, with no client counted yet, whose
// table holds at most maxClients clients (SWW_WAF_MAX_TRACKED_CLIENTS). Past
// it, the least recently seen client is dropped, with its history, and
// starts afresh if it comes back.
func New(limits Limits, maxClients int) *Limiter {
clients, err := simplelru.NewLRU[netip.Prefix, *Client](maxClients, nil)
if err != nil {
panic(err) // NewLRU fails only for a size below one
}
limits := params.Limits
return &Limiter{
windows: [3]window{
{
+39 -63
View File
@@ -33,10 +33,10 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
client := netip.MustParsePrefix("203.0.113.9/32")
now := midnight()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
MaxClients: tableSize,
})
limiter := ratelimit.New(
ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
tableSize,
)
if limiter.OverARateLimit(client, now) {
t.Error("a client never seen is over a rate limit")
@@ -72,7 +72,7 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
wantCount(t, limiter, client, now, minute)
// A limit that is off is never gone over.
off := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
off := ratelimit.New(ratelimit.Limits{}, tableSize)
off.Count(client, now, whole)
if off.OverARateLimit(client, now) {
@@ -95,10 +95,7 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
t.Run(tc.window, func(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: tc.limits,
MaxClients: tableSize,
})
limiter := ratelimit.New(tc.limits, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
quarter := tc.length / 4
@@ -129,10 +126,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit, PerHour: limit},
MaxClients: tableSize,
})
limiter := ratelimit.New(
ratelimit.Limits{PerMinute: limit, PerHour: limit},
tableSize,
)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -162,13 +159,10 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{
PerMinute: 5,
BytesPerDay: math.MaxInt64,
},
MaxClients: tableSize,
})
limiter := ratelimit.New(
ratelimit.Limits{PerMinute: 5, BytesPerDay: math.MaxInt64},
tableSize,
)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -209,10 +203,10 @@ func TestZeroPercentIsAZeroAllowanceAndALimitOffStaysOff(t *testing.T) {
t.Parallel()
// Only the hour has limits: the minute's and the day's are off.
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
MaxClients: tableSize,
})
limiter := ratelimit.New(
ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
tableSize,
)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -259,10 +253,7 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
t.Run(tc.window, func(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: tc.limits,
MaxClients: tableSize,
})
limiter := ratelimit.New(tc.limits, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
// 600 bytes are within the limit, 600 more over it.
@@ -296,10 +287,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
MaxClients: tableSize,
})
limiter := ratelimit.New(
ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
tableSize,
)
client := netip.MustParsePrefix("203.0.113.9/32")
other := netip.MustParsePrefix("203.0.113.10/32")
start := midnight()
@@ -323,7 +314,7 @@ func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
func TestCountGivesTheBytesInEachWindow(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -349,10 +340,7 @@ func TestCountGivesTheBytesInEachWindow(t *testing.T) {
func TestResetSetsTheBytesBackToZero(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{BytesPerDay: 1000},
MaxClients: tableSize,
})
limiter := ratelimit.New(ratelimit.Limits{BytesPerDay: 1000}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -368,7 +356,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -423,7 +411,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -450,10 +438,10 @@ func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
func TestResetSetsTheCountsBackToZero(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit},
MaxClients: tableSize,
})
limiter := ratelimit.New(
ratelimit.Limits{PerMinute: limit, PerDay: limit},
tableSize,
)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -475,10 +463,7 @@ func TestResetSetsTheCountsBackToZero(t *testing.T) {
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -497,10 +482,10 @@ func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
func TestRefusedRequestsCount(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
MaxClients: tableSize,
})
limiter := ratelimit.New(
ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
tableSize,
)
refused := netip.MustParsePrefix("203.0.113.9/32")
within := netip.MustParsePrefix("203.0.113.10/32")
start := midnight()
@@ -533,10 +518,7 @@ func TestRefusedRequestsCount(t *testing.T) {
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit},
MaxClients: tableSize,
})
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -552,10 +534,7 @@ func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
t.Parallel()
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
@@ -583,10 +562,7 @@ func TestKeepsAtMostMaxClientsDroppingTheLeastRecentlySeen(t *testing.T) {
const maxClients = 3
limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: 1},
MaxClients: maxClients,
})
limiter := ratelimit.New(ratelimit.Limits{PerMinute: 1}, maxClients)
now := midnight()
clients := make([]netip.Prefix, maxClients+1)
+6 -12
View File
@@ -19,7 +19,7 @@ func TestSnapshotListsTheClientsByAddress(t *testing.T) {
"2001:db8::/64",
}
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
for _, i := range []int{2, 3, 0, 1} {
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
}
@@ -52,10 +52,7 @@ func TestLoadedCountsCarryOn(t *testing.T) {
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
before := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
before := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
for range limit {
wantCount(t, before, client, start, "")
}
@@ -63,10 +60,7 @@ func TestLoadedCountsCarryOn(t *testing.T) {
// Loaded into a new limiter, as across a restart, the client has no
// fresh allowance.
later := start.Add(time.Minute)
after := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
after := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
after.Load(before.Snapshot(), later)
wantCount(t, after, client, later, hour)
}
@@ -77,7 +71,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight()
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
limiter.Count(client, start, whole)
limiter.CountBytes(client, start, 5, whole)
limiter.CountRefusal(client, start, limit)
@@ -86,7 +80,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
loaded := func(now time.Time) ratelimit.Client {
t.Helper()
after := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
after := ratelimit.New(ratelimit.Limits{}, tableSize)
after.Load(limiter.Snapshot(), now)
return after.Snapshot()[0]
@@ -148,7 +142,7 @@ func TestLoadDropsTheLeastRecentlySeenFirst(t *testing.T) {
addr = addr.Next()
}
limiter := ratelimit.New(ratelimit.Params{MaxClients: maxClients})
limiter := ratelimit.New(ratelimit.Limits{}, maxClients)
limiter.Load(clients, midnight())
got := limiter.Snapshot()
+16 -48
View File
@@ -110,12 +110,8 @@ type AbuseIPDB struct {
// scores are by client. Each is added as it is fetched and never moved
// up, so that the one fetched longest ago is the first dropped.
scores *simplelru.LRU[netip.Prefix, Score]
// checking are the clients whose check is under way, each with the
// function that ends it. stopped is true once StopChecking has been
// called, and checkGoroutines waits for the goroutine of each check.
checking map[netip.Prefix]context.CancelFunc
stopped bool
checkGoroutines sync.WaitGroup
// checking are the clients whose check is under way.
checking map[netip.Prefix]bool
// day is the day, in UTC, of the checks spent counts.
day civil.Date
spent int
@@ -138,7 +134,7 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
params: params,
httpClient: &http.Client{},
scores: scores,
checking: map[netip.Prefix]context.CancelFunc{},
checking: map[netip.Prefix]bool{},
}
}
@@ -148,11 +144,11 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
// addresses its request comes from. A client without one is checked in
// the background, by addr, the address its request came from, if
// offender, if it has committed an offence, unless its check is under
// way, a check failed less than failureDelay ago, the day's checks have
// used up DailyBudget, or StopChecking has been called; Hit never waits
// for a check. The check that uses the budget up is logged and raised as a
// source_failure alert. ctx is the context of the client's request, and a
// check goes on after the request ends, until StopChecking.
// way, a check failed less than failureDelay ago, or the day's checks
// have used up DailyBudget; Hit never waits for a check. The check that
// uses the budget up is logged and raised as a source_failure alert. ctx
// is the context of the client's request, and a check goes on after the
// request ends.
func (a *AbuseIPDB) Hit(
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
) (int64, bool) {
@@ -171,21 +167,14 @@ func (a *AbuseIPDB) Hit(
a.day, a.spent = today, 0
}
_, checking := a.checking[client]
check := offender && !checking && !a.stopped &&
!now.Before(a.retryAt) && a.spent < a.params.DailyBudget
check := offender && !a.checking[client] && !now.Before(a.retryAt) &&
a.spent < a.params.DailyBudget
if check {
checkCtx, end := context.WithCancel(context.WithoutCancel(ctx))
a.checking[client] = end
a.checking[client] = true
a.checks++
a.spent++
a.checkGoroutines.Go(func() {
defer end()
a.check(checkCtx, client, addr)
})
go a.check(context.WithoutCancel(ctx), client, addr)
}
usedUp := check && a.spent == a.params.DailyBudget
@@ -200,24 +189,6 @@ func (a *AbuseIPDB) Hit(
return 0, false
}
// StopChecking ends the checks under way, and returns once their goroutines
// have. A check it ends gives no score and is no failure: it is neither
// counted, logged nor raised as an alert, and checks are not left alone
// after it. The ended check stays spent from the day's budget. After
// StopChecking, Hit checks no client.
func (a *AbuseIPDB) StopChecking() {
a.mu.Lock()
a.stopped = true
for _, end := range a.checking {
end()
}
a.mu.Unlock()
a.checkGoroutines.Wait()
}
// Checked returns how many checks were made.
func (a *AbuseIPDB) Checked() int {
a.mu.Lock()
@@ -344,8 +315,7 @@ func (a *AbuseIPDB) RunBlacklist(ctx context.Context) {
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
// the score as client's, and notes the check as no longer under way. A
// check that fails gives no score: it is counted, logged and raised as a
// source_failure alert, and no client is checked for failureDelay. A check
// ended as ctx ends, by StopChecking, is no failure.
// source_failure alert, and no client is checked for failureDelay.
func (a *AbuseIPDB) check(
ctx context.Context,
client netip.Prefix,
@@ -353,26 +323,24 @@ func (a *AbuseIPDB) check(
) {
score, err := a.ask(ctx, addr)
now := a.params.Now()
isFailure := err != nil && ctx.Err() == nil
a.mu.Lock()
delete(a.checking, client)
switch {
case err == nil:
if err == nil {
a.scores.Add(
client,
Score{Client: client, Score: score, Fetched: now},
)
case isFailure:
} else {
a.failures++
a.retryAt = now.Add(failureDelay)
}
a.mu.Unlock()
if isFailure {
if err != nil {
a.alert("checking a client with AbuseIPDB failed", err)
}
}
-37
View File
@@ -394,43 +394,6 @@ func TestCheckNotAnsweredWithinTheTimeoutFailsAndHitNeverWaits(
})
}
func TestStopCheckingEndsTheCheckUnderWayAsNoFailureAndChecksNoMore(
t *testing.T,
) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
queue := newQueue()
p := abuseIPDBParams()
p.Alerts = queue
abuseIPDB := &abuseIPDBStandIn{hanging: true}
checker := newAbuseIPDB(abuseIPDB, p)
wantScore(t, checker, suspect, true, 0, false)
synctest.Wait()
// The check is ended, not waited out, and is no failure.
began := time.Now()
checker.StopChecking()
if waited := time.Since(began); waited != 0 {
t.Errorf("StopChecking returned after %s, want at once", waited)
}
wantFailures(t, checker, 0)
if got := waiting(queue); len(got) != 0 {
t.Errorf("alerts waiting %+v, want none", got)
}
// An offender without a score is then not checked.
wantScore(t, checker, other, true, 0, false)
synctest.Wait()
wantChecked(t, abuseIPDB, suspect)
})
}
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
t.Parallel()
+13 -44
View File
@@ -82,12 +82,8 @@ type DNSBL struct {
// verdicts are by query. Each is added as it is fetched and never moved
// up, so that the one fetched longest ago is the first dropped.
verdicts *simplelru.LRU[query, Verdict]
// asking are the queries under way, each with the function that ends
// it. stopped is true once StopAsking has been called, and
// queryGoroutines waits for the goroutine of each query.
asking map[query]context.CancelFunc
stopped bool
queryGoroutines sync.WaitGroup
// asking are the queries under way.
asking map[query]bool
// queries and failures count, by zone, the queries made and those that
// failed, and retryAt is when a zone whose last query failed may be
// asked again.
@@ -124,7 +120,7 @@ func NewDNSBL(params DNSBLParams) *DNSBL {
params: params,
resolver: resolver,
verdicts: verdicts,
asking: map[query]context.CancelFunc{},
asking: map[query]bool{},
queries: map[string]int{},
failures: map[string]int{},
retryAt: map[string]time.Time{},
@@ -142,10 +138,9 @@ func (d *DNSBL) Zones() []string {
// log, the alerts and the metrics. A verdict is used until CacheTTL has
// passed since it was fetched. Each zone without one is asked about addr
// in the background, unless a query about addr to it is under way, the
// zone is left alone after a failure, maxQueries are under way, or
// StopAsking has been called; ListedBy never waits for a query. ctx is the
// context of the client's request, and a query goes on after the request
// ends, until StopAsking.
// zone is left alone after a failure, or maxQueries are under way;
// ListedBy never waits for a query. ctx is the context of the client's
// request, and a query goes on after the request ends.
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
d.mu.Lock()
defer d.mu.Unlock()
@@ -158,47 +153,23 @@ func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
q := query{zone: zone, client: addr}
kept, found := d.verdicts.Peek(q)
_, asking := d.asking[q]
switch {
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
if kept.Listed {
listedBy = append(listedBy, config.MaskZoneKey(zone))
}
case !asking && !d.stopped && !now.Before(d.retryAt[zone]) &&
len(d.asking) < maxQueries:
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
d.asking[q] = end
case !d.asking[q] && !now.Before(d.retryAt[zone]) && len(d.asking) < maxQueries:
d.asking[q] = true
d.queries[zone]++
d.queryGoroutines.Go(func() {
defer end()
d.ask(queryCtx, q)
})
go d.ask(context.WithoutCancel(ctx), q)
}
}
return listedBy
}
// StopAsking ends the queries under way, and returns once their
// goroutines have. A query it ends gives no verdict and is no failure: it
// is neither counted, logged nor raised as an alert, and the zone is not
// left alone after it. After StopAsking, ListedBy asks no zone.
func (d *DNSBL) StopAsking() {
d.mu.Lock()
d.stopped = true
for _, end := range d.asking {
end()
}
d.mu.Unlock()
d.queryGoroutines.Wait()
}
// Queries returns how many queries were made to zone.
func (d *DNSBL) Queries(zone string) int {
d.mu.Lock()
@@ -267,29 +238,27 @@ func (d *DNSBL) Load(verdicts []Verdict) {
// query as no longer under way. A query that fails gives no verdict: it
// is counted, logged and raised as a source_failure alert, which show the
// zone with its key masked, and the zone is not asked again for
// failureDelay. A query ended as ctx ends, by StopAsking, is no failure.
// failureDelay.
func (d *DNSBL) ask(ctx context.Context, q query) {
listed, err := d.lookUp(ctx, q)
now := d.params.Now()
isFailure := err != nil && ctx.Err() == nil
d.mu.Lock()
delete(d.asking, q)
switch {
case err == nil:
if err == nil {
d.verdicts.Add(q, Verdict{
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
})
case isFailure:
} else {
d.failures[q.zone]++
d.retryAt[q.zone] = now.Add(failureDelay)
}
d.mu.Unlock()
if isFailure {
if err != nil {
const failed = "asking a DNSBL zone failed"
shown := config.MaskZoneKey(q.zone)
-39
View File
@@ -210,45 +210,6 @@ func TestQueryNotAnsweredWithinTheTimeoutFails(t *testing.T) {
})
}
//
//nolint:paralleltest // one at a time, as the comment at the top of this file says
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
t *testing.T,
) {
synctest.Test(t, func(t *testing.T) {
queue := newQueue()
p := dnsblParams(zone)
p.Alerts = queue
resolver := &resolverStandIn{hanging: true}
dnsbl := newDNSBL(resolver, p)
wantZones(t, dnsbl, listed)
synctest.Wait()
// The query is ended, not waited out, and is no failure.
began := time.Now()
dnsbl.StopAsking()
if waited := time.Since(began); waited != 0 {
t.Errorf("StopAsking returned after %s, want at once", waited)
}
wantQueries(t, dnsbl, 1, 0)
if got := waiting(queue); len(got) != 0 {
t.Errorf("alerts waiting %+v, want none", got)
}
// A client without a verdict is then asked about in no zone.
wantZones(t, dnsbl, unlisted)
synctest.Wait()
wantQueries(t, dnsbl, 1, 0)
wantAsked(t, resolver, listedName)
waitForTheResolver()
})
}
//
//nolint:paralleltest // one at a time, as the comment at the top of this file says
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(
+33 -4
View File
@@ -1,7 +1,7 @@
// Package requestlog writes the request log sww prints on stdout: JSON
// lines marked "type":"request", at most two from waf and a short one
// from rproxy for each request. Package processlog writes the process's
// own messages.
// Package requestlog writes the lines sww prints on stdout: one
// JSON object per request, two from waf and a short one from rproxy,
// marked "type":"request", and the process's own messages as JSON lines marked
// "type":"process".
package requestlog
import (
@@ -9,6 +9,7 @@ import (
"encoding/json"
"fmt"
"io"
"log/slog"
"time"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
@@ -299,3 +300,31 @@ func Milliseconds(d time.Duration) float64 {
time.Millisecond/time.Microsecond,
)
}
// NewProcessLogger returns the logger for the process's own messages:
// JSON lines on w, marked "type":"process", with the time in the same form
// as a request line's, and instanceName, SWW_WAF_INSTANCE_NAME, as instance.
// It writes only the messages at level, SWW_WAF_LOG_LEVEL, or more severe;
// the request lines Write writes are never held back.
func NewProcessLogger(
w io.Writer,
instanceName string,
level slog.Level,
) *slog.Logger {
handler := slog.NewJSONHandler(w, &slog.HandlerOptions{
Level: level,
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
if attr.Key == slog.TimeKey && len(groups) == 0 {
return slog.String(
slog.TimeKey,
FormatTime(attr.Value.Time()),
)
}
return attr
},
})
return slog.New(handler).
With("type", "process", "instance", instanceName)
}
+87
View File
@@ -3,6 +3,8 @@ package requestlog_test
import (
"bytes"
"encoding/json"
"log/slog"
"slices"
"strings"
"testing"
"time"
@@ -132,3 +134,88 @@ func TestWriteRProxyLineWritesOneJSONLineWithOnlyItsFields(t *testing.T) {
t.Errorf("wrote %s, want %s", out.String(), want)
}
}
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
t.Parallel()
var out bytes.Buffer
requestlog.NewProcessLogger(&out, "host1/blog", slog.LevelInfo).
Info("starting",
"version", "v1")
var fields map[string]any
err := json.Unmarshal(out.Bytes(), &fields)
if err != nil {
t.Fatalf("decode %q: %v", out.String(), err)
}
if fields["type"] != "process" ||
fields["instance"] != "host1/blog" ||
fields["msg"] != "starting" ||
fields["level"] != "INFO" ||
fields["version"] != "v1" {
t.Errorf("process line %v", fields)
}
timeText, _ := fields["time"].(string)
logged, err := time.Parse(time.RFC3339, timeText)
if err != nil || !strings.HasSuffix(timeText, "Z") ||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
time.Since(logged) > time.Minute {
t.Errorf(
"process line time %q, want now in UTC with milliseconds",
timeText,
)
}
}
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
t.Parallel()
levels := []slog.Level{
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
}
for i, level := range levels {
t.Run(level.String(), func(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processLog := requestlog.NewProcessLogger(
&out,
"host1/blog",
level,
)
for _, at := range levels {
processLog.Log(t.Context(), at, "message")
}
var got, want []string
for line := range strings.Lines(out.String()) {
var fields struct {
Level string `json:"level"`
}
err := json.Unmarshal([]byte(line), &fields)
if err != nil {
t.Fatalf("decode %q: %v", line, err)
}
got = append(got, fields.Level)
}
for _, written := range levels[i:] {
want = append(want, written.String())
}
if !slices.Equal(got, want) {
t.Errorf("lines at %v, want %v", got, want)
}
})
}
}
+12 -43
View File
@@ -99,12 +99,8 @@ type Client struct {
// answers are by client. Each is added as it comes and never moved up,
// so that the one that came longest ago is the first dropped.
answers *simplelru.LRU[netip.Prefix, Answer]
// asking are the clients whose query is under way, each with the
// function that ends it. stopped is true once StopAsking has been
// called, and queryGoroutines waits for the goroutine of each query.
asking map[netip.Prefix]context.CancelFunc
stopped bool
queryGoroutines sync.WaitGroup
// asking are the clients whose query is under way.
asking map[netip.Prefix]bool
// retryAt is when routewatch may be asked again after a query failed.
retryAt time.Time
}
@@ -120,7 +116,7 @@ func New(params Params) *Client {
params: params,
httpClient: &http.Client{},
answers: answers,
asking: map[netip.Prefix]context.CancelFunc{},
asking: map[netip.Prefix]bool{},
}
}
@@ -128,9 +124,8 @@ func New(params Params) *Client {
// IPv6 group, or why it has no route. It never waits for a query. A client
// without an answer is asked about in the background, by addr, the address its
// request came from, unless its query is under way, a query failed less than
// failureKept ago, maxQueries are under way, or StopAsking has been called. ctx
// is the context of the client's request, and a query goes on after the request
// ends, until StopAsking.
// failureKept ago, or maxQueries are under way. ctx is the context of the
// client's request, and a query goes on after the request ends.
func (c *Client) Route(
ctx context.Context, client netip.Prefix, addr netip.Addr,
) (Route, Missing) {
@@ -152,38 +147,15 @@ func (c *Client) Route(
return Route{}, RequestFailed
}
_, asking := c.asking[client]
if !asking && !c.stopped && len(c.asking) < maxQueries {
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
c.asking[client] = end
if !c.asking[client] && len(c.asking) < maxQueries {
c.asking[client] = true
c.queryGoroutines.Go(func() {
defer end()
c.ask(queryCtx, client, addr)
})
go c.ask(context.WithoutCancel(ctx), client, addr)
}
return Route{}, NoAnswerYet
}
// StopAsking ends the queries under way, and returns once their goroutines
// have. A query it ends gives no answer and is no failure: it is not logged,
// and routewatch is not left alone after it. After StopAsking, Route asks
// routewatch nothing.
func (c *Client) StopAsking() {
c.mu.Lock()
c.stopped = true
for _, end := range c.asking {
end()
}
c.mu.Unlock()
c.queryGoroutines.Wait()
}
// Snapshot returns every answer still in use, sorted by client, as
// routewatch.json lists them.
func (c *Client) Snapshot() []Answer {
@@ -233,8 +205,7 @@ func (c *Client) Load(answers []Answer) {
// ask asks routewatch about addr, keeps the answer as client's, and notes the
// query as no longer under way. A query that fails gives no answer, and
// routewatch is not asked again for failureKept. Only the first failure of
// those under way is logged. A query ended as ctx ends, by StopAsking, is no
// failure.
// those under way is logged.
func (c *Client) ask(
ctx context.Context,
client netip.Prefix,
@@ -242,20 +213,18 @@ func (c *Client) ask(
) {
route, err := c.query(ctx, addr)
now := c.params.Now()
isFailure := err != nil && ctx.Err() == nil
c.mu.Lock()
delete(c.asking, client)
firstFailure := isFailure && !now.Before(c.retryAt)
firstFailure := err != nil && !now.Before(c.retryAt)
switch {
case err == nil:
if err == nil {
c.answers.Add(client, Answer{
Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
})
case isFailure:
} else {
c.retryAt = now.Add(failureKept)
}
-48
View File
@@ -361,54 +361,6 @@ func TestHangingRoutewatchNeverDelaysAndFailsTheQuery(t *testing.T) {
})
}
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
t *testing.T,
) {
t.Parallel()
// In a synctest bubble, as TestHangingRoutewatchNeverDelaysAndFailsTheQuery
// says.
synctest.Test(t, func(t *testing.T) {
f := &fixture{t: t, log: &syncBuffer{}}
f.client = routewatch.New(routewatch.Params{
URL: silentURL, Now: time.Now,
ProcessLog: slog.New(slog.NewTextHandler(f.log, nil)),
})
f.route()
synctest.Wait()
// The query is ended, not waited out.
began := time.Now()
f.client.StopAsking()
if waited := time.Since(began); waited != 0 {
t.Errorf("StopAsking returned after %s, want at once", waited)
}
// A query started now would fail within waitLimit. None is, and the
// one ended is no failure: nothing is logged, and routewatch is not
// left alone.
for range 2 {
if _, missing := f.route(); missing != routewatch.NoAnswerYet {
t.Errorf(
"Route gave %q, want %q",
missing,
routewatch.NoAnswerYet,
)
}
time.Sleep(waitLimit)
synctest.Wait()
}
if f.log.String() != "" {
t.Errorf("process log %q, want nothing", f.log.String())
}
})
}
func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
t.Parallel()
+6 -6
View File
@@ -18,7 +18,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
@@ -51,11 +51,11 @@ func TestServerHasTheFixedLimits(t *testing.T) {
server := rproxy.New(rproxy.Params{
Config: cfg,
RequestLog: io.Discard,
ProcessLog: processlog.New(processlog.Params{
Output: io.Discard,
InstanceName: "test",
Level: slog.LevelInfo,
}),
ProcessLog: requestlog.NewProcessLogger(
io.Discard,
"test",
slog.LevelInfo,
),
})
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
+6 -6
View File
@@ -21,7 +21,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
@@ -370,11 +370,11 @@ func startServerLogging(
server := rproxy.New(rproxy.Params{
Config: cfg,
RequestLog: requestLog,
ProcessLog: processlog.New(processlog.Params{
Output: processLog,
InstanceName: "test",
Level: slog.LevelInfo,
}),
ProcessLog: requestlog.NewProcessLogger(
processLog,
"test",
slog.LevelInfo,
),
})
listener, err := (&net.ListenConfig{}).Listen(
+1 -1
View File
@@ -2448,7 +2448,7 @@ func newParams(dir string) state.Params {
AttackBanDuration: 7 * 24 * time.Hour,
MaxBans: 5000,
}),
Limiter: ratelimit.New(ratelimit.Params{MaxClients: 20000}),
Limiter: ratelimit.New(ratelimit.Limits{}, 20000),
GeoJS: geojs.New(geojs.Params{Now: midnight}),
Lists: reputation.New(reputation.Params{
BlocklistURLs: []string{
+6 -6
View File
@@ -10,7 +10,7 @@ import (
"go.uber.org/fx"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy"
)
@@ -23,11 +23,11 @@ import (
// setting for either.
func RunRProxy(ctx context.Context, params Params) int {
hostname, _ := os.Hostname() // "" when the host has no name to give
processLog := processlog.New(processlog.Params{
Output: params.Stdout,
InstanceName: hostname,
Level: slog.LevelInfo,
})
processLog := requestlog.NewProcessLogger(
params.Stdout,
hostname,
slog.LevelInfo,
)
cfg, err := config.RProxyFromEnvironment(params.Viper)
if err != nil {
+1 -3
View File
@@ -660,9 +660,7 @@ func TestCoreRuleSetExclusionsFileThatCannotBeUsedStopsTheStartWhileTheCoreRuleS
status := run(t.Context(), map[string]string{
listenAddr: localhost + ":0", stateDir: t.TempDir(), ruleFilesDir: t.TempDir(),
"SWW_WAF_CORE_RULE_SET_MODE": string(
config.CoreRuleSetModeOff,
),
"SWW_WAF_CORE_RULE_SET_MODE": config.CoreRuleSetModeOff,
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE": path,
}, out)
if status != 1 {
+32 -87
View File
@@ -2,7 +2,6 @@ package sww
import (
"context"
"fmt"
"io"
"log/slog"
"net"
@@ -19,10 +18,10 @@ import (
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/remotelog"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/routewatch"
"sneak.berlin/go/smallwebwaf/internal/rules"
"sneak.berlin/go/smallwebwaf/internal/state"
@@ -43,11 +42,8 @@ const remoteLogStopTimeout = 2 * time.Second
func RunWAF(ctx context.Context, params Params) int {
// Until the settings are read, the one message is an invalid setting's
// error, which every SWW_WAF_LOG_LEVEL lets through.
processLog := processlog.New(processlog.Params{
Output: params.Stdout,
InstanceName: config.InstanceName(params.Viper),
Level: slog.LevelError,
})
processLog := requestlog.NewProcessLogger(params.Stdout,
config.InstanceName(params.Viper), slog.LevelError)
cfg, err := config.WAFFromEnvironment(params.Viper)
if err != nil {
@@ -66,11 +62,11 @@ func RunWAF(ctx context.Context, params Params) int {
stdout = io.MultiWriter(params.Stdout, remote)
}
processLog = processlog.New(processlog.Params{
Output: stdout,
InstanceName: cfg.InstanceName,
Level: cfg.LogLevel,
})
processLog = requestlog.NewProcessLogger(
stdout,
cfg.InstanceName,
cfg.LogLevel,
)
if remote != nil {
stopSending := startSending(ctx, remote, processLog)
@@ -121,8 +117,7 @@ func RunWAF(ctx context.Context, params Params) int {
"settings", cfg)
// The last write follows the stop of every hook, and so the end of every
// loop that changes the state or raises an alert, and of every ask of
// GeoJS, the DNSBL zones, AbuseIPDB and routewatch. A hook of the state
// loop that changes the state or raises an alert. A hook of the state
// files would stop before the loops of the parts built ahead of them.
return run(ctx, app, failed, processLog, func() error {
err := files.WriteAll()
@@ -180,35 +175,11 @@ func newRuleFiles(
return ruleFiles, nil
}
// newCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
// coreruleset.New refuses is an error in every mode. While the mode is off, a
// set file is loaded with the Core Rule Set only to check it.
// newCoreRuleSet returns the Core Rule Set, as waf.NewCoreRuleSet does.
func newCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
cfg.CoreRuleSetExclusionsFile == "" {
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
}
crs, err := coreruleset.New(coreruleset.Params{
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
DisabledRules: cfg.CoreRuleSetDisabledRules,
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
Apps: cfg.Apps,
})
crs, err := waf.NewCoreRuleSet(cfg)
if err != nil {
return nil, startError{
"cannot use the Core Rule Set",
fmt.Errorf("SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w", err),
}
}
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
return nil, startError{"cannot use the Core Rule Set", err}
}
return crs, nil
@@ -221,7 +192,7 @@ func newLookupFile(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
alertQueue *alerts.Queue,
) (*lookup.File, error) {
if cfg.LookupSource != config.LookupSourceFile {
if cfg.LookupSource != "file" {
return nil, nil //nolint:nilnil // no lookup database, and no error
}
@@ -242,17 +213,14 @@ func newLookupFile(
// newLimiter returns the table of clients, with the rate and byte limits.
func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
return ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{
PerMinute: cfg.RateLimitPerMinute,
PerHour: cfg.RateLimitPerHour,
PerDay: cfg.RateLimitPerDay,
BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerHour: cfg.BytesLimitPerHour,
BytesPerDay: cfg.BytesLimitPerDay,
},
MaxClients: cfg.MaxTrackedClients,
})
return ratelimit.New(ratelimit.Limits{
PerMinute: cfg.RateLimitPerMinute,
PerHour: cfg.RateLimitPerHour,
PerDay: cfg.RateLimitPerDay,
BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerHour: cfg.BytesLimitPerHour,
BytesPerDay: cfg.BytesLimitPerDay,
}, cfg.MaxTrackedClients)
}
// newLedger returns the ledger of the bans, with the settings for them.
@@ -304,13 +272,11 @@ func newLists(
return lists
}
// newDNSBL returns what asks the DNSBL zones, none asked yet. At the stop it
// ends the queries under way, as reputation.DNSBL.StopAsking says.
// newDNSBL returns what asks the DNSBL zones, none asked yet.
func newDNSBL(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
alertQueue *alerts.Queue,
cfg *config.WAF, processLog *slog.Logger, alertQueue *alerts.Queue,
) *reputation.DNSBL {
dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{
return reputation.NewDNSBL(reputation.DNSBLParams{
Zones: cfg.DNSBLZones,
Resolver: cfg.DNSBLResolver,
CacheTTL: cfg.IPReputationCacheTTL,
@@ -319,17 +285,12 @@ func newDNSBL(
ProcessLog: processLog,
Alerts: alertQueue,
})
lc.Append(fx.StopHook(dnsbl.StopAsking))
return dnsbl
}
// newAbuseIPDB returns what checks clients with AbuseIPDB while
// SWW_WAF_ABUSEIPDB_KEY is set, none checked yet. Until waf stops, it
// fetches AbuseIPDB's blacklist as it is due, and bans the clients on it in
// ledger, as waf.BanAbuseIPDBBlacklist says. At the stop it ends the checks
// under way, as reputation.AbuseIPDB.StopChecking says.
// ledger, as waf.BanAbuseIPDBBlacklist says.
func newAbuseIPDB(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
alertQueue *alerts.Queue, ledger *bans.Ledger,
@@ -355,30 +316,24 @@ func newAbuseIPDB(
})
lc.Append(background(abuseIPDB.RunBlacklist))
lc.Append(fx.StopHook(abuseIPDB.StopChecking))
return abuseIPDB
}
// newRouteWatch returns what asks routewatch about clients, or nil while
// SWW_WAF_ROUTEWATCH_URL is unset. At the stop it ends the queries under
// way, as routewatch.Client.StopAsking says.
// SWW_WAF_ROUTEWATCH_URL is unset.
func newRouteWatch(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
cfg *config.WAF, processLog *slog.Logger,
) *routewatch.Client {
if cfg.RouteWatchURL == "" {
return nil
}
routeWatch := routewatch.New(routewatch.Params{
return routewatch.New(routewatch.Params{
URL: cfg.RouteWatchURL,
Now: utcNow,
ProcessLog: processLog,
})
lc.Append(fx.StopHook(routeWatch.StopAsking))
return routeWatch
}
// newMetrics returns the metrics, with those read from the other parts as
@@ -416,18 +371,12 @@ func newMetrics(
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
// answer goes to the limiter and the ledger, as waf.AddLookup says, and each
// failure is handled as waf.GeoJSFailed says. At the stop it ends the request
// under way, as geojs.GeoJS.StopAsking says.
// failure is handled as waf.GeoJSFailed says.
func newGeoJS(
lc fx.Lifecycle,
cfg *config.WAF,
m *metrics.Metrics,
limiter *ratelimit.Limiter,
ledger *bans.Ledger,
alertQueue *alerts.Queue,
processLog *slog.Logger,
cfg *config.WAF, m *metrics.Metrics, limiter *ratelimit.Limiter,
ledger *bans.Ledger, alertQueue *alerts.Queue, processLog *slog.Logger,
) *geojs.GeoJS {
geoJS := geojs.New(geojs.Params{
return geojs.New(geojs.Params{
URL: geojs.URL,
Answered: func(answer geojs.Answer) {
waf.AddLookup(cfg, limiter, ledger, answer)
@@ -439,10 +388,6 @@ func newGeoJS(
},
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
})
lc.Append(fx.StopHook(geoJS.StopAsking))
return geoJS
}
// newWAFServer returns the server waf runs, with its parts and the time
+1 -1
View File
@@ -54,7 +54,7 @@ func (abuseIPDB) check(
"scored by AbuseIPDB at or over SWW_WAF_ABUSEIPDB_MIN_SCORE",
)
if rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
if rq.h.config.DNSBLAbuseIPDBAction != deny {
return nil
}
+90 -145
View File
@@ -2,7 +2,6 @@ package waf
import (
"bytes"
"crypto/subtle"
"encoding/json"
"errors"
"fmt"
@@ -15,7 +14,6 @@ import (
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/state"
)
@@ -41,80 +39,23 @@ var (
errNotAddress = errors.New("is not an address, such as 203.0.113.9")
)
// answerHealth answers a request for the health endpoint.
func (rq *request) answerHealth() {
rq.line.Action = requestlog.ActionOwnEndpoint
// Set here rather than left to Go's server, which would set it only
// after the log line has taken the response's headers.
rq.out.Header().Set("Content-Type", "text/plain; charset=utf-8")
_, _ = io.WriteString(rq.out, "ok\n")
}
// answerAdmin answers a request for one of waf's own endpoints. Each
// needs a token, sent as Authorization: Bearer <token>: the metrics
// SWW_WAF_METRICS_TOKEN, the others SWW_WAF_ADMIN_TOKEN. A request without it is
// refused with 401. An endpoint whose token is unset answers 404, as any other
// path does.
func (rq *request) answerAdmin() {
rq.line.Action = requestlog.ActionOwnEndpoint
token, answer := rq.endpoint()
switch {
case token == "":
http.Error(
rq.out,
http.StatusText(http.StatusNotFound),
http.StatusNotFound,
)
case !hasToken(rq.in, token):
rq.out.Header().Set("WWW-Authenticate", "Bearer")
http.Error(
rq.out,
http.StatusText(http.StatusUnauthorized),
http.StatusUnauthorized,
)
default:
answer()
// handleHealth answers GET HealthPath.
func handleHealth() http.HandlerFunc {
return func(w http.ResponseWriter, _ *http.Request) {
// Set here rather than left to Go's server, which would set it only
// after the log line has taken the response's headers.
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
_, _ = io.WriteString(w, "ok\n")
}
}
// endpoint returns the token the request's endpoint needs, and what
// answers the request there; "" when there is no such endpoint.
func (rq *request) endpoint() (string, func()) {
cfg := rq.h.config
method, path := rq.in.Method, rq.in.URL.Path
switch {
case method == http.MethodGet && path == MetricsPath:
return cfg.MetricsToken, func() { rq.h.metrics.ServeHTTP(rq.out, rq.in) }
case method == http.MethodGet && path == BansPath:
return cfg.AdminToken, rq.listBans
case method == http.MethodPost && path == BansPath:
return cfg.AdminToken, rq.addBan
case method == http.MethodDelete && strings.HasPrefix(path, BansPath+"/"):
return cfg.AdminToken, rq.liftBans
case method == http.MethodGet && strings.HasPrefix(path, ClientsPath):
return cfg.AdminToken, rq.showClient
default:
return "", nil
// handleListBans answers GET BansPath with every ban held.
func (h *handler) handleListBans() http.HandlerFunc {
return func(w http.ResponseWriter, _ *http.Request) {
h.answerBans(w, h.ledger.Snapshot())
}
}
// hasToken reports whether r carries token, as Authorization: Bearer
// <token>.
func hasToken(r *http.Request, token string) bool {
scheme, sent, _ := strings.Cut(r.Header.Get("Authorization"), " ")
return strings.EqualFold(scheme, "Bearer") &&
subtle.ConstantTimeCompare([]byte(sent), []byte(token)) == 1
}
// listBans answers GET BansPath with every ban held.
func (rq *request) listBans() {
rq.answerBans(rq.h.ledger.Snapshot())
}
// banToAdd is the body of POST BansPath.
type banToAdd struct {
// Netblock is a netblock, or a client's address, which stands for the
@@ -126,44 +67,44 @@ type banToAdd struct {
Reason string `json:"reason"`
}
// addBan answers POST BansPath: it bans the netblock the body names, as
// an admin, from now for the duration the body gives, with its reason,
// handleAddBan answers POST BansPath: it bans the netblock the body names,
// as an admin, from now for the duration the body gives, with its reason,
// and answers with that ban.
func (rq *request) addBan() {
toAdd, err := rq.readBanToAdd()
func (h *handler) handleAddBan() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
toAdd, err := readBanToAdd(r)
var (
netblock netip.Prefix
expires time.Time
now = rq.h.now()
)
var (
netblock netip.Prefix
expires time.Time
now = h.now()
)
if err == nil {
netblock, err = rq.h.parseBanNetblock(toAdd.Netblock)
if err == nil {
netblock, err = h.parseBanNetblock(toAdd.Netblock)
}
if err == nil {
expires, err = expiry(toAdd.Duration, now)
}
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
ban := h.ledger.BanForAdmin(netblock, now, expires, toAdd.Reason)
h.answerBans(w, []bans.Ban{ban})
}
if err == nil {
expires, err = expiry(toAdd.Duration, now)
}
if err != nil {
http.Error(rq.out, err.Error(), http.StatusBadRequest)
return
}
ban := rq.h.ledger.BanForAdmin(netblock, now, expires, toAdd.Reason)
rq.answerBans([]bans.Ban{ban})
}
// readBanToAdd reads the body of POST BansPath: a JSON object with
// nothing but whitespace after it, in at most banBodyMaxBytes.
func (rq *request) readBanToAdd() (banToAdd, error) {
func readBanToAdd(r *http.Request) (banToAdd, error) {
data, err := io.ReadAll(
http.MaxBytesReader(nil, rq.in.Body, banBodyMaxBytes),
http.MaxBytesReader(nil, r.Body, banBodyMaxBytes),
)
rq.requestBytes = int64(len(data))
if err != nil {
return banToAdd{}, fmt.Errorf("%w: %w", errNotBanToAdd, err)
}
@@ -246,29 +187,31 @@ func expiry(duration string, now time.Time) (time.Time, error) {
return now.Add(length), nil
}
// liftBans answers DELETE BansPath/<client>: it lifts every ban active on
// a netblock the client's address is in, and answers with those bans, or
// with 404 when none is active.
func (rq *request) liftBans() {
client, err := pathAddress(rq.in.URL.Path, BansPath+"/")
if err != nil {
http.Error(rq.out, err.Error(), http.StatusBadRequest)
// handleLiftBans answers DELETE BansPath/<client>: it lifts every ban
// active on a netblock the client's address is in, and answers with those
// bans, or with 404 when none is active.
func (h *handler) handleLiftBans() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
client, err := pathAddress(r.URL.Path, BansPath+"/")
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
return
}
lifted := h.ledger.Lift(client, h.now())
if len(lifted) == 0 {
http.Error(
w,
"no ban is active on "+client.String(),
http.StatusNotFound,
)
return
}
h.answerBans(w, lifted)
}
lifted := rq.h.ledger.Lift(client, rq.h.now())
if len(lifted) == 0 {
http.Error(
rq.out,
"no ban is active on "+client.String(),
http.StatusNotFound,
)
return
}
rq.answerBans(lifted)
}
// clientAnswer is the answer to GET ClientsPath<ip>: the client the
@@ -280,27 +223,29 @@ type clientAnswer struct {
Bans []state.BanEntry `json:"bans"`
}
// showClient answers GET ClientsPath<ip> with what waf knows of the client:
// its counters, its history, which holds its country as last looked up and its
// offences, and its bans with their notes.
func (rq *request) showClient() {
addr, err := pathAddress(rq.in.URL.Path, ClientsPath)
if err != nil {
http.Error(rq.out, err.Error(), http.StatusBadRequest)
// handleShowClient answers GET ClientsPath<ip> with what waf knows of the
// client: its counters, its history, which holds its country as last looked
// up and its offences, and its bans with their notes.
func (h *handler) handleShowClient() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
addr, err := pathAddress(r.URL.Path, ClientsPath)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
return
}
answer := clientAnswer{
Bans: state.BanEntries(h.ledger.Covering(addr)),
}
client, seen := h.limiter.Client(clientGroup(h.config, addr))
if seen {
answer.Client = &client
}
h.answerJSON(w, answer)
}
answer := clientAnswer{
Bans: state.BanEntries(rq.h.ledger.Covering(addr)),
}
client, seen := rq.h.limiter.Client(clientGroup(rq.h.config, addr))
if seen {
answer.Client = &client
}
rq.answerJSON(answer)
}
// pathAddress reads the client's address that follows prefix in path.
@@ -316,27 +261,27 @@ func pathAddress(path, prefix string) (netip.Addr, error) {
}
// answerBans answers with held under bans, as bans.json lists them.
func (rq *request) answerBans(held []bans.Ban) {
rq.answerJSON(struct {
func (h *handler) answerBans(w http.ResponseWriter, held []bans.Ban) {
h.answerJSON(w, struct {
Bans []state.BanEntry `json:"bans"`
}{state.BanEntries(held)})
}
// answerJSON answers with value as indented JSON.
func (rq *request) answerJSON(value any) {
func (h *handler) answerJSON(w http.ResponseWriter, value any) {
body, err := json.MarshalIndent(value, "", " ")
if err != nil {
rq.h.processLog.Error(
h.processLog.Error(
"encoding an answer failed",
"error",
err.Error(),
)
http.Error(rq.out, http.StatusText(http.StatusInternalServerError),
http.Error(w, http.StatusText(http.StatusInternalServerError),
http.StatusInternalServerError)
return
}
rq.out.Header().Set("Content-Type", "application/json")
_, _ = rq.out.Write(append(body, '\n'))
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write(append(body, '\n'))
}
+33
View File
@@ -333,6 +333,39 @@ func TestBanToAddThatCannotBeReadIsRefused(t *testing.T) {
}
}
func TestBanToAddIsLoggedWithTheBytesOfItsBody(t *testing.T) {
t.Parallel()
s, _, _ := startWithClock(
t,
"",
map[string]string{adminToken: adminSecret},
)
s.admin(http.MethodPost, waf.BansPath, banOtherClient, http.StatusOK)
line := s.out.endedLines(t, s.sent)[s.sent-1]
if line.RequestBytes != int64(len(banOtherClient)) {
t.Errorf(
"the ban to add was logged with request_bytes %d, want %d",
line.RequestBytes,
len(banOtherClient),
)
}
// The health endpoint reads none of a body sent to it.
s.ownEndpointRequest("", http.MethodGet, waf.HealthPath, "ok",
http.StatusOK)
line = s.out.endedLines(t, s.sent)[s.sent-1]
if line.RequestBytes != 0 {
t.Errorf(
"the health request was logged with request_bytes %d, want 0",
line.RequestBytes,
)
}
}
func TestClientEndpointShowsTheClientAndItsBans(t *testing.T) {
t.Parallel()
+4 -5
View File
@@ -86,14 +86,13 @@ func (rq *request) countedBytes() int64 {
response, request := rq.out.bytes, rq.requestBytes
switch rq.h.config.BytesCounted {
case config.BytesCountedResponse:
case "response":
return response
case config.BytesCountedRequest:
case "request":
return request
case config.BytesCountedBoth:
default: // both
return response + request
}
return response + request
}
// banForLimit bans the client's netblock at now for a broken limit, the
+2 -3
View File
@@ -63,8 +63,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
}
blocklisted := percentage{percent: whole}
if rq.blocklisted &&
cfg.BlocklistAction == config.ReputationActionLimit {
if rq.blocklisted && cfg.BlocklistAction == "limit" {
blocklisted = percentage{
cfg.BlocklistLimitPercent,
"SWW_WAF_BLOCKLIST_ACTION",
@@ -73,7 +72,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
dnsblAbuseIPDBListed := percentage{percent: whole}
if (rq.dnsblListed || rq.abuseIPDBHit) &&
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit {
cfg.DNSBLAbuseIPDBAction == "limit" {
dnsblAbuseIPDBListed = percentage{
cfg.DNSBLAbuseIPDBLimitPercent,
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
+1 -3
View File
@@ -7,7 +7,6 @@ import (
"context"
"time"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
@@ -42,6 +41,5 @@ func (blocklists) check(
// client, given listedBy, the blocklists that list it: it does when one lists
// it and SWW_WAF_BLOCKLIST_ACTION is deny.
func (rq *request) refusedByBlocklists(listedBy []string) bool {
return len(listedBy) > 0 && rq.h.config.BlocklistAction ==
config.ReputationActionDeny
return len(listedBy) > 0 && rq.h.config.BlocklistAction == deny
}
+2 -2
View File
@@ -47,7 +47,7 @@ func (coreRuleSet) check(
rq.line.CoreRuleSetScore = &result.Score
for _, id := range result.RuleIDs {
rq.h.metrics.CoreRuleSetMatched(string(cfg.CoreRuleSetMode), id)
rq.h.metrics.CoreRuleSetMatched(cfg.CoreRuleSetMode, id)
}
threshold := cfg.CoreRuleSetScoreThreshold
@@ -99,7 +99,7 @@ func (rq *request) alertCoreRuleSetMatch(result coreruleset.Result) {
switch {
case rq.h.config.CoreRuleSetMode == config.CoreRuleSetModeDetect:
detail["mode"] = string(config.CoreRuleSetModeDetect)
detail["mode"] = config.CoreRuleSetModeDetect
case rq.h.config.Observe:
detail["mode"] = "observe"
}
+1 -2
View File
@@ -32,8 +32,7 @@ func (dnsbl) check(ctx context.Context, rq *request, _ time.Time) *verdict {
rq.addTag(config.DNSBLTagPrefix + strings.ToLower(zone))
}
if !rq.dnsblListed ||
rq.h.config.DNSBLAbuseIPDBAction != config.ReputationActionDeny {
if !rq.dnsblListed || rq.h.config.DNSBLAbuseIPDBAction != deny {
return nil
}
+11
View File
@@ -53,3 +53,14 @@ func TestHealthEndpointCountsTowardNoLimit(t *testing.T) {
}
}
}
func TestHealthEndpointIsFoundAtItsPathPercentEncoded(t *testing.T) {
t.Parallel()
// %68 is h. An endpoint is found by the decoded path.
s, _, _ := startWithClock(t, "", nil)
got := s.ownEndpointRequest("", http.MethodGet, "/_sww/%68ealthz", "",
http.StatusOK)
wantBody(t, got, "ok\n")
}
+2 -3
View File
@@ -49,8 +49,7 @@ func (rq *request) startLookUp(
) <-chan lookupResult {
result := make(chan lookupResult, 1)
if rq.h.config.LookupSource == config.LookupSourceOff ||
!canBePlaced(rq.client) {
if rq.h.config.LookupSource == "off" || !canBePlaced(rq.client) {
result <- lookupResult{}
return result
@@ -58,7 +57,7 @@ func (rq *request) startLookUp(
client := clientGroup(rq.h.config, rq.client)
if rq.h.config.LookupSource == config.LookupSourceFile {
if rq.h.config.LookupSource == "file" {
result <- lookupResult{
lookedUp: true, answer: rq.h.lookupFile.LookUp(client),
}
+44 -3
View File
@@ -6,6 +6,7 @@ package waf
import (
"cmp"
"fmt"
"io"
"log/slog"
"net/http"
@@ -82,8 +83,8 @@ type Params struct {
// LookupFile is the lookup database they are looked up in while
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
LookupFile *lookup.File
// CoreRuleSet is what inspects requests while
// SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
// CoreRuleSet is what NewCoreRuleSet returns: what inspects requests
// while SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
CoreRuleSet CoreRuleSetInspector
// Now tells the time by which requests are counted for the rate
// limits, bans are made and run out, and GeoJS's answers are kept,
@@ -140,7 +141,7 @@ func newHandler(params Params) *handler {
panic(err) // New fails only for a size below one
}
return &handler{
h := &handler{
config: params.Config,
requestLog: params.RequestLog,
processLog: params.ProcessLog,
@@ -171,6 +172,43 @@ func newHandler(params Params) *handler {
besideLookupTimeLimit,
),
}
h.ownEndpoints = h.routes()
return h
}
// NewCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
// coreruleset.New refuses is an error in every mode. While the mode is off, a
// set file is loaded with the Core Rule Set only to check it.
func NewCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
cfg.CoreRuleSetExclusionsFile == "" {
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
}
coreRuleSet, err := coreruleset.New(coreruleset.Params{
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
DisabledRules: cfg.CoreRuleSetDisabledRules,
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
Apps: cfg.Apps,
})
if err != nil {
return nil, fmt.Errorf(
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w",
err,
)
}
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
}
return coreRuleSet, nil
}
// handler is waf's handler. It holds what every request shares; what
@@ -202,4 +240,7 @@ type handler struct {
// awaitingReport holds the decision on each answered question by its id
// until its report comes, and nil for a question still being handled.
awaitingReport *lru.Cache[string, *decision]
// ownEndpoints answers every request that is not a question, as routes
// lists the endpoints.
ownEndpoints http.Handler
}
+31 -56
View File
@@ -20,11 +20,9 @@ import (
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
@@ -399,11 +397,11 @@ func startRProxy(
server := rproxy.New(rproxy.Params{
Config: cfg,
RequestLog: io.Discard,
ProcessLog: processlog.New(processlog.Params{
Output: io.Discard,
InstanceName: "rproxy",
Level: slog.LevelInfo,
}),
ProcessLog: requestlog.NewProcessLogger(
io.Discard,
"rproxy",
slog.LevelInfo,
),
})
listener, err := (&net.ListenConfig{}).Listen(
@@ -472,11 +470,11 @@ func newWAFWith(
}
out := &output{}
processLog := processlog.New(processlog.Params{
Output: out,
InstanceName: cfg.InstanceName,
Level: cfg.LogLevel,
})
processLog := requestlog.NewProcessLogger(
out,
cfg.InstanceName,
cfg.LogLevel,
)
ruleFiles, err := rules.Load(rules.Params{
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
@@ -506,20 +504,24 @@ func newWAFWith(
}
}
coreRuleSet, err := waf.NewCoreRuleSet(cfg)
if err != nil {
t.Fatalf("Core Rule Set: %v", err)
}
params := waf.Params{
Config: cfg,
RequestLog: out,
ProcessLog: processLog,
LookupFile: lookupFile,
Now: now,
Rules: ruleFiles,
Alerts: alertQueue,
Config: cfg,
RequestLog: out,
ProcessLog: processLog,
LookupFile: lookupFile,
CoreRuleSet: coreRuleSet,
Now: now,
Rules: ruleFiles,
Alerts: alertQueue,
// The tests of the limit set it back, and hold the Core Rule Set
// with a stand-in.
BesideLookupTimeLimit: time.Hour,
}
addCoreRuleSet(t, &params)
addCounters(&params)
addOutsideSources(&params, geojsURL, abuseIPDBCheckURL)
@@ -528,30 +530,6 @@ func newWAFWith(
return server, out, alertQueue
}
// addCoreRuleSet adds to params the Core Rule Set, with the settings for it,
// while SWW_WAF_CORE_RULE_SET_MODE is not off.
func addCoreRuleSet(t *testing.T, params *waf.Params) {
t.Helper()
cfg := params.Config
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
return
}
coreRuleSet, err := coreruleset.New(coreruleset.Params{
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
DisabledRules: cfg.CoreRuleSetDisabledRules,
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
Apps: cfg.Apps,
})
if err != nil {
t.Fatalf("Core Rule Set: %v", err)
}
params.CoreRuleSet = coreRuleSet
}
// addCounters adds to params the metrics, the table of clients, the ledger
// and the anomaly counters, as sww builds them from params' settings. The
// metrics read the ledger, the table of clients and the rules.
@@ -562,17 +540,14 @@ func addCounters(params *waf.Params) {
cfg.MetricsBusiestCountriesAndASNs,
cfg.InstanceName,
)
params.Limiter = ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{
PerMinute: cfg.RateLimitPerMinute,
PerHour: cfg.RateLimitPerHour,
PerDay: cfg.RateLimitPerDay,
BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerHour: cfg.BytesLimitPerHour,
BytesPerDay: cfg.BytesLimitPerDay,
},
MaxClients: cfg.MaxTrackedClients,
})
params.Limiter = ratelimit.New(ratelimit.Limits{
PerMinute: cfg.RateLimitPerMinute,
PerHour: cfg.RateLimitPerHour,
PerDay: cfg.RateLimitPerDay,
BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerHour: cfg.BytesLimitPerHour,
BytesPerDay: cfg.BytesLimitPerDay,
}, cfg.MaxTrackedClients)
params.Ledger = bans.New(bans.Params{
LimitBanDuration: cfg.LimitBanDuration,
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
+4
View File
@@ -7,6 +7,10 @@ import (
"sneak.berlin/go/smallwebwaf/internal/bans"
)
// deny is the SWW_WAF_BLOCKLIST_ACTION and the SWW_WAF_DNSBL_ABUSEIPDB_ACTION that
// refuses the requests of a client a source lists.
const deny = "deny"
// noteListed notes each of sources, the URLs of the blocklists or the
// DNSBL zones, their keys masked, that list the client, as noteHit does,
// with reason.
+1 -1
View File
@@ -421,7 +421,7 @@ func (rq *request) answerAtTheEnd() (geojs.Answer, bool) {
return geojs.Answer{}, false
}
if rq.h.config.LookupSource == config.LookupSourceFile {
if rq.h.config.LookupSource == "file" {
return rq.lookupAnswer, true
}
+141
View File
@@ -0,0 +1,141 @@
package waf
import (
"crypto/subtle"
"io"
"net/http"
"strings"
"github.com/go-chi/chi/v5"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
// routes returns the router of waf's own endpoints. A question never reaches
// it, since ServeHTTP answers a question first. No check runs on a request
// here: waf's address is on the private network, where the peers are rproxy
// and the admin's tools, not visitors.
func (h *handler) routes() http.Handler {
r := chi.NewRouter()
r.Use(matchDecodedPath)
// The report writes no log line of its own: it finishes the line of the
// request it reports.
r.Post(ReportPath, h.handleReport())
r.Group(func(r chi.Router) {
r.Use(h.logOwnEndpoint)
// The answer to a path with no endpoint, and to a method the path has
// no endpoint for, is set in this group, so that it is logged too.
r.NotFound(answerNotFound)
r.MethodNotAllowed(answerNotFound)
r.Get(HealthPath, handleHealth())
r.Group(func(r chi.Router) {
r.Use(requireToken(h.config.MetricsToken))
r.Get(MetricsPath, h.metrics.ServeHTTP)
})
// The handlers under BansPath/ and ClientsPath read the address from
// the path themselves, so that one that is not an address gets 400
// after the token check.
r.Group(func(r chi.Router) {
r.Use(requireToken(h.config.AdminToken))
r.Get(BansPath, h.handleListBans())
r.Post(BansPath, h.handleAddBan())
r.Delete(BansPath+"/*", h.handleLiftBans())
r.Get(ClientsPath+"*", h.handleShowClient())
})
})
return r
}
// matchDecodedPath has the router match the decoded path, which the handlers
// read. chi matches the path as sent when it is encoded in an unusual way,
// which would make /_sww/%68ealthz a path with no endpoint.
func matchDecodedPath(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
chi.RouteContext(r.Context()).RoutePath = r.URL.Path
next.ServeHTTP(w, r)
})
}
// logOwnEndpoint writes one log line for each request, with the action
// own_endpoint. The line's request_bytes is how much of the body the handler
// read.
func (h *handler) logOwnEndpoint(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
peer := peerAddress(r)
rq := h.startRequest(w, r, peer, peer, false)
defer rq.finish()
rq.line.Action = requestlog.ActionOwnEndpoint
counted := r.Clone(r.Context())
counted.Body = &countedBody{
ReadCloser: r.Body,
read: &rq.requestBytes,
}
next.ServeHTTP(rq.out, counted)
})
}
// countedBody is a request's body that adds the bytes read of it to read.
type countedBody struct {
io.ReadCloser
read *int64
}
// Read reads part of the body.
func (b *countedBody) Read(p []byte) (int, error) {
n, err := b.ReadCloser.Read(p)
*b.read += int64(n)
return n, err
}
// requireToken returns middleware that lets a request through only when it
// carries token, as Authorization: Bearer <token>. It answers 401 to a request
// without it. While token is unset, it answers 404, as a path with no endpoint
// does.
func requireToken(token string) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
switch {
case token == "":
answerNotFound(w, r)
case !hasToken(r, token):
w.Header().Set("WWW-Authenticate", "Bearer")
http.Error(
w,
http.StatusText(http.StatusUnauthorized),
http.StatusUnauthorized,
)
default:
next.ServeHTTP(w, r)
}
},
)
}
}
// hasToken reports whether r carries token, as Authorization: Bearer
// <token>.
func hasToken(r *http.Request, token string) bool {
scheme, sent, _ := strings.Cut(r.Header.Get("Authorization"), " ")
return strings.EqualFold(scheme, "Bearer") &&
subtle.ConstantTimeCompare([]byte(sent), []byte(token)) == 1
}
// answerNotFound answers 404, for a path with no endpoint, and for a method
// the path has no endpoint for.
func answerNotFound(w http.ResponseWriter, _ *http.Request) {
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
}
+22 -43
View File
@@ -132,8 +132,8 @@ func New(params Params) *Server {
}
// ServeHTTP handles one request to waf: a request carrying
// ClientHeader is rproxy's question, and any other is for waf
// itself.
// ClientHeader is rproxy's question, and any other is for one of waf's
// own endpoints.
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if len(r.Header.Values(ClientHeader)) > 0 {
h.answerQuestion(w, r)
@@ -141,7 +141,7 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
h.answerOwnEndpoint(w, r)
h.ownEndpoints.ServeHTTP(w, r)
}
// refusedWhateverTheAnswer reports whether the request is one that does not wait
@@ -358,50 +358,29 @@ func (rq *request) deny(r refusal) {
http.Error(rq.out, http.StatusText(r.status), r.status)
}
// answerOwnEndpoint answers a request to waf's own address. No check
// runs on it: the address is on the private network, where the peers are
// rproxy and the admin's tools, not visitors.
func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPost && r.URL.Path == ReportPath {
h.takeReport(w, r)
return
}
peer := peerAddress(r)
rq := h.startRequest(w, r, peer, peer, false)
defer rq.finish()
if r.Method == http.MethodGet && r.URL.Path == HealthPath {
rq.answerHealth()
return
}
rq.answerAdmin()
}
// takeReport answers POST ReportPath: it finishes the request the report
// handleReport answers POST ReportPath: it finishes the request the report
// names, if it holds that request's decision, and ignores the report
// otherwise.
func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) {
report, err := readReport(w, r)
if err != nil {
h.processLog.Warn("a report is refused", "error", err.Error())
http.Error(w, "the body is not a report", http.StatusBadRequest)
func (h *handler) handleReport() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
report, err := readReport(w, r)
if err != nil {
h.processLog.Warn("a report is refused", "error", err.Error())
http.Error(w, "the body is not a report", http.StatusBadRequest)
return
return
}
held, found := h.awaitingReport.Peek(report.RequestID)
// Remove reports true to one report only, of several that name the
// id. A question still being handled is held as nil.
if found && held != nil &&
h.awaitingReport.Remove(report.RequestID) {
h.finishWith(r.Context(), held, report)
}
w.WriteHeader(http.StatusNoContent)
}
held, found := h.awaitingReport.Peek(report.RequestID)
// Remove reports true to one report only, of several that name the id. A
// question still being handled is held as nil.
if found && held != nil && h.awaitingReport.Remove(report.RequestID) {
h.finishWith(r.Context(), held, report)
}
w.WriteHeader(http.StatusNoContent)
}
// readReport reads a report from r's body, which must be one JSON object with