Compare commits

...
Author SHA1 Message Date
sneak 643773546b waf: Server exports no parts (closes #436)
check / check (push) Failing after 3m41s
waf.Server held its parts as public fields only for the tests to read.
The tests now keep the Params the server was built from in a test-only
wafAndParts, which embeds both, so every assertion reads as before.
newWAFWith takes a change to the Params instead of a build function, so
the Params kept are the ones the server was built from.

Model: opus-5-5
2026-11-19 23:29:58 +00:00
clawbot abf6c8d517 rproxy: the two time limits get a type and a file of their own (closes #432)
check / check (push) Failing after 3m36s
timeLimits in internal/rproxy/timelimits.go owns the mutex, the two
timers and the end of the request; request holds one field of it. It
is given refuse, whether the visitor's body is being waited on, and a
function that ends that read. Tests of the type alone run in a
synctest bubble.

Model: opus-5-5
2026-11-20 00:28:55 +01:00
clawbot 17400f68b8 waf: heuristics pass facts through typed fields of decision (closes #442)
check / check (push) Failing after 1m35s
The tags, what observe mode would have done, the broken limit and the
three reputation flags are fields of decision, each naming what sets it.
The log line is written from them and from the lookup answer, and no
heuristic, ban note or alert reads it back. A ban's notes take the
request's method, Host and User-Agent from decision, since the report's
steps have no request to read them from.

Model: opus-5-5
2026-11-19 23:43:00 +01:00
clawbot 9d008ed31a waf: a heuristic says itself that it refuses whatever the lookup answers (closes #446)
check / check (push) Failing after 1m37s
refusedWhateverTheAnswer asks each heuristic of afterLookup that
implements the new refusesWithoutLookup interface, instead of naming
blocklists and rate_limits. The comment on adding a heuristic names the
places outside its file that may change with it.

Model: opus-5-5
2026-11-19 23:28:52 +01:00
clawbot 86bb33bf07 waf: list its own endpoints on a chi router in routes.go (closes #426)
check / check (push) Successful in 7m7s
routes.go lists every own endpoint. One middleware checks a token and
another writes the log line. answerOwnEndpoint, answerAdmin and
endpoint() are gone; the handlers return http.HandlerFunc. The router
matches the decoded path, as the old code did. A test checks the
request_bytes of an own endpoint's log line.

Model: opus-5-5
2026-11-19 23:13:24 +01:00
40 changed files with 904 additions and 543 deletions

No files matched your search

+4 -3
View File
@@ -74,9 +74,9 @@ never as a time.
`go.uber.org/fx`. fx builds the parts of `waf` and `rproxy`, and starts
and stops their background work.
- `internal/config`: reads the settings, the one place they are read.
- `internal/waf`: `waf`'s server. It runs the checks in `check`, and answers
`waf`'s own endpoints in `answerOwnEndpoint`. It counts bytes, refusals
and anomalies, and writes the log lines.
- `internal/waf`: `waf`'s server. It runs the checks in `check`. Its
`routes.go` lists `waf`'s own endpoints. It counts bytes, refusals and
anomalies, and writes the log lines.
- `internal/rproxy`: the reverse proxy in front of one app that asks `waf`
about each request. It passes a request to the app with the standard
library's `httputil.ReverseProxy`, within the time and size limits.
@@ -128,6 +128,7 @@ never as a time.
default `SWW_WAF_CORE_RULE_SET_DISABLED_RULES`, are built for that
version. They are checked again before the version changes, since rule
ids, and what each rule matches, change between versions;
- `github.com/go-chi/chi/v5` to route the requests to `waf`'s own endpoints;
- `github.com/hashicorp/golang-lru/v2` for the bounded tables;
- `github.com/oschwald/maxminddb-golang/v2` to read the lookup database;
- `github.com/fsnotify/fsnotify` to notice edited files;
+1
View File
@@ -7,6 +7,7 @@ require (
github.com/corazawaf/coraza-coreruleset/v4 v4.25.0
github.com/corazawaf/coraza/v3 v3.8.2-0.20261022102553-d028be399c68
github.com/fsnotify/fsnotify v1.10.1
github.com/go-chi/chi/v5 v5.5.0
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/maxmind/mmdbwriter v1.2.0
github.com/oschwald/maxminddb-golang/v2 v2.7.0
+2
View File
@@ -21,6 +21,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
github.com/go-chi/chi/v5 v5.5.0 h1:xdKQ1dJmVJsmgahrXSKhUx8NyVPzpwdlyrrQFpMPJgg=
github.com/go-chi/chi/v5 v5.5.0/go.mod h1:d1IG7GtKPSjPyBP365PH8sAGVHk3qfB/IhjFNRH1wWM=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
+30 -162
View File
@@ -12,7 +12,6 @@ import (
"net/url"
"os"
"strconv"
"sync"
"sync/atomic"
"time"
@@ -82,17 +81,9 @@ type request struct {
// upgraded is the connection to the app once the app has switched protocols,
// as for a WebSocket, and nil otherwise.
upgraded *upgradedConn
// mu guards what follows. The time limits run on goroutines of their own,
// and the transport starts and stops them, and notes requestEnd, from its
// own; once timersStopped is set, neither time limit acts any more.
mu sync.Mutex
timersStopped bool
requestTimer *time.Timer
responseTimer *time.Timer
// requestEnd is when the app had been sent the whole request, or the answer
// started, if that was sooner.
requestEnd time.Time
// limits are the request's SWW_RPROXY_MAX_REQUEST_TIME_MS and
// SWW_RPROXY_MAX_RESPONSE_TIME_MS.
limits *timeLimits
}
// newRequest starts handling r: it notes the time, works out the visitor from
@@ -127,12 +118,25 @@ func (h *handler) newRequest(
}
}
rq.limits = &timeLimits{
maxRequestTime: h.config.MaxRequestTime,
maxResponseTime: h.config.MaxResponseTime,
start: rq.start,
refuse: rq.refuse,
waitingForVisitor: func() bool {
return rq.body != nil && rq.body.waiting.Load()
},
endVisitorRead: func() {
_ = rq.rc.SetReadDeadline(time.Now())
},
}
return rq
}
// answerHealth answers the health check.
func (rq *request) answerHealth() {
rq.prepareOwnAnswer(rq.requestDeadline())
rq.prepareOwnAnswer(rq.limits.requestDeadline())
rq.out.Header().Set("Content-Type", plainText)
rq.send(http.StatusOK, []byte("ok\n"))
}
@@ -148,7 +152,7 @@ func (rq *request) readFirstPartOfBody() bool {
return true
}
_ = rq.rc.SetReadDeadline(rq.requestDeadline())
_ = rq.rc.SetReadDeadline(rq.limits.requestDeadline())
held, err := io.ReadAll(
io.LimitReader(rq.body, wafprotocol.QuestionBodyMaxBytes),
)
@@ -188,7 +192,7 @@ func (rq *request) forward(ctx context.Context) {
rq.cancel = cancel
ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{
WroteRequest: rq.wroteRequest,
WroteRequest: rq.limits.wroteRequest,
})
out := rq.in.WithContext(ctx)
@@ -205,7 +209,7 @@ func (rq *request) forward(ctx context.Context) {
ErrorHandler: rq.answerError,
}
rq.startRequestTimer()
rq.limits.startRequestTimer()
rq.upstreamStart = time.Now()
reverseProxy.ServeHTTP(rq.out, out)
}
@@ -248,7 +252,7 @@ func (rq *request) modifyResponse(res *http.Response) error {
// switch, as when the app's Upgrade is not the visitor's, so the status
// is noted only once it has taken over the connection, by rq.out's
// Hijack.
rq.stopTimers()
rq.limits.stop()
conn, ok := res.Body.(io.ReadWriteCloser)
if ok {
@@ -259,7 +263,7 @@ func (rq *request) modifyResponse(res *http.Response) error {
return nil
}
requestEnd := rq.endRequest()
requestEnd := rq.limits.endRequest()
// An answer that arrives once SWW_RPROXY_MAX_RESPONSE_TIME_MS has run out is not
// passed on: the visitor is answered 504.
@@ -289,12 +293,12 @@ func (rq *request) modifyResponse(res *http.Response) error {
// An answer that arrives before the app has been sent the whole request ends
// the request, and SWW_RPROXY_MAX_RESPONSE_TIME_MS starts.
rq.startResponseTimer(requestEnd)
rq.limits.startResponseTimer(requestEnd)
res.Body = &responseBody{
body: limitBody(res.Body, maxBytes),
refuse: rq.refuse,
responseReceived: rq.responseReceived,
responseReceived: rq.limits.stop,
ctx: rq.in.Context(),
}
rq.out.afterStatus = rq.setResponseDeadline
@@ -347,7 +351,7 @@ func (rq *request) answer(status int) {
// A visitor found too slow is read no more; any other may go on sending until
// its time is up, so that the rest of the body is read and the request ends
// cleanly: by Go's server, or by readRestOfBody.
deadline := rq.requestDeadline()
deadline := rq.limits.requestDeadline()
if status == http.StatusRequestTimeout {
deadline = time.Now()
}
@@ -393,7 +397,7 @@ func (rq *request) closeAfterAnswer() {
// used that time up first, and at least minSendTime.
func (rq *request) prepareOwnAnswer(deadline time.Time) {
rq.stopReadingBody(deadline)
rq.endRequest()
rq.limits.endRequest()
limit := rq.h.config.MaxResponseTime
if limit > 0 {
@@ -414,7 +418,7 @@ func (rq *request) refuse(status int) {
// finish ends the request's time limits, writes its log line and, if waf
// answered the question, sends the report. ctx is the request's own context.
func (rq *request) finish(ctx context.Context) {
rq.stopTimers()
rq.limits.stop()
// After a cut-off answer, the reading of a body that has not arrived whole
// ends now. Go's server would otherwise keep the connection open while it
@@ -432,10 +436,10 @@ func (rq *request) finish(ctx context.Context) {
errors.Is(rq.rc.Flush(), os.ErrDeadlineExceeded):
rq.stopPassingOnBody(time.Now())
case rq.refused.Load() == 0:
rq.stopPassingOnBody(rq.requestDeadline())
rq.stopPassingOnBody(rq.limits.requestDeadline())
}
requestEnd := rq.endRequest()
requestEnd := rq.limits.endRequest()
end := time.Now()
requestBytes, responseBytes := rq.bytesEachWay()
@@ -509,17 +513,6 @@ func (rq *request) bytesEachWay() (int64, int64) {
return requestBytes, responseBytes
}
// requestDeadline is when the visitor must have sent its whole request, or zero
// when SWW_RPROXY_MAX_REQUEST_TIME_MS is off.
func (rq *request) requestDeadline() time.Time {
limit := rq.h.config.MaxRequestTime
if limit == 0 {
return time.Time{}
}
return rq.start.Add(limit)
}
// stopReadingBody ends, at deadline, the reading of a visitor's body that has not
// arrived whole: Go's server then reads no more of it, and closes the connection
// after the answer.
@@ -576,100 +569,6 @@ func (rq *request) stopPassingOnBody(deadline time.Time) {
rq.stopReadingBody(deadline)
}
// startRequestTimer starts SWW_RPROXY_MAX_REQUEST_TIME_MS, which runs from the
// request's headers being read until the app has been sent the whole request.
func (rq *request) startRequestTimer() {
deadline := rq.requestDeadline()
if deadline.IsZero() {
return
}
rq.mu.Lock()
defer rq.mu.Unlock()
rq.requestTimer = time.AfterFunc(
time.Until(deadline),
rq.requestTimedOut,
)
}
// requestTimedOut is called when SWW_RPROXY_MAX_REQUEST_TIME_MS runs out while the
// request is still on its way to the app. The answer names the side rproxy was
// waiting on at that moment: 408 when it was waiting for the visitor to send more
// of its body, 504 when it was waiting for the app to be reached or to take what
// it had.
func (rq *request) requestTimedOut() {
rq.mu.Lock()
defer rq.mu.Unlock()
if rq.timersStopped {
return
}
if rq.body == nil || !rq.body.waiting.Load() {
rq.refuse(http.StatusGatewayTimeout)
return
}
rq.refuse(http.StatusRequestTimeout)
// The transport gives up on the app only once its Read of the visitor's body
// returns, so that Read is ended now. The lock keeps this from reaching the
// connection after the request is handled.
_ = rq.rc.SetReadDeadline(time.Now())
}
// wroteRequest is called once the app has been sent the whole request:
// SWW_RPROXY_MAX_REQUEST_TIME_MS ends and SWW_RPROXY_MAX_RESPONSE_TIME_MS starts.
func (rq *request) wroteRequest(info httptrace.WroteRequestInfo) {
if info.Err != nil {
return // the transport gives up, or tries again
}
requestEnd := rq.endRequest()
rq.mu.Lock()
stopTimer(rq.requestTimer)
rq.mu.Unlock()
rq.startResponseTimer(requestEnd)
}
// startResponseTimer starts SWW_RPROXY_MAX_RESPONSE_TIME_MS, which runs from
// requestEnd, unless it has started already.
func (rq *request) startResponseTimer(requestEnd time.Time) {
limit := rq.h.config.MaxResponseTime
if limit == 0 {
return
}
rq.mu.Lock()
defer rq.mu.Unlock()
if rq.timersStopped || rq.responseTimer != nil {
return
}
rq.responseTimer = time.AfterFunc(time.Until(requestEnd.Add(limit)),
rq.responseTimedOut)
}
// responseTimedOut is called when SWW_RPROXY_MAX_RESPONSE_TIME_MS runs out before the
// app has sent its whole answer.
func (rq *request) responseTimedOut() {
rq.mu.Lock()
defer rq.mu.Unlock()
if !rq.timersStopped {
rq.refuse(http.StatusGatewayTimeout)
}
}
// responseReceived is called once the app has sent its whole answer.
func (rq *request) responseReceived() {
rq.stopTimers()
}
// setResponseDeadline sends the status and headers of the app's answer, with
// at least minSendTime, before ReverseProxy reads the answer's body, and then
// the rest has until SWW_RPROXY_MAX_RESPONSE_TIME_MS after the end of the request.
@@ -679,7 +578,7 @@ func (rq *request) setResponseDeadline() {
return
}
deadline := rq.endRequest().Add(limit)
deadline := rq.limits.endRequest().Add(limit)
_ = rq.rc.SetWriteDeadline(
time.Now().Add(max(time.Until(deadline), minSendTime)),
@@ -687,34 +586,3 @@ func (rq *request) setResponseDeadline() {
_ = rq.rc.Flush()
_ = rq.rc.SetWriteDeadline(deadline)
}
// endRequest notes now as the end of the request, unless an end is noted, and
// returns the end noted.
func (rq *request) endRequest() time.Time {
rq.mu.Lock()
defer rq.mu.Unlock()
if rq.requestEnd.IsZero() {
rq.requestEnd = time.Now()
}
return rq.requestEnd
}
// stopTimers stops the request's time limits and keeps any from starting later:
// the app's answer is complete, the connection upgraded, or the request handled.
func (rq *request) stopTimers() {
rq.mu.Lock()
defer rq.mu.Unlock()
rq.timersStopped = true
stopTimer(rq.requestTimer)
stopTimer(rq.responseTimer)
}
// stopTimer stops t, which is nil when its time limit is off.
func stopTimer(t *time.Timer) {
if t != nil {
t.Stop()
}
}
+165
View File
@@ -0,0 +1,165 @@
package rproxy
import (
"net/http"
"net/http/httptrace"
"sync"
"time"
)
// timeLimits are one request's two time limits: SWW_RPROXY_MAX_REQUEST_TIME_MS,
// which runs from the request's headers being read until the app has been sent
// the whole request, and SWW_RPROXY_MAX_RESPONSE_TIME_MS, which runs from then
// until the app has sent its whole answer. A limit of zero is off. Each limit
// runs on a goroutine of its own, and the transport starts and stops them from
// its own.
type timeLimits struct {
maxRequestTime time.Duration
maxResponseTime time.Duration
// start is when the request's headers had been read.
start time.Time
// refuse refuses the request with 408 or 504 when a limit runs out.
refuse func(status int)
// waitingForVisitor reports whether a read of the visitor's body is waiting
// for the visitor to send more.
waitingForVisitor func() bool
// endVisitorRead ends at once a read of the visitor's body that is waiting.
endVisitorRead func()
// mu guards what follows. Once stopped is set, neither limit acts any more.
mu sync.Mutex
stopped bool
requestTimer *time.Timer
responseTimer *time.Timer
// requestEnd is when the app had been sent the whole request, or the answer
// started, if that was sooner.
requestEnd time.Time
}
// requestDeadline is when the visitor must have sent its whole request, or zero
// when SWW_RPROXY_MAX_REQUEST_TIME_MS is off.
func (l *timeLimits) requestDeadline() time.Time {
if l.maxRequestTime == 0 {
return time.Time{}
}
return l.start.Add(l.maxRequestTime)
}
// startRequestTimer starts SWW_RPROXY_MAX_REQUEST_TIME_MS.
func (l *timeLimits) startRequestTimer() {
deadline := l.requestDeadline()
if deadline.IsZero() {
return
}
l.mu.Lock()
defer l.mu.Unlock()
l.requestTimer = time.AfterFunc(time.Until(deadline), l.requestTimedOut)
}
// requestTimedOut is called when SWW_RPROXY_MAX_REQUEST_TIME_MS runs out while
// the request is still on its way to the app. The answer names the side rproxy
// was waiting on at that moment: 408 when it was waiting for the visitor to send
// more of its body, 504 when it was waiting for the app to be reached or to take
// what it had.
func (l *timeLimits) requestTimedOut() {
l.mu.Lock()
defer l.mu.Unlock()
if l.stopped {
return
}
if !l.waitingForVisitor() {
l.refuse(http.StatusGatewayTimeout)
return
}
l.refuse(http.StatusRequestTimeout)
// The transport gives up on the app only once its Read of the visitor's body
// returns, so that Read is ended now. The lock keeps this from reaching the
// connection after the request is handled.
l.endVisitorRead()
}
// wroteRequest is called once the app has been sent the whole request:
// SWW_RPROXY_MAX_REQUEST_TIME_MS ends and SWW_RPROXY_MAX_RESPONSE_TIME_MS
// starts.
func (l *timeLimits) wroteRequest(info httptrace.WroteRequestInfo) {
if info.Err != nil {
return // the transport gives up, or tries again
}
requestEnd := l.endRequest()
l.mu.Lock()
stopTimer(l.requestTimer)
l.mu.Unlock()
l.startResponseTimer(requestEnd)
}
// startResponseTimer starts SWW_RPROXY_MAX_RESPONSE_TIME_MS, which runs from
// requestEnd, unless it has started already.
func (l *timeLimits) startResponseTimer(requestEnd time.Time) {
if l.maxResponseTime == 0 {
return
}
l.mu.Lock()
defer l.mu.Unlock()
if l.stopped || l.responseTimer != nil {
return
}
l.responseTimer = time.AfterFunc(
time.Until(requestEnd.Add(l.maxResponseTime)),
l.responseTimedOut,
)
}
// responseTimedOut is called when SWW_RPROXY_MAX_RESPONSE_TIME_MS runs out
// before the app has sent its whole answer.
func (l *timeLimits) responseTimedOut() {
l.mu.Lock()
defer l.mu.Unlock()
if !l.stopped {
l.refuse(http.StatusGatewayTimeout)
}
}
// endRequest notes now as the end of the request, unless an end is noted, and
// returns the end noted.
func (l *timeLimits) endRequest() time.Time {
l.mu.Lock()
defer l.mu.Unlock()
if l.requestEnd.IsZero() {
l.requestEnd = time.Now()
}
return l.requestEnd
}
// stop stops both limits and keeps either from starting later: the app's
// answer is complete, the connection upgraded, or the request handled.
func (l *timeLimits) stop() {
l.mu.Lock()
defer l.mu.Unlock()
l.stopped = true
stopTimer(l.requestTimer)
stopTimer(l.responseTimer)
}
// stopTimer stops t, which is nil when its limit is off.
func stopTimer(t *time.Timer) {
if t != nil {
t.Stop()
}
}
+171
View File
@@ -0,0 +1,171 @@
package rproxy
import (
"net/http"
"net/http/httptrace"
"sync/atomic"
"testing"
"testing/synctest"
"time"
)
// The tests below run timeLimits in a synctest bubble, where time is a clock of
// the test's own: time.Sleep moves it on at once, and synctest.Wait returns once
// a limit that ran out has refused the request.
const (
testRequestTime = 3 * time.Second
testResponseTime = 5 * time.Second
)
// limitsSeen is what a timeLimits under test did to its request.
type limitsSeen struct {
// refused is the status of the last refusal, or zero.
refused atomic.Int64
// readEnded is true once the read of the visitor's body was ended.
readEnded atomic.Bool
// waiting is what waitingForVisitor reports.
waiting atomic.Bool
}
// newTestLimits returns time limits that started now, with what they do noted
// in seen.
func newTestLimits(seen *limitsSeen) *timeLimits {
return &timeLimits{
maxRequestTime: testRequestTime,
maxResponseTime: testResponseTime,
start: time.Now(),
refuse: func(status int) {
seen.refused.Store(int64(status))
},
waitingForVisitor: seen.waiting.Load,
endVisitorRead: func() {
seen.readEnded.Store(true)
},
}
}
// sleepThenWait moves the bubble's clock on by d and waits for what is then
// due.
func sleepThenWait(d time.Duration) {
time.Sleep(d)
synctest.Wait()
}
func TestRequestTimeRunningOutWhileTheVisitorIsWaitedOnRefuses408(
t *testing.T,
) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
seen := &limitsSeen{}
seen.waiting.Store(true)
limits := newTestLimits(seen)
limits.startRequestTimer()
sleepThenWait(testRequestTime - time.Millisecond)
if got := seen.refused.Load(); got != 0 {
t.Fatalf(
"refused %d before the limit ran out, want nothing",
got,
)
}
sleepThenWait(time.Millisecond)
if got := seen.refused.Load(); got != http.StatusRequestTimeout {
t.Errorf("refused %d, want 408", got)
}
if !seen.readEnded.Load() {
t.Error("the read of the visitor's body goes on, want it ended")
}
})
}
func TestRequestTimeRunningOutWhileTheAppIsWaitedOnRefuses504(
t *testing.T,
) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
seen := &limitsSeen{}
limits := newTestLimits(seen)
limits.startRequestTimer()
sleepThenWait(testRequestTime)
if got := seen.refused.Load(); got != http.StatusGatewayTimeout {
t.Errorf("refused %d, want 504", got)
}
if seen.readEnded.Load() {
t.Error("the read of the visitor's body was ended, want it not")
}
})
}
func TestResponseTimeRunsFromTheEndOfTheRequest(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
seen := &limitsSeen{}
limits := newTestLimits(seen)
limits.startRequestTimer()
sleepThenWait(time.Second)
end := time.Now()
limits.wroteRequest(httptrace.WroteRequestInfo{})
if got := limits.endRequest(); !got.Equal(end) {
t.Errorf("the request ended at %v, want %v", got, end)
}
// Past SWW_RPROXY_MAX_REQUEST_TIME_MS, which the end of the request
// stopped, and just short of SWW_RPROXY_MAX_RESPONSE_TIME_MS.
sleepThenWait(testResponseTime - time.Millisecond)
if got := seen.refused.Load(); got != 0 {
t.Fatalf(
"refused %d before the limit ran out, want nothing",
got,
)
}
sleepThenWait(time.Millisecond)
if got := seen.refused.Load(); got != http.StatusGatewayTimeout {
t.Errorf("refused %d, want 504", got)
}
})
}
func TestStoppedLimitsRefuseNothing(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
seen := &limitsSeen{}
seen.waiting.Store(true)
limits := newTestLimits(seen)
limits.startRequestTimer()
limits.stop()
limits.startResponseTimer(limits.endRequest())
sleepThenWait(testRequestTime + testResponseTime)
if got := seen.refused.Load(); got != 0 {
t.Errorf("refused %d, want nothing", got)
}
if seen.readEnded.Load() {
t.Error("the read of the visitor's body was ended, want it not")
}
})
}
+1 -1
View File
@@ -188,7 +188,7 @@ func canBeAsked(r *http.Request) bool {
// answerWith gives the visitor waf's answer, less its disposition.
func (rq *request) answerWith(answer *wafAnswer) {
rq.prepareOwnAnswer(rq.requestDeadline())
rq.prepareOwnAnswer(rq.limits.requestDeadline())
header := rq.out.Header()
maps.Copy(header, answer.header)
+2 -2
View File
@@ -54,7 +54,7 @@ func TestClientsOnTheAbuseIPDBBlacklistAreBannedForTheBanDays(t *testing.T) {
})
// Two clients in one netblock make one ban, with the higher score.
server.BanAbuseIPDBBlacklist([]reputation.BlacklistEntry{
server.banAbuseIPDBBlacklist([]reputation.BlacklistEntry{
{Addr: netip.MustParseAddr(client), Score: 95},
{Addr: netip.MustParseAddr(otherClient), Score: 100},
{Addr: netip.MustParseAddr("2001:db8:5::1"), Score: 90},
@@ -88,7 +88,7 @@ func TestAbuseIPDBBlacklistMakesNoBanInObserveMode(t *testing.T) {
map[string]string{mode: "observe"},
)
server.BanAbuseIPDBBlacklist([]reputation.BlacklistEntry{
server.banAbuseIPDBBlacklist([]reputation.BlacklistEntry{
{Addr: netip.MustParseAddr(client), Score: 100},
})
+92 -147
View File
@@ -2,7 +2,6 @@ package waf
import (
"bytes"
"crypto/subtle"
"encoding/json"
"errors"
"fmt"
@@ -15,7 +14,6 @@ import (
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/state"
)
@@ -41,80 +39,23 @@ var (
errNotAddress = errors.New("is not an address, such as 203.0.113.9")
)
// answerHealth answers a request for the health endpoint.
func (rq *request) answerHealth() {
rq.line.Action = requestlog.ActionOwnEndpoint
// Set here rather than left to Go's server, which would set it only
// after the log line has taken the response's headers.
rq.out.Header().Set("Content-Type", "text/plain; charset=utf-8")
_, _ = io.WriteString(rq.out, "ok\n")
}
// answerAdmin answers a request for one of waf's own endpoints. Each
// needs a token, sent as Authorization: Bearer <token>: the metrics
// SWW_WAF_METRICS_TOKEN, the others SWW_WAF_ADMIN_TOKEN. A request without it is
// refused with 401. An endpoint whose token is unset answers 404, as any other
// path does.
func (rq *request) answerAdmin() {
rq.line.Action = requestlog.ActionOwnEndpoint
token, answer := rq.endpoint()
switch {
case token == "":
http.Error(
rq.out,
http.StatusText(http.StatusNotFound),
http.StatusNotFound,
)
case !hasToken(rq.in, token):
rq.out.Header().Set("WWW-Authenticate", "Bearer")
http.Error(
rq.out,
http.StatusText(http.StatusUnauthorized),
http.StatusUnauthorized,
)
default:
answer()
// handleHealth answers GET wafprotocol.HealthPath.
func handleHealth() http.HandlerFunc {
return func(w http.ResponseWriter, _ *http.Request) {
// Set here rather than left to Go's server, which would set it only
// after the log line has taken the response's headers.
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
_, _ = io.WriteString(w, "ok\n")
}
}
// endpoint returns the token the request's endpoint needs, and what
// answers the request there; "" when there is no such endpoint.
func (rq *request) endpoint() (string, func()) {
cfg := rq.h.config
method, path := rq.in.Method, rq.in.URL.Path
switch {
case method == http.MethodGet && path == MetricsPath:
return cfg.MetricsToken, func() { rq.h.metrics.ServeHTTP(rq.out, rq.in) }
case method == http.MethodGet && path == BansPath:
return cfg.AdminToken, rq.listBans
case method == http.MethodPost && path == BansPath:
return cfg.AdminToken, rq.addBan
case method == http.MethodDelete && strings.HasPrefix(path, BansPath+"/"):
return cfg.AdminToken, rq.liftBans
case method == http.MethodGet && strings.HasPrefix(path, ClientsPath):
return cfg.AdminToken, rq.showClient
default:
return "", nil
// handleListBans answers GET BansPath with every ban held.
func (h *handler) handleListBans() http.HandlerFunc {
return func(w http.ResponseWriter, _ *http.Request) {
h.answerBans(w, h.ledger.Snapshot())
}
}
// hasToken reports whether r carries token, as Authorization: Bearer
// <token>.
func hasToken(r *http.Request, token string) bool {
scheme, sent, _ := strings.Cut(r.Header.Get("Authorization"), " ")
return strings.EqualFold(scheme, "Bearer") &&
subtle.ConstantTimeCompare([]byte(sent), []byte(token)) == 1
}
// listBans answers GET BansPath with every ban held.
func (rq *request) listBans() {
rq.answerBans(rq.h.ledger.Snapshot())
}
// banToAdd is the body of POST BansPath.
type banToAdd struct {
// Netblock is a netblock, or a client's address, which stands for the
@@ -126,44 +67,44 @@ type banToAdd struct {
Reason string `json:"reason"`
}
// addBan answers POST BansPath: it bans the netblock the body names, as
// an admin, from now for the duration the body gives, with its reason,
// handleAddBan answers POST BansPath: it bans the netblock the body names,
// as an admin, from now for the duration the body gives, with its reason,
// and answers with that ban.
func (rq *request) addBan() {
toAdd, err := rq.readBanToAdd()
func (h *handler) handleAddBan() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
toAdd, err := readBanToAdd(r)
var (
netblock netip.Prefix
expires time.Time
now = rq.h.now()
)
var (
netblock netip.Prefix
expires time.Time
now = h.now()
)
if err == nil {
netblock, err = rq.h.parseBanNetblock(toAdd.Netblock)
if err == nil {
netblock, err = h.parseBanNetblock(toAdd.Netblock)
}
if err == nil {
expires, err = expiry(toAdd.Duration, now)
}
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
ban := h.ledger.BanForAdmin(netblock, now, expires, toAdd.Reason)
h.answerBans(w, []bans.Ban{ban})
}
if err == nil {
expires, err = expiry(toAdd.Duration, now)
}
if err != nil {
http.Error(rq.out, err.Error(), http.StatusBadRequest)
return
}
ban := rq.h.ledger.BanForAdmin(netblock, now, expires, toAdd.Reason)
rq.answerBans([]bans.Ban{ban})
}
// readBanToAdd reads the body of POST BansPath: a JSON object with
// nothing but whitespace after it, in at most banBodyMaxBytes.
func (rq *request) readBanToAdd() (banToAdd, error) {
func readBanToAdd(r *http.Request) (banToAdd, error) {
data, err := io.ReadAll(
http.MaxBytesReader(nil, rq.in.Body, banBodyMaxBytes),
http.MaxBytesReader(nil, r.Body, banBodyMaxBytes),
)
rq.requestBytes = int64(len(data))
if err != nil {
return banToAdd{}, fmt.Errorf("%w: %w", errNotBanToAdd, err)
}
@@ -250,29 +191,31 @@ func expiry(duration string, now time.Time) (time.Time, error) {
return now.Add(length), nil
}
// liftBans answers DELETE BansPath/<client>: it lifts every ban active on
// a netblock the client's address is in, and answers with those bans, or
// with 404 when none is active.
func (rq *request) liftBans() {
client, err := pathAddress(rq.in.URL.Path, BansPath+"/")
if err != nil {
http.Error(rq.out, err.Error(), http.StatusBadRequest)
// handleLiftBans answers DELETE BansPath/<client>: it lifts every ban
// active on a netblock the client's address is in, and answers with those
// bans, or with 404 when none is active.
func (h *handler) handleLiftBans() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
client, err := pathAddress(r.URL.Path, BansPath+"/")
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
return
}
lifted := h.ledger.Lift(client, h.now())
if len(lifted) == 0 {
http.Error(
w,
"no ban is active on "+client.String(),
http.StatusNotFound,
)
return
}
h.answerBans(w, lifted)
}
lifted := rq.h.ledger.Lift(client, rq.h.now())
if len(lifted) == 0 {
http.Error(
rq.out,
"no ban is active on "+client.String(),
http.StatusNotFound,
)
return
}
rq.answerBans(lifted)
}
// clientAnswer is the answer to GET ClientsPath<ip>: the client the
@@ -284,29 +227,31 @@ type clientAnswer struct {
Bans []state.BanEntry `json:"bans"`
}
// showClient answers GET ClientsPath<ip> with what waf knows of the client:
// its counters, its history, which holds its country as last looked up and its
// offences, and its bans with their notes.
func (rq *request) showClient() {
addr, err := pathAddress(rq.in.URL.Path, ClientsPath)
if err != nil {
http.Error(rq.out, err.Error(), http.StatusBadRequest)
// handleShowClient answers GET ClientsPath<ip> with what waf knows of the
// client: its counters, its history, which holds its country as last looked
// up and its offences, and its bans with their notes.
func (h *handler) handleShowClient() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
addr, err := pathAddress(r.URL.Path, ClientsPath)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
return
}
answer := clientAnswer{
Bans: state.BanEntries(h.ledger.Covering(addr)),
}
client, seen := h.limiter.Client(
clientGroup(h.config.IPv6GroupPrefix, addr),
)
if seen {
answer.Client = &client
}
h.answerJSON(w, answer)
}
answer := clientAnswer{
Bans: state.BanEntries(rq.h.ledger.Covering(addr)),
}
client, seen := rq.h.limiter.Client(
clientGroup(rq.h.config.IPv6GroupPrefix, addr),
)
if seen {
answer.Client = &client
}
rq.answerJSON(answer)
}
// pathAddress reads the client's address that follows prefix in path.
@@ -322,27 +267,27 @@ func pathAddress(path, prefix string) (netip.Addr, error) {
}
// answerBans answers with held under bans, as bans.json lists them.
func (rq *request) answerBans(held []bans.Ban) {
rq.answerJSON(struct {
func (h *handler) answerBans(w http.ResponseWriter, held []bans.Ban) {
h.answerJSON(w, struct {
Bans []state.BanEntry `json:"bans"`
}{state.BanEntries(held)})
}
// answerJSON answers with value as indented JSON.
func (rq *request) answerJSON(value any) {
func (h *handler) answerJSON(w http.ResponseWriter, value any) {
body, err := json.MarshalIndent(value, "", " ")
if err != nil {
rq.h.processLog.Error(
h.processLog.Error(
"encoding an answer failed",
"error",
err.Error(),
)
http.Error(rq.out, http.StatusText(http.StatusInternalServerError),
http.Error(w, http.StatusText(http.StatusInternalServerError),
http.StatusInternalServerError)
return
}
rq.out.Header().Set("Content-Type", "application/json")
_, _ = rq.out.Write(append(body, '\n'))
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write(append(body, '\n'))
}
+34
View File
@@ -14,6 +14,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/state"
"sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
const (
@@ -333,6 +334,39 @@ func TestBanToAddThatCannotBeReadIsRefused(t *testing.T) {
}
}
func TestBanToAddIsLoggedWithTheBytesOfItsBody(t *testing.T) {
t.Parallel()
s, _, _ := startWithClock(
t,
"",
map[string]string{adminToken: adminSecret},
)
s.admin(http.MethodPost, waf.BansPath, banOtherClient, http.StatusOK)
line := s.out.endedLines(t, s.sent)[s.sent-1]
if line.RequestBytes != int64(len(banOtherClient)) {
t.Errorf(
"the ban to add was logged with request_bytes %d, want %d",
line.RequestBytes,
len(banOtherClient),
)
}
// The health endpoint reads none of a body sent to it.
s.ownEndpointRequest("", http.MethodGet, wafprotocol.HealthPath, "ok",
http.StatusOK)
line = s.out.endedLines(t, s.sent)[s.sent-1]
if line.RequestBytes != 0 {
t.Errorf(
"the health request was logged with request_bytes %d, want 0",
line.RequestBytes,
)
}
}
func TestClientEndpointShowsTheClientAndItsBans(t *testing.T) {
t.Parallel()
+2 -3
View File
@@ -11,7 +11,6 @@ import (
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
const (
@@ -250,7 +249,7 @@ func TestObserveModeWorksOutABanOnlyWhenItsAlertWouldBeSent(t *testing.T) {
// never sent them, and returns the queue they wait in as well.
func startWithAlerts(
t *testing.T, env map[string]string,
) (*sender, *clock, *waf.Server, *alerts.Queue) {
) (*sender, *clock, *wafAndParts, *alerts.Queue) {
t.Helper()
return startAppWithAlerts(
@@ -263,7 +262,7 @@ func startWithAlerts(
// startAppWithAlerts is startWithAlerts in front of the app handler.
func startAppWithAlerts(
t *testing.T, handler http.HandlerFunc, env map[string]string,
) (*sender, *clock, *waf.Server, *alerts.Queue) {
) (*sender, *clock, *wafAndParts, *alerts.Queue) {
t.Helper()
app := startApp(t, handler)
+1 -2
View File
@@ -15,7 +15,6 @@ import (
"sneak.berlin/go/smallwebwaf/internal/anomaly"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
// The anomaly thresholds: the prefix of a scope followed by the end of a
@@ -332,7 +331,7 @@ func TestRequestCountsForTheASNumberGeoJSGivesBeforeItEnds(t *testing.T) {
// kept.
geojsURL, _, release := startHeldGeoJS(t)
var server atomic.Pointer[waf.Server]
var server atomic.Pointer[wafAndParts]
app := startApp(t, func(http.ResponseWriter, *http.Request) {
release()
+13 -13
View File
@@ -439,7 +439,7 @@ func TestTheHeaderLimitsComeBeforeEveryCheckThatRefuses(t *testing.T) {
env map[string]string
path string
// prepare gets the server ready, at now, before the question.
prepare func(t *testing.T, server *waf.Server, now time.Time)
prepare func(t *testing.T, server *wafAndParts, now time.Time)
}{
{"SWW_WAF_DENY_NETS", map[string]string{denyNets: client}, "/", leaveAsItIs},
{"a ban", nil, "/", banClient},
@@ -454,21 +454,21 @@ func TestTheHeaderLimitsComeBeforeEveryCheckThatRefuses(t *testing.T) {
"a DNS blocklist", map[string]string{
dnsblZones: dnsblZone, dnsblResolver: noResolver, dnsblAbuseIPDBAction: actionDeny,
},
"/", func(_ *testing.T, server *waf.Server, _ time.Time) {
"/", func(_ *testing.T, server *wafAndParts, _ time.Time) {
loadVerdicts(server, map[string][]string{client: {dnsblZone}})
},
},
{
"AbuseIPDB",
map[string]string{abuseIPDBKey: accountKey, dnsblAbuseIPDBAction: actionDeny},
"/", func(_ *testing.T, server *waf.Server, _ time.Time) {
"/", func(_ *testing.T, server *wafAndParts, _ time.Time) {
// Over SWW_WAF_ABUSEIPDB_MIN_SCORE, 75 by default.
loadScores(server, map[string]int64{client: 80})
},
},
{
"SWW_WAF_TAG_DENY", map[string]string{tagLists: "vpn=" + vpnList, tagDeny: "vpn"},
"/", func(t *testing.T, server *waf.Server, _ time.Time) {
"/", func(t *testing.T, server *wafAndParts, _ time.Time) {
t.Helper()
loadLists(t, server, map[string][]string{vpnList: {client}})
},
@@ -685,7 +685,7 @@ func TestObserveModeAsksGeoJSNothingForARequestOverTheHeaderLimits(
// for env. No request to it may use the network.
func newWAFInBubble(
t *testing.T, env map[string]string,
) (*waf.Server, *output) {
) (*wafAndParts, *output) {
t.Helper()
settings := map[string]string{tagDeny: "country=KP"}
@@ -699,7 +699,7 @@ func newWAFInBubble(
// askDirectly hands a question about a request from client straight to server,
// as change leaves it, and returns the answer's status and how long it took.
func askDirectly(
t *testing.T, server *waf.Server, change func(*http.Request),
t *testing.T, server *wafAndParts, change func(*http.Request),
) (int, time.Duration) {
t.Helper()
@@ -723,7 +723,7 @@ func askDirectly(
}
// banClient bans client at now.
func banClient(_ *testing.T, server *waf.Server, now time.Time) {
func banClient(_ *testing.T, server *wafAndParts, now time.Time) {
server.Ledger.Ban(
netip.MustParsePrefix(client+"/32"),
now,
@@ -732,7 +732,7 @@ func banClient(_ *testing.T, server *waf.Server, now time.Time) {
}
// countClient counts one request from client at now for the rate limits.
func countClient(_ *testing.T, server *waf.Server, now time.Time) {
func countClient(_ *testing.T, server *wafAndParts, now time.Time) {
server.Limiter.Count(
netip.MustParsePrefix(client+"/32"),
now,
@@ -741,7 +741,7 @@ func countClient(_ *testing.T, server *waf.Server, now time.Time) {
}
// listClient puts client on the blocklist at dropURL.
func listClient(t *testing.T, server *waf.Server, now time.Time) {
func listClient(t *testing.T, server *wafAndParts, now time.Time) {
t.Helper()
err := server.Lists.Load([]reputation.List{{
@@ -753,7 +753,7 @@ func listClient(t *testing.T, server *waf.Server, now time.Time) {
}
// leaveAsItIs prepares nothing.
func leaveAsItIs(*testing.T, *waf.Server, time.Time) {}
func leaveAsItIs(*testing.T, *wafAndParts, time.Time) {}
func TestARefusedRequestDoesNotWaitForASlowGeoJS(t *testing.T) {
t.Parallel()
@@ -764,7 +764,7 @@ func TestARefusedRequestDoesNotWaitForASlowGeoJS(t *testing.T) {
name string
env map[string]string
// prepare gets the server ready, at now, before the question.
prepare func(t *testing.T, server *waf.Server, now time.Time)
prepare func(t *testing.T, server *wafAndParts, now time.Time)
// cookie is the request's Cookie header.
cookie string
status int
@@ -849,7 +849,7 @@ func TestAQuestionWaitsForASlowGeoJSOnlyAsLongAsTheDefaultSays(
name string
env map[string]string
// prepare gets the server ready, at now, before the question.
prepare func(t *testing.T, server *waf.Server, now time.Time)
prepare func(t *testing.T, server *wafAndParts, now time.Time)
status int
action string
}{
@@ -915,7 +915,7 @@ func TestWAFAnswersWithinTheTimeLimitWhileEveryOutsideSourceHangs(
held, _, _ := startHeldGeoJS(t)
resolver, releaseResolver := startHeldResolver(t)
server, out, _ := newWAFWith(t, waf.New,
server, out, _ := newWAFWith(t, func(*waf.Params) {},
silentGeoJSURL, silentAbuseIPDBURL, time.Now, map[string]string{
tagDeny: "country=KP", coreRuleSetMode: block,
dnsblZones: dnsblZone, dnsblResolver: resolver,
+12 -8
View File
@@ -45,8 +45,11 @@ func (a activeBan) check(
}
ban, banned, madePermanent := check(rq.client, now)
var expires string
if banned {
rq.line.BanExpires = banExpires(ban)
expires = banExpires(ban)
rq.line.BanExpires = expires
}
if madePermanent {
@@ -61,7 +64,7 @@ func (a activeBan) check(
return &verdict{
action: banAction(ban.Permanent()),
figures: map[string]any{
figureNetblock: ban.Netblock.String(), "ban_expires": rq.line.BanExpires,
figureNetblock: ban.Netblock.String(), "ban_expires": expires,
},
}
}
@@ -148,6 +151,7 @@ func (b banMaker) banForLimit(
rq.line.LimitHit = hit.Window
}
rq.brokeLimit = true
rq.line.Offence = requestlog.OffenceLimit
b.metrics.LimitHit(hit)
@@ -259,9 +263,9 @@ func (b banMaker) withClientNotes(
status int,
netblock netip.Prefix,
) bans.Notes {
notes.ASN = rq.line.ASN
notes.ASName = rq.line.ASName
notes.Country = rq.line.Country
notes.ASN = rq.lookupAnswer.ASN
notes.ASName = rq.lookupAnswer.ASName
notes.Country = rq.lookupAnswer.Country
notes.Reputation = rq.reputation
notes.Request = rq.noted(now, status)
notes.Requests = b.netblockRequests(netblock)
@@ -328,11 +332,11 @@ func (b banMaker) netblockRequests(netblock netip.Prefix) int64 {
func (rq *request) noted(now time.Time, status int) bans.Request {
return bans.Request{
Time: now,
Method: rq.line.Method,
Host: rq.line.Host,
Method: rq.method,
Host: rq.site,
Path: rq.requestURI,
Status: status,
UserAgent: rq.line.UserAgent,
UserAgent: rq.userAgent,
}
}
+1 -2
View File
@@ -16,7 +16,6 @@ import (
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
const (
@@ -466,7 +465,7 @@ func (c *clock) advance(d time.Duration) {
// window.
func startWithClock(
t *testing.T, geojsURL string, env map[string]string,
) (*sender, *clock, *waf.Server) {
) (*sender, *clock, *wafAndParts) {
t.Helper()
app := startApp(t, func(http.ResponseWriter, *http.Request) {})
+3 -5
View File
@@ -62,7 +62,7 @@ func coreRuleSetMatch() coreruleset.Result {
// at its default but for env. No request to it may use the network.
func newWAFHoldingTheCoreRuleSet(
t *testing.T, crs *heldCoreRuleSet, env map[string]string,
) (*waf.Server, *output, *alerts.Queue) {
) (*wafAndParts, *output, *alerts.Queue) {
t.Helper()
settings := map[string]string{
@@ -71,11 +71,9 @@ func newWAFHoldingTheCoreRuleSet(
}
maps.Copy(settings, env)
server, out, queue := newWAFWith(t, func(params waf.Params) *waf.Server {
server, out, queue := newWAFWith(t, func(params *waf.Params) {
params.CoreRuleSet = crs
params.BesideLookupTimeLimit = waf.BesideLookupTimeLimit
return waf.New(params)
}, silentGeoJSURL, abuseIPDBURL, time.Now, settings)
return server, out, queue
@@ -105,7 +103,7 @@ func wantAnswer(
// left no trace: line has none of its fields, and server has counted no match
// of it and raised no alert for one.
func wantNoCoreRuleSetTrace(
t *testing.T, server *waf.Server, queue *alerts.Queue, line logLine,
t *testing.T, server *wafAndParts, queue *alerts.Queue, line logLine,
) {
t.Helper()
+2 -2
View File
@@ -68,7 +68,7 @@ type biasedThresholds struct {
func (b biasedThresholds) limitPercentages(
rq *request,
) (percentage, percentage) {
asn, country := rq.line.ASN, rq.line.Country
asn, country := rq.lookupAnswer.ASN, rq.lookupAnswer.Country
unknown := percentage{percent: whole}
if country == "" && !rq.skippedLookupWait {
@@ -115,7 +115,7 @@ func (b biasedThresholds) limitPercentages(
otherTags := percentage{percent: whole}
for _, tag := range rq.line.Tags {
for _, tag := range rq.tags {
if tag != asnTag(asn) && tag != countryTag(country) {
otherTags = lowest(
otherTags,
+3 -4
View File
@@ -18,7 +18,6 @@ import (
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
@@ -530,7 +529,7 @@ func TestBanForALoweredLimitGivesThePercentageInItsNotesAndItsAlert(
// clock.
func startWithLookups(
t *testing.T, env map[string]string,
) (*sender, *waf.Server, *alerts.Queue) {
) (*sender, *wafAndParts, *alerts.Queue) {
t.Helper()
s, _, server, queue := startWithLookupsAndClock(t, env)
@@ -545,7 +544,7 @@ func startWithLookups(
// country, and no other address.
func startWithLookupsAndClock(
t *testing.T, env map[string]string,
) (*sender, *clock, *waf.Server, *alerts.Queue) {
) (*sender, *clock, *wafAndParts, *alerts.Queue) {
t.Helper()
path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
@@ -585,7 +584,7 @@ func (s *sender) uploadFrom(from string) logLine {
// straight to server's handler, without the network, and returns once it is
// answered.
func serveFromDE(
t *testing.T, server *waf.Server, method string, body io.Reader,
t *testing.T, server *wafAndParts, method string, body io.Reader,
) {
t.Helper()
+9
View File
@@ -43,6 +43,15 @@ func (b blocklists) check(
}
}
// refusesWhateverTheLookupAnswers reports whether blocklists refuses rq, as
// refusedByBlocklists says. The lookup's answer plays no part in it.
func (b blocklists) refusesWhateverTheLookupAnswers(
rq *request,
_ time.Time,
) bool {
return b.refusedByBlocklists(b.lists.ListedBy(rq.client))
}
// refusedByBlocklists reports whether the heuristic blocklists refuses a
// client, given listedBy, the blocklists that list it: it does when one lists
// it and SWW_WAF_BLOCKLIST_ACTION is deny.
+1 -1
View File
@@ -33,7 +33,7 @@ func (b byteLimits) check(
rq *request,
now time.Time,
) *verdict {
if !rq.counted || rq.line.WouldAction != "" {
if !rq.counted || rq.wouldAction != "" {
return nil
}
+3 -3
View File
@@ -121,9 +121,9 @@ func (c coreRuleSet) alertCoreRuleSetMatch(
Host: rq.site,
Client: rq.client,
Netblock: clientGroup(c.ipv6GroupPrefix, rq.client),
ASN: rq.line.ASN,
ASName: rq.line.ASName,
Country: rq.line.Country,
ASN: rq.lookupAnswer.ASN,
ASName: rq.lookupAnswer.ASName,
Country: rq.lookupAnswer.Country,
Reason: "scored by the Core Rule Set at or over " +
"SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD",
Detail: detail,
+1 -1
View File
@@ -27,7 +27,7 @@ func (c countries) check(
_ time.Time,
) *verdict {
allowed := c.allowed
country := rq.line.Country
country := rq.lookupAnswer.Country
if len(allowed) == 0 || slices.Contains(allowed, country) {
return nil
-9
View File
@@ -1,13 +1,4 @@
package waf
import "sneak.berlin/go/smallwebwaf/internal/reputation"
// BesideLookupTimeLimit is besideLookupTimeLimit.
const BesideLookupTimeLimit = besideLookupTimeLimit
// BanAbuseIPDBBlacklist does on s what a fetch of AbuseIPDB's blacklist with
// the clients on listed does.
func (s *Server) BanAbuseIPDBBlacklist(listed []reputation.BlacklistEntry) {
h := s.Handler.(*handler) //nolint:forcetypeassert // New makes it one
BanAbuseIPDBBlacklist(h.config, h.ledger, h.processLog, h.now(), listed)
}
+11
View File
@@ -58,3 +58,14 @@ func TestHealthEndpointCountsTowardNoLimit(t *testing.T) {
}
}
}
func TestHealthEndpointIsFoundAtItsPathPercentEncoded(t *testing.T) {
t.Parallel()
// %68 is h. An endpoint is found by the decoded path.
s, _, _ := startWithClock(t, "", nil)
got := s.ownEndpointRequest("", http.MethodGet, "/_sww/%68ealthz", "",
http.StatusOK)
wantBody(t, got, "ok\n")
}
+19
View File
@@ -9,6 +9,16 @@ import (
// and holds the settings and parts it reads. Add one line for it to one of the
// lists that newHandler builds, which hands it those settings and parts. Its
// settings go in config.WAF, and the comment at the top of its file names them.
// A heuristic of afterLookup that can refuse a request whatever the lookup
// answers also implements refusesWithoutLookup.
//
// Three places outside its file may change with it:
// - waitForGeoJS in newHandler lists the settings that need the lookup's
// answer;
// - requestCanGetTag in internal/config/waf.go holds which setting can give a
// request which tag;
// - findBesideLookup does the work of each heuristic of besideLookup, by
// name.
// heuristic is one of the checks that decide what waf does with a
// request.
@@ -20,6 +30,15 @@ type heuristic interface {
check(ctx context.Context, rq *request, now time.Time) *verdict
}
// refusesWithoutLookup is a heuristic of afterLookup that can tell, before the
// lookup answers, that it will refuse a request whatever the answer.
type refusesWithoutLookup interface {
// refusesWhateverTheLookupAnswers reports whether the heuristic will refuse
// rq at now, whatever the lookup answers about its client. It changes
// nothing.
refusesWhateverTheLookupAnswers(rq *request, now time.Time) bool
}
// verdict is a heuristic refusing a request, or banning its client: the action
// the log line names, one of requestlog's Action constants, and the figures the
// heuristic acted on, by name, such as the count and the limit.
+1 -1
View File
@@ -151,7 +151,7 @@ func TestPathUnderTheOwnPrefixIsNeitherForwardedNorRefused(t *testing.T) {
// historyOf returns the history of the client at addr.
func historyOf(
t *testing.T, server *waf.Server, addr string,
t *testing.T, server *wafAndParts, addr string,
) ratelimit.History {
t.Helper()
+2 -2
View File
@@ -94,11 +94,11 @@ func (rq *request) noteLookup(result lookupResult) {
rq.line.ASName = rq.lookupAnswer.ASName
rq.line.Country = rq.lookupAnswer.Country
if tag := asnTag(rq.line.ASN); tag != "" {
if tag := asnTag(rq.lookupAnswer.ASN); tag != "" {
rq.addTag(tag)
}
if tag := countryTag(rq.line.Country); tag != "" {
if tag := countryTag(rq.lookupAnswer.Country); tag != "" {
rq.addTag(tag)
}
}
+31 -44
View File
@@ -107,21 +107,9 @@ type CoreRuleSetInspector interface {
) (coreruleset.Result, []byte, error)
}
// Server is the server waf runs, with the parts of its Params. The tests
// read them.
// Server is the server waf runs.
type Server struct {
*http.Server
Ledger *bans.Ledger
Limiter *ratelimit.Limiter
GeoJS *geojs.GeoJS
Anomalies *anomaly.Counters
LookupFile *lookup.File
Lists *reputation.Lists
DNSBL *reputation.DNSBL
AbuseIPDB *reputation.AbuseIPDB
RouteWatch *routewatch.Client
Metrics *metrics.Metrics
}
// newHandler returns the handler with what every request shares, the parts
@@ -177,30 +165,6 @@ func newHandler(params Params) *handler {
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
}
h.blocklists = blocklists{
lists: params.Lists,
action: cfg.BlocklistAction,
hitNoter: hitNoter,
}
h.rateLimits = rateLimits{
limiter: params.Limiter,
exemptNets: cfg.RateLimitExemptNets,
exemptPaths: cfg.RateLimitExemptPaths,
banResponse: cfg.BanResponse,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
biasedThresholds: biasedThresholds{
lists: params.Lists,
tagLimitPercent: cfg.TagLimitPercent,
asnBytesLimitPercent: cfg.ASNBytesLimitPercent,
countryBytesLimitPercent: cfg.CountryBytesLimitPercent,
unknownCountryLimitPercent: cfg.UnknownCountryLimitPercent,
blocklistAction: cfg.BlocklistAction,
blocklistLimitPercent: cfg.BlocklistLimitPercent,
dnsblAbuseIPDBAction: cfg.DNSBLAbuseIPDBAction,
dnsblAbuseIPDBLimitPercent: cfg.DNSBLAbuseIPDBLimitPercent,
},
banMaker: sharedBanMaker,
}
h.trapPaths = trapPaths{paths: cfg.TrapPaths, banMaker: sharedBanMaker}
h.ruleFiles = ruleFiles{
rules: params.Rules,
@@ -230,7 +194,11 @@ func newHandler(params Params) *handler {
}
h.afterLookup = []heuristic{
countries{allowed: cfg.ExclusivelyAllowedCountries},
h.blocklists,
blocklists{
lists: params.Lists,
action: cfg.BlocklistAction,
hitNoter: hitNoter,
},
dnsbl{
dnsbl: params.DNSBL,
action: cfg.DNSBLAbuseIPDBAction,
@@ -246,7 +214,25 @@ func newHandler(params Params) *handler {
hitNoter: hitNoter,
},
tagDeny{tags: cfg.TagDeny},
h.rateLimits,
rateLimits{
limiter: params.Limiter,
exemptNets: cfg.RateLimitExemptNets,
exemptPaths: cfg.RateLimitExemptPaths,
banResponse: cfg.BanResponse,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
biasedThresholds: biasedThresholds{
lists: params.Lists,
tagLimitPercent: cfg.TagLimitPercent,
asnBytesLimitPercent: cfg.ASNBytesLimitPercent,
countryBytesLimitPercent: cfg.CountryBytesLimitPercent,
unknownCountryLimitPercent: cfg.UnknownCountryLimitPercent,
blocklistAction: cfg.BlocklistAction,
blocklistLimitPercent: cfg.BlocklistLimitPercent,
dnsblAbuseIPDBAction: cfg.DNSBLAbuseIPDBAction,
dnsblAbuseIPDBLimitPercent: cfg.DNSBLAbuseIPDBLimitPercent,
},
banMaker: sharedBanMaker,
},
}
h.besideLookup = []heuristic{h.trapPaths, h.ruleFiles, h.coreRuleSet}
h.afterBesideLookup = []heuristic{tagDeny{tags: cfg.TagDeny}}
@@ -268,6 +254,8 @@ func newHandler(params Params) *handler {
},
}
h.ownEndpoints = h.routes()
return h
}
@@ -293,6 +281,9 @@ type handler struct {
// awaitingReport holds the decision on each answered question by its id
// until its report comes, and nil for a question still being handled.
awaitingReport *lru.Cache[string, *decision]
// ownEndpoints answers every request that is not a question, as routes
// lists the endpoints.
ownEndpoints http.Handler
// The lists of heuristics, each in the order its heuristics run. A client
// in SWW_WAF_ALLOW_NETS skips them all.
@@ -323,11 +314,7 @@ type handler struct {
// afterAnswer, they only ban.
onReport []heuristic
// The heuristics of the lists that other steps of a request use too:
// refusedWhateverTheAnswer asks blocklists and rateLimits, and
// findBesideLookup does the work of trapPaths, ruleFiles and coreRuleSet.
blocklists blocklists
rateLimits rateLimits
// The heuristics of besideLookup, whose work findBesideLookup does.
trapPaths trapPaths
ruleFiles ruleFiles
coreRuleSet coreRuleSet
+36 -12
View File
@@ -316,7 +316,7 @@ func startProxyWithGeoJS(
func startProxyWithClock(
t *testing.T, appURL, geojsURL string, now func() time.Time,
env map[string]string,
) (string, *output, *waf.Server) {
) (string, *output, *wafAndParts) {
t.Helper()
addr, out, server, _ := startProxyWithAlerts(
@@ -335,7 +335,7 @@ func startProxyWithClock(
func startProxyWithAlerts(
t *testing.T, appURL, geojsURL string, now func() time.Time,
env map[string]string,
) (string, *output, *waf.Server, *alerts.Queue) {
) (string, *output, *wafAndParts, *alerts.Queue) {
t.Helper()
server, out, alertQueue := newWAF(t, geojsURL, now, env)
@@ -346,7 +346,7 @@ func startProxyWithAlerts(
// serve serves server on a port of its own until the test ends, and sets its
// Addr to where it listens.
func serve(t *testing.T, server *waf.Server) {
func serve(t *testing.T, server *wafAndParts) {
t.Helper()
listener, err := (&net.ListenConfig{}).Listen(
@@ -422,8 +422,29 @@ func startRProxy(
return listener.Addr().String()
}
// newWAF makes waf's server, without starting it, and
// returns it, what it writes, and the queue of the alerts it raises, as the
// wafAndParts is waf's server with the Params it was built from, so that a
// test can read the parts the server uses.
type wafAndParts struct {
*waf.Server
waf.Params
}
// banAbuseIPDBBlacklist does on w what a fetch of AbuseIPDB's blacklist with
// the clients on listed does.
func (w *wafAndParts) banAbuseIPDBBlacklist(
listed []reputation.BlacklistEntry,
) {
waf.BanAbuseIPDBBlacklist(
w.Config,
w.Ledger,
w.ProcessLog,
w.Now(),
listed,
)
}
// newWAF makes waf's server, without starting it, and returns it with its
// parts, what it writes, and the queue of the alerts it raises, as the
// settings in env make it. No alert is sent from the queue: they wait in it,
// for the test to look at.
// With no geojsURL, there is no stand-in for GeoJS to look clients up at, and
@@ -438,19 +459,21 @@ func newWAF(
geojsURL string,
now func() time.Time,
env map[string]string,
) (*waf.Server, *output, *alerts.Queue) {
) (*wafAndParts, *output, *alerts.Queue) {
t.Helper()
return newWAFWith(t, waf.New, geojsURL, abuseIPDBURL, now, env)
return newWAFWith(
t, func(*waf.Params) {}, geojsURL, abuseIPDBURL, now, env,
)
}
// newWAFWith is newWAF for the server build makes, with clients checked with
// AbuseIPDB at abuseIPDBCheckURL.
// newWAFWith is newWAF with change applied to the Params before the server is
// built from them, and clients checked with AbuseIPDB at abuseIPDBCheckURL.
func newWAFWith(
t *testing.T, build func(waf.Params) *waf.Server,
t *testing.T, change func(*waf.Params),
geojsURL, abuseIPDBCheckURL string, now func() time.Time,
env map[string]string,
) (*waf.Server, *output, *alerts.Queue) {
) (*wafAndParts, *output, *alerts.Queue) {
t.Helper()
settings := map[string]string{
@@ -518,8 +541,9 @@ func newWAFWith(
addCoreRuleSet(t, &params)
addCounters(&params)
addOutsideSources(&params, geojsURL, abuseIPDBCheckURL)
change(&params)
server := build(params)
server := &wafAndParts{Server: waf.New(params), Params: params}
return server, out, alertQueue
}
+8 -5
View File
@@ -67,11 +67,14 @@ func (r rateLimits) check(
}
}
// overAWholeRateLimit reports whether the heuristic rate_limits will refuse rq
// at now, whatever its client's limit percentages: the rate limits count it,
// and one more request from its client takes it over a whole rate limit. It
// counts nothing.
func (r rateLimits) overAWholeRateLimit(rq *request, now time.Time) bool {
// refusesWhateverTheLookupAnswers reports whether the heuristic rate_limits
// will refuse rq at now, whatever its client's limit percentages, which the
// lookup's answer can lower: the rate limits count it, and one more request
// from its client takes it over a whole rate limit. It counts nothing.
func (r rateLimits) refusesWhateverTheLookupAnswers(
rq *request,
now time.Time,
) bool {
return r.countedByRateLimits(rq) &&
r.limiter.OverARateLimit(
clientGroup(r.ipv6GroupPrefix, rq.client),
+1 -1
View File
@@ -54,7 +54,7 @@ func (r refusalLimit) check(
// What the client was sent, or in observe mode would have been.
status := rq.out.status
switch rq.line.WouldAction {
switch rq.wouldAction {
case requestlog.ActionBlockedByBlockRule,
requestlog.ActionBlockedByCoreRuleSet:
status = http.StatusForbidden
+1 -2
View File
@@ -12,7 +12,6 @@ import (
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
const refusalsLimitPerMinute = "SWW_WAF_REFUSALS_LIMIT_PER_MINUTE"
@@ -52,7 +51,7 @@ func (r refused) send(s *sender, from string) logLine {
// SWW_WAF_REFUSALS_LIMIT_PER_MINUTE at limit, and the settings in env.
func startForRefusalLimit(
t *testing.T, limit string, env map[string]string,
) (*sender, *clock, *waf.Server) {
) (*sender, *clock, *wafAndParts) {
t.Helper()
settings := map[string]string{
+3 -3
View File
@@ -47,9 +47,9 @@ func (n reputationHitNoter) noteHit(
Host: rq.site,
Client: rq.client,
Netblock: clientGroup(n.ipv6GroupPrefix, rq.client),
ASN: rq.line.ASN,
ASName: rq.line.ASName,
Country: rq.line.Country,
ASN: rq.lookupAnswer.ASN,
ASName: rq.lookupAnswer.ASName,
Country: rq.lookupAnswer.Country,
Reason: reason,
Detail: detail,
})
+11 -12
View File
@@ -18,7 +18,6 @@ import (
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
// The reputation settings.
@@ -139,13 +138,13 @@ func TestReputationRefusalNamesItsHeuristicAndWhatListedTheClient(
for _, tc := range []struct {
heuristic string
env map[string]string
load func(*testing.T, *waf.Server)
load func(*testing.T, *wafAndParts)
figures string
}{
{
"blocklists",
map[string]string{blocklistURLs: dropURL, blocklistAction: actionDeny},
func(t *testing.T, server *waf.Server) {
func(t *testing.T, server *wafAndParts) {
t.Helper()
loadLists(t, server, map[string][]string{dropURL: {client}})
},
@@ -157,7 +156,7 @@ func TestReputationRefusalNamesItsHeuristicAndWhatListedTheClient(
dnsblZones: dnsblZone, dnsblResolver: noResolver,
dnsblAbuseIPDBAction: actionDeny,
},
func(_ *testing.T, server *waf.Server) {
func(_ *testing.T, server *wafAndParts) {
loadVerdicts(server, map[string][]string{client: {dnsblZone}})
},
`{"zones": ["` + dnsblZone + `"]}`,
@@ -165,7 +164,7 @@ func TestReputationRefusalNamesItsHeuristicAndWhatListedTheClient(
{
"abuseipdb",
map[string]string{abuseIPDBKey: accountKey, dnsblAbuseIPDBAction: actionDeny},
func(_ *testing.T, server *waf.Server) {
func(_ *testing.T, server *wafAndParts) {
loadScores(server, map[string]int64{client: 80})
},
`{"score": 80, "min_score": 75}`,
@@ -560,7 +559,7 @@ func TestReputationSourcesTagTheRequestWhateverTheirAction(t *testing.T) {
for _, tc := range []struct {
name string
env map[string]string
load func(*waf.Server)
load func(*wafAndParts)
want []string
}{
{
@@ -568,7 +567,7 @@ func TestReputationSourcesTagTheRequestWhateverTheirAction(t *testing.T) {
map[string]string{
dnsblZones: dnsblZone, dnsblResolver: noResolver, dnsblAbuseIPDBAction: actionLog,
},
func(server *waf.Server) {
func(server *wafAndParts) {
loadVerdicts(server, map[string][]string{fromDE: {dnsblZone}})
},
[]string{"asn=64496", "country=DE", "dnsbl=" + dnsblZone},
@@ -576,7 +575,7 @@ func TestReputationSourcesTagTheRequestWhateverTheirAction(t *testing.T) {
{
abuseipdb,
map[string]string{abuseIPDBKey: accountKey, dnsblAbuseIPDBAction: actionLog},
func(server *waf.Server) {
func(server *wafAndParts) {
loadScores(server, map[string]int64{fromDE: 80})
},
[]string{abuseipdb, "asn=64496", "country=DE"},
@@ -1256,7 +1255,7 @@ func verdictsFetched() time.Time {
// verdictsFetched, as dnsbl.json would at start: one that lists the
// client from each zone listedBy gives for it, and one that does not from
// each other zone.
func loadVerdicts(server *waf.Server, listedBy map[string][]string) {
func loadVerdicts(server *wafAndParts, listedBy map[string][]string) {
verdicts := make(
[]reputation.Verdict,
0,
@@ -1281,7 +1280,7 @@ func loadVerdicts(server *waf.Server, listedBy map[string][]string) {
// loadScores puts into server's AbuseIPDB the score scores gives each
// client, an IPv4 address, fetched at verdictsFetched, as
// abuseipdb-scores.json would at start.
func loadScores(server *waf.Server, scores map[string]int64) {
func loadScores(server *wafAndParts, scores map[string]int64) {
kept := make([]reputation.Score, 0, len(scores))
for client, score := range scores {
kept = append(kept, reputation.Score{
@@ -1295,7 +1294,7 @@ func loadScores(server *waf.Server, scores map[string]int64) {
// wantAbuseIPDBChecks checks how many clients server has checked with
// AbuseIPDB.
func wantAbuseIPDBChecks(t *testing.T, server *waf.Server, want int) {
func wantAbuseIPDBChecks(t *testing.T, server *wafAndParts, want int) {
t.Helper()
if got := server.AbuseIPDB.Checked(); got != want {
@@ -1307,7 +1306,7 @@ func wantAbuseIPDBChecks(t *testing.T, server *waf.Server, want int) {
// its lines, fetched at listsFetched, as lists.json would at start.
func loadLists(
t *testing.T,
server *waf.Server,
server *wafAndParts,
copies map[string][]string,
) {
t.Helper()
+42 -20
View File
@@ -24,42 +24,64 @@ type refusal struct {
action string
}
// decision is what the checks note of a request: its log line, and what
// counting the request needs once it has ended. waf keeps a copy of it
// until the request's report comes. It holds no credentials, since the log line
// never does, and neither the request's objects nor their bodies.
// decision is what the checks note of a request: its log line, what the
// heuristics tell each other, and what counting the request needs once it has
// ended. Nothing reads the log line back to decide. waf keeps a copy of it
// until the request's report comes. It holds no credentials, since the log
// line never does, and neither the request's objects nor their bodies.
type decision struct {
line requestlog.Line
client netip.Addr
// requestURI is the path with the query, as a ban's notes keep it.
requestURI string
// method and userAgent are the request's method and User-Agent, and
// requestURI its path with the query, as a ban's notes keep them.
// startRequest sets them.
method, userAgent, requestURI string
// tags are the request's tags, sorted, each once. addTag adds them, for
// tagLists, routeWatchTags, noteLookup, dnsbl, abuseIPDB, ruleFiles and
// coreRuleSet.
tags []string
// site is, for a question, the host the visitor asked for, which the
// alerts name, since one waf serves many sites.
// alerts and a ban's notes name, since one waf serves many sites.
// answerQuestion sets it.
site string
// allowed is true for a question answered with DispositionAllow.
allowed bool
// lookedUp is true once the client's AS number and country have been
// looked up, whether or not an answer was there, and lookupAnswer is
// what the lookup gave then, the zero Answer while GeoJS had given none.
// noteLookup sets them.
lookedUp bool
lookupAnswer geojs.Answer
// skippedLookupWait is true for a request that did not wait for GeoJS and
// found no answer kept. SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT does not apply to it,
// so that not waiting never lowers its client's limits.
// so that not waiting never lowers its client's limits. noteLookup sets
// it.
skippedLookupWait bool
// counted is true for a request the rate limits counted, whose bytes
// the byte limits count once it has ended. rateLimitPercent and
// bytesLimitPercent are then its client's limit percentages for the rate
// limits and for the byte limits.
// limits and for the byte limits. rateLimits sets them.
counted bool
rateLimitPercent, bytesLimitPercent percentage
// attack is true for a request that matched a ban rule or asked for a
// trap path, blockedByBlockRule for one a block rule refused, and
// blockedByCoreRuleSet for one the Core Rule Set refused, each an offence
// its client's history counts.
// its client's history counts. trapPaths and ruleFiles set attack,
// ruleFiles blockedByBlockRule, and coreRuleSet blockedByCoreRuleSet.
attack, blockedByBlockRule, blockedByCoreRuleSet bool
// wouldAction is, in observe mode, the action enforce mode would have
// refused the request with, one of requestlog's Action constants, and ""
// otherwise. check sets it.
wouldAction string
// brokeLimit is true for a request that broke a limit, an offence its
// client's history counts. banForLimit sets it.
brokeLimit bool
// blocklisted is true once blocklists finds a blocklist that lists the
// client, dnsblListed once dnsbl finds a DNSBL zone's verdict that does,
// and abuseIPDBHit once abuseIPDB finds AbuseIPDB's score of it a hit.
blocklisted, dnsblListed, abuseIPDBHit bool
// reputation is the reputation sources that list the client, for the
// notes of a ban the request makes.
// notes of a ban the request makes. noteHit adds them.
reputation []bans.ReputationHit
}
@@ -72,11 +94,7 @@ type request struct {
in *http.Request
out *responseWriter
// blocklisted is true once a blocklist is found to list the client,
// dnsblListed once a DNSBL zone's verdict is, and abuseIPDBHit once
// AbuseIPDB's score of it is a hit.
blocklisted, dnsblListed, abuseIPDBHit bool
start time.Time
start time.Time
// checked is when the checks were done.
checked time.Time
// refused is what the request was refused with, nil if it was not.
@@ -106,6 +124,8 @@ func (h *handler) startRequest(
rq := &request{
decision: decision{
client: client,
method: r.Method,
userAgent: r.UserAgent(),
requestURI: r.URL.RequestURI(),
line: requestlog.Line{
Time: requestlog.FormatTime(start),
@@ -151,11 +171,12 @@ func (h *handler) startRequest(
}
// addTag adds tag to the request's tags, which stay sorted and hold each tag
// once.
// once, and to its log line.
func (rq *request) addTag(tag string) {
at, found := slices.BinarySearch(rq.line.Tags, tag)
at, found := slices.BinarySearch(rq.tags, tag)
if !found {
rq.line.Tags = slices.Insert(rq.line.Tags, at, tag)
rq.tags = slices.Insert(rq.tags, at, tag)
rq.line.Tags = rq.tags
}
}
@@ -196,6 +217,7 @@ func (rq *request) check(ctx context.Context) *refusal {
switch {
case rq.h.config.Observe:
rq.wouldAction = action
rq.line.WouldAction = action
return nil
@@ -371,7 +393,7 @@ func (rq *request) addToHistory() {
Status: rq.out.status,
RequestBytes: rq.requestBytes,
ResponseBytes: rq.out.bytes,
BrokeLimit: rq.line.Offence == requestlog.OffenceLimit,
BrokeLimit: rq.brokeLimit,
Attack: rq.attack,
BlockedByBlockRule: rq.blockedByBlockRule,
BlockedByCoreRuleSet: rq.blockedByCoreRuleSet,
+142
View File
@@ -0,0 +1,142 @@
package waf
import (
"crypto/subtle"
"io"
"net/http"
"strings"
"github.com/go-chi/chi/v5"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
// routes returns the router of waf's own endpoints. A question never reaches
// it, since ServeHTTP answers a question first. No check runs on a request
// here: waf's address is on the private network, where the peers are rproxy
// and the admin's tools, not visitors.
func (h *handler) routes() http.Handler {
r := chi.NewRouter()
r.Use(matchDecodedPath)
// The report writes no log line of its own: it finishes the line of the
// request it reports.
r.Post(wafprotocol.ReportPath, h.handleReport())
r.Group(func(r chi.Router) {
r.Use(h.logOwnEndpoint)
// The answer to a path with no endpoint, and to a method the path has
// no endpoint for, is set in this group, so that it is logged too.
r.NotFound(answerNotFound)
r.MethodNotAllowed(answerNotFound)
r.Get(wafprotocol.HealthPath, handleHealth())
r.Group(func(r chi.Router) {
r.Use(requireToken(h.config.MetricsToken))
r.Get(MetricsPath, h.metrics.ServeHTTP)
})
// The handlers under BansPath/ and ClientsPath read the address from
// the path themselves, so that one that is not an address gets 400
// after the token check.
r.Group(func(r chi.Router) {
r.Use(requireToken(h.config.AdminToken))
r.Get(BansPath, h.handleListBans())
r.Post(BansPath, h.handleAddBan())
r.Delete(BansPath+"/*", h.handleLiftBans())
r.Get(ClientsPath+"*", h.handleShowClient())
})
})
return r
}
// matchDecodedPath has the router match the decoded path, which the handlers
// read. chi matches the path as sent when it is encoded in an unusual way,
// which would make /_sww/%68ealthz a path with no endpoint.
func matchDecodedPath(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
chi.RouteContext(r.Context()).RoutePath = r.URL.Path
next.ServeHTTP(w, r)
})
}
// logOwnEndpoint writes one log line for each request, with the action
// own_endpoint. The line's request_bytes is how much of the body the handler
// read.
func (h *handler) logOwnEndpoint(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
peer := peerAddress(r)
rq := h.startRequest(w, r, peer, peer, false)
defer rq.finish()
rq.line.Action = requestlog.ActionOwnEndpoint
counted := r.Clone(r.Context())
counted.Body = &countedBody{
ReadCloser: r.Body,
read: &rq.requestBytes,
}
next.ServeHTTP(rq.out, counted)
})
}
// countedBody is a request's body that adds the bytes read of it to read.
type countedBody struct {
io.ReadCloser
read *int64
}
// Read reads part of the body.
func (b *countedBody) Read(p []byte) (int, error) {
n, err := b.ReadCloser.Read(p)
*b.read += int64(n)
return n, err
}
// requireToken returns middleware that lets a request through only when it
// carries token, as Authorization: Bearer <token>. It answers 401 to a request
// without it. While token is unset, it answers 404, as a path with no endpoint
// does.
func requireToken(token string) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
switch {
case token == "":
answerNotFound(w, r)
case !hasToken(r, token):
w.Header().Set("WWW-Authenticate", "Bearer")
http.Error(
w,
http.StatusText(http.StatusUnauthorized),
http.StatusUnauthorized,
)
default:
next.ServeHTTP(w, r)
}
},
)
}
}
// hasToken reports whether r carries token, as Authorization: Bearer
// <token>.
func hasToken(r *http.Request, token string) bool {
scheme, sent, _ := strings.Cut(r.Header.Get("Authorization"), " ")
return strings.EqualFold(scheme, "Bearer") &&
subtle.ConstantTimeCompare([]byte(sent), []byte(token)) == 1
}
// answerNotFound answers 404, for a path with no endpoint, and for a method
// the path has no endpoint for.
func answerNotFound(w http.ResponseWriter, _ *http.Request) {
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
}
+1 -2
View File
@@ -11,7 +11,6 @@ import (
"time"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
// routeWatchURL is SWW_WAF_ROUTEWATCH_URL, and tagsMissing the log line's field
@@ -58,7 +57,7 @@ func startRouteWatch(t *testing.T, routes map[string]standInRoute) string {
// waitForRouteWatch waits until routewatch's answer about from, in the client
// group of the given length, is kept, so that the next request gets its tags.
func waitForRouteWatch(
t *testing.T, server *waf.Server, from string, bits int,
t *testing.T, server *wafAndParts, from string, bits int,
) {
t.Helper()
+1 -1
View File
@@ -23,7 +23,7 @@ func (t tagDeny) check(
rq *request,
_ time.Time,
) *verdict {
for _, tag := range rq.line.Tags {
for _, tag := range rq.tags {
if slices.Contains(t.tags, tag) {
return &verdict{
action: requestlog.ActionClientDenied,
+37 -66
View File
@@ -30,12 +30,10 @@ const (
// is far above rproxy's, and a question refused for its size would have rproxy
// let the request through.
func New(params Params) *Server {
h := newHandler(params)
return &Server{
Server: &http.Server{
Addr: params.Config.ListenAddr,
Handler: h,
Handler: newHandler(params),
ErrorLog: slog.NewLogLogger(
params.ProcessLog.Handler(),
slog.LevelWarn,
@@ -44,22 +42,12 @@ func New(params Params) *Server {
WriteTimeout: wafTimeout,
IdleTimeout: wafIdleTimeout,
},
Ledger: h.ledger,
Limiter: h.limiter,
GeoJS: h.geojs,
Anomalies: h.anomalies,
LookupFile: h.lookupFile,
Lists: params.Lists,
DNSBL: params.DNSBL,
AbuseIPDB: params.AbuseIPDB,
RouteWatch: params.RouteWatch,
Metrics: h.metrics,
}
}
// ServeHTTP handles one request to waf: a request carrying
// wafprotocol.ClientHeader is rproxy's question, and any other is for waf
// itself.
// wafprotocol.ClientHeader is rproxy's question, and any other is for one of
// waf's own endpoints.
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if len(r.Header.Values(wafprotocol.ClientHeader)) > 0 {
h.answerQuestion(w, r)
@@ -67,23 +55,27 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
h.answerOwnEndpoint(w, r)
h.ownEndpoints.ServeHTTP(w, r)
}
// refusedWhateverTheAnswer reports whether the request is one that does not wait
// for GeoJS, since the checks will refuse it at now whatever GeoJS answers about
// its client: the heuristic blocklists or rate_limits will refuse it, as a
// function in that heuristic's file says. In observe mode, which refuses
// nothing, none is. rproxy lets a request through when waf answers too late.
// its client: a heuristic of afterLookup that implements refusesWithoutLookup
// says it will refuse it. In observe mode, which refuses nothing, none is.
// rproxy lets a request through when waf answers too late.
func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
if rq.h.config.Observe {
return false
}
listedBy := rq.h.blocklists.lists.ListedBy(rq.client)
for _, each := range rq.h.afterLookup {
knows, ok := each.(refusesWithoutLookup)
if ok && knows.refusesWhateverTheLookupAnswers(rq, now) {
return true
}
}
return rq.h.blocklists.refusedByBlocklists(listedBy) ||
rq.h.rateLimits.overAWholeRateLimit(rq, now)
return false
}
// answerQuestion runs the checks on the visitor's request a question
@@ -202,8 +194,8 @@ func (rq *request) allow() {
answer := wafprotocol.Answer{Disposition: wafprotocol.DispositionAllow}
if rq.h.config.AddLookupHeaders {
answer.ASN = rq.line.ASN
answer.Country = rq.line.Country
answer.ASN = rq.lookupAnswer.ASN
answer.Country = rq.lookupAnswer.Country
}
wafprotocol.SetAnswerHeaders(rq.out.Header(), answer)
@@ -240,50 +232,29 @@ func (rq *request) deny(r refusal) {
http.Error(rq.out, http.StatusText(r.status), r.status)
}
// answerOwnEndpoint answers a request to waf's own address. No check
// runs on it: the address is on the private network, where the peers are
// rproxy and the admin's tools, not visitors.
func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPost && r.URL.Path == wafprotocol.ReportPath {
h.takeReport(w, r)
// handleReport answers POST wafprotocol.ReportPath: it finishes the request
// the report names, if it holds that request's decision, and ignores the
// report otherwise.
func (h *handler) handleReport() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
report, err := wafprotocol.ReadReport(w, r)
if err != nil {
h.processLog.Warn("a report is refused", "error", err.Error())
http.Error(w, "the body is not a report", http.StatusBadRequest)
return
return
}
held, found := h.awaitingReport.Peek(report.RequestID)
// Remove reports true to one report only, of several that name the
// id. A question still being handled is held as nil.
if found && held != nil &&
h.awaitingReport.Remove(report.RequestID) {
h.finishWith(r.Context(), held, report)
}
w.WriteHeader(http.StatusNoContent)
}
peer := peerAddress(r)
rq := h.startRequest(w, r, peer, peer, false)
defer rq.finish()
if r.Method == http.MethodGet && r.URL.Path == wafprotocol.HealthPath {
rq.answerHealth()
return
}
rq.answerAdmin()
}
// takeReport answers POST wafprotocol.ReportPath: it finishes the request the
// report names, if it holds that request's decision, and ignores the report
// otherwise.
func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) {
report, err := wafprotocol.ReadReport(w, r)
if err != nil {
h.processLog.Warn("a report is refused", "error", err.Error())
http.Error(w, "the body is not a report", http.StatusBadRequest)
return
}
held, found := h.awaitingReport.Peek(report.RequestID)
// Remove reports true to one report only, of several that name the id. A
// question still being handled is held as nil.
if found && held != nil && h.awaitingReport.Remove(report.RequestID) {
h.finishWith(r.Context(), held, report)
}
w.WriteHeader(http.StatusNoContent)
}
// finishWith finishes a request with what its report says, and writes the
+4 -4
View File
@@ -58,7 +58,7 @@ type WAF struct {
t *testing.T
addr string
out *output
server *waf.Server
server *wafAndParts
clock *clock
queue *alerts.Queue
}
@@ -70,14 +70,14 @@ func startWAF(
) *WAF {
t.Helper()
return startWAFWith(t, geojsURL, env, func(*waf.Server) {})
return startWAFWith(t, geojsURL, env, func(*wafAndParts) {})
}
// startWAFWith is startWAF with change applied to the server before
// it starts.
func startWAFWith(
t *testing.T, geojsURL string, env map[string]string,
change func(*waf.Server),
change func(*wafAndParts),
) *WAF {
t.Helper()
@@ -703,7 +703,7 @@ func TestQuestionWhoseBodyStopsArrivingGetsNoAnswer(t *testing.T) {
t,
"",
map[string]string{metricsToken: token},
func(server *waf.Server) { server.ReadTimeout = readTimeout },
func(server *wafAndParts) { server.ReadTimeout = readTimeout },
)
conn := dial(t, s.addr)