Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f0ac70618b |
No files matched your search
+162
-30
@@ -12,6 +12,7 @@ import (
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
@@ -81,9 +82,17 @@ type request struct {
|
||||
// upgraded is the connection to the app once the app has switched protocols,
|
||||
// as for a WebSocket, and nil otherwise.
|
||||
upgraded *upgradedConn
|
||||
// limits are the request's SWW_RPROXY_MAX_REQUEST_TIME_MS and
|
||||
// SWW_RPROXY_MAX_RESPONSE_TIME_MS.
|
||||
limits *timeLimits
|
||||
|
||||
// mu guards what follows. The time limits run on goroutines of their own,
|
||||
// and the transport starts and stops them, and notes requestEnd, from its
|
||||
// own; once timersStopped is set, neither time limit acts any more.
|
||||
mu sync.Mutex
|
||||
timersStopped bool
|
||||
requestTimer *time.Timer
|
||||
responseTimer *time.Timer
|
||||
// requestEnd is when the app had been sent the whole request, or the answer
|
||||
// started, if that was sooner.
|
||||
requestEnd time.Time
|
||||
}
|
||||
|
||||
// newRequest starts handling r: it notes the time, works out the visitor from
|
||||
@@ -118,25 +127,12 @@ func (h *handler) newRequest(
|
||||
}
|
||||
}
|
||||
|
||||
rq.limits = &timeLimits{
|
||||
maxRequestTime: h.config.MaxRequestTime,
|
||||
maxResponseTime: h.config.MaxResponseTime,
|
||||
start: rq.start,
|
||||
refuse: rq.refuse,
|
||||
waitingForVisitor: func() bool {
|
||||
return rq.body != nil && rq.body.waiting.Load()
|
||||
},
|
||||
endVisitorRead: func() {
|
||||
_ = rq.rc.SetReadDeadline(time.Now())
|
||||
},
|
||||
}
|
||||
|
||||
return rq
|
||||
}
|
||||
|
||||
// answerHealth answers the health check.
|
||||
func (rq *request) answerHealth() {
|
||||
rq.prepareOwnAnswer(rq.limits.requestDeadline())
|
||||
rq.prepareOwnAnswer(rq.requestDeadline())
|
||||
rq.out.Header().Set("Content-Type", plainText)
|
||||
rq.send(http.StatusOK, []byte("ok\n"))
|
||||
}
|
||||
@@ -152,7 +148,7 @@ func (rq *request) readFirstPartOfBody() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
_ = rq.rc.SetReadDeadline(rq.limits.requestDeadline())
|
||||
_ = rq.rc.SetReadDeadline(rq.requestDeadline())
|
||||
held, err := io.ReadAll(
|
||||
io.LimitReader(rq.body, wafprotocol.QuestionBodyMaxBytes),
|
||||
)
|
||||
@@ -192,7 +188,7 @@ func (rq *request) forward(ctx context.Context) {
|
||||
|
||||
rq.cancel = cancel
|
||||
ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{
|
||||
WroteRequest: rq.limits.wroteRequest,
|
||||
WroteRequest: rq.wroteRequest,
|
||||
})
|
||||
|
||||
out := rq.in.WithContext(ctx)
|
||||
@@ -209,7 +205,7 @@ func (rq *request) forward(ctx context.Context) {
|
||||
ErrorHandler: rq.answerError,
|
||||
}
|
||||
|
||||
rq.limits.startRequestTimer()
|
||||
rq.startRequestTimer()
|
||||
rq.upstreamStart = time.Now()
|
||||
reverseProxy.ServeHTTP(rq.out, out)
|
||||
}
|
||||
@@ -252,7 +248,7 @@ func (rq *request) modifyResponse(res *http.Response) error {
|
||||
// switch, as when the app's Upgrade is not the visitor's, so the status
|
||||
// is noted only once it has taken over the connection, by rq.out's
|
||||
// Hijack.
|
||||
rq.limits.stop()
|
||||
rq.stopTimers()
|
||||
|
||||
conn, ok := res.Body.(io.ReadWriteCloser)
|
||||
if ok {
|
||||
@@ -263,7 +259,7 @@ func (rq *request) modifyResponse(res *http.Response) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
requestEnd := rq.limits.endRequest()
|
||||
requestEnd := rq.endRequest()
|
||||
|
||||
// An answer that arrives once SWW_RPROXY_MAX_RESPONSE_TIME_MS has run out is not
|
||||
// passed on: the visitor is answered 504.
|
||||
@@ -293,12 +289,12 @@ func (rq *request) modifyResponse(res *http.Response) error {
|
||||
|
||||
// An answer that arrives before the app has been sent the whole request ends
|
||||
// the request, and SWW_RPROXY_MAX_RESPONSE_TIME_MS starts.
|
||||
rq.limits.startResponseTimer(requestEnd)
|
||||
rq.startResponseTimer(requestEnd)
|
||||
|
||||
res.Body = &responseBody{
|
||||
body: limitBody(res.Body, maxBytes),
|
||||
refuse: rq.refuse,
|
||||
responseReceived: rq.limits.stop,
|
||||
responseReceived: rq.responseReceived,
|
||||
ctx: rq.in.Context(),
|
||||
}
|
||||
rq.out.afterStatus = rq.setResponseDeadline
|
||||
@@ -351,7 +347,7 @@ func (rq *request) answer(status int) {
|
||||
// A visitor found too slow is read no more; any other may go on sending until
|
||||
// its time is up, so that the rest of the body is read and the request ends
|
||||
// cleanly: by Go's server, or by readRestOfBody.
|
||||
deadline := rq.limits.requestDeadline()
|
||||
deadline := rq.requestDeadline()
|
||||
if status == http.StatusRequestTimeout {
|
||||
deadline = time.Now()
|
||||
}
|
||||
@@ -397,7 +393,7 @@ func (rq *request) closeAfterAnswer() {
|
||||
// used that time up first, and at least minSendTime.
|
||||
func (rq *request) prepareOwnAnswer(deadline time.Time) {
|
||||
rq.stopReadingBody(deadline)
|
||||
rq.limits.endRequest()
|
||||
rq.endRequest()
|
||||
|
||||
limit := rq.h.config.MaxResponseTime
|
||||
if limit > 0 {
|
||||
@@ -418,7 +414,7 @@ func (rq *request) refuse(status int) {
|
||||
// finish ends the request's time limits, writes its log line and, if waf
|
||||
// answered the question, sends the report. ctx is the request's own context.
|
||||
func (rq *request) finish(ctx context.Context) {
|
||||
rq.limits.stop()
|
||||
rq.stopTimers()
|
||||
|
||||
// After a cut-off answer, the reading of a body that has not arrived whole
|
||||
// ends now. Go's server would otherwise keep the connection open while it
|
||||
@@ -436,10 +432,10 @@ func (rq *request) finish(ctx context.Context) {
|
||||
errors.Is(rq.rc.Flush(), os.ErrDeadlineExceeded):
|
||||
rq.stopPassingOnBody(time.Now())
|
||||
case rq.refused.Load() == 0:
|
||||
rq.stopPassingOnBody(rq.limits.requestDeadline())
|
||||
rq.stopPassingOnBody(rq.requestDeadline())
|
||||
}
|
||||
|
||||
requestEnd := rq.limits.endRequest()
|
||||
requestEnd := rq.endRequest()
|
||||
end := time.Now()
|
||||
requestBytes, responseBytes := rq.bytesEachWay()
|
||||
|
||||
@@ -513,6 +509,17 @@ func (rq *request) bytesEachWay() (int64, int64) {
|
||||
return requestBytes, responseBytes
|
||||
}
|
||||
|
||||
// requestDeadline is when the visitor must have sent its whole request, or zero
|
||||
// when SWW_RPROXY_MAX_REQUEST_TIME_MS is off.
|
||||
func (rq *request) requestDeadline() time.Time {
|
||||
limit := rq.h.config.MaxRequestTime
|
||||
if limit == 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
return rq.start.Add(limit)
|
||||
}
|
||||
|
||||
// stopReadingBody ends, at deadline, the reading of a visitor's body that has not
|
||||
// arrived whole: Go's server then reads no more of it, and closes the connection
|
||||
// after the answer.
|
||||
@@ -569,6 +576,100 @@ func (rq *request) stopPassingOnBody(deadline time.Time) {
|
||||
rq.stopReadingBody(deadline)
|
||||
}
|
||||
|
||||
// startRequestTimer starts SWW_RPROXY_MAX_REQUEST_TIME_MS, which runs from the
|
||||
// request's headers being read until the app has been sent the whole request.
|
||||
func (rq *request) startRequestTimer() {
|
||||
deadline := rq.requestDeadline()
|
||||
if deadline.IsZero() {
|
||||
return
|
||||
}
|
||||
|
||||
rq.mu.Lock()
|
||||
defer rq.mu.Unlock()
|
||||
|
||||
rq.requestTimer = time.AfterFunc(
|
||||
time.Until(deadline),
|
||||
rq.requestTimedOut,
|
||||
)
|
||||
}
|
||||
|
||||
// requestTimedOut is called when SWW_RPROXY_MAX_REQUEST_TIME_MS runs out while the
|
||||
// request is still on its way to the app. The answer names the side rproxy was
|
||||
// waiting on at that moment: 408 when it was waiting for the visitor to send more
|
||||
// of its body, 504 when it was waiting for the app to be reached or to take what
|
||||
// it had.
|
||||
func (rq *request) requestTimedOut() {
|
||||
rq.mu.Lock()
|
||||
defer rq.mu.Unlock()
|
||||
|
||||
if rq.timersStopped {
|
||||
return
|
||||
}
|
||||
|
||||
if rq.body == nil || !rq.body.waiting.Load() {
|
||||
rq.refuse(http.StatusGatewayTimeout)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
rq.refuse(http.StatusRequestTimeout)
|
||||
// The transport gives up on the app only once its Read of the visitor's body
|
||||
// returns, so that Read is ended now. The lock keeps this from reaching the
|
||||
// connection after the request is handled.
|
||||
_ = rq.rc.SetReadDeadline(time.Now())
|
||||
}
|
||||
|
||||
// wroteRequest is called once the app has been sent the whole request:
|
||||
// SWW_RPROXY_MAX_REQUEST_TIME_MS ends and SWW_RPROXY_MAX_RESPONSE_TIME_MS starts.
|
||||
func (rq *request) wroteRequest(info httptrace.WroteRequestInfo) {
|
||||
if info.Err != nil {
|
||||
return // the transport gives up, or tries again
|
||||
}
|
||||
|
||||
requestEnd := rq.endRequest()
|
||||
|
||||
rq.mu.Lock()
|
||||
stopTimer(rq.requestTimer)
|
||||
rq.mu.Unlock()
|
||||
|
||||
rq.startResponseTimer(requestEnd)
|
||||
}
|
||||
|
||||
// startResponseTimer starts SWW_RPROXY_MAX_RESPONSE_TIME_MS, which runs from
|
||||
// requestEnd, unless it has started already.
|
||||
func (rq *request) startResponseTimer(requestEnd time.Time) {
|
||||
limit := rq.h.config.MaxResponseTime
|
||||
if limit == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
rq.mu.Lock()
|
||||
defer rq.mu.Unlock()
|
||||
|
||||
if rq.timersStopped || rq.responseTimer != nil {
|
||||
return
|
||||
}
|
||||
|
||||
rq.responseTimer = time.AfterFunc(time.Until(requestEnd.Add(limit)),
|
||||
rq.responseTimedOut)
|
||||
}
|
||||
|
||||
// responseTimedOut is called when SWW_RPROXY_MAX_RESPONSE_TIME_MS runs out before the
|
||||
// app has sent its whole answer.
|
||||
func (rq *request) responseTimedOut() {
|
||||
rq.mu.Lock()
|
||||
defer rq.mu.Unlock()
|
||||
|
||||
if !rq.timersStopped {
|
||||
rq.refuse(http.StatusGatewayTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
// responseReceived is called once the app has sent its whole answer.
|
||||
func (rq *request) responseReceived() {
|
||||
rq.stopTimers()
|
||||
}
|
||||
|
||||
// setResponseDeadline sends the status and headers of the app's answer, with
|
||||
// at least minSendTime, before ReverseProxy reads the answer's body, and then
|
||||
// the rest has until SWW_RPROXY_MAX_RESPONSE_TIME_MS after the end of the request.
|
||||
@@ -578,7 +679,7 @@ func (rq *request) setResponseDeadline() {
|
||||
return
|
||||
}
|
||||
|
||||
deadline := rq.limits.endRequest().Add(limit)
|
||||
deadline := rq.endRequest().Add(limit)
|
||||
|
||||
_ = rq.rc.SetWriteDeadline(
|
||||
time.Now().Add(max(time.Until(deadline), minSendTime)),
|
||||
@@ -586,3 +687,34 @@ func (rq *request) setResponseDeadline() {
|
||||
_ = rq.rc.Flush()
|
||||
_ = rq.rc.SetWriteDeadline(deadline)
|
||||
}
|
||||
|
||||
// endRequest notes now as the end of the request, unless an end is noted, and
|
||||
// returns the end noted.
|
||||
func (rq *request) endRequest() time.Time {
|
||||
rq.mu.Lock()
|
||||
defer rq.mu.Unlock()
|
||||
|
||||
if rq.requestEnd.IsZero() {
|
||||
rq.requestEnd = time.Now()
|
||||
}
|
||||
|
||||
return rq.requestEnd
|
||||
}
|
||||
|
||||
// stopTimers stops the request's time limits and keeps any from starting later:
|
||||
// the app's answer is complete, the connection upgraded, or the request handled.
|
||||
func (rq *request) stopTimers() {
|
||||
rq.mu.Lock()
|
||||
defer rq.mu.Unlock()
|
||||
|
||||
rq.timersStopped = true
|
||||
stopTimer(rq.requestTimer)
|
||||
stopTimer(rq.responseTimer)
|
||||
}
|
||||
|
||||
// stopTimer stops t, which is nil when its time limit is off.
|
||||
func stopTimer(t *time.Timer) {
|
||||
if t != nil {
|
||||
t.Stop()
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
@@ -111,7 +112,11 @@ func New(params Params) *Server {
|
||||
// which SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS bounds. Any report after this call
|
||||
// is dropped: that of a WebSocket that closes while rproxy stops, for one.
|
||||
func (s *Server) WaitForReports() {
|
||||
s.h.reports.stopAndWait()
|
||||
s.h.reportsMu.Lock()
|
||||
s.h.reportsStopped = true
|
||||
s.h.reportsMu.Unlock()
|
||||
|
||||
s.h.reports.Wait()
|
||||
}
|
||||
|
||||
// handler is the proxy. It holds what every request shares; what belongs to
|
||||
@@ -131,8 +136,11 @@ type handler struct {
|
||||
// the time.
|
||||
breaker breaker
|
||||
breakerClock func() time.Time
|
||||
// reports are the goroutines sending reports to waf.
|
||||
reports reportGoroutines
|
||||
// reports waits for the reports being sent. reportsMu guards starting one
|
||||
// against reportsStopped, which is set once nothing is to wait for another.
|
||||
reports sync.WaitGroup
|
||||
reportsMu sync.Mutex
|
||||
reportsStopped bool
|
||||
}
|
||||
|
||||
// newTransport returns what carries requests to the app. It never goes through
|
||||
|
||||
@@ -1,165 +0,0 @@
|
||||
package rproxy
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptrace"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// timeLimits are one request's two time limits: SWW_RPROXY_MAX_REQUEST_TIME_MS,
|
||||
// which runs from the request's headers being read until the app has been sent
|
||||
// the whole request, and SWW_RPROXY_MAX_RESPONSE_TIME_MS, which runs from then
|
||||
// until the app has sent its whole answer. A limit of zero is off. Each limit
|
||||
// runs on a goroutine of its own, and the transport starts and stops them from
|
||||
// its own.
|
||||
type timeLimits struct {
|
||||
maxRequestTime time.Duration
|
||||
maxResponseTime time.Duration
|
||||
// start is when the request's headers had been read.
|
||||
start time.Time
|
||||
// refuse refuses the request with 408 or 504 when a limit runs out.
|
||||
refuse func(status int)
|
||||
// waitingForVisitor reports whether a read of the visitor's body is waiting
|
||||
// for the visitor to send more.
|
||||
waitingForVisitor func() bool
|
||||
// endVisitorRead ends at once a read of the visitor's body that is waiting.
|
||||
endVisitorRead func()
|
||||
|
||||
// mu guards what follows. Once stopped is set, neither limit acts any more.
|
||||
mu sync.Mutex
|
||||
stopped bool
|
||||
requestTimer *time.Timer
|
||||
responseTimer *time.Timer
|
||||
// requestEnd is when the app had been sent the whole request, or the answer
|
||||
// started, if that was sooner.
|
||||
requestEnd time.Time
|
||||
}
|
||||
|
||||
// requestDeadline is when the visitor must have sent its whole request, or zero
|
||||
// when SWW_RPROXY_MAX_REQUEST_TIME_MS is off.
|
||||
func (l *timeLimits) requestDeadline() time.Time {
|
||||
if l.maxRequestTime == 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
return l.start.Add(l.maxRequestTime)
|
||||
}
|
||||
|
||||
// startRequestTimer starts SWW_RPROXY_MAX_REQUEST_TIME_MS.
|
||||
func (l *timeLimits) startRequestTimer() {
|
||||
deadline := l.requestDeadline()
|
||||
if deadline.IsZero() {
|
||||
return
|
||||
}
|
||||
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
l.requestTimer = time.AfterFunc(time.Until(deadline), l.requestTimedOut)
|
||||
}
|
||||
|
||||
// requestTimedOut is called when SWW_RPROXY_MAX_REQUEST_TIME_MS runs out while
|
||||
// the request is still on its way to the app. The answer names the side rproxy
|
||||
// was waiting on at that moment: 408 when it was waiting for the visitor to send
|
||||
// more of its body, 504 when it was waiting for the app to be reached or to take
|
||||
// what it had.
|
||||
func (l *timeLimits) requestTimedOut() {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
if l.stopped {
|
||||
return
|
||||
}
|
||||
|
||||
if !l.waitingForVisitor() {
|
||||
l.refuse(http.StatusGatewayTimeout)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
l.refuse(http.StatusRequestTimeout)
|
||||
// The transport gives up on the app only once its Read of the visitor's body
|
||||
// returns, so that Read is ended now. The lock keeps this from reaching the
|
||||
// connection after the request is handled.
|
||||
l.endVisitorRead()
|
||||
}
|
||||
|
||||
// wroteRequest is called once the app has been sent the whole request:
|
||||
// SWW_RPROXY_MAX_REQUEST_TIME_MS ends and SWW_RPROXY_MAX_RESPONSE_TIME_MS
|
||||
// starts.
|
||||
func (l *timeLimits) wroteRequest(info httptrace.WroteRequestInfo) {
|
||||
if info.Err != nil {
|
||||
return // the transport gives up, or tries again
|
||||
}
|
||||
|
||||
requestEnd := l.endRequest()
|
||||
|
||||
l.mu.Lock()
|
||||
stopTimer(l.requestTimer)
|
||||
l.mu.Unlock()
|
||||
|
||||
l.startResponseTimer(requestEnd)
|
||||
}
|
||||
|
||||
// startResponseTimer starts SWW_RPROXY_MAX_RESPONSE_TIME_MS, which runs from
|
||||
// requestEnd, unless it has started already.
|
||||
func (l *timeLimits) startResponseTimer(requestEnd time.Time) {
|
||||
if l.maxResponseTime == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
if l.stopped || l.responseTimer != nil {
|
||||
return
|
||||
}
|
||||
|
||||
l.responseTimer = time.AfterFunc(
|
||||
time.Until(requestEnd.Add(l.maxResponseTime)),
|
||||
l.responseTimedOut,
|
||||
)
|
||||
}
|
||||
|
||||
// responseTimedOut is called when SWW_RPROXY_MAX_RESPONSE_TIME_MS runs out
|
||||
// before the app has sent its whole answer.
|
||||
func (l *timeLimits) responseTimedOut() {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
if !l.stopped {
|
||||
l.refuse(http.StatusGatewayTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
// endRequest notes now as the end of the request, unless an end is noted, and
|
||||
// returns the end noted.
|
||||
func (l *timeLimits) endRequest() time.Time {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
if l.requestEnd.IsZero() {
|
||||
l.requestEnd = time.Now()
|
||||
}
|
||||
|
||||
return l.requestEnd
|
||||
}
|
||||
|
||||
// stop stops both limits and keeps either from starting later: the app's
|
||||
// answer is complete, the connection upgraded, or the request handled.
|
||||
func (l *timeLimits) stop() {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
|
||||
l.stopped = true
|
||||
stopTimer(l.requestTimer)
|
||||
stopTimer(l.responseTimer)
|
||||
}
|
||||
|
||||
// stopTimer stops t, which is nil when its limit is off.
|
||||
func stopTimer(t *time.Timer) {
|
||||
if t != nil {
|
||||
t.Stop()
|
||||
}
|
||||
}
|
||||
@@ -1,171 +0,0 @@
|
||||
package rproxy
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptrace"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The tests below run timeLimits in a synctest bubble, where time is a clock of
|
||||
// the test's own: time.Sleep moves it on at once, and synctest.Wait returns once
|
||||
// a limit that ran out has refused the request.
|
||||
|
||||
const (
|
||||
testRequestTime = 3 * time.Second
|
||||
testResponseTime = 5 * time.Second
|
||||
)
|
||||
|
||||
// limitsSeen is what a timeLimits under test did to its request.
|
||||
type limitsSeen struct {
|
||||
// refused is the status of the last refusal, or zero.
|
||||
refused atomic.Int64
|
||||
// readEnded is true once the read of the visitor's body was ended.
|
||||
readEnded atomic.Bool
|
||||
// waiting is what waitingForVisitor reports.
|
||||
waiting atomic.Bool
|
||||
}
|
||||
|
||||
// newTestLimits returns time limits that started now, with what they do noted
|
||||
// in seen.
|
||||
func newTestLimits(seen *limitsSeen) *timeLimits {
|
||||
return &timeLimits{
|
||||
maxRequestTime: testRequestTime,
|
||||
maxResponseTime: testResponseTime,
|
||||
start: time.Now(),
|
||||
refuse: func(status int) {
|
||||
seen.refused.Store(int64(status))
|
||||
},
|
||||
waitingForVisitor: seen.waiting.Load,
|
||||
endVisitorRead: func() {
|
||||
seen.readEnded.Store(true)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// sleepThenWait moves the bubble's clock on by d and waits for what is then
|
||||
// due.
|
||||
func sleepThenWait(d time.Duration) {
|
||||
time.Sleep(d)
|
||||
synctest.Wait()
|
||||
}
|
||||
|
||||
func TestRequestTimeRunningOutWhileTheVisitorIsWaitedOnRefuses408(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
seen := &limitsSeen{}
|
||||
seen.waiting.Store(true)
|
||||
|
||||
limits := newTestLimits(seen)
|
||||
limits.startRequestTimer()
|
||||
|
||||
sleepThenWait(testRequestTime - time.Millisecond)
|
||||
|
||||
if got := seen.refused.Load(); got != 0 {
|
||||
t.Fatalf(
|
||||
"refused %d before the limit ran out, want nothing",
|
||||
got,
|
||||
)
|
||||
}
|
||||
|
||||
sleepThenWait(time.Millisecond)
|
||||
|
||||
if got := seen.refused.Load(); got != http.StatusRequestTimeout {
|
||||
t.Errorf("refused %d, want 408", got)
|
||||
}
|
||||
|
||||
if !seen.readEnded.Load() {
|
||||
t.Error("the read of the visitor's body goes on, want it ended")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRequestTimeRunningOutWhileTheAppIsWaitedOnRefuses504(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
seen := &limitsSeen{}
|
||||
|
||||
limits := newTestLimits(seen)
|
||||
limits.startRequestTimer()
|
||||
|
||||
sleepThenWait(testRequestTime)
|
||||
|
||||
if got := seen.refused.Load(); got != http.StatusGatewayTimeout {
|
||||
t.Errorf("refused %d, want 504", got)
|
||||
}
|
||||
|
||||
if seen.readEnded.Load() {
|
||||
t.Error("the read of the visitor's body was ended, want it not")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestResponseTimeRunsFromTheEndOfTheRequest(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
seen := &limitsSeen{}
|
||||
|
||||
limits := newTestLimits(seen)
|
||||
limits.startRequestTimer()
|
||||
|
||||
sleepThenWait(time.Second)
|
||||
|
||||
end := time.Now()
|
||||
|
||||
limits.wroteRequest(httptrace.WroteRequestInfo{})
|
||||
|
||||
if got := limits.endRequest(); !got.Equal(end) {
|
||||
t.Errorf("the request ended at %v, want %v", got, end)
|
||||
}
|
||||
|
||||
// Past SWW_RPROXY_MAX_REQUEST_TIME_MS, which the end of the request
|
||||
// stopped, and just short of SWW_RPROXY_MAX_RESPONSE_TIME_MS.
|
||||
sleepThenWait(testResponseTime - time.Millisecond)
|
||||
|
||||
if got := seen.refused.Load(); got != 0 {
|
||||
t.Fatalf(
|
||||
"refused %d before the limit ran out, want nothing",
|
||||
got,
|
||||
)
|
||||
}
|
||||
|
||||
sleepThenWait(time.Millisecond)
|
||||
|
||||
if got := seen.refused.Load(); got != http.StatusGatewayTimeout {
|
||||
t.Errorf("refused %d, want 504", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestStoppedLimitsRefuseNothing(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
seen := &limitsSeen{}
|
||||
seen.waiting.Store(true)
|
||||
|
||||
limits := newTestLimits(seen)
|
||||
limits.startRequestTimer()
|
||||
limits.stop()
|
||||
limits.startResponseTimer(limits.endRequest())
|
||||
|
||||
sleepThenWait(testRequestTime + testResponseTime)
|
||||
|
||||
if got := seen.refused.Load(); got != 0 {
|
||||
t.Errorf("refused %d, want nothing", got)
|
||||
}
|
||||
|
||||
if seen.readEnded.Load() {
|
||||
t.Error("the read of the visitor's body was ended, want it not")
|
||||
}
|
||||
})
|
||||
}
|
||||
+9
-34
@@ -8,7 +8,6 @@ import (
|
||||
"maps"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||
@@ -189,7 +188,7 @@ func canBeAsked(r *http.Request) bool {
|
||||
|
||||
// answerWith gives the visitor waf's answer, less its disposition.
|
||||
func (rq *request) answerWith(answer *wafAnswer) {
|
||||
rq.prepareOwnAnswer(rq.limits.requestDeadline())
|
||||
rq.prepareOwnAnswer(rq.requestDeadline())
|
||||
|
||||
header := rq.out.Header()
|
||||
maps.Copy(header, answer.header)
|
||||
@@ -211,7 +210,14 @@ func (h *handler) sendReport(
|
||||
ctx context.Context,
|
||||
report *wafprotocol.Report,
|
||||
) {
|
||||
h.reports.startUnlessStopped(func() {
|
||||
h.reportsMu.Lock()
|
||||
defer h.reportsMu.Unlock()
|
||||
|
||||
if h.reportsStopped {
|
||||
return
|
||||
}
|
||||
|
||||
h.reports.Go(func() {
|
||||
err := h.postReport(context.WithoutCancel(ctx), report)
|
||||
if err != nil {
|
||||
h.processLog.Warn("a report is dropped", "error", err.Error())
|
||||
@@ -219,37 +225,6 @@ func (h *handler) sendReport(
|
||||
})
|
||||
}
|
||||
|
||||
// reportGoroutines are the goroutines sending reports to waf. None starts once
|
||||
// stopAndWait has been called.
|
||||
type reportGoroutines struct {
|
||||
mu sync.Mutex
|
||||
stopped bool
|
||||
running sync.WaitGroup
|
||||
}
|
||||
|
||||
// startUnlessStopped runs send on a goroutine of its own, or drops it once
|
||||
// stopAndWait has been called.
|
||||
func (g *reportGoroutines) startUnlessStopped(send func()) {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
|
||||
if g.stopped {
|
||||
return
|
||||
}
|
||||
|
||||
g.running.Go(send)
|
||||
}
|
||||
|
||||
// stopAndWait stops any more goroutines from starting, and returns once those
|
||||
// running have ended.
|
||||
func (g *reportGoroutines) stopAndWait() {
|
||||
g.mu.Lock()
|
||||
g.stopped = true
|
||||
g.mu.Unlock()
|
||||
|
||||
g.running.Wait()
|
||||
}
|
||||
|
||||
// postReport posts report to waf, and waits for its answer for up to
|
||||
// SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS.
|
||||
func (h *handler) postReport(
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
package waf
|
||||
|
||||
// The heuristic abuseipdb. It reads SWW_WAF_ABUSEIPDB_KEY,
|
||||
// SWW_WAF_ABUSEIPDB_MIN_SCORE, SWW_WAF_DNSBL_ABUSEIPDB_ACTION and
|
||||
// SWW_WAF_BAN_RESPONSE.
|
||||
// SWW_WAF_ABUSEIPDB_MIN_SCORE and SWW_WAF_DNSBL_ABUSEIPDB_ACTION.
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -17,11 +16,10 @@ import (
|
||||
|
||||
// abuseIPDB adds the tag abuseipdb and notes AbuseIPDB, as noteHit does, with
|
||||
// the score, when its score of the client is a hit, and refuses the request
|
||||
// with SWW_WAF_BAN_RESPONSE when SWW_WAF_DNSBL_ABUSEIPDB_ACTION is deny, as
|
||||
// dnsbl does for a zone. While SWW_WAF_ABUSEIPDB_KEY is unset it does nothing.
|
||||
// A client without a score is checked in the background, by the request's
|
||||
// address, if its history counts an offence, and the request does not wait for
|
||||
// the answer. The score is then
|
||||
// when SWW_WAF_DNSBL_ABUSEIPDB_ACTION is deny, as dnsbl does for a zone. While
|
||||
// SWW_WAF_ABUSEIPDB_KEY is unset it does nothing. A client without a score is
|
||||
// checked in the background, by the request's address, if its history counts
|
||||
// an offence, and the request does not wait for the answer. The score is then
|
||||
// used for each address of the client. Its figures are the score and
|
||||
// SWW_WAF_ABUSEIPDB_MIN_SCORE.
|
||||
type abuseIPDB struct {
|
||||
@@ -32,7 +30,6 @@ type abuseIPDB struct {
|
||||
keySet bool // SWW_WAF_ABUSEIPDB_KEY is set
|
||||
minScore int64 // SWW_WAF_ABUSEIPDB_MIN_SCORE
|
||||
action config.ReputationAction // SWW_WAF_DNSBL_ABUSEIPDB_ACTION
|
||||
banResponse int // SWW_WAF_BAN_RESPONSE
|
||||
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
||||
hitNoter reputationHitNoter
|
||||
}
|
||||
@@ -74,7 +71,6 @@ func (a abuseIPDB) check(
|
||||
|
||||
return &verdict{
|
||||
action: requestlog.ActionClientDenied,
|
||||
status: a.banResponse,
|
||||
figures: map[string]any{
|
||||
figureScore: score, "min_score": a.minScore,
|
||||
},
|
||||
|
||||
@@ -54,7 +54,7 @@ func TestClientsOnTheAbuseIPDBBlacklistAreBannedForTheBanDays(t *testing.T) {
|
||||
})
|
||||
|
||||
// Two clients in one netblock make one ban, with the higher score.
|
||||
server.banAbuseIPDBBlacklist([]reputation.BlacklistEntry{
|
||||
server.BanAbuseIPDBBlacklist([]reputation.BlacklistEntry{
|
||||
{Addr: netip.MustParseAddr(client), Score: 95},
|
||||
{Addr: netip.MustParseAddr(otherClient), Score: 100},
|
||||
{Addr: netip.MustParseAddr("2001:db8:5::1"), Score: 90},
|
||||
@@ -88,7 +88,7 @@ func TestAbuseIPDBBlacklistMakesNoBanInObserveMode(t *testing.T) {
|
||||
map[string]string{mode: "observe"},
|
||||
)
|
||||
|
||||
server.banAbuseIPDBBlacklist([]reputation.BlacklistEntry{
|
||||
server.BanAbuseIPDBBlacklist([]reputation.BlacklistEntry{
|
||||
{Addr: netip.MustParseAddr(client), Score: 100},
|
||||
})
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -249,7 +250,7 @@ func TestObserveModeWorksOutABanOnlyWhenItsAlertWouldBeSent(t *testing.T) {
|
||||
// never sent them, and returns the queue they wait in as well.
|
||||
func startWithAlerts(
|
||||
t *testing.T, env map[string]string,
|
||||
) (*sender, *clock, *wafAndParts, *alerts.Queue) {
|
||||
) (*sender, *clock, *waf.Server, *alerts.Queue) {
|
||||
t.Helper()
|
||||
|
||||
return startAppWithAlerts(
|
||||
@@ -262,7 +263,7 @@ func startWithAlerts(
|
||||
// startAppWithAlerts is startWithAlerts in front of the app handler.
|
||||
func startAppWithAlerts(
|
||||
t *testing.T, handler http.HandlerFunc, env map[string]string,
|
||||
) (*sender, *clock, *wafAndParts, *alerts.Queue) {
|
||||
) (*sender, *clock, *waf.Server, *alerts.Queue) {
|
||||
t.Helper()
|
||||
|
||||
app := startApp(t, handler)
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
)
|
||||
|
||||
// The anomaly thresholds: the prefix of a scope followed by the end of a
|
||||
@@ -331,7 +332,7 @@ func TestRequestCountsForTheASNumberGeoJSGivesBeforeItEnds(t *testing.T) {
|
||||
// kept.
|
||||
geojsURL, _, release := startHeldGeoJS(t)
|
||||
|
||||
var server atomic.Pointer[wafAndParts]
|
||||
var server atomic.Pointer[waf.Server]
|
||||
|
||||
app := startApp(t, func(http.ResponseWriter, *http.Request) {
|
||||
release()
|
||||
|
||||
@@ -439,7 +439,7 @@ func TestTheHeaderLimitsComeBeforeEveryCheckThatRefuses(t *testing.T) {
|
||||
env map[string]string
|
||||
path string
|
||||
// prepare gets the server ready, at now, before the question.
|
||||
prepare func(t *testing.T, server *wafAndParts, now time.Time)
|
||||
prepare func(t *testing.T, server *waf.Server, now time.Time)
|
||||
}{
|
||||
{"SWW_WAF_DENY_NETS", map[string]string{denyNets: client}, "/", leaveAsItIs},
|
||||
{"a ban", nil, "/", banClient},
|
||||
@@ -454,21 +454,21 @@ func TestTheHeaderLimitsComeBeforeEveryCheckThatRefuses(t *testing.T) {
|
||||
"a DNS blocklist", map[string]string{
|
||||
dnsblZones: dnsblZone, dnsblResolver: noResolver, dnsblAbuseIPDBAction: actionDeny,
|
||||
},
|
||||
"/", func(_ *testing.T, server *wafAndParts, _ time.Time) {
|
||||
"/", func(_ *testing.T, server *waf.Server, _ time.Time) {
|
||||
loadVerdicts(server, map[string][]string{client: {dnsblZone}})
|
||||
},
|
||||
},
|
||||
{
|
||||
"AbuseIPDB",
|
||||
map[string]string{abuseIPDBKey: accountKey, dnsblAbuseIPDBAction: actionDeny},
|
||||
"/", func(_ *testing.T, server *wafAndParts, _ time.Time) {
|
||||
"/", func(_ *testing.T, server *waf.Server, _ time.Time) {
|
||||
// Over SWW_WAF_ABUSEIPDB_MIN_SCORE, 75 by default.
|
||||
loadScores(server, map[string]int64{client: 80})
|
||||
},
|
||||
},
|
||||
{
|
||||
"SWW_WAF_TAG_DENY", map[string]string{tagLists: "vpn=" + vpnList, tagDeny: "vpn"},
|
||||
"/", func(t *testing.T, server *wafAndParts, _ time.Time) {
|
||||
"/", func(t *testing.T, server *waf.Server, _ time.Time) {
|
||||
t.Helper()
|
||||
loadLists(t, server, map[string][]string{vpnList: {client}})
|
||||
},
|
||||
@@ -685,7 +685,7 @@ func TestObserveModeAsksGeoJSNothingForARequestOverTheHeaderLimits(
|
||||
// for env. No request to it may use the network.
|
||||
func newWAFInBubble(
|
||||
t *testing.T, env map[string]string,
|
||||
) (*wafAndParts, *output) {
|
||||
) (*waf.Server, *output) {
|
||||
t.Helper()
|
||||
|
||||
settings := map[string]string{tagDeny: "country=KP"}
|
||||
@@ -699,7 +699,7 @@ func newWAFInBubble(
|
||||
// askDirectly hands a question about a request from client straight to server,
|
||||
// as change leaves it, and returns the answer's status and how long it took.
|
||||
func askDirectly(
|
||||
t *testing.T, server *wafAndParts, change func(*http.Request),
|
||||
t *testing.T, server *waf.Server, change func(*http.Request),
|
||||
) (int, time.Duration) {
|
||||
t.Helper()
|
||||
|
||||
@@ -723,7 +723,7 @@ func askDirectly(
|
||||
}
|
||||
|
||||
// banClient bans client at now.
|
||||
func banClient(_ *testing.T, server *wafAndParts, now time.Time) {
|
||||
func banClient(_ *testing.T, server *waf.Server, now time.Time) {
|
||||
server.Ledger.Ban(
|
||||
netip.MustParsePrefix(client+"/32"),
|
||||
now,
|
||||
@@ -732,7 +732,7 @@ func banClient(_ *testing.T, server *wafAndParts, now time.Time) {
|
||||
}
|
||||
|
||||
// countClient counts one request from client at now for the rate limits.
|
||||
func countClient(_ *testing.T, server *wafAndParts, now time.Time) {
|
||||
func countClient(_ *testing.T, server *waf.Server, now time.Time) {
|
||||
server.Limiter.Count(
|
||||
netip.MustParsePrefix(client+"/32"),
|
||||
now,
|
||||
@@ -741,7 +741,7 @@ func countClient(_ *testing.T, server *wafAndParts, now time.Time) {
|
||||
}
|
||||
|
||||
// listClient puts client on the blocklist at dropURL.
|
||||
func listClient(t *testing.T, server *wafAndParts, now time.Time) {
|
||||
func listClient(t *testing.T, server *waf.Server, now time.Time) {
|
||||
t.Helper()
|
||||
|
||||
err := server.Lists.Load([]reputation.List{{
|
||||
@@ -753,7 +753,7 @@ func listClient(t *testing.T, server *wafAndParts, now time.Time) {
|
||||
}
|
||||
|
||||
// leaveAsItIs prepares nothing.
|
||||
func leaveAsItIs(*testing.T, *wafAndParts, time.Time) {}
|
||||
func leaveAsItIs(*testing.T, *waf.Server, time.Time) {}
|
||||
|
||||
func TestARefusedRequestDoesNotWaitForASlowGeoJS(t *testing.T) {
|
||||
t.Parallel()
|
||||
@@ -764,7 +764,7 @@ func TestARefusedRequestDoesNotWaitForASlowGeoJS(t *testing.T) {
|
||||
name string
|
||||
env map[string]string
|
||||
// prepare gets the server ready, at now, before the question.
|
||||
prepare func(t *testing.T, server *wafAndParts, now time.Time)
|
||||
prepare func(t *testing.T, server *waf.Server, now time.Time)
|
||||
// cookie is the request's Cookie header.
|
||||
cookie string
|
||||
status int
|
||||
@@ -849,7 +849,7 @@ func TestAQuestionWaitsForASlowGeoJSOnlyAsLongAsTheDefaultSays(
|
||||
name string
|
||||
env map[string]string
|
||||
// prepare gets the server ready, at now, before the question.
|
||||
prepare func(t *testing.T, server *wafAndParts, now time.Time)
|
||||
prepare func(t *testing.T, server *waf.Server, now time.Time)
|
||||
status int
|
||||
action requestlog.Action
|
||||
}{
|
||||
@@ -915,7 +915,7 @@ func TestWAFAnswersWithinTheTimeLimitWhileEveryOutsideSourceHangs(
|
||||
held, _, _ := startHeldGeoJS(t)
|
||||
resolver, releaseResolver := startHeldResolver(t)
|
||||
|
||||
server, out, _ := newWAFWith(t, func(*waf.Params) {},
|
||||
server, out, _ := newWAFWith(t, waf.New,
|
||||
silentGeoJSURL, silentAbuseIPDBURL, time.Now, map[string]string{
|
||||
tagDeny: "country=KP", coreRuleSetMode: block,
|
||||
dnsblZones: dnsblZone, dnsblResolver: resolver,
|
||||
|
||||
+13
-11
@@ -1,8 +1,7 @@
|
||||
package waf
|
||||
|
||||
// The heuristic active_ban, and banMaker, which makes the bans of several
|
||||
// heuristics. active_ban reads SWW_WAF_MODE and SWW_WAF_BAN_RESPONSE; the bans
|
||||
// are the ban ledger's.
|
||||
// heuristics. active_ban reads SWW_WAF_MODE; the bans are the ban ledger's.
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -17,11 +16,16 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// activeBan refuses a client a ban covers, with SWW_WAF_BAN_RESPONSE: a ban on
|
||||
// a netblock the client is in, in force at now. It notes for the log line when
|
||||
// that ban ends. A request that makes the ban permanent, or in observe mode
|
||||
// would have, raises the alert for it. Its figures are the ban's netblock and
|
||||
// when it ends.
|
||||
// banResponse is a refusal answered with SWW_WAF_BAN_RESPONSE, and logged
|
||||
// with action.
|
||||
func (rq *request) banResponse(action requestlog.Action) *refusal {
|
||||
return &refusal{status: rq.h.config.BanResponse, action: action}
|
||||
}
|
||||
|
||||
// activeBan refuses a client a ban covers: a ban on a netblock the client is
|
||||
// in, in force at now. It notes for the log line when that ban ends. A request
|
||||
// that makes the ban permanent, or in observe mode would have, raises the alert
|
||||
// for it. Its figures are the ban's netblock and when it ends.
|
||||
type activeBan struct {
|
||||
banMaker banMaker
|
||||
}
|
||||
@@ -59,7 +63,6 @@ func (a activeBan) check(
|
||||
|
||||
return &verdict{
|
||||
action: banAction(ban.Permanent()),
|
||||
status: a.banMaker.banResponse,
|
||||
figures: map[string]any{
|
||||
figureNetblock: ban.Netblock.String(), "ban_expires": expires,
|
||||
},
|
||||
@@ -115,9 +118,8 @@ type banMaker struct {
|
||||
alerts *alerts.Queue
|
||||
// observe is true in observe mode (SWW_WAF_MODE).
|
||||
observe bool
|
||||
// banResponse is SWW_WAF_BAN_RESPONSE: the status of a request refused
|
||||
// under a ban or for a clear sign of attack, which the notes of a ban for
|
||||
// a clear sign of attack give.
|
||||
// banResponse is SWW_WAF_BAN_RESPONSE, the status the notes of a ban for a
|
||||
// clear sign of attack give.
|
||||
banResponse int
|
||||
// banScopeV4Prefix is SWW_WAF_BAN_SCOPE_V4_PREFIX and ipv6GroupPrefix
|
||||
// SWW_WAF_IPV6_GROUP_PREFIX, which set the netblock a ban covers.
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -465,7 +466,7 @@ func (c *clock) advance(d time.Duration) {
|
||||
// window.
|
||||
func startWithClock(
|
||||
t *testing.T, geojsURL string, env map[string]string,
|
||||
) (*sender, *clock, *wafAndParts) {
|
||||
) (*sender, *clock, *waf.Server) {
|
||||
t.Helper()
|
||||
|
||||
app := startApp(t, func(http.ResponseWriter, *http.Request) {})
|
||||
|
||||
@@ -4,7 +4,6 @@ package waf
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
@@ -162,7 +161,6 @@ func (rq *request) applyFindings(
|
||||
func (rq *request) timedOut(running string) *verdict {
|
||||
v := &verdict{
|
||||
action: requestlog.ActionChecksTimedOut,
|
||||
status: http.StatusServiceUnavailable,
|
||||
figures: map[string]any{
|
||||
"time_limit_ms": rq.h.besideLookupLimit.Milliseconds(),
|
||||
},
|
||||
|
||||
@@ -62,7 +62,7 @@ func coreRuleSetMatch() coreruleset.Result {
|
||||
// at its default but for env. No request to it may use the network.
|
||||
func newWAFHoldingTheCoreRuleSet(
|
||||
t *testing.T, crs *heldCoreRuleSet, env map[string]string,
|
||||
) (*wafAndParts, *output, *alerts.Queue) {
|
||||
) (*waf.Server, *output, *alerts.Queue) {
|
||||
t.Helper()
|
||||
|
||||
settings := map[string]string{
|
||||
@@ -71,9 +71,11 @@ func newWAFHoldingTheCoreRuleSet(
|
||||
}
|
||||
maps.Copy(settings, env)
|
||||
|
||||
server, out, queue := newWAFWith(t, func(params *waf.Params) {
|
||||
server, out, queue := newWAFWith(t, func(params waf.Params) *waf.Server {
|
||||
params.CoreRuleSet = crs
|
||||
params.BesideLookupTimeLimit = waf.BesideLookupTimeLimit
|
||||
|
||||
return waf.New(params)
|
||||
}, silentGeoJSURL, abuseIPDBURL, time.Now, settings)
|
||||
|
||||
return server, out, queue
|
||||
@@ -103,7 +105,7 @@ func wantAnswer(
|
||||
// left no trace: line has none of its fields, and server has counted no match
|
||||
// of it and raised no alert for one.
|
||||
func wantNoCoreRuleSetTrace(
|
||||
t *testing.T, server *wafAndParts, queue *alerts.Queue, line logLine,
|
||||
t *testing.T, server *waf.Server, queue *alerts.Queue, line logLine,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||
)
|
||||
|
||||
@@ -529,7 +530,7 @@ func TestBanForALoweredLimitGivesThePercentageInItsNotesAndItsAlert(
|
||||
// clock.
|
||||
func startWithLookups(
|
||||
t *testing.T, env map[string]string,
|
||||
) (*sender, *wafAndParts, *alerts.Queue) {
|
||||
) (*sender, *waf.Server, *alerts.Queue) {
|
||||
t.Helper()
|
||||
|
||||
s, _, server, queue := startWithLookupsAndClock(t, env)
|
||||
@@ -544,7 +545,7 @@ func startWithLookups(
|
||||
// country, and no other address.
|
||||
func startWithLookupsAndClock(
|
||||
t *testing.T, env map[string]string,
|
||||
) (*sender, *clock, *wafAndParts, *alerts.Queue) {
|
||||
) (*sender, *clock, *waf.Server, *alerts.Queue) {
|
||||
t.Helper()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
|
||||
@@ -584,7 +585,7 @@ func (s *sender) uploadFrom(from string) logLine {
|
||||
// straight to server's handler, without the network, and returns once it is
|
||||
// answered.
|
||||
func serveFromDE(
|
||||
t *testing.T, server *wafAndParts, method string, body io.Reader,
|
||||
t *testing.T, server *waf.Server, method string, body io.Reader,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package waf
|
||||
|
||||
// The heuristic blocklists. It reads SWW_WAF_BLOCKLIST_ACTION and
|
||||
// SWW_WAF_BAN_RESPONSE, and the blocklists through the reputation lists.
|
||||
// The heuristic blocklists. It reads SWW_WAF_BLOCKLIST_ACTION, and the blocklists
|
||||
// through the reputation lists.
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -13,15 +13,13 @@ import (
|
||||
)
|
||||
|
||||
// blocklists notes the blocklists that list the client, as noteListed does,
|
||||
// and refuses the request with SWW_WAF_BAN_RESPONSE when
|
||||
// SWW_WAF_BLOCKLIST_ACTION is deny. Being limit, it lowers the client's limits
|
||||
// instead (see limitPercentages), and being log, it does nothing more. Its
|
||||
// figure is the blocklists that list the client.
|
||||
// and refuses the request when SWW_WAF_BLOCKLIST_ACTION is deny. Being limit, it
|
||||
// lowers the client's limits instead (see limitPercentages), and being log, it
|
||||
// does nothing more. Its figure is the blocklists that list the client.
|
||||
type blocklists struct {
|
||||
lists *reputation.Lists
|
||||
action config.ReputationAction // SWW_WAF_BLOCKLIST_ACTION
|
||||
banResponse int // SWW_WAF_BAN_RESPONSE
|
||||
hitNoter reputationHitNoter
|
||||
lists *reputation.Lists
|
||||
action config.ReputationAction // SWW_WAF_BLOCKLIST_ACTION
|
||||
hitNoter reputationHitNoter
|
||||
}
|
||||
|
||||
func (blocklists) name() string { return "blocklists" }
|
||||
@@ -41,7 +39,6 @@ func (b blocklists) check(
|
||||
|
||||
return &verdict{
|
||||
action: requestlog.ActionClientDenied,
|
||||
status: b.banResponse,
|
||||
figures: map[string]any{"lists": listedBy},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ package waf
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
@@ -78,7 +77,6 @@ func (c coreRuleSet) check(
|
||||
|
||||
return &verdict{
|
||||
action: requestlog.ActionBlockedByCoreRuleSet,
|
||||
status: http.StatusForbidden,
|
||||
figures: map[string]any{
|
||||
figureScore: result.Score,
|
||||
"threshold": threshold,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package waf
|
||||
|
||||
// The heuristic countries. It reads SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES and
|
||||
// SWW_WAF_BAN_RESPONSE.
|
||||
// The heuristic countries. It reads SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES.
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -14,12 +13,10 @@ import (
|
||||
|
||||
// countries refuses a request when SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES is set and
|
||||
// does not hold the client's country as it was looked up. A client without a
|
||||
// country, or whose country cannot be found, is refused. It refuses with
|
||||
// SWW_WAF_BAN_RESPONSE. Its figures are the country and the setting that
|
||||
// refused it.
|
||||
// country, or whose country cannot be found, is refused. Its figures are the
|
||||
// country and the setting that refused it.
|
||||
type countries struct {
|
||||
allowed []string // SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES
|
||||
banResponse int // SWW_WAF_BAN_RESPONSE
|
||||
allowed []string // SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES
|
||||
}
|
||||
|
||||
func (countries) name() string { return "countries" }
|
||||
@@ -38,7 +35,6 @@ func (c countries) check(
|
||||
|
||||
return &verdict{
|
||||
action: requestlog.ActionCountryDenied,
|
||||
status: c.banResponse,
|
||||
figures: map[string]any{
|
||||
"country": country,
|
||||
"setting": config.ExclusivelyAllowedCountriesSetting,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package waf
|
||||
|
||||
// The heuristic deny_nets. It reads SWW_WAF_DENY_NETS and SWW_WAF_BAN_RESPONSE.
|
||||
// The heuristic deny_nets. It reads SWW_WAF_DENY_NETS.
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -10,11 +10,10 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// denyNets refuses a client in SWW_WAF_DENY_NETS, with SWW_WAF_BAN_RESPONSE.
|
||||
// Its figure is the netblock of the setting the client is in.
|
||||
// denyNets refuses a client in SWW_WAF_DENY_NETS. Its figure is the netblock of
|
||||
// the setting the client is in.
|
||||
type denyNets struct {
|
||||
netblocks []netip.Prefix // SWW_WAF_DENY_NETS
|
||||
banResponse int // SWW_WAF_BAN_RESPONSE
|
||||
netblocks []netip.Prefix // SWW_WAF_DENY_NETS
|
||||
}
|
||||
|
||||
func (denyNets) name() string { return "deny_nets" }
|
||||
@@ -28,7 +27,6 @@ func (d denyNets) check(
|
||||
if netblock.Contains(rq.client) {
|
||||
return &verdict{
|
||||
action: requestlog.ActionClientDenied,
|
||||
status: d.banResponse,
|
||||
figures: map[string]any{figureNetblock: netblock.String()},
|
||||
}
|
||||
}
|
||||
|
||||
+10
-14
@@ -1,8 +1,7 @@
|
||||
package waf
|
||||
|
||||
// The heuristic dnsbl. It reads SWW_WAF_DNSBL_ABUSEIPDB_ACTION and
|
||||
// SWW_WAF_BAN_RESPONSE, and the DNSBL zones' verdicts through the DNSBL
|
||||
// checker.
|
||||
// The heuristic dnsbl. It reads SWW_WAF_DNSBL_ABUSEIPDB_ACTION, and the DNSBL zones'
|
||||
// verdicts through the DNSBL checker.
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -16,17 +15,15 @@ import (
|
||||
|
||||
// dnsbl notes the DNSBL zones whose verdict lists the client, as noteListed
|
||||
// does, adds the tag dnsbl=<zone> for each, the zone's key masked, and refuses
|
||||
// the request with SWW_WAF_BAN_RESPONSE when SWW_WAF_DNSBL_ABUSEIPDB_ACTION is
|
||||
// deny. Being limit, it lowers the client's limits instead (see
|
||||
// limitPercentages), and being log, it does nothing more. A zone without a
|
||||
// verdict on the client is asked about it in the background, and the request
|
||||
// does not wait for the answer. Its figure is the zones that list the client,
|
||||
// their keys masked.
|
||||
// the request when SWW_WAF_DNSBL_ABUSEIPDB_ACTION is deny. Being
|
||||
// limit, it lowers the client's limits instead (see limitPercentages), and
|
||||
// being log, it does nothing more. A zone without a verdict on the client is
|
||||
// asked about it in the background, and the request does not wait for the
|
||||
// answer. Its figure is the zones that list the client, their keys masked.
|
||||
type dnsbl struct {
|
||||
dnsbl *reputation.DNSBL
|
||||
action config.ReputationAction // SWW_WAF_DNSBL_ABUSEIPDB_ACTION
|
||||
banResponse int // SWW_WAF_BAN_RESPONSE
|
||||
hitNoter reputationHitNoter
|
||||
dnsbl *reputation.DNSBL
|
||||
action config.ReputationAction // SWW_WAF_DNSBL_ABUSEIPDB_ACTION
|
||||
hitNoter reputationHitNoter
|
||||
}
|
||||
|
||||
func (dnsbl) name() string { return "dnsbl" }
|
||||
@@ -50,7 +47,6 @@ func (d dnsbl) check(
|
||||
|
||||
return &verdict{
|
||||
action: requestlog.ActionClientDenied,
|
||||
status: d.banResponse,
|
||||
figures: map[string]any{"zones": listedBy},
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,13 @@
|
||||
package waf
|
||||
|
||||
import "sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
|
||||
// BesideLookupTimeLimit is besideLookupTimeLimit.
|
||||
const BesideLookupTimeLimit = besideLookupTimeLimit
|
||||
|
||||
// BanAbuseIPDBBlacklist does on s what a fetch of AbuseIPDB's blacklist with
|
||||
// the clients on listed does.
|
||||
func (s *Server) BanAbuseIPDBBlacklist(listed []reputation.BlacklistEntry) {
|
||||
h := s.Handler.(*handler) //nolint:forcetypeassert // New makes it one
|
||||
BanAbuseIPDBBlacklist(h.config, h.ledger, h.processLog, h.now(), listed)
|
||||
}
|
||||
@@ -4,7 +4,6 @@ package waf
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -40,7 +39,6 @@ func (headerLimits) check(
|
||||
if requestLine > maxLineBytes {
|
||||
return &verdict{
|
||||
action: requestlog.ActionRequestLineTooLong,
|
||||
status: http.StatusRequestURITooLong,
|
||||
figures: map[string]any{
|
||||
"request_line_bytes": requestLine,
|
||||
"request_line_bytes_limit": maxLineBytes,
|
||||
@@ -58,7 +56,6 @@ func (headerLimits) check(
|
||||
|
||||
return &verdict{
|
||||
action: requestlog.ActionRequestHeaderTooLarge,
|
||||
status: http.StatusRequestHeaderFieldsTooLarge,
|
||||
figures: map[string]any{
|
||||
"longest_header_line_bytes": longest,
|
||||
"header_line_bytes_limit": maxLineBytes,
|
||||
|
||||
@@ -42,13 +42,10 @@ type refusesWithoutLookup interface {
|
||||
}
|
||||
|
||||
// verdict is a heuristic refusing a request, or banning its client: the action
|
||||
// the log line names, the status the visitor is answered with, and the figures
|
||||
// the heuristic acted on, by name, such as the count and the limit. A status of
|
||||
// 0 closes the visitor's connection without an answer. A verdict of afterAnswer
|
||||
// or onReport refuses nothing, and has neither action nor status.
|
||||
// the log line names, and the figures the heuristic acted on, by name, such as
|
||||
// the count and the limit.
|
||||
type verdict struct {
|
||||
action requestlog.Action
|
||||
status int
|
||||
figures map[string]any
|
||||
}
|
||||
|
||||
|
||||
@@ -151,7 +151,7 @@ func TestPathUnderTheOwnPrefixIsNeitherForwardedNorRefused(t *testing.T) {
|
||||
|
||||
// historyOf returns the history of the client at addr.
|
||||
func historyOf(
|
||||
t *testing.T, server *wafAndParts, addr string,
|
||||
t *testing.T, server *waf.Server, addr string,
|
||||
) ratelimit.History {
|
||||
t.Helper()
|
||||
|
||||
|
||||
+24
-19
@@ -107,9 +107,21 @@ type CoreRuleSetInspector interface {
|
||||
) (coreruleset.Result, []byte, error)
|
||||
}
|
||||
|
||||
// Server is the server waf runs.
|
||||
// Server is the server waf runs, with the parts of its Params. The tests
|
||||
// read them.
|
||||
type Server struct {
|
||||
*http.Server
|
||||
|
||||
Ledger *bans.Ledger
|
||||
Limiter *ratelimit.Limiter
|
||||
GeoJS *geojs.GeoJS
|
||||
Anomalies *anomaly.Counters
|
||||
LookupFile *lookup.File
|
||||
Lists *reputation.Lists
|
||||
DNSBL *reputation.DNSBL
|
||||
AbuseIPDB *reputation.AbuseIPDB
|
||||
RouteWatch *routewatch.Client
|
||||
Metrics *metrics.Metrics
|
||||
}
|
||||
|
||||
// newHandler returns the handler with what every request shares, the parts
|
||||
@@ -189,25 +201,20 @@ func newHandler(params Params) *handler {
|
||||
routeWatch: params.RouteWatch,
|
||||
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||
},
|
||||
denyNets{netblocks: cfg.DenyNets, banResponse: cfg.BanResponse},
|
||||
denyNets{netblocks: cfg.DenyNets},
|
||||
activeBan{banMaker: sharedBanMaker},
|
||||
}
|
||||
h.afterLookup = []heuristic{
|
||||
countries{
|
||||
allowed: cfg.ExclusivelyAllowedCountries,
|
||||
banResponse: cfg.BanResponse,
|
||||
},
|
||||
countries{allowed: cfg.ExclusivelyAllowedCountries},
|
||||
blocklists{
|
||||
lists: params.Lists,
|
||||
action: cfg.BlocklistAction,
|
||||
banResponse: cfg.BanResponse,
|
||||
hitNoter: hitNoter,
|
||||
lists: params.Lists,
|
||||
action: cfg.BlocklistAction,
|
||||
hitNoter: hitNoter,
|
||||
},
|
||||
dnsbl{
|
||||
dnsbl: params.DNSBL,
|
||||
action: cfg.DNSBLAbuseIPDBAction,
|
||||
banResponse: cfg.BanResponse,
|
||||
hitNoter: hitNoter,
|
||||
dnsbl: params.DNSBL,
|
||||
action: cfg.DNSBLAbuseIPDBAction,
|
||||
hitNoter: hitNoter,
|
||||
},
|
||||
abuseIPDB{
|
||||
abuseIPDB: params.AbuseIPDB,
|
||||
@@ -215,11 +222,10 @@ func newHandler(params Params) *handler {
|
||||
keySet: cfg.AbuseIPDBKey != "",
|
||||
minScore: cfg.AbuseIPDBMinScore,
|
||||
action: cfg.DNSBLAbuseIPDBAction,
|
||||
banResponse: cfg.BanResponse,
|
||||
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||
hitNoter: hitNoter,
|
||||
},
|
||||
tagDeny{tags: cfg.TagDeny, banResponse: cfg.BanResponse},
|
||||
tagDeny{tags: cfg.TagDeny},
|
||||
rateLimits{
|
||||
limiter: params.Limiter,
|
||||
exemptNets: cfg.RateLimitExemptNets,
|
||||
@@ -241,13 +247,12 @@ func newHandler(params Params) *handler {
|
||||
},
|
||||
}
|
||||
h.besideLookup = []heuristic{h.trapPaths, h.ruleFiles, h.coreRuleSet}
|
||||
h.afterBesideLookup = []heuristic{
|
||||
tagDeny{tags: cfg.TagDeny, banResponse: cfg.BanResponse},
|
||||
}
|
||||
h.afterBesideLookup = []heuristic{tagDeny{tags: cfg.TagDeny}}
|
||||
h.afterAnswer = []heuristic{
|
||||
refusalLimit{
|
||||
limiter: params.Limiter,
|
||||
perMinute: cfg.RefusalsLimitPerMinute,
|
||||
banResponse: cfg.BanResponse,
|
||||
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
||||
banMaker: sharedBanMaker,
|
||||
},
|
||||
|
||||
+12
-36
@@ -316,7 +316,7 @@ func startProxyWithGeoJS(
|
||||
func startProxyWithClock(
|
||||
t *testing.T, appURL, geojsURL string, now func() time.Time,
|
||||
env map[string]string,
|
||||
) (string, *output, *wafAndParts) {
|
||||
) (string, *output, *waf.Server) {
|
||||
t.Helper()
|
||||
|
||||
addr, out, server, _ := startProxyWithAlerts(
|
||||
@@ -335,7 +335,7 @@ func startProxyWithClock(
|
||||
func startProxyWithAlerts(
|
||||
t *testing.T, appURL, geojsURL string, now func() time.Time,
|
||||
env map[string]string,
|
||||
) (string, *output, *wafAndParts, *alerts.Queue) {
|
||||
) (string, *output, *waf.Server, *alerts.Queue) {
|
||||
t.Helper()
|
||||
|
||||
server, out, alertQueue := newWAF(t, geojsURL, now, env)
|
||||
@@ -346,7 +346,7 @@ func startProxyWithAlerts(
|
||||
|
||||
// serve serves server on a port of its own until the test ends, and sets its
|
||||
// Addr to where it listens.
|
||||
func serve(t *testing.T, server *wafAndParts) {
|
||||
func serve(t *testing.T, server *waf.Server) {
|
||||
t.Helper()
|
||||
|
||||
listener, err := (&net.ListenConfig{}).Listen(
|
||||
@@ -422,29 +422,8 @@ func startRProxy(
|
||||
return listener.Addr().String()
|
||||
}
|
||||
|
||||
// wafAndParts is waf's server with the Params it was built from, so that a
|
||||
// test can read the parts the server uses.
|
||||
type wafAndParts struct {
|
||||
*waf.Server
|
||||
waf.Params
|
||||
}
|
||||
|
||||
// banAbuseIPDBBlacklist does on w what a fetch of AbuseIPDB's blacklist with
|
||||
// the clients on listed does.
|
||||
func (w *wafAndParts) banAbuseIPDBBlacklist(
|
||||
listed []reputation.BlacklistEntry,
|
||||
) {
|
||||
waf.BanAbuseIPDBBlacklist(
|
||||
w.Config,
|
||||
w.Ledger,
|
||||
w.ProcessLog,
|
||||
w.Now(),
|
||||
listed,
|
||||
)
|
||||
}
|
||||
|
||||
// newWAF makes waf's server, without starting it, and returns it with its
|
||||
// parts, what it writes, and the queue of the alerts it raises, as the
|
||||
// newWAF makes waf's server, without starting it, and
|
||||
// returns it, what it writes, and the queue of the alerts it raises, as the
|
||||
// settings in env make it. No alert is sent from the queue: they wait in it,
|
||||
// for the test to look at.
|
||||
// With no geojsURL, there is no stand-in for GeoJS to look clients up at, and
|
||||
@@ -459,21 +438,19 @@ func newWAF(
|
||||
geojsURL string,
|
||||
now func() time.Time,
|
||||
env map[string]string,
|
||||
) (*wafAndParts, *output, *alerts.Queue) {
|
||||
) (*waf.Server, *output, *alerts.Queue) {
|
||||
t.Helper()
|
||||
|
||||
return newWAFWith(
|
||||
t, func(*waf.Params) {}, geojsURL, abuseIPDBURL, now, env,
|
||||
)
|
||||
return newWAFWith(t, waf.New, geojsURL, abuseIPDBURL, now, env)
|
||||
}
|
||||
|
||||
// newWAFWith is newWAF with change applied to the Params before the server is
|
||||
// built from them, and clients checked with AbuseIPDB at abuseIPDBCheckURL.
|
||||
// newWAFWith is newWAF for the server build makes, with clients checked with
|
||||
// AbuseIPDB at abuseIPDBCheckURL.
|
||||
func newWAFWith(
|
||||
t *testing.T, change func(*waf.Params),
|
||||
t *testing.T, build func(waf.Params) *waf.Server,
|
||||
geojsURL, abuseIPDBCheckURL string, now func() time.Time,
|
||||
env map[string]string,
|
||||
) (*wafAndParts, *output, *alerts.Queue) {
|
||||
) (*waf.Server, *output, *alerts.Queue) {
|
||||
t.Helper()
|
||||
|
||||
settings := map[string]string{
|
||||
@@ -541,9 +518,8 @@ func newWAFWith(
|
||||
addCoreRuleSet(t, ¶ms)
|
||||
addCounters(¶ms)
|
||||
addOutsideSources(¶ms, geojsURL, abuseIPDBCheckURL)
|
||||
change(¶ms)
|
||||
|
||||
server := &wafAndParts{Server: waf.New(params), Params: params}
|
||||
server := build(params)
|
||||
|
||||
return server, out, alertQueue
|
||||
}
|
||||
|
||||
@@ -17,9 +17,9 @@ import (
|
||||
// countedByRateLimits exempts it, by the client's limit percentages, which it
|
||||
// notes for the log line with the client's counts. A request that takes the
|
||||
// client over a rate limit, as its limit percentage lowers it, breaks it: it is
|
||||
// refused with SWW_WAF_BAN_RESPONSE, and bans the client. A request exempt
|
||||
// from the rate limits is exempt from the byte limits too. Its figures are the
|
||||
// window, the count and the limit.
|
||||
// refused, and bans the client. A request exempt from the rate limits is exempt
|
||||
// from the byte limits too. Its figures are the window, the count and the
|
||||
// limit.
|
||||
type rateLimits struct {
|
||||
limiter *ratelimit.Limiter
|
||||
exemptNets []netip.Prefix // SWW_WAF_RATE_LIMIT_EXEMPT_NETS
|
||||
@@ -63,9 +63,7 @@ func (r rateLimits) check(
|
||||
r.banMaker.banForLimit(rq, now, hit, r.banResponse)
|
||||
|
||||
return &verdict{
|
||||
action: requestlog.ActionClientIsRateLimited,
|
||||
status: r.banResponse,
|
||||
figures: limitFigures(hit),
|
||||
action: requestlog.ActionClientIsRateLimited, figures: limitFigures(hit),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
package waf
|
||||
|
||||
// The heuristic refusal_limit. It reads SWW_WAF_REFUSALS_LIMIT_PER_MINUTE.
|
||||
// The heuristic refusal_limit. It reads SWW_WAF_REFUSALS_LIMIT_PER_MINUTE and
|
||||
// SWW_WAF_BAN_RESPONSE.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// refusalLimit counts the request once it has been answered, if waf
|
||||
@@ -15,11 +18,11 @@ import (
|
||||
// SWW_WAF_REFUSALS_LIMIT_PER_MINUTE such refusals of the client within a minute break
|
||||
// a limit, which bans the client. A client in SWW_WAF_ALLOW_NETS, which the
|
||||
// heuristics skip, is not counted, and nothing is while the limit is off.
|
||||
// The ban's notes give the status the client was sent, or in observe mode
|
||||
// would have been. Its figures are the window, the count and the limit.
|
||||
// Its figures are the window, the count and the limit.
|
||||
type refusalLimit struct {
|
||||
limiter *ratelimit.Limiter
|
||||
perMinute int64 // SWW_WAF_REFUSALS_LIMIT_PER_MINUTE
|
||||
banResponse int // SWW_WAF_BAN_RESPONSE
|
||||
ipv6GroupPrefix int // SWW_WAF_IPV6_GROUP_PREFIX
|
||||
banMaker banMaker
|
||||
}
|
||||
@@ -48,7 +51,27 @@ func (r refusalLimit) check(
|
||||
return nil
|
||||
}
|
||||
|
||||
r.banMaker.banForLimit(rq, now, hit, rq.refusalStatus)
|
||||
// What the client was sent, or in observe mode would have been.
|
||||
status := rq.out.status
|
||||
|
||||
switch rq.wouldAction {
|
||||
case requestlog.ActionBlockedByBlockRule,
|
||||
requestlog.ActionBlockedByCoreRuleSet:
|
||||
status = http.StatusForbidden
|
||||
case requestlog.ActionTemporarilyBanned,
|
||||
requestlog.ActionPermanentlyBanned:
|
||||
status = r.banResponse
|
||||
case requestlog.ActionForward,
|
||||
requestlog.ActionRequestLineTooLong,
|
||||
requestlog.ActionRequestHeaderTooLarge,
|
||||
requestlog.ActionClientIsRateLimited,
|
||||
requestlog.ActionClientDenied,
|
||||
requestlog.ActionCountryDenied,
|
||||
requestlog.ActionChecksTimedOut,
|
||||
requestlog.ActionOwnEndpoint:
|
||||
}
|
||||
|
||||
r.banMaker.banForLimit(rq, now, hit, status)
|
||||
|
||||
return &verdict{figures: limitFigures(hit)}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
)
|
||||
|
||||
const refusalsLimitPerMinute = "SWW_WAF_REFUSALS_LIMIT_PER_MINUTE"
|
||||
@@ -51,7 +52,7 @@ func (r refused) send(s *sender, from string) logLine {
|
||||
// SWW_WAF_REFUSALS_LIMIT_PER_MINUTE at limit, and the settings in env.
|
||||
func startForRefusalLimit(
|
||||
t *testing.T, limit string, env map[string]string,
|
||||
) (*sender, *clock, *wafAndParts) {
|
||||
) (*sender, *clock, *waf.Server) {
|
||||
t.Helper()
|
||||
|
||||
settings := map[string]string{
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
)
|
||||
|
||||
// The reputation settings.
|
||||
@@ -138,13 +139,13 @@ func TestReputationRefusalNamesItsHeuristicAndWhatListedTheClient(
|
||||
for _, tc := range []struct {
|
||||
heuristic string
|
||||
env map[string]string
|
||||
load func(*testing.T, *wafAndParts)
|
||||
load func(*testing.T, *waf.Server)
|
||||
figures string
|
||||
}{
|
||||
{
|
||||
"blocklists",
|
||||
map[string]string{blocklistURLs: dropURL, blocklistAction: actionDeny},
|
||||
func(t *testing.T, server *wafAndParts) {
|
||||
func(t *testing.T, server *waf.Server) {
|
||||
t.Helper()
|
||||
loadLists(t, server, map[string][]string{dropURL: {client}})
|
||||
},
|
||||
@@ -156,7 +157,7 @@ func TestReputationRefusalNamesItsHeuristicAndWhatListedTheClient(
|
||||
dnsblZones: dnsblZone, dnsblResolver: noResolver,
|
||||
dnsblAbuseIPDBAction: actionDeny,
|
||||
},
|
||||
func(_ *testing.T, server *wafAndParts) {
|
||||
func(_ *testing.T, server *waf.Server) {
|
||||
loadVerdicts(server, map[string][]string{client: {dnsblZone}})
|
||||
},
|
||||
`{"zones": ["` + dnsblZone + `"]}`,
|
||||
@@ -164,7 +165,7 @@ func TestReputationRefusalNamesItsHeuristicAndWhatListedTheClient(
|
||||
{
|
||||
"abuseipdb",
|
||||
map[string]string{abuseIPDBKey: accountKey, dnsblAbuseIPDBAction: actionDeny},
|
||||
func(_ *testing.T, server *wafAndParts) {
|
||||
func(_ *testing.T, server *waf.Server) {
|
||||
loadScores(server, map[string]int64{client: 80})
|
||||
},
|
||||
`{"score": 80, "min_score": 75}`,
|
||||
@@ -559,7 +560,7 @@ func TestReputationSourcesTagTheRequestWhateverTheirAction(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
env map[string]string
|
||||
load func(*wafAndParts)
|
||||
load func(*waf.Server)
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
@@ -567,7 +568,7 @@ func TestReputationSourcesTagTheRequestWhateverTheirAction(t *testing.T) {
|
||||
map[string]string{
|
||||
dnsblZones: dnsblZone, dnsblResolver: noResolver, dnsblAbuseIPDBAction: actionLog,
|
||||
},
|
||||
func(server *wafAndParts) {
|
||||
func(server *waf.Server) {
|
||||
loadVerdicts(server, map[string][]string{fromDE: {dnsblZone}})
|
||||
},
|
||||
[]string{"asn=64496", "country=DE", "dnsbl=" + dnsblZone},
|
||||
@@ -575,7 +576,7 @@ func TestReputationSourcesTagTheRequestWhateverTheirAction(t *testing.T) {
|
||||
{
|
||||
abuseipdb,
|
||||
map[string]string{abuseIPDBKey: accountKey, dnsblAbuseIPDBAction: actionLog},
|
||||
func(server *wafAndParts) {
|
||||
func(server *waf.Server) {
|
||||
loadScores(server, map[string]int64{fromDE: 80})
|
||||
},
|
||||
[]string{abuseipdb, "asn=64496", "country=DE"},
|
||||
@@ -1255,7 +1256,7 @@ func verdictsFetched() time.Time {
|
||||
// verdictsFetched, as dnsbl.json would at start: one that lists the
|
||||
// client from each zone listedBy gives for it, and one that does not from
|
||||
// each other zone.
|
||||
func loadVerdicts(server *wafAndParts, listedBy map[string][]string) {
|
||||
func loadVerdicts(server *waf.Server, listedBy map[string][]string) {
|
||||
verdicts := make(
|
||||
[]reputation.Verdict,
|
||||
0,
|
||||
@@ -1280,7 +1281,7 @@ func loadVerdicts(server *wafAndParts, listedBy map[string][]string) {
|
||||
// loadScores puts into server's AbuseIPDB the score scores gives each
|
||||
// client, an IPv4 address, fetched at verdictsFetched, as
|
||||
// abuseipdb-scores.json would at start.
|
||||
func loadScores(server *wafAndParts, scores map[string]int64) {
|
||||
func loadScores(server *waf.Server, scores map[string]int64) {
|
||||
kept := make([]reputation.Score, 0, len(scores))
|
||||
for client, score := range scores {
|
||||
kept = append(kept, reputation.Score{
|
||||
@@ -1294,7 +1295,7 @@ func loadScores(server *wafAndParts, scores map[string]int64) {
|
||||
|
||||
// wantAbuseIPDBChecks checks how many clients server has checked with
|
||||
// AbuseIPDB.
|
||||
func wantAbuseIPDBChecks(t *testing.T, server *wafAndParts, want int) {
|
||||
func wantAbuseIPDBChecks(t *testing.T, server *waf.Server, want int) {
|
||||
t.Helper()
|
||||
|
||||
if got := server.AbuseIPDB.Checked(); got != want {
|
||||
@@ -1306,7 +1307,7 @@ func wantAbuseIPDBChecks(t *testing.T, server *wafAndParts, want int) {
|
||||
// its lines, fetched at listsFetched, as lists.json would at start.
|
||||
func loadLists(
|
||||
t *testing.T,
|
||||
server *wafAndParts,
|
||||
server *waf.Server,
|
||||
copies map[string][]string,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
+25
-12
@@ -69,10 +69,6 @@ type decision struct {
|
||||
// its client's history counts. trapPaths and ruleFiles set attack,
|
||||
// ruleFiles blockedByBlockRule, and coreRuleSet blockedByCoreRuleSet.
|
||||
attack, blockedByBlockRule, blockedByCoreRuleSet bool
|
||||
// refusalStatus is the status of the verdict that refused the request, or
|
||||
// in observe mode would have: what the visitor was answered with, or would
|
||||
// have been. check sets it.
|
||||
refusalStatus int
|
||||
// wouldAction is, in observe mode, the action enforce mode would have
|
||||
// refused the request with, one of requestlog's Action constants, and ""
|
||||
// otherwise. check sets it.
|
||||
@@ -207,8 +203,9 @@ func requestHeaders(r *http.Request, names []string) map[string]string {
|
||||
|
||||
// check is the one place where a request can be refused. It returns nil to let
|
||||
// the request through. The heuristics of checkClient decide: a request they
|
||||
// refuse is answered with the status and logged with the action of the verdict
|
||||
// that refused it. In observe mode nothing is refused, and the log line names
|
||||
// refuse is answered with SWW_WAF_BAN_RESPONSE, 403 for a block rule or the Core
|
||||
// Rule Set, 503 for heuristics that ran out of time, or 414 or 431 for the
|
||||
// header limits. In observe mode nothing is refused, and the log line names
|
||||
// what enforce mode would have done. ctx is the request's own context.
|
||||
func (rq *request) check(ctx context.Context) *refusal {
|
||||
v := rq.checkClient(ctx)
|
||||
@@ -216,16 +213,32 @@ func (rq *request) check(ctx context.Context) *refusal {
|
||||
return nil
|
||||
}
|
||||
|
||||
rq.refusalStatus = v.status
|
||||
action := v.action
|
||||
|
||||
if rq.h.config.Observe {
|
||||
rq.wouldAction = v.action
|
||||
rq.line.WouldAction = v.action
|
||||
switch {
|
||||
case rq.h.config.Observe:
|
||||
rq.wouldAction = action
|
||||
rq.line.WouldAction = action
|
||||
|
||||
return nil
|
||||
case action == requestlog.ActionBlockedByBlockRule ||
|
||||
action == requestlog.ActionBlockedByCoreRuleSet:
|
||||
return &refusal{status: http.StatusForbidden, action: action}
|
||||
case action == requestlog.ActionChecksTimedOut:
|
||||
return &refusal{
|
||||
status: http.StatusServiceUnavailable,
|
||||
action: action,
|
||||
}
|
||||
case action == requestlog.ActionRequestLineTooLong:
|
||||
return &refusal{status: http.StatusRequestURITooLong, action: action}
|
||||
case action == requestlog.ActionRequestHeaderTooLarge:
|
||||
return &refusal{
|
||||
status: http.StatusRequestHeaderFieldsTooLarge,
|
||||
action: action,
|
||||
}
|
||||
default:
|
||||
return rq.banResponse(action)
|
||||
}
|
||||
|
||||
return &refusal{status: v.status, action: v.action}
|
||||
}
|
||||
|
||||
// checkClient runs the heuristics on the request, and returns the first
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
)
|
||||
|
||||
// routeWatchURL is SWW_WAF_ROUTEWATCH_URL, and tagsMissing the log line's field
|
||||
@@ -57,7 +58,7 @@ func startRouteWatch(t *testing.T, routes map[string]standInRoute) string {
|
||||
// waitForRouteWatch waits until routewatch's answer about from, in the client
|
||||
// group of the given length, is kept, so that the next request gets its tags.
|
||||
func waitForRouteWatch(
|
||||
t *testing.T, server *wafAndParts, from string, bits int,
|
||||
t *testing.T, server *waf.Server, from string, bits int,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
package waf
|
||||
|
||||
// The heuristic rule_files. It reads the rules of the rule files, which
|
||||
// SWW_WAF_RULE_FILES_DIR and SWW_WAF_RULE_FILES_ENABLED name, and
|
||||
// SWW_WAF_BAN_RESPONSE.
|
||||
// SWW_WAF_RULE_FILES_DIR and SWW_WAF_RULE_FILES_ENABLED name.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||
@@ -19,9 +17,8 @@ import (
|
||||
// ruleFiles takes the rules of the rule files the request matched, which
|
||||
// findBesideLookup found, notes their ids in the log line, adds the tag
|
||||
// exploitsignature when it matches any, and refuses the request
|
||||
// when the last rule it matches refuses it: with 403 and
|
||||
// ActionBlockedByBlockRule for a block rule, and for a ban rule with
|
||||
// SWW_WAF_BAN_RESPONSE and the action for the ban it makes. A ban
|
||||
// when the last rule it matches refuses it: with ActionBlockedByBlockRule for a
|
||||
// block rule, and for a ban rule with the action for the ban it makes. A ban
|
||||
// rule bans the client's netblock for a clear sign of attack, or in observe
|
||||
// mode raises the alert for the ban
|
||||
// it would have made. Either rule's match is noted as an offence, for the
|
||||
@@ -62,7 +59,6 @@ func (r ruleFiles) check(
|
||||
|
||||
return &verdict{
|
||||
action: requestlog.ActionBlockedByBlockRule,
|
||||
status: http.StatusForbidden,
|
||||
figures: figures,
|
||||
}
|
||||
case rules.ActionBan:
|
||||
@@ -74,7 +70,6 @@ func (r ruleFiles) check(
|
||||
now,
|
||||
bans.Notes{RuleID: last.ID, Target: last.Target},
|
||||
),
|
||||
status: r.banMaker.banResponse,
|
||||
figures: figures,
|
||||
}
|
||||
case rules.ActionLog:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package waf
|
||||
|
||||
// The heuristic tag_deny. It reads SWW_WAF_TAG_DENY and SWW_WAF_BAN_RESPONSE.
|
||||
// The heuristic tag_deny. It reads SWW_WAF_TAG_DENY.
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -10,11 +10,10 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
// tagDeny refuses a request that carries a tag in SWW_WAF_TAG_DENY, with
|
||||
// SWW_WAF_BAN_RESPONSE. Its figure is the first such tag, in sorted order.
|
||||
// tagDeny refuses a request that carries a tag in SWW_WAF_TAG_DENY. Its figure is
|
||||
// the first such tag, in sorted order.
|
||||
type tagDeny struct {
|
||||
tags []string // SWW_WAF_TAG_DENY
|
||||
banResponse int // SWW_WAF_BAN_RESPONSE
|
||||
tags []string // SWW_WAF_TAG_DENY
|
||||
}
|
||||
|
||||
func (tagDeny) name() string { return "tag_deny" }
|
||||
@@ -28,7 +27,6 @@ func (t tagDeny) check(
|
||||
if slices.Contains(t.tags, tag) {
|
||||
return &verdict{
|
||||
action: requestlog.ActionClientDenied,
|
||||
status: t.banResponse,
|
||||
figures: map[string]any{"tag": tag},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package waf
|
||||
|
||||
// The heuristic trap_paths. It reads SWW_WAF_TRAP_PATHS and
|
||||
// SWW_WAF_BAN_RESPONSE.
|
||||
// The heuristic trap_paths. It reads SWW_WAF_TRAP_PATHS.
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -13,10 +12,10 @@ import (
|
||||
)
|
||||
|
||||
// trapPaths refuses a request for a path in SWW_WAF_TRAP_PATHS, as trapPath
|
||||
// finds it, with SWW_WAF_BAN_RESPONSE. Such a request is a clear sign of
|
||||
// attack, as a ban rule's match is: it bans the client's netblock, or in
|
||||
// observe mode raises the alert for the ban it would have made. The ban's
|
||||
// notes give the path as sent, and so does its figure.
|
||||
// finds it. Such a request is a clear sign of attack, as a ban rule's match is:
|
||||
// it bans the client's netblock, or in observe mode raises the alert for the
|
||||
// ban it would have made. The ban's notes give the path as sent, and so does
|
||||
// its figure.
|
||||
type trapPaths struct {
|
||||
paths []string // SWW_WAF_TRAP_PATHS
|
||||
banMaker banMaker
|
||||
@@ -42,7 +41,6 @@ func (t trapPaths) check(
|
||||
now,
|
||||
bans.Notes{TrapPath: path},
|
||||
),
|
||||
status: t.banMaker.banResponse,
|
||||
figures: map[string]any{"path": path},
|
||||
}
|
||||
}
|
||||
|
||||
+13
-1
@@ -30,10 +30,12 @@ const (
|
||||
// is far above rproxy's, and a question refused for its size would have rproxy
|
||||
// let the request through.
|
||||
func New(params Params) *Server {
|
||||
h := newHandler(params)
|
||||
|
||||
return &Server{
|
||||
Server: &http.Server{
|
||||
Addr: params.Config.ListenAddr,
|
||||
Handler: newHandler(params),
|
||||
Handler: h,
|
||||
ErrorLog: slog.NewLogLogger(
|
||||
params.ProcessLog.Handler(),
|
||||
slog.LevelWarn,
|
||||
@@ -42,6 +44,16 @@ func New(params Params) *Server {
|
||||
WriteTimeout: wafTimeout,
|
||||
IdleTimeout: wafIdleTimeout,
|
||||
},
|
||||
Ledger: h.ledger,
|
||||
Limiter: h.limiter,
|
||||
GeoJS: h.geojs,
|
||||
Anomalies: h.anomalies,
|
||||
LookupFile: h.lookupFile,
|
||||
Lists: params.Lists,
|
||||
DNSBL: params.DNSBL,
|
||||
AbuseIPDB: params.AbuseIPDB,
|
||||
RouteWatch: params.RouteWatch,
|
||||
Metrics: h.metrics,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -58,7 +58,7 @@ type WAF struct {
|
||||
t *testing.T
|
||||
addr string
|
||||
out *output
|
||||
server *wafAndParts
|
||||
server *waf.Server
|
||||
clock *clock
|
||||
queue *alerts.Queue
|
||||
}
|
||||
@@ -70,14 +70,14 @@ func startWAF(
|
||||
) *WAF {
|
||||
t.Helper()
|
||||
|
||||
return startWAFWith(t, geojsURL, env, func(*wafAndParts) {})
|
||||
return startWAFWith(t, geojsURL, env, func(*waf.Server) {})
|
||||
}
|
||||
|
||||
// startWAFWith is startWAF with change applied to the server before
|
||||
// it starts.
|
||||
func startWAFWith(
|
||||
t *testing.T, geojsURL string, env map[string]string,
|
||||
change func(*wafAndParts),
|
||||
change func(*waf.Server),
|
||||
) *WAF {
|
||||
t.Helper()
|
||||
|
||||
@@ -703,7 +703,7 @@ func TestQuestionWhoseBodyStopsArrivingGetsNoAnswer(t *testing.T) {
|
||||
t,
|
||||
"",
|
||||
map[string]string{metricsToken: token},
|
||||
func(server *wafAndParts) { server.ReadTimeout = readTimeout },
|
||||
func(server *waf.Server) { server.ReadTimeout = readTimeout },
|
||||
)
|
||||
|
||||
conn := dial(t, s.addr)
|
||||
|
||||
Reference in new issue
Block a user