From 298e850e9edb78931d99cd35020e046175f91102 Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 19 Nov 2026 16:46:33 +0000 Subject: [PATCH] Move what rproxy and waf send each other into wafprotocol (closes #425) internal/wafprotocol writes and reads the question, the answer and the report, and declares once each header, path and limit both daemons use. rproxy no longer imports internal/waf. Nothing on the wire changes. Model: opus-5-5 --- README.md | 2 + SPEC.md | 11 +- internal/rproxy/client.go | 10 - internal/rproxy/limits_test.go | 6 +- internal/rproxy/question_test.go | 26 +-- internal/rproxy/request.go | 24 +- internal/rproxy/rproxy.go | 22 +- internal/rproxy/rproxy_test.go | 36 +-- internal/rproxy/waf.go | 116 +++------- internal/sww/healthcheck.go | 4 +- internal/sww/rproxy_test.go | 23 +- internal/sww/sww_test.go | 4 +- internal/sww/waf_test.go | 22 +- internal/waf/answertime_test.go | 84 +++++-- internal/waf/biased_test.go | 3 +- internal/waf/client.go | 6 +- internal/waf/coreruleset_test.go | 6 +- internal/waf/health_test.go | 11 +- internal/waf/history_test.go | 3 +- internal/waf/lookup.go | 22 -- internal/waf/metrics_test.go | 3 +- internal/waf/proxy.go | 8 - internal/waf/requestlog_test.go | 4 +- internal/waf/waf.go | 280 +++++----------------- internal/waf/waf_test.go | 281 ++++++++++++++++------- internal/wafprotocol/answer.go | 46 ++++ internal/wafprotocol/question.go | 189 +++++++++++++++ internal/wafprotocol/report.go | 123 ++++++++++ internal/wafprotocol/wafprotocol.go | 61 +++++ internal/wafprotocol/wafprotocol_test.go | 204 ++++++++++++++++ 30 files changed, 1083 insertions(+), 557 deletions(-) create mode 100644 internal/wafprotocol/answer.go create mode 100644 internal/wafprotocol/question.go create mode 100644 internal/wafprotocol/report.go create mode 100644 internal/wafprotocol/wafprotocol.go create mode 100644 internal/wafprotocol/wafprotocol_test.go diff --git a/README.md b/README.md index 9ab4147..5d40162 100644 --- a/README.md +++ b/README.md @@ -902,6 +902,8 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)). - `internal/rproxy`: the reverse proxy in front of one app. - `internal/waf`: `waf`'s server, which runs the checks and answers its own endpoints. +- `internal/wafprotocol`: the question, the answer and the report that `rproxy` + and `waf` send each other. - `internal/ratelimit`, `internal/bans`, `internal/rules`, `internal/coreruleset`, `internal/geojs`, `internal/lookup`, `internal/reputation`, `internal/routewatch`, `internal/anomaly` and diff --git a/SPEC.md b/SPEC.md index 5849f66..4f46fff 100644 --- a/SPEC.md +++ b/SPEC.md @@ -73,13 +73,16 @@ never as a time. `go.uber.org/fx`. fx builds the parts of `waf` and `rproxy`, and starts and stops their background work. - `internal/config`: reads the settings, the one place they are read. - - `internal/waf`: `waf`'s server. It runs the checks in `check`, and holds - `waf`'s question, answer and report. It answers `waf`'s own endpoints in - `answerOwnEndpoint`. It counts bytes, refusals and anomalies, and writes - the log lines. + - `internal/waf`: `waf`'s server. It runs the checks in `check`, and answers + `waf`'s own endpoints in `answerOwnEndpoint`. It counts bytes, refusals + and anomalies, and writes the log lines. - `internal/rproxy`: the reverse proxy in front of one app that asks `waf` about each request. It passes a request to the app with the standard library's `httputil.ReverseProxy`, within the time and size limits. + - `internal/wafprotocol`: the question, the answer and the report that + `rproxy` and `waf` send each other (see "`waf`"). It writes and reads each + one, and declares every header, path and limit both sides use. + `internal/rproxy` imports it, not `internal/waf`. - `internal/ratelimit`: the table of clients. It counts requests, bytes and refusals, and keeps each client's history. - `internal/bans`: the bans. It holds each netblock's bans with their notes, diff --git a/internal/rproxy/client.go b/internal/rproxy/client.go index 9b13e41..8452c09 100644 --- a/internal/rproxy/client.go +++ b/internal/rproxy/client.go @@ -7,16 +7,6 @@ import ( "strings" ) -// The headers rproxy sets on the requests it passes to the app. Go writes every -// header name in this form, as it sends it and as it receives it, so -// X-Client-ASN arrives as X-Client-Asn, and Del removes a visitor's own whatever -// their case; header names are not case-sensitive. -const ( - requestIDHeader = "X-Request-ID" - asnHeader = "X-Client-Asn" - countryHeader = "X-Client-Country" -) - // peerAddress is the address of the request's TCP peer, normally traefik, without // any IPv6 zone, which waf refuses. func peerAddress(r *http.Request) netip.Addr { diff --git a/internal/rproxy/limits_test.go b/internal/rproxy/limits_test.go index 8035e4e..d168006 100644 --- a/internal/rproxy/limits_test.go +++ b/internal/rproxy/limits_test.go @@ -20,7 +20,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/config/configtest" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/rproxy" - "sneak.berlin/go/smallwebwaf/internal/waf" + "sneak.berlin/go/smallwebwaf/internal/wafprotocol" ) // post is the start of a POST whose headers end with what follows it. @@ -492,7 +492,7 @@ func TestASettingOfOffSetsNoLimit(t *testing.T) { strings.NewReader("a body"))), ) - if got := get(t, addr, waf.HealthPath); got.status != http.StatusOK { + if got := get(t, addr, wafprotocol.HealthPath); got.status != http.StatusOK { t.Errorf("answered %+v, want rproxy's own 200", got) } }) @@ -1471,7 +1471,7 @@ func TestAVisitorThatStopsALittleShortGetsAnAnswerRProxyGivesAtOnce( {"a deny", post, http.StatusForbidden}, { "the health check", - "GET " + waf.HealthPath + " HTTP/1.1\r\nHost: site.example\r\n", + "GET " + wafprotocol.HealthPath + " HTTP/1.1\r\nHost: site.example\r\n", http.StatusOK, }, } { diff --git a/internal/rproxy/question_test.go b/internal/rproxy/question_test.go index bc265a7..88a6a53 100644 --- a/internal/rproxy/question_test.go +++ b/internal/rproxy/question_test.go @@ -12,11 +12,11 @@ import ( "strings" "testing" - "sneak.berlin/go/smallwebwaf/internal/waf" + "sneak.berlin/go/smallwebwaf/internal/wafprotocol" ) // questionBodyMaxBytes is the most of a body a question carries. -const questionBodyMaxBytes = waf.QuestionBodyMaxBytes +const questionBodyMaxBytes = wafprotocol.QuestionBodyMaxBytes // chunked is the encoding of a body whose length the visitor did not announce. const chunked = "chunked" @@ -42,9 +42,9 @@ func TestQuestionIsTheVisitorsRequestWithRProxysHeaders(t *testing.T) { req.Header.Set(forwardedForHeader, "198.51.100.1") req.Header.Set(forwardedProtoHeader, "gopher") req.Header.Set(requestIDHeader, "the-visitor's") - req.Header.Set(waf.ClientHeader, "198.51.100.2") - req.Header.Set(waf.PeerHeader, "198.51.100.3") - req.Header.Set(waf.ContentLengthHeader, "12345") + req.Header.Set(wafprotocol.ClientHeader, "198.51.100.2") + req.Header.Set(wafprotocol.PeerHeader, "198.51.100.3") + req.Header.Set(wafprotocol.ContentLengthHeader, "12345") // Go's client then sends no User-Agent, so that one in the question would be // Go's own. req.Header["User-Agent"] = nil @@ -73,8 +73,8 @@ func TestQuestionIsTheVisitorsRequestWithRProxysHeaders(t *testing.T) { want.Set(forwardedForHeader, "198.51.100.1") want.Set(forwardedProtoHeader, "http") want.Set(requestIDHeader, asked.header.Get(requestIDHeader)) - want.Set(waf.ClientHeader, localhost) - want.Set(waf.PeerHeader, localhost) + want.Set(wafprotocol.ClientHeader, localhost) + want.Set(wafprotocol.PeerHeader, localhost) if !maps.EqualFunc(asked.header, want, slices.Equal) { t.Errorf( @@ -124,8 +124,8 @@ func TestQuestionNamesTheVisitorATrustedProxySpeaksFor(t *testing.T) { wantGreeting(t, visit(t, req)) asked := standIn.question(t) - if asked.header.Get(waf.ClientHeader) != tc.client || - asked.header.Get(waf.PeerHeader) != localhost || + if asked.header.Get(wafprotocol.ClientHeader) != tc.client || + asked.header.Get(wafprotocol.PeerHeader) != localhost || asked.header.Get(forwardedForHeader) != visitor || asked.header.Get(forwardedProtoHeader) != tc.proto { t.Errorf("the question's headers are %v", asked.header) @@ -191,7 +191,7 @@ func TestQuestionNamesTheClientAsTheForwardedAddressesSay(t *testing.T) { req.Header[forwardedForHeader] = tc.header wantGreeting(t, visit(t, req)) - if got := standIn.question(t).header.Get(waf.ClientHeader); got != + if got := standIn.question(t).header.Get(wafprotocol.ClientHeader); got != tc.client { t.Errorf("the question names %s, want %s", got, tc.client) } @@ -282,7 +282,7 @@ func wantQuestionBody( ) } - values := asked.header.Values(waf.ContentLengthHeader) + values := asked.header.Values(wafprotocol.ContentLengthHeader) if announced { if !slices.Equal(values, []string{strconv.Itoa(size)}) || @@ -586,8 +586,8 @@ func TestAQuestionNamesNoZone(t *testing.T) { server.Handler.ServeHTTP(httptest.NewRecorder(), req) asked := standIn.question(t) - if asked.header.Get(waf.ClientHeader) != tc.client || - asked.header.Get(waf.PeerHeader) != tc.peer { + if asked.header.Get(wafprotocol.ClientHeader) != tc.client || + asked.header.Get(wafprotocol.PeerHeader) != tc.peer { t.Errorf("the question's headers are %v", asked.header) } diff --git a/internal/rproxy/request.go b/internal/rproxy/request.go index 2246bd7..cf24c93 100644 --- a/internal/rproxy/request.go +++ b/internal/rproxy/request.go @@ -18,17 +18,13 @@ import ( "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/rules" - "sneak.berlin/go/smallwebwaf/internal/waf" + "sneak.berlin/go/smallwebwaf/internal/wafprotocol" ) // flushAfterEachWrite has ReverseProxy pass on each part of the app's answer as // soon as it arrives. const flushAfterEachWrite time.Duration = -1 -// reportStatusMax is the highest status waf takes in a report. An app can -// send a higher one, up to 999, and the visitor gets that as it is. -const reportStatusMax = 599 - // plainText is the Content-Type of the answers rproxy gives itself. const plainText = "text/plain; charset=utf-8" @@ -154,7 +150,7 @@ func (rq *request) readFirstPartOfBody() bool { _ = rq.rc.SetReadDeadline(rq.requestDeadline()) held, err := io.ReadAll( - io.LimitReader(rq.body, waf.QuestionBodyMaxBytes), + io.LimitReader(rq.body, wafprotocol.QuestionBodyMaxBytes), ) // From here on, the request timer ends a read of the rest of the body, so that // the answer can name the side rproxy was waiting on. @@ -228,16 +224,17 @@ func (rq *request) rewrite(pr *httputil.ProxyRequest) { setPathAsSent(pr.Out.URL, pr.In) setForwardedHeaders(pr.In, pr.Out, rq.peer, rq.peerTrusted) - pr.Out.Header.Set(requestIDHeader, rq.id) - pr.Out.Header.Del(asnHeader) - pr.Out.Header.Del(countryHeader) + pr.Out.Header.Set(wafprotocol.RequestIDHeader, rq.id) + // Del removes a visitor's own, whatever their case. + pr.Out.Header.Del(wafprotocol.ASNHeader) + pr.Out.Header.Del(wafprotocol.CountryHeader) if rq.asn != "" { - pr.Out.Header.Set(asnHeader, rq.asn) + pr.Out.Header.Set(wafprotocol.ASNHeader, rq.asn) } if rq.country != "" { - pr.Out.Header.Set(countryHeader, rq.country) + pr.Out.Header.Set(wafprotocol.CountryHeader, rq.country) } } @@ -470,9 +467,10 @@ func (rq *request) finish(ctx context.Context) { return } - report := &waf.Report{ + // An app can send a status up to 999, and the visitor gets it as it is. + report := &wafprotocol.Report{ RequestID: rq.id, - Status: min(rq.out.status, reportStatusMax), + Status: min(rq.out.status, wafprotocol.ReportStatusMax), RequestBytes: requestBytes, ResponseBytes: responseBytes, DurationTotalMS: requestlog.Milliseconds(end.Sub(rq.start)), diff --git a/internal/rproxy/rproxy.go b/internal/rproxy/rproxy.go index 489c4c6..0da3415 100644 --- a/internal/rproxy/rproxy.go +++ b/internal/rproxy/rproxy.go @@ -24,7 +24,7 @@ import ( "time" "sneak.berlin/go/smallwebwaf/internal/config" - "sneak.berlin/go/smallwebwaf/internal/waf" + "sneak.berlin/go/smallwebwaf/internal/wafprotocol" ) // How rproxy keeps connections to the app and to waf open between @@ -47,10 +47,6 @@ const ( idleTimeout = 120 * time.Second ) -// ownEndpointPrefix starts the path of every request for sww itself, which -// never reaches the app. -const ownEndpointPrefix = "/_sww/" - // allowedMethods lists the methods rproxy passes on, as an Allow header gives // them. It answers any other method 405 itself. // The WebDAV ones are for Nextcloud. @@ -92,7 +88,9 @@ func New(params Params) *Server { transport: newTransport(), wafTransport: newWAFTransport(), reportURL: (&url.URL{ - Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath, + Scheme: backend.Scheme, + Host: backend.Host, + Path: wafprotocol.ReportPath, }).String(), } @@ -176,7 +174,7 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { defer rq.readRestOfBody() defer rq.finish(r.Context()) - if r.Method == http.MethodGet && r.URL.Path == waf.HealthPath { + if r.Method == http.MethodGet && r.URL.Path == wafprotocol.HealthPath { rq.answerHealth() return @@ -209,9 +207,9 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { rq.noteAnswer(answer) switch rq.disposition { - case waf.DispositionAllow: + case wafprotocol.DispositionAllow: rq.passToApp(r.Context()) - case waf.DispositionClose: + case wafprotocol.DispositionClose: // This panic has Go's server close the connection without an answer, // and log nothing; the deferred finish still runs. panic(http.ErrAbortHandler) @@ -221,8 +219,8 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // passToApp passes the request on to the app, unless a limit or its path keeps -// it from the app. waf answers a path under ownEndpointPrefix itself, so -// such a request gets here only when +// it from the app. waf answers a path under wafprotocol.OwnEndpointPrefix +// itself, so such a request gets here only when // SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION allowed it. // ctx is the request's own context. func (rq *request) passToApp(ctx context.Context) { @@ -231,7 +229,7 @@ func (rq *request) passToApp(ctx context.Context) { switch { case rq.refused.Load() != 0 || (maxBytes > 0 && rq.in.ContentLength > maxBytes): rq.answer(http.StatusRequestEntityTooLarge) - case strings.HasPrefix(rq.in.URL.Path, ownEndpointPrefix): + case strings.HasPrefix(rq.in.URL.Path, wafprotocol.OwnEndpointPrefix): rq.answer(http.StatusNotFound) default: rq.forward(ctx) diff --git a/internal/rproxy/rproxy_test.go b/internal/rproxy/rproxy_test.go index 74e8eae..8fbe494 100644 --- a/internal/rproxy/rproxy_test.go +++ b/internal/rproxy/rproxy_test.go @@ -23,7 +23,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/config/configtest" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/rproxy" - "sneak.berlin/go/smallwebwaf/internal/waf" + "sneak.berlin/go/smallwebwaf/internal/wafprotocol" ) const ( @@ -56,7 +56,7 @@ const ( // The headers the tests set and look at. const ( - dispositionHeader = waf.DispositionHeader + dispositionHeader = wafprotocol.DispositionHeader forwardedForHeader = "X-Forwarded-For" forwardedProtoHeader = "X-Forwarded-Proto" requestIDHeader = "X-Request-ID" @@ -72,9 +72,9 @@ const ( // What the stand-in for waf answers. const ( - dispositionAllow = waf.DispositionAllow - dispositionDeny = waf.DispositionDeny - dispositionClose = waf.DispositionClose + dispositionAllow = wafprotocol.DispositionAllow + dispositionDeny = wafprotocol.DispositionDeny + dispositionClose = wafprotocol.DispositionClose // deniedBody is the body of a deny. deniedBody = "refused by waf" ) @@ -175,7 +175,7 @@ type WAF struct { *httptest.Server questions chan question - reports chan waf.Report + reports chan wafprotocol.Report } // startWAF starts a stand-in for waf. @@ -184,11 +184,11 @@ func startWAF(t *testing.T, answer http.HandlerFunc) *WAF { s := &WAF{ questions: make(chan question, 10), - reports: make(chan waf.Report, 10), + reports: make(chan wafprotocol.Report, 10), } s.Server = httptest.NewServer(http.HandlerFunc( func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get(waf.ClientHeader) == "" { + if r.Header.Get(wafprotocol.ClientHeader) == "" { s.takeReport(t, w, r) return @@ -217,11 +217,11 @@ func (s *WAF) takeReport( ) { t.Helper() - var report waf.Report + var report wafprotocol.Report err := json.NewDecoder(r.Body).Decode(&report) if err != nil || r.Method != http.MethodPost || - r.URL.Path != waf.ReportPath { + r.URL.Path != wafprotocol.ReportPath { t.Errorf("%s %s is no report: %v", r.Method, r.URL.Path, err) } @@ -242,7 +242,7 @@ func (s *WAF) wantNoReport(t *testing.T) { } // report waits for a report. -func (s *WAF) report(t *testing.T) waf.Report { +func (s *WAF) report(t *testing.T) wafprotocol.Report { t.Helper() select { @@ -251,7 +251,7 @@ func (s *WAF) report(t *testing.T) waf.Report { case <-time.After(waitLimit): t.Fatal("no report") - return waf.Report{} + return wafprotocol.Report{} } } @@ -279,7 +279,7 @@ func startWAFAnsweringReports( backend := httptest.NewServer(http.HandlerFunc( func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get(waf.ClientHeader) != "" { + if r.Header.Get(wafprotocol.ClientHeader) != "" { answer(w, r) return @@ -811,7 +811,7 @@ func TestAnAnswerRProxyHoldsWholeIsSentBeforeItsLogLine(t *testing.T) { }{ {"a deny", "GET /" + host, deniedBody, http.StatusForbidden}, {"rproxy's own answer", "GET x:admin" + host, "Bad Request\n", http.StatusBadRequest}, - {"the health check", "GET " + waf.HealthPath + host, "ok\n", http.StatusOK}, + {"the health check", "GET " + wafprotocol.HealthPath + host, "ok\n", http.StatusOK}, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -1576,19 +1576,19 @@ func TestHealthCheckIsAnsweredWhileWAFIsDown(t *testing.T) { app := startApp(t, greet) addr, out := startRProxy(t, nowhere, app.URL, nil) - got := get(t, addr, waf.HealthPath) + got := get(t, addr, wafprotocol.HealthPath) if got.status != http.StatusOK || got.body != "ok\n" { t.Errorf("answered %+v, want ok", got) } line := out.requestLine(t) - if line["path"] != waf.HealthPath || line["disposition"] != nil { + if line["path"] != wafprotocol.HealthPath || line["disposition"] != nil { t.Errorf("line %v, want one with no disposition", line) } // With waf up too, it is not asked. addr, _ = startRProxy(t, standIn.URL, app.URL, nil) - get(t, addr, waf.HealthPath) + get(t, addr, wafprotocol.HealthPath) if len(standIn.questions) != 0 { t.Error("waf was asked about the health check") @@ -1749,7 +1749,7 @@ func TestWaitForReportsWaitsForAReportBeingSent(t *testing.T) { release := make(chan struct{}) backend := httptest.NewServer(http.HandlerFunc( func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get(waf.ClientHeader) != "" { + if r.Header.Get(wafprotocol.ClientHeader) != "" { allow(w, r) return diff --git a/internal/rproxy/waf.go b/internal/rproxy/waf.go index 630329b..5b31003 100644 --- a/internal/rproxy/waf.go +++ b/internal/rproxy/waf.go @@ -1,9 +1,7 @@ package rproxy import ( - "bytes" "context" - "encoding/json" "errors" "fmt" "io" @@ -12,7 +10,8 @@ import ( "strings" "time" - "sneak.berlin/go/smallwebwaf/internal/waf" + "sneak.berlin/go/smallwebwaf/internal/rules" + "sneak.berlin/go/smallwebwaf/internal/wafprotocol" ) // answerBodyMaxBytes is the most of the body of waf's answer that is read. @@ -41,15 +40,12 @@ type wafAnswer struct { // notes nothing, and no report is due for it: waf holds no decision to report // on. func (rq *request) noteAnswer(answer *wafAnswer) { - disposition := answer.header.Get(waf.DispositionHeader) - - switch disposition { - case waf.DispositionAllow, waf.DispositionDeny, - waf.DispositionClose: - rq.disposition = disposition + read, valid := wafprotocol.ReadAnswer(answer.header) + if valid { + rq.disposition = read.Disposition rq.answered = true - rq.asn = answer.header.Get(asnHeader) - rq.country = answer.header.Get(countryHeader) + rq.asn = read.ASN + rq.country = read.Country } } @@ -80,7 +76,7 @@ func (rq *request) handleNoUsableAnswer(ctx context.Context, err error) { rq.disposition = rq.h.config.WAFBackendNoAnswerDisposition rq.wafNoAnswer = true - if rq.disposition == waf.DispositionAllow { + if rq.disposition == wafprotocol.DispositionAllow { rq.passToApp(ctx) } else { rq.answer(http.StatusServiceUnavailable) @@ -120,7 +116,21 @@ func (rq *request) ask(ctx context.Context) (*wafAnswer, error) { ctx, cancel := context.WithTimeout(ctx, rq.h.config.WAFBackendTimeout) defer cancel() - question, err := rq.newQuestion(ctx) + var held []byte + if rq.body != nil { + held = rq.body.held + } + + question, err := wafprotocol.NewQuestion(ctx, wafprotocol.QuestionParams{ + Visitor: rq.in, + PathAsSent: rules.Path(rq.in), + Body: held, + WAFBackend: rq.h.config.WAFBackend, + Client: rq.client, + Peer: rq.peer, + RequestID: rq.id, + Scheme: scheme(rq.in, rq.peerTrusted), + }) if err != nil { return nil, err } @@ -177,69 +187,13 @@ func canBeAsked(r *http.Request) bool { len(r.Header.Values("User-Agent")) <= 1 } -// newQuestion returns the question about the request, as "The question" in -// SPEC.md gives it: the visitor's request as the visitor sent it, with the part -// of the body it holds and rproxy's headers. -func (rq *request) newQuestion(ctx context.Context) (*http.Request, error) { - in := rq.in - - var held []byte - if rq.body != nil { - held = rq.body.held - } - - question, err := http.NewRequestWithContext( //nolint:gosec // a setting's URL - ctx, - in.Method, - rq.h.config.WAFBackend.String(), - bytes.NewReader(held), - ) - if err != nil { - return nil, fmt.Errorf("make the question: %w", err) - } - - backend := question.URL - target := *in.URL - target.Scheme = backend.Scheme - target.Host = backend.Host - setPathAsSent(&target, in) - question.URL = &target - question.Host = in.Host - - // The length NewRequestWithContext set is that of what is held. This line makes - // a body whose length the visitor did not announce go chunked, as "The question" - // in SPEC.md requires. - question.TransferEncoding = in.TransferEncoding - - question.Header = in.Header.Clone() - question.Header.Set(waf.ClientHeader, rq.client.String()) - question.Header.Set(waf.PeerHeader, rq.peer.String()) - question.Header.Del(waf.ContentLengthHeader) - - announced := in.Header.Get("Content-Length") - if announced != "" { - question.Header.Set(waf.ContentLengthHeader, announced) - } - - question.Header.Set(requestIDHeader, rq.id) - question.Header.Set("X-Forwarded-Proto", scheme(in, rq.peerTrusted)) - - // Go's client adds a User-Agent of its own to a request without the key. - _, sent := question.Header["User-Agent"] - if !sent { - question.Header["User-Agent"] = nil - } - - return question, nil -} - // answerWith gives the visitor waf's answer, less its disposition. func (rq *request) answerWith(answer *wafAnswer) { rq.prepareOwnAnswer(rq.requestDeadline()) header := rq.out.Header() maps.Copy(header, answer.header) - header.Del(waf.DispositionHeader) + header.Del(wafprotocol.DispositionHeader) // Go's server adds a Content-Type it guesses from the body to a response whose // headers have no Content-Type key. The key with no value sends none. @@ -253,7 +207,10 @@ func (rq *request) answerWith(answer *wafAnswer) { // sendReport sends waf report on a goroutine of its own, so that the // visitor does not wait for it. The report goes on after ctx, the request's own // context, has ended. -func (h *handler) sendReport(ctx context.Context, report *waf.Report) { +func (h *handler) sendReport( + ctx context.Context, + report *wafprotocol.Report, +) { h.reportsMu.Lock() defer h.reportsMu.Unlock() @@ -271,23 +228,18 @@ func (h *handler) sendReport(ctx context.Context, report *waf.Report) { // postReport posts report to waf, and waits for its answer for up to // SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS. -func (h *handler) postReport(ctx context.Context, report *waf.Report) error { - body, err := json.Marshal(report) - if err != nil { - return fmt.Errorf("encode the report: %w", err) - } - +func (h *handler) postReport( + ctx context.Context, + report *wafprotocol.Report, +) error { ctx, cancel := context.WithTimeout(ctx, h.config.WAFBackendTimeout) defer cancel() - req, err := http.NewRequestWithContext( //nolint:gosec // a setting's URL - ctx, http.MethodPost, h.reportURL, bytes.NewReader(body)) + req, err := wafprotocol.NewReportRequest(ctx, h.reportURL, report) if err != nil { - return fmt.Errorf("make the request: %w", err) + return err } - req.Header.Set("Content-Type", "application/json") - res, err := h.wafTransport.RoundTrip(req) if err != nil { return fmt.Errorf("send the report: %w", err) diff --git a/internal/sww/healthcheck.go b/internal/sww/healthcheck.go index 32b76ac..38aa3f4 100644 --- a/internal/sww/healthcheck.go +++ b/internal/sww/healthcheck.go @@ -13,7 +13,7 @@ import ( "github.com/spf13/viper" "sneak.berlin/go/smallwebwaf/internal/config" - "sneak.berlin/go/smallwebwaf/internal/waf" + "sneak.berlin/go/smallwebwaf/internal/wafprotocol" ) // healthCheckTimeout bounds the whole health check. @@ -116,7 +116,7 @@ func askHealthEndpoint(ctx context.Context, listenAddr string) error { health := "http://" + net.JoinHostPort( "127.0.0.1", port, - ) + waf.HealthPath + ) + wafprotocol.HealthPath req, err := http.NewRequestWithContext( ctx, diff --git a/internal/sww/rproxy_test.go b/internal/sww/rproxy_test.go index 8d54d66..832a4bc 100644 --- a/internal/sww/rproxy_test.go +++ b/internal/sww/rproxy_test.go @@ -21,7 +21,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/config/configtest" "sneak.berlin/go/smallwebwaf/internal/sww" - "sneak.berlin/go/smallwebwaf/internal/waf" + "sneak.berlin/go/smallwebwaf/internal/wafprotocol" ) const ( @@ -164,7 +164,7 @@ func TestABanThroughOneSiteRefusesTheClientOnTheOther(t *testing.T) { // One request's three lines carry one id, rproxy's with waf's answer. line := firstOut.line(t, "path", bannedProbe) - if line["disposition"] != waf.DispositionDeny || + if line["disposition"] != wafprotocol.DispositionDeny || line["client_ip"] != placed { t.Errorf("rproxy's line is %v", line) } @@ -249,7 +249,7 @@ func TestTheSiteAnswersWhileWAFIsStoppedAndStartedAgain(t *testing.T) { wantStatus(t, site+"/after-stop", placed, http.StatusOK) line := out.line(t, "path", "/after-stop") - if line["disposition"] != waf.DispositionAllow || + if line["disposition"] != wafprotocol.DispositionAllow || line["waf_backend_no_answer"] != true { t.Errorf("rproxy's line is %v", line) } @@ -437,7 +437,7 @@ func TestAChunkedBodyStaysABodyAtWAF(t *testing.T) { const smuggledID = "smuggled-request" smuggled := "GET " + bannedProbe + " HTTP/1.1\r\nHost: site.example\r\n" + - waf.ClientHeader + ": " + otherClient + "\r\n" + + wafprotocol.ClientHeader + ": " + otherClient + "\r\n" + "X-Request-ID: " + smuggledID + "\r\n\r\n" status := sendRaw( @@ -605,10 +605,11 @@ func TestRProxyAsksAWAFThatSpeaksHTTP2OverHTTP1(t *testing.T) { protocols := make(chan string, 10) backend := httptest.NewUnstartedServer(http.HandlerFunc( func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get(waf.ClientHeader) != "" { + if r.Header.Get(wafprotocol.ClientHeader) != "" { protocols <- r.Proto - w.Header().Set(waf.DispositionHeader, waf.DispositionDeny) + w.Header(). + Set(wafprotocol.DispositionHeader, wafprotocol.DispositionDeny) w.WriteHeader(http.StatusForbidden) } })) @@ -690,8 +691,9 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) { release := make(chan struct{}) backend := httptest.NewServer(http.HandlerFunc( func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get(waf.ClientHeader) != "" { - w.Header().Set(waf.DispositionHeader, waf.DispositionAllow) + if r.Header.Get(wafprotocol.ClientHeader) != "" { + w.Header(). + Set(wafprotocol.DispositionHeader, wafprotocol.DispositionAllow) return } @@ -802,7 +804,8 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) { backend := httptest.NewServer(http.HandlerFunc( func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set(waf.DispositionHeader, waf.DispositionAllow) + w.Header(). + Set(wafprotocol.DispositionHeader, wafprotocol.DispositionAllow) })) t.Cleanup(backend.Close) @@ -935,7 +938,7 @@ func TestMainRunsRProxy(t *testing.T) { addr, stop := startRProxyProcess(t, wafBackend+"=http://127.0.0.1:9") req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, - "http://"+addr+waf.HealthPath, http.NoBody) + "http://"+addr+wafprotocol.HealthPath, http.NoBody) if err != nil { t.Fatalf("new request: %v", err) } diff --git a/internal/sww/sww_test.go b/internal/sww/sww_test.go index dbe7571..74937de 100644 --- a/internal/sww/sww_test.go +++ b/internal/sww/sww_test.go @@ -23,7 +23,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/config/configtest" "sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest" "sneak.berlin/go/smallwebwaf/internal/sww" - "sneak.berlin/go/smallwebwaf/internal/waf" + "sneak.berlin/go/smallwebwaf/internal/wafprotocol" ) const ( @@ -1811,7 +1811,7 @@ func statusFrom(t *testing.T, url, from string) int { } req.Header.Set("X-Forwarded-For", from) - req.Header.Set(waf.ClientHeader, from) + req.Header.Set(wafprotocol.ClientHeader, from) req.Header.Set("X-Request-ID", rand.Text()) transport := &http.Transport{} diff --git a/internal/sww/waf_test.go b/internal/sww/waf_test.go index 8ad703c..0599341 100644 --- a/internal/sww/waf_test.go +++ b/internal/sww/waf_test.go @@ -20,7 +20,7 @@ import ( _ "time/tzdata" "sneak.berlin/go/smallwebwaf/internal/sww" - "sneak.berlin/go/smallwebwaf/internal/waf" + "sneak.berlin/go/smallwebwaf/internal/wafprotocol" ) // commandForMain, while set in the environment, has this test binary run Main @@ -106,7 +106,7 @@ func TestMainRunsWAF(t *testing.T) { starting := out.lineBeforeExit(t, "starting", exited) addr, _ := starting["address"].(string) - ask(t, addr, "/", http.StatusOK, waf.DispositionAllow) + ask(t, addr, "/", http.StatusOK, wafprotocol.DispositionAllow) // The files are written as sww stops. err = cmd.Process.Signal(syscall.SIGTERM) @@ -217,7 +217,7 @@ func ask(t *testing.T, addr, path string, status int, disposition string) { t.Fatalf("new request: %v", err) } - req.Header.Set(waf.ClientHeader, placed) + req.Header.Set(wafprotocol.ClientHeader, placed) req.Header.Set("X-Request-ID", rand.Text()) transport := &http.Transport{} @@ -232,11 +232,11 @@ func ask(t *testing.T, addr, path string, status int, disposition string) { _ = res.Body.Close() if res.StatusCode != status || - res.Header.Get(waf.DispositionHeader) != disposition { + res.Header.Get(wafprotocol.DispositionHeader) != disposition { t.Errorf( "answered %d with disposition %q, want %d with %q", res.StatusCode, - res.Header.Get(waf.DispositionHeader), + res.Header.Get(wafprotocol.DispositionHeader), status, disposition, ) @@ -287,12 +287,18 @@ func TestWAFKeepsItsBansAcrossARestartAndNeedsNoApp(t *testing.T) { } runWAF(t, env, func(addr string) { - ask(t, addr, "/", http.StatusOK, waf.DispositionAllow) + ask(t, addr, "/", http.StatusOK, wafprotocol.DispositionAllow) // The default rule file bans a probe for this. - ask(t, addr, "/.env", http.StatusForbidden, waf.DispositionDeny) + ask( + t, + addr, + "/.env", + http.StatusForbidden, + wafprotocol.DispositionDeny, + ) }) runWAF(t, env, func(addr string) { - ask(t, addr, "/", http.StatusForbidden, waf.DispositionDeny) + ask(t, addr, "/", http.StatusForbidden, wafprotocol.DispositionDeny) }) } diff --git a/internal/waf/answertime_test.go b/internal/waf/answertime_test.go index c3e16b5..73cb643 100644 --- a/internal/waf/answertime_test.go +++ b/internal/waf/answertime_test.go @@ -26,6 +26,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/waf" + "sneak.berlin/go/smallwebwaf/internal/wafprotocol" ) const ( @@ -84,7 +85,7 @@ func TestSixteenThousandCookiesAreRefusedBeforeTheCoreRuleSetReadsThem( got := do(t, req) - wantDisposition(t, got, tooLarge, waf.DispositionDeny) + wantDisposition(t, got, tooLarge, wafprotocol.DispositionDeny) line := s.out.requestLine(t) wantLine(t, line, tooLarge, requestlog.ActionRequestHeaderTooLarge) @@ -164,9 +165,14 @@ func TestARequestLineOverTheLimitIsRefusedWith414BeforeAnyOtherCheck( line := s.out.requestLine(t) if tc.status == http.StatusOK { - wantDisposition(t, got, tc.status, waf.DispositionAllow) + wantDisposition( + t, + got, + tc.status, + wafprotocol.DispositionAllow, + ) } else { - wantDisposition(t, got, tc.status, waf.DispositionDeny) + wantDisposition(t, got, tc.status, wafprotocol.DispositionDeny) } wantLine(t, line, tc.status, tc.action) @@ -219,9 +225,14 @@ func TestAHeaderLineOverTheLimitIsRefusedWith431(t *testing.T) { got := do(t, req) if tc.refused { - wantDisposition(t, got, tooLarge, waf.DispositionDeny) + wantDisposition( + t, + got, + tooLarge, + wafprotocol.DispositionDeny, + ) } else { - wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) + wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow) } }) } @@ -244,10 +255,7 @@ func TestRequestLineRefusalNamesItsHeuristicLengthAndLimit(t *testing.T) { func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) { t.Parallel() - repeated := make([]string, headerFieldLimit) - for i := range repeated { - repeated[i] = "1" - } + repeated := slices.Repeat([]string{"1"}, headerFieldLimit) for _, tc := range []struct { name string @@ -294,7 +302,12 @@ func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) { line := s.out.requestLine(t) if tc.refused { - wantDisposition(t, got, tooLarge, waf.DispositionDeny) + wantDisposition( + t, + got, + tooLarge, + wafprotocol.DispositionDeny, + ) wantLine( t, line, @@ -305,7 +318,12 @@ func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) { return } - wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) + wantDisposition( + t, + got, + http.StatusOK, + wafprotocol.DispositionAllow, + ) wantLine(t, line, http.StatusOK, requestlog.ActionForward) if line.DurationCoreRuleSetMS == nil { @@ -332,7 +350,12 @@ func TestHeaderLimitsHoldWhateverTheCoreRuleSetSettingsSay(t *testing.T) { req := s.question(siteA, client, "/") req.Header.Set(cookieHeader, cookies(cookieLimit+1)) - wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny) + wantDisposition( + t, + do(t, req), + tooLarge, + wafprotocol.DispositionDeny, + ) }) } } @@ -348,7 +371,12 @@ func TestClientInTheAllowNetsSkipsTheHeaderLimits(t *testing.T) { req := s.question(siteA, client, "/") req.Header.Set(cookieHeader, cookies(cookieLimit+1)) - wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow) + wantDisposition( + t, + do(t, req), + http.StatusOK, + wafprotocol.DispositionAllow, + ) } func TestObserveModeAllowsARequestOverTheHeaderLimitsUnread(t *testing.T) { @@ -364,7 +392,7 @@ func TestObserveModeAllowsARequestOverTheHeaderLimitsUnread(t *testing.T) { got := do(t, req) - wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) + wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow) line := s.out.requestLine(t) if line.WouldAction != requestlog.ActionRequestHeaderTooLarge || @@ -386,14 +414,14 @@ func TestTheRateLimitsDoNotCountRequestsOverTheHeaderLimits(t *testing.T) { for range 2 { req := s.question(siteA, client, "/") req.Header.Set(cookieHeader, cookies(cookieLimit+1)) - wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny) + wantDisposition(t, do(t, req), tooLarge, wafprotocol.DispositionDeny) } wantDisposition( t, s.ask(client, "/"), http.StatusOK, - waf.DispositionAllow, + wafprotocol.DispositionAllow, ) } @@ -466,7 +494,12 @@ func TestTheHeaderLimitsComeBeforeEveryCheckThatRefuses(t *testing.T) { req := s.question(siteA, client, tc.path) req.Header.Set(cookieHeader, cookies(cookieLimit+1)) - wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny) + wantDisposition( + t, + do(t, req), + tooLarge, + wafprotocol.DispositionDeny, + ) wantLine( t, s.out.requestLine(t), @@ -503,11 +536,16 @@ func TestARequestOverTheHeaderLimitsRunsNoOtherCheck(t *testing.T) { over := s.question(siteA, client, "/") over.Header.Set(cookieHeader, cookies(cookieLimit+1)) - wantDisposition(t, do(t, over), tooLarge, waf.DispositionDeny) + wantDisposition(t, do(t, over), tooLarge, wafprotocol.DispositionDeny) within := s.question(siteA, client, "/") within.Header.Set(cookieHeader, cookies(cookieLimit)) - wantDisposition(t, do(t, within), http.StatusOK, waf.DispositionAllow) + wantDisposition( + t, + do(t, within), + http.StatusOK, + wafprotocol.DispositionAllow, + ) lines := s.out.requestLines(t, 2) @@ -564,7 +602,7 @@ func TestNotWaitingForGeoJSNeverLowersAClientsLimit(t *testing.T) { t, s.ask(fromDE, "/"), http.StatusForbidden, - waf.DispositionDeny, + wafprotocol.DispositionDeny, ) line := s.out.requestLine(t) @@ -601,7 +639,7 @@ func TestAClientGeoJSPlacedInNoCountryCountsAsUnknownWithoutWaiting( t, s.ask(client, "/"), http.StatusForbidden, - waf.DispositionDeny, + wafprotocol.DispositionDeny, ) line := s.out.requestLine(t) @@ -630,7 +668,7 @@ func TestObserveModeAsksGeoJSNothingForARequestOverTheHeaderLimits( req.Header.Set(cookieHeader, cookies(cookieLimit+1)) got := do(t, req) - wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) + wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow) if len(asked()) != 0 || got.header.Get("X-Client-Asn") != "" { t.Errorf( @@ -672,7 +710,7 @@ func askDirectly( http.NoBody, ) req.RemoteAddr = net.JoinHostPort(rproxyPeer, "1234") - req.Header.Set(waf.ClientHeader, client) + req.Header.Set(wafprotocol.ClientHeader, client) req.Header.Set(requestIDHeader, "the-id") change(req) diff --git a/internal/waf/biased_test.go b/internal/waf/biased_test.go index 0835a4f..aadffab 100644 --- a/internal/waf/biased_test.go +++ b/internal/waf/biased_test.go @@ -19,6 +19,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/waf" + "sneak.berlin/go/smallwebwaf/internal/wafprotocol" ) // The biased thresholds. @@ -589,7 +590,7 @@ func serveFromDE( t.Helper() req := httptest.NewRequestWithContext(t.Context(), method, "/", body) - req.Header.Set(waf.ClientHeader, fromDE) + req.Header.Set(wafprotocol.ClientHeader, fromDE) req.Header.Set(requestIDHeader, "the-id") server.Handler.ServeHTTP(httptest.NewRecorder(), req) diff --git a/internal/waf/client.go b/internal/waf/client.go index 8381f0a..cef59a6 100644 --- a/internal/waf/client.go +++ b/internal/waf/client.go @@ -7,6 +7,7 @@ import ( "slices" "sneak.berlin/go/smallwebwaf/internal/config" + "sneak.berlin/go/smallwebwaf/internal/wafprotocol" ) // peerAddress is the address of the request's TCP peer. @@ -19,14 +20,11 @@ func peerAddress(r *http.Request) netip.Addr { return addrPort.Addr().Unmap() } -// requestIDHeader carries the request's id, from rproxy. -const requestIDHeader = "X-Request-ID" - // requestID is the request's id: the one a question gives, or for any other // request a new random one. func requestID(r *http.Request, isQuestion bool) string { if isQuestion { - return r.Header.Get(requestIDHeader) + return r.Header.Get(wafprotocol.RequestIDHeader) } return rand.Text() diff --git a/internal/waf/coreruleset_test.go b/internal/waf/coreruleset_test.go index 39829ef..54707c3 100644 --- a/internal/waf/coreruleset_test.go +++ b/internal/waf/coreruleset_test.go @@ -12,7 +12,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/requestlog" - "sneak.berlin/go/smallwebwaf/internal/waf" + "sneak.berlin/go/smallwebwaf/internal/wafprotocol" ) // The Core Rule Set's settings the tests set, besides SWW_WAF_CORE_RULE_SET_MODE, and @@ -395,9 +395,9 @@ func TestGiteasFixesApplyOnlyToAHostnameThatServesGitea(t *testing.T) { }) wantDisposition(t, do(t, s.question(siteA, client, fileName)), - http.StatusOK, waf.DispositionAllow) + http.StatusOK, wafprotocol.DispositionAllow) wantDisposition(t, do(t, s.question(siteB, client, fileName)), - http.StatusForbidden, waf.DispositionDeny) + http.StatusForbidden, wafprotocol.DispositionDeny) lines := s.out.requestLines(t, 2) wantLine(t, lines[0], http.StatusOK, requestlog.ActionForward) diff --git a/internal/waf/health_test.go b/internal/waf/health_test.go index 70d3532..83ca9ad 100644 --- a/internal/waf/health_test.go +++ b/internal/waf/health_test.go @@ -5,7 +5,7 @@ import ( "testing" "sneak.berlin/go/smallwebwaf/internal/requestlog" - "sneak.berlin/go/smallwebwaf/internal/waf" + "sneak.berlin/go/smallwebwaf/internal/wafprotocol" ) func TestHealthEndpointCountsTowardNoLimit(t *testing.T) { @@ -25,8 +25,13 @@ func TestHealthEndpointCountsTowardNoLimit(t *testing.T) { ) for range healthChecks { - got := s.ownEndpointRequest("", http.MethodGet, waf.HealthPath, "", - http.StatusOK) + got := s.ownEndpointRequest( + "", + http.MethodGet, + wafprotocol.HealthPath, + "", + http.StatusOK, + ) if string(got.body) != "ok\n" || got.header.Get("Content-Type") != contentType { diff --git a/internal/waf/history_test.go b/internal/waf/history_test.go index 0254273..3973c4f 100644 --- a/internal/waf/history_test.go +++ b/internal/waf/history_test.go @@ -11,6 +11,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/waf" + "sneak.berlin/go/smallwebwaf/internal/wafprotocol" ) func TestHistoryKeepsEachRequestOfTheClient(t *testing.T) { @@ -118,7 +119,7 @@ func TestOwnEndpointsAreNotInTheHistory(t *testing.T) { s.ownEndpointRequest( "", http.MethodGet, - waf.HealthPath, + wafprotocol.HealthPath, "", http.StatusOK, ) diff --git a/internal/waf/lookup.go b/internal/waf/lookup.go index a6fc5e4..aca7bbe 100644 --- a/internal/waf/lookup.go +++ b/internal/waf/lookup.go @@ -3,7 +3,6 @@ package waf import ( "context" "log/slog" - "net/http" "net/netip" "strings" "time" @@ -16,15 +15,6 @@ import ( "sneak.berlin/go/smallwebwaf/internal/ratelimit" ) -// The headers in which an answer of allow gives rproxy the client's AS number -// and country, for the app, while SWW_WAF_ADD_LOOKUP_HEADERS is true. Go writes -// every header name in this form, as it sends it and as it receives it, so -// X-Client-ASN arrives as X-Client-Asn; header names are not case-sensitive. -const ( - asnHeader = "X-Client-Asn" - countryHeader = "X-Client-Country" -) - // lookupResult is what the lookup gave for a request: whether its client was // looked up, the answer, and whether the request skipped the wait for GeoJS. type lookupResult struct { @@ -166,18 +156,6 @@ func GeoJSFailed( m.GeoJSFailures.Inc() } -// setLookupHeaders sets the headers that give the client's AS number and -// country, leaving out one that is unknown. -func setLookupHeaders(header http.Header, asn, country string) { - if asn != "" { - header.Set(asnHeader, asn) - } - - if country != "" { - header.Set(countryHeader, country) - } -} - // canBePlaced reports whether a lookup can place addr: private, loopback // and link-local addresses have no AS number or country. func canBePlaced(addr netip.Addr) bool { diff --git a/internal/waf/metrics_test.go b/internal/waf/metrics_test.go index dac7175..a4c5d5e 100644 --- a/internal/waf/metrics_test.go +++ b/internal/waf/metrics_test.go @@ -16,6 +16,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/geojs" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/waf" + "sneak.berlin/go/smallwebwaf/internal/wafprotocol" ) const ( @@ -66,7 +67,7 @@ func TestMetricsNeedTheTokenAndOtherPathsAreNotFound(t *testing.T) { {http.MethodPost, waf.MetricsPath, bearer, http.StatusNotFound}, {http.MethodGet, waf.MetricsPath + "/", bearer, http.StatusNotFound}, {http.MethodGet, "/_sww/bans", bearer, http.StatusNotFound}, - {http.MethodPost, waf.HealthPath, "", http.StatusNotFound}, + {http.MethodPost, wafprotocol.HealthPath, "", http.StatusNotFound}, } { got := s.ownEndpointRequest( tc.authorization, diff --git a/internal/waf/proxy.go b/internal/waf/proxy.go index 966fe8b..121c72d 100644 --- a/internal/waf/proxy.go +++ b/internal/waf/proxy.go @@ -29,14 +29,6 @@ import ( "sneak.berlin/go/smallwebwaf/internal/rules" ) -// ownEndpointPrefix starts the path of each of waf's own endpoints. The app -// never sees a path under it. -const ownEndpointPrefix = "/_sww/" - -// HealthPath is the health endpoint of waf and of rproxy, which the -// container's health check asks. -const HealthPath = "/_sww/healthz" - // MetricsPath is where the metrics are, for a request that carries // SWW_WAF_METRICS_TOKEN. const MetricsPath = "/_sww/metrics" diff --git a/internal/waf/requestlog_test.go b/internal/waf/requestlog_test.go index 871e7e3..9ede1bf 100644 --- a/internal/waf/requestlog_test.go +++ b/internal/waf/requestlog_test.go @@ -13,7 +13,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/requestlog" - "sneak.berlin/go/smallwebwaf/internal/waf" + "sneak.berlin/go/smallwebwaf/internal/wafprotocol" ) const ( @@ -248,7 +248,7 @@ func TestTimingsAreInOrder(t *testing.T) { wantStatus(t, do(t, req), http.StatusForbidden) refused := out.endedLines(t, 2)[1] - wantStatus(t, get(t, server.Addr, waf.HealthPath), http.StatusOK) + wantStatus(t, get(t, server.Addr, wafprotocol.HealthPath), http.StatusOK) health := out.endedLines(t, 3)[2] // A request passed to the app has every timing; one refused, none of diff --git a/internal/waf/waf.go b/internal/waf/waf.go index ec8d9ea..9ba74d0 100644 --- a/internal/waf/waf.go +++ b/internal/waf/waf.go @@ -1,75 +1,21 @@ package waf import ( - "bytes" "context" - "encoding/json" "errors" - "fmt" - "io" "log/slog" "net/http" - "net/netip" - "strconv" "strings" "time" - "unicode/utf8" "sneak.berlin/go/smallwebwaf/internal/requestlog" + "sneak.berlin/go/smallwebwaf/internal/wafprotocol" ) -// The headers of rproxy's question and of waf's answer, as -// "waf" in SPEC.md gives them. -const ( - // ClientHeader marks a request as rproxy's question, and gives the - // visitor's address. - ClientHeader = "X-Sww-Client" - // PeerHeader gives the address of rproxy's own TCP peer. - PeerHeader = "X-Sww-Peer" - // ContentLengthHeader gives the body length the visitor announced. - ContentLengthHeader = "X-Sww-Content-Length" - // DispositionHeader gives waf's answer. - DispositionHeader = "X-Sww-Disposition" -) - -// The values of DispositionHeader. -const ( - // DispositionAllow tells rproxy to pass the request to the app. - DispositionAllow = "allow" - // DispositionDeny tells rproxy to give the visitor the answer's status, - // headers and body. - DispositionDeny = "deny" - // DispositionClose tells rproxy to close the visitor's connection - // without an answer, which SWW_WAF_BAN_RESPONSE asks for when it is close. - // waf closing the question's connection instead would read as - // waf being down, and rproxy would then let the request through. - DispositionClose = "close" -) - -// ReportPath is where rproxy sends the report of each request waf -// answered. -const ReportPath = "/_sww/report" - -// QuestionBodyMaxBytes is the most of a visitor's body a question carries. -const QuestionBodyMaxBytes = 64 << 10 - const ( // maxAwaitingReport is the most answered requests held until their // reports come. Past it, the request answered longest ago is dropped. maxAwaitingReport = 10000 - // maxRequestIDBytes is the longest id a question may give. - maxRequestIDBytes = 128 - // reportBodyMaxBytes is the most of a report's body that is read. It is - // far above any report's. - reportBodyMaxBytes = 1 << 20 - // maxHTTPStatus is the highest status a report may give. - maxHTTPStatus = 599 - // maxReportedBytes is the most bytes a report may give for each way, 1 - // PiB, so that adding them up cannot overflow. - maxReportedBytes = 1 << 50 - // maxReportedMilliseconds is the longest duration a report may give, - // about 31 years, so that it fits a time.Duration. - maxReportedMilliseconds = 1e12 // wafTimeout bounds how long a request to waf takes to // arrive, and its answer to leave, and wafIdleTimeout how long a // kept-open connection waits for its next request. @@ -77,26 +23,6 @@ const ( wafIdleTimeout = 120 * time.Second ) -var ( - errReportFieldMissing = errors.New("a field is missing") - errReportValue = errors.New("a value is out of range") -) - -// Report is the JSON object rproxy posts to ReportPath once the visitor's -// answer has ended, its fields named as the request log's of the same -// meaning. RequestID is the question's id. Status is what the visitor got, 0 -// if nothing. The durations are in milliseconds. -// -//nolint:tagliatelle // docs/request-log.md names the fields in snake_case -type Report struct { - RequestID string `json:"request_id"` - Status int `json:"status"` - RequestBytes int64 `json:"request_bytes"` - ResponseBytes int64 `json:"response_bytes"` - DurationTotalMS float64 `json:"duration_total_ms"` - DurationUpstreamTotalMS float64 `json:"duration_upstream_total_ms"` -} - // New returns the server `sww waf` runs. It answers // rproxy's questions, takes rproxy's reports and answers its own endpoints. It // passes nothing to an app. rproxy applies the size and time limits to the @@ -132,10 +58,10 @@ func New(params Params) *Server { } // ServeHTTP handles one request to waf: a request carrying -// ClientHeader is rproxy's question, and any other is for waf +// wafprotocol.ClientHeader is rproxy's question, and any other is for waf // itself. func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if len(r.Header.Values(ClientHeader)) > 0 { + if len(r.Header.Values(wafprotocol.ClientHeader)) > 0 { h.answerQuestion(w, r) return @@ -162,19 +88,16 @@ func (rq *request) refusedWhateverTheAnswer(now time.Time) bool { // carries, and answers rproxy what to do with it. A denied request's counting // is finished here, and an allowed one's by its report. func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) { - client, body, ok := h.readQuestion(w, r) + question, ok := h.readQuestion(w, r) if !ok { return } - // One that cannot be read leaves the log line's peer_ip invalid. - peer, _ := netip.ParseAddr(r.Header.Get(PeerHeader)) - rq := h.startRequest( w, - asTheVisitorSentIt(r, body), - client.Unmap(), - peer.Unmap(), + question.Visitor, + question.Client, + question.Peer, true, ) // The last deferred runs first, so that what is held has the finished log @@ -187,7 +110,10 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) { rq.line.WAFLine = requestlog.WAFLineDecision refused := rq.check(r.Context()) rq.checked = time.Now() - underOwnPrefix := strings.HasPrefix(r.URL.Path, ownEndpointPrefix) + underOwnPrefix := strings.HasPrefix( + r.URL.Path, + wafprotocol.OwnEndpointPrefix, + ) if refused == nil && !underOwnPrefix { // In observe mode, what enforce mode would have refused counts here. @@ -213,7 +139,10 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) { // The app never sees this prefix. waf's own endpoints are at its // own address, so this answers as a path with no endpoint does. rq.line.Action = requestlog.ActionOwnEndpoint - rq.out.Header().Set(DispositionHeader, DispositionDeny) + wafprotocol.SetAnswerHeaders( + rq.out.Header(), + wafprotocol.Answer{Disposition: wafprotocol.DispositionDeny}, + ) http.Error( rq.out, http.StatusText(http.StatusNotFound), @@ -221,45 +150,19 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) { ) } -// readQuestion returns the visitor's address and the body a question carries, -// and whether waf takes the question. It answers a question it does not -// take with 400 and no disposition. rproxy passes that 400 to the visitor, -// and SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION does not apply. The id of -// a question waf takes is held from here on, without a decision yet, so -// that another question with that id is refused while this one is being -// handled. +// readQuestion reads a question, and reports whether waf takes it. It answers +// a question it does not take with 400 and no disposition. rproxy passes that +// 400 to the visitor, and SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION does +// not apply. The id of a question waf takes is held from here on, without a +// decision yet, so that another question with that id is refused while this +// one is being handled. // -// The body is read before any check, since Go's server does not send an answer -// until it has read the body. A question whose body stops arriving gets no -// answer, and is not looked at. +// A question whose body stops arriving gets no answer, and is not looked at. func (h *handler) readQuestion( w http.ResponseWriter, r *http.Request, -) (netip.Addr, []byte, bool) { - client, err := netip.ParseAddr(r.Header.Get(ClientHeader)) - if err != nil || client.Zone() != "" { - h.refuseQuestion(w, ClientHeader+" is not an address without a zone") - - return netip.Addr{}, nil, false - } - - id := r.Header.Get(requestIDHeader) - if id == "" || len(id) > maxRequestIDBytes || !utf8.ValidString(id) { - h.refuseQuestion(w, requestIDHeader+" is not a valid id") - - return netip.Addr{}, nil, false - } - - body, err := io.ReadAll( - http.MaxBytesReader(w, r.Body, QuestionBodyMaxBytes), - ) - - if _, tooLarge := errors.AsType[*http.MaxBytesError](err); tooLarge { - h.refuseQuestion(w, "the body is over 64 KiB") - - return netip.Addr{}, nil, false - } - - if err != nil { +) (wafprotocol.Question, bool) { + question, err := wafprotocol.ReadQuestion(w, r) + if errors.Is(err, wafprotocol.ErrBodyStoppedArriving) { h.processLog.Warn( "a question's body stopped arriving", "error", @@ -270,17 +173,23 @@ func (h *handler) readQuestion( panic(http.ErrAbortHandler) } - held, _ := h.awaitingReport.ContainsOrAdd(id, nil) + if err != nil { + h.refuseQuestion(w, err.Error()) + + return question, false + } + + held, _ := h.awaitingReport.ContainsOrAdd(question.RequestID, nil) if held { h.refuseQuestion( w, - requestIDHeader+" is the id of a request waf holds", + wafprotocol.RequestIDHeader+" is the id of a request waf holds", ) - return netip.Addr{}, nil, false + return question, false } - return client, body, true + return question, true } // refuseQuestion answers a question waf does not take, for reason. @@ -289,48 +198,18 @@ func (h *handler) refuseQuestion(w http.ResponseWriter, reason string) { http.Error(w, reason, http.StatusBadRequest) } -// asTheVisitorSentIt returns a copy of r, a question, as the checks and the -// log line are to see it: without the headers rproxy adds for waf, -// with body, which has been read of r, and with the length the visitor -// announced, as -1 when it announced none. The question's own length covers -// only the part of the body it carries. -func asTheVisitorSentIt(r *http.Request, body []byte) *http.Request { - sent := r.Clone(r.Context()) - announced := sent.Header.Get(ContentLengthHeader) - - sent.Header.Del(ClientHeader) - sent.Header.Del(PeerHeader) - sent.Header.Del(ContentLengthHeader) - sent.Header.Del("Content-Length") - - sent.ContentLength = -1 - - length, err := strconv.ParseInt(announced, 10, 64) - if err == nil && length >= 0 { - sent.ContentLength = length - sent.Header.Set("Content-Length", announced) - } - - sent.Body = http.NoBody - if len(body) > 0 { - sent.Body = io.NopCloser(bytes.NewReader(body)) - } - - return sent -} - -// allow answers a question with DispositionAllow and, while +// allow answers a question with wafprotocol.DispositionAllow and, while // SWW_WAF_ADD_LOOKUP_HEADERS is true, the client's AS number and country. func (rq *request) allow() { rq.allowed = true - header := rq.out.Header() - header.Set(DispositionHeader, DispositionAllow) - + answer := wafprotocol.Answer{Disposition: wafprotocol.DispositionAllow} if rq.h.config.AddLookupHeaders { - setLookupHeaders(header, rq.line.ASN, rq.line.Country) + answer.ASN = rq.line.ASN + answer.Country = rq.line.Country } + wafprotocol.SetAnswerHeaders(rq.out.Header(), answer) rq.out.WriteHeader(http.StatusOK) } @@ -348,13 +227,19 @@ func (rq *request) deny(r refusal) { if r.status == 0 { // SWW_WAF_BAN_RESPONSE is close. Written past rq.out, since the log line's // status stays 0: the visitor is sent nothing. - rq.out.Header().Set(DispositionHeader, DispositionClose) + wafprotocol.SetAnswerHeaders( + rq.out.Header(), + wafprotocol.Answer{Disposition: wafprotocol.DispositionClose}, + ) rq.out.ResponseWriter.WriteHeader(http.StatusOK) return } - rq.out.Header().Set(DispositionHeader, DispositionDeny) + wafprotocol.SetAnswerHeaders( + rq.out.Header(), + wafprotocol.Answer{Disposition: wafprotocol.DispositionDeny}, + ) http.Error(rq.out, http.StatusText(r.status), r.status) } @@ -362,7 +247,7 @@ func (rq *request) deny(r refusal) { // runs on it: the address is on the private network, where the peers are // rproxy and the admin's tools, not visitors. func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodPost && r.URL.Path == ReportPath { + if r.Method == http.MethodPost && r.URL.Path == wafprotocol.ReportPath { h.takeReport(w, r) return @@ -373,7 +258,7 @@ func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) { rq := h.startRequest(w, r, peer, peer, false) defer rq.finish() - if r.Method == http.MethodGet && r.URL.Path == HealthPath { + if r.Method == http.MethodGet && r.URL.Path == wafprotocol.HealthPath { rq.answerHealth() return @@ -382,11 +267,11 @@ func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) { rq.answerAdmin() } -// takeReport answers POST ReportPath: it finishes the request the report -// names, if it holds that request's decision, and ignores the report +// takeReport answers POST wafprotocol.ReportPath: it finishes the request the +// report names, if it holds that request's decision, and ignores the report // otherwise. func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) { - report, err := readReport(w, r) + report, err := wafprotocol.ReadReport(w, r) if err != nil { h.processLog.Warn("a report is refused", "error", err.Error()) http.Error(w, "the body is not a report", http.StatusBadRequest) @@ -404,63 +289,6 @@ func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) } -// readReport reads a report from r's body, which must be one JSON object with -// all of Report's fields, in the ranges SPEC.md gives. -func readReport(w http.ResponseWriter, r *http.Request) (Report, error) { - var report Report - - body, err := io.ReadAll( - http.MaxBytesReader(w, r.Body, reportBodyMaxBytes), - ) - if err != nil { - return report, fmt.Errorf("read the body: %w", err) - } - - var fields map[string]json.RawMessage - - err = json.Unmarshal(body, &fields) - if err != nil { - return report, fmt.Errorf("decode the body: %w", err) - } - - for _, name := range []string{ - "request_id", "status", "request_bytes", "response_bytes", - "duration_total_ms", "duration_upstream_total_ms", - } { - value, there := fields[name] - if !there || string(value) == "null" { - return report, fmt.Errorf("%w: %s", errReportFieldMissing, name) - } - } - - err = json.Unmarshal(body, &report) - if err != nil { - return report, fmt.Errorf("decode the body: %w", err) - } - - return report, report.checkRanges() -} - -// checkRanges returns an error for a value of the report that cannot be true. -func (report *Report) checkRanges() error { - switch { - case report.Status != 0 && - (report.Status < http.StatusContinue || report.Status > maxHTTPStatus): - return fmt.Errorf("%w: status", errReportValue) - case report.RequestBytes < 0 || report.RequestBytes > maxReportedBytes: - return fmt.Errorf("%w: request_bytes", errReportValue) - case report.ResponseBytes < 0 || report.ResponseBytes > maxReportedBytes: - return fmt.Errorf("%w: response_bytes", errReportValue) - case report.DurationTotalMS < 0 || report.DurationTotalMS > maxReportedMilliseconds: - return fmt.Errorf("%w: duration_total_ms", errReportValue) - case report.DurationUpstreamTotalMS < 0 || - report.DurationUpstreamTotalMS > maxReportedMilliseconds: - return fmt.Errorf("%w: duration_upstream_total_ms", errReportValue) - } - - return nil -} - // finishWith finishes a request with what its report says, and writes the // request's last log line, which holds its decision's fields and the report's. // For an allowed request it first runs the heuristics of onReport, and adds the @@ -470,7 +298,7 @@ func (report *Report) checkRanges() error { func (h *handler) finishWith( ctx context.Context, held *decision, - report Report, + report wafprotocol.Report, ) { rq := &request{ decision: *held, diff --git a/internal/waf/waf_test.go b/internal/waf/waf_test.go index 93a0e15..b319504 100644 --- a/internal/waf/waf_test.go +++ b/internal/waf/waf_test.go @@ -22,6 +22,7 @@ import ( "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/waf" + "sneak.berlin/go/smallwebwaf/internal/wafprotocol" ) const ( @@ -106,8 +107,8 @@ func (s *WAF) questionWith( req := newRequest(s.t, method, s.addr, path, body) req.Host = host - req.Header.Set(waf.ClientHeader, from) - req.Header.Set(waf.PeerHeader, rproxyPeer) + req.Header.Set(wafprotocol.ClientHeader, from) + req.Header.Set(wafprotocol.PeerHeader, rproxyPeer) req.Header.Set(requestIDHeader, rand.Text()) return req @@ -134,13 +135,18 @@ func (s *WAF) allowed(from string) string { s.t.Helper() req := s.question(siteA, from, "/") - wantDisposition(s.t, do(s.t, req), http.StatusOK, waf.DispositionAllow) + wantDisposition( + s.t, + do(s.t, req), + http.StatusOK, + wafprotocol.DispositionAllow, + ) return req.Header.Get(requestIDHeader) } // report sends rproxy's report and returns the answer. -func (s *WAF) report(report waf.Report) answer { +func (s *WAF) report(report wafprotocol.Report) answer { s.t.Helper() body, err := json.Marshal(report) @@ -155,8 +161,11 @@ func (s *WAF) report(report waf.Report) answer { func (s *WAF) reportBody(body string) answer { s.t.Helper() - return do(s.t, newRequest(s.t, http.MethodPost, s.addr, waf.ReportPath, - strings.NewReader(body))) + return do( + s.t, + newRequest(s.t, http.MethodPost, s.addr, wafprotocol.ReportPath, + strings.NewReader(body)), + ) } // passedOn asks about a request from the visitor at from on siteA, and reports @@ -164,7 +173,7 @@ func (s *WAF) reportBody(body string) answer { func (s *WAF) passedOn(from string) { s.t.Helper() - got := s.report(waf.Report{ + got := s.report(wafprotocol.Report{ RequestID: s.allowed(from), Status: http.StatusOK, RequestBytes: bodyBytes, @@ -210,7 +219,7 @@ func wantDisposition( t.Helper() wantStatus(t, got, status) - if answered := got.header.Get(waf.DispositionHeader); answered != + if answered := got.header.Get(wafprotocol.DispositionHeader); answered != disposition { t.Errorf( "answered with disposition %q, want %q", @@ -272,10 +281,10 @@ func TestQuestionAllowedIsAnsweredAndLogged(t *testing.T) { req := s.question(siteA, client, "/page?q=1&token=visible") req.Header.Set(requestIDHeader, "the-id") req.Header.Set("X-Forwarded-Proto", "https") - req.Header.Set(waf.ContentLengthHeader, "1234") + req.Header.Set(wafprotocol.ContentLengthHeader, "1234") got := do(t, req) - wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) + wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow) wantBody(t, got, "") if id := got.header.Get(requestIDHeader); id != "" { @@ -308,7 +317,12 @@ func TestIPv6VisitorIsCountedInItsGroup(t *testing.T) { s.allowed("2001:db8::1") got := s.ask("2001:db8::2", "/") - wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny) + wantDisposition( + t, + got, + http.StatusForbidden, + wafprotocol.DispositionDeny, + ) } func TestIPv4VisitorInIPv6FormIsTheIPv4Client(t *testing.T) { @@ -317,9 +331,14 @@ func TestIPv4VisitorInIPv6FormIsTheIPv4Client(t *testing.T) { s := startWAF(t, "", map[string]string{denyNets: client}) req := s.question(siteA, "::ffff:"+client, "/") - req.Header.Set(waf.PeerHeader, "::ffff:"+rproxyPeer) + req.Header.Set(wafprotocol.PeerHeader, "::ffff:"+rproxyPeer) - wantDisposition(t, do(t, req), http.StatusForbidden, waf.DispositionDeny) + wantDisposition( + t, + do(t, req), + http.StatusForbidden, + wafprotocol.DispositionDeny, + ) line := s.out.requestLine(t) if line.ClientIP != client || line.PeerIP != rproxyPeer { @@ -339,13 +358,13 @@ func TestMalformedQuestionGets400AndAWarningAndNoLine(t *testing.T) { change func(*http.Request) }{ {"client not an address", func(req *http.Request) { - req.Header.Set(waf.ClientHeader, "not an address") + req.Header.Set(wafprotocol.ClientHeader, "not an address") }}, {"client empty", func(req *http.Request) { - req.Header.Set(waf.ClientHeader, "") + req.Header.Set(wafprotocol.ClientHeader, "") }}, {"client address with a zone", func(req *http.Request) { - req.Header.Set(waf.ClientHeader, "2001:db8::1%eth0") + req.Header.Set(wafprotocol.ClientHeader, "2001:db8::1%eth0") }}, {"id absent", func(req *http.Request) { req.Header.Del(requestIDHeader) @@ -390,7 +409,12 @@ func TestLongestValidIDIsTaken(t *testing.T) { req := s.question(siteA, client, "/") req.Header.Set(requestIDHeader, strings.Repeat("x", 128)) - wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow) + wantDisposition( + t, + do(t, req), + http.StatusOK, + wafprotocol.DispositionAllow, + ) } func TestQuestionWithABodyOver64KiBGets400(t *testing.T) { @@ -399,16 +423,18 @@ func TestQuestionWithABodyOver64KiBGets400(t *testing.T) { s := startWAF(t, "", nil) over := strings.NewReader( - strings.Repeat("x", waf.QuestionBodyMaxBytes+1), + strings.Repeat("x", wafprotocol.QuestionBodyMaxBytes+1), ) got := do(t, s.questionWith(http.MethodPost, siteA, client, "/", over)) wantDisposition(t, got, http.StatusBadRequest, "") wantProcessMessages(t, s, "a question is refused", 1) wantNoRequestLines(t, s) - whole := strings.NewReader(strings.Repeat("x", waf.QuestionBodyMaxBytes)) + whole := strings.NewReader( + strings.Repeat("x", wafprotocol.QuestionBodyMaxBytes), + ) got = do(t, s.questionWith(http.MethodPost, siteA, client, "/", whole)) - wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) + wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow) } func TestQuestionWithAHeldIDIsRefused(t *testing.T) { @@ -424,12 +450,12 @@ func TestQuestionWithAHeldIDIsRefused(t *testing.T) { wantProcessMessages(t, s, "a question is refused", 1) // A denied request's id is held as well. - denied := s.question(siteA, otherClient, waf.HealthPath) + denied := s.question(siteA, otherClient, wafprotocol.HealthPath) wantDisposition( t, do(t, denied), http.StatusNotFound, - waf.DispositionDeny, + wafprotocol.DispositionDeny, ) again = s.question(siteA, client, "/") @@ -437,7 +463,9 @@ func TestQuestionWithAHeldIDIsRefused(t *testing.T) { wantDisposition(t, do(t, again), http.StatusBadRequest, "") // The report ends the holding, and the refused questions changed nothing. - got := s.report(waf.Report{RequestID: id, ResponseBytes: answerBytes}) + got := s.report( + wafprotocol.Report{RequestID: id, ResponseBytes: answerBytes}, + ) wantStatus(t, got, http.StatusNoContent) if history := s.history(); history.ResponseBytes != answerBytes { @@ -446,7 +474,12 @@ func TestQuestionWithAHeldIDIsRefused(t *testing.T) { again = s.question(siteA, client, "/") again.Header.Set(requestIDHeader, id) - wantDisposition(t, do(t, again), http.StatusOK, waf.DispositionAllow) + wantDisposition( + t, + do(t, again), + http.StatusOK, + wafprotocol.DispositionAllow, + ) } func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) { @@ -481,7 +514,7 @@ func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) { // A report for a question not yet answered is ignored, and that question's // id is taken. - got := s.report(waf.Report{RequestID: id, ResponseBytes: 1000}) + got := s.report(wafprotocol.Report{RequestID: id, ResponseBytes: 1000}) wantStatus(t, got, http.StatusNoContent) // This would wait on the same lookup were the id not taken. @@ -496,7 +529,7 @@ func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) { select { case got := <-answered: - wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) + wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow) case <-time.After(waitLimit): t.Fatal("the question got no answer") } @@ -515,7 +548,9 @@ func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) { } // The true report comes after, and is the one that counts. - got = s.report(waf.Report{RequestID: id, ResponseBytes: answerBytes}) + got = s.report( + wafprotocol.Report{RequestID: id, ResponseBytes: answerBytes}, + ) wantStatus(t, got, http.StatusNoContent) if history := s.history(); history.ResponseBytes != answerBytes { @@ -533,17 +568,17 @@ func TestPeerAbsentOrUnreadableIsAnsweredAndLoggedInvalid(t *testing.T) { s := startWAF(t, "", nil) req := s.question(siteA, client, "/") - req.Header.Del(waf.PeerHeader) + req.Header.Del(wafprotocol.PeerHeader) if peer != "" { - req.Header.Set(waf.PeerHeader, peer) + req.Header.Set(wafprotocol.PeerHeader, peer) } wantDisposition( t, do(t, req), http.StatusOK, - waf.DispositionAllow, + wafprotocol.DispositionAllow, ) if line := s.out.requestLine(t); line.PeerIP != "invalid IP" { @@ -571,12 +606,12 @@ func TestContentLengthUnreadableOrNegativeCountsAsAbsent(t *testing.T) { }) req := s.question(siteA, client, "/") - req.Header.Set(waf.ContentLengthHeader, announced) + req.Header.Set(wafprotocol.ContentLengthHeader, announced) wantDisposition( t, do(t, req), http.StatusOK, - waf.DispositionAllow, + wafprotocol.DispositionAllow, ) if line := s.out.requestLine(t); line.ContentLength != 0 { @@ -595,13 +630,18 @@ func TestHeaderSentTwiceUsesItsFirstValue(t *testing.T) { s := startWAF(t, "", nil) req := s.question(siteA, client, "/") - req.Header.Add(waf.ClientHeader, otherClient) - req.Header.Add(waf.PeerHeader, "172.18.0.3") + req.Header.Add(wafprotocol.ClientHeader, otherClient) + req.Header.Add(wafprotocol.PeerHeader, "172.18.0.3") req.Header.Set(requestIDHeader, "first") req.Header.Add(requestIDHeader, "second") - req.Header.Set(waf.ContentLengthHeader, "1234") - req.Header.Add(waf.ContentLengthHeader, "5678") - wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow) + req.Header.Set(wafprotocol.ContentLengthHeader, "1234") + req.Header.Add(wafprotocol.ContentLengthHeader, "5678") + wantDisposition( + t, + do(t, req), + http.StatusOK, + wafprotocol.DispositionAllow, + ) line := s.out.requestLine(t) if line.ClientIP != client || line.PeerIP != rproxyPeer || @@ -630,7 +670,7 @@ func sendQuestionWithPartOfItsBody(t *testing.T, conn net.Conn) { _, err := fmt.Fprintf(conn, "POST / HTTP/1.1\r\nHost: %s\r\n%s: %s\r\n%s: an-id\r\n"+ "Content-Length: 100\r\n\r\npart", - siteA, waf.ClientHeader, client, requestIDHeader) + siteA, wafprotocol.ClientHeader, client, requestIDHeader) if err != nil { t.Fatalf("send the question: %v", err) } @@ -699,7 +739,7 @@ func TestQuestionWhoseBodyStopsArrivingGetsNoAnswer(t *testing.T) { t, do(t, again), http.StatusOK, - waf.DispositionAllow, + wafprotocol.DispositionAllow, ) }) } @@ -717,7 +757,7 @@ func TestQuestionDeniedGetsTheBanResponse(t *testing.T) { }) got := s.ask(client, "/") - wantDisposition(t, got, status, waf.DispositionDeny) + wantDisposition(t, got, status, wafprotocol.DispositionDeny) wantBody(t, got, http.StatusText(status)+"\n") wantLine( t, @@ -737,7 +777,7 @@ func TestBanResponseCloseIsAnAnswerOfItsOwn(t *testing.T) { }) got := s.ask(client, "/") - wantDisposition(t, got, http.StatusOK, waf.DispositionClose) + wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionClose) wantBody(t, got, "") // The visitor is sent nothing. wantLine(t, s.out.requestLine(t), 0, requestlog.ActionClientDenied) @@ -761,12 +801,22 @@ func TestBlockRuleIsDeniedWith403AndABanRuleWithTheBanResponse( }) got := s.ask(client, "/blocked") - wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny) + wantDisposition( + t, + got, + http.StatusForbidden, + wafprotocol.DispositionDeny, + ) wantLine(t, s.out.requestLine(t), http.StatusForbidden, requestlog.ActionBlockedByBlockRule) got = s.ask(client, "/.env") - wantDisposition(t, got, http.StatusTooManyRequests, waf.DispositionDeny) + wantDisposition( + t, + got, + http.StatusTooManyRequests, + wafprotocol.DispositionDeny, + ) wantLine(t, s.out.requestLines(t, 2)[1], http.StatusTooManyRequests, requestlog.ActionTemporarilyBanned) } @@ -781,10 +831,15 @@ func TestQuestionUnderTheOwnPrefixIsDeniedWith404AfterTheChecks( }) for _, path := range []string{ - waf.HealthPath, waf.MetricsPath, waf.ReportPath, + wafprotocol.HealthPath, waf.MetricsPath, wafprotocol.ReportPath, } { got := s.ask(client, path) - wantDisposition(t, got, http.StatusNotFound, waf.DispositionDeny) + wantDisposition( + t, + got, + http.StatusNotFound, + wafprotocol.DispositionDeny, + ) } wantLine( @@ -795,7 +850,12 @@ func TestQuestionUnderTheOwnPrefixIsDeniedWith404AfterTheChecks( ) got := s.ask(otherClient, waf.MetricsPath) - wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny) + wantDisposition( + t, + got, + http.StatusForbidden, + wafprotocol.DispositionDeny, + ) } func TestObserveModeAllowsAndLogsWhatEnforceModeWouldHaveDone(t *testing.T) { @@ -804,7 +864,7 @@ func TestObserveModeAllowsAndLogsWhatEnforceModeWouldHaveDone(t *testing.T) { s := startWAF(t, "", map[string]string{mode: observe, denyNets: client}) got := s.ask(client, "/") - wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) + wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow) line := s.out.requestLine(t) wantLine(t, line, http.StatusOK, requestlog.ActionForward) @@ -822,7 +882,7 @@ func TestObserveModeRefusalAnsweredAllowCountsForTheRefusalLimit( for range 2 { got := s.ask(client, "/blocked") - wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) + wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow) } lines := s.out.requestLines(t, 2) @@ -846,7 +906,12 @@ func TestDeniedQuestionCountsForTheRefusalLimit(t *testing.T) { for range 2 { got := s.ask(client, "/blocked") - wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny) + wantDisposition( + t, + got, + http.StatusForbidden, + wafprotocol.DispositionDeny, + ) } lines := s.out.requestLines(t, 2) @@ -882,7 +947,12 @@ func TestAllowCarriesTheLookupHeadersWhileTheSettingIsSet(t *testing.T) { }) got := s.ask(fromDE, "/") - wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) + wantDisposition( + t, + got, + http.StatusOK, + wafprotocol.DispositionAllow, + ) asn, country := got.header.Get( "X-Client-Asn", @@ -916,7 +986,7 @@ func TestReportCountsTowardAByteLimitWhichThenBans(t *testing.T) { } // 100 bytes are over the limit of 99. - got := s.report(waf.Report{ + got := s.report(wafprotocol.Report{ RequestID: id, Status: http.StatusBadGateway, RequestBytes: bodyBytes, ResponseBytes: answerBytes, }) @@ -940,7 +1010,12 @@ func TestReportCountsTowardAByteLimitWhichThenBans(t *testing.T) { } got = s.ask(client, "/") - wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny) + wantDisposition( + t, + got, + http.StatusForbidden, + wafprotocol.DispositionDeny, + ) } func TestReportCountsAtThePercentageItsQuestionWasGiven(t *testing.T) { @@ -958,12 +1033,12 @@ func TestReportCountsAtThePercentageItsQuestionWasGiven(t *testing.T) { t, s.ask(fromKP, "/"), http.StatusOK, - waf.DispositionAllow, + wafprotocol.DispositionAllow, ) s.passedOn(fromDE) wantDisposition(t, s.ask(fromDE, "/"), http.StatusForbidden, - waf.DispositionDeny) + wafprotocol.DispositionDeny) } func TestReportAddsTheRequestToItsClientsHistoryAndTheAnomalyCounters( @@ -989,7 +1064,7 @@ func TestReportAddsTheRequestToItsClientsHistoryAndTheAnomalyCounters( ) } - got := s.report(waf.Report{ + got := s.report(wafprotocol.Report{ RequestID: id, Status: http.StatusBadGateway, RequestBytes: bodyBytes, ResponseBytes: answerBytes, }) @@ -1021,7 +1096,7 @@ func TestDeniedRequestIsFinishedAtItsDecision(t *testing.T) { t, do(t, denied), http.StatusForbidden, - waf.DispositionDeny, + wafprotocol.DispositionDeny, ) counters := s.server.Anomalies.Snapshot() @@ -1039,7 +1114,7 @@ func TestDeniedRequestIsFinishedAtItsDecision(t *testing.T) { wantMetric(t, s.metrics(), series, 1) - got := s.report(waf.Report{ + got := s.report(wafprotocol.Report{ RequestID: denied.Header.Get( requestIDHeader, ), Status: http.StatusForbidden, @@ -1082,7 +1157,7 @@ func TestMetricsCountAnAllowedRequestFromItsReport(t *testing.T) { 1, ) // the scrape itself - got := s.report(waf.Report{ + got := s.report(wafprotocol.Report{ RequestID: id, Status: http.StatusBadGateway, RequestBytes: bodyBytes, ResponseBytes: answerBytes, DurationTotalMS: 5, DurationUpstreamTotalMS: 4, @@ -1100,7 +1175,7 @@ func TestMetricsCountAnAllowedRequestFromItsReport(t *testing.T) { // check needs a server that has not been scraped yet. unscraped := startWAF(t, "", map[string]string{metricsToken: token}) - got = unscraped.report(waf.Report{ + got = unscraped.report(wafprotocol.Report{ RequestID: unscraped.allowed(client), Status: http.StatusBadGateway, DurationTotalMS: 5, }) @@ -1118,7 +1193,7 @@ func TestReportForARequestNotHeldIsIgnored(t *testing.T) { s := startWAF(t, "", map[string]string{bytesLimitPerMinute: byteLimit}) - got := s.report(waf.Report{ + got := s.report(wafprotocol.Report{ RequestID: "no-such-request", ResponseBytes: 1000, }) wantStatus(t, got, http.StatusNoContent) @@ -1137,7 +1212,9 @@ func TestReportForARequestNotHeldIsIgnored(t *testing.T) { // A second report of the same request does not count it twice. id := s.allowed(client) for range 2 { - got = s.report(waf.Report{RequestID: id, ResponseBytes: answerBytes}) + got = s.report( + wafprotocol.Report{RequestID: id, ResponseBytes: answerBytes}, + ) wantStatus(t, got, http.StatusNoContent) } @@ -1347,7 +1424,7 @@ func TestBytesAtTheirMostEachWayBanRatherThanOverflow(t *testing.T) { s := startWAF(t, "", map[string]string{bytesLimitPerMinute: byteLimit}) - got := s.report(waf.Report{ + got := s.report(wafprotocol.Report{ RequestID: s.allowed(client), Status: http.StatusOK, RequestBytes: reportedBytesLimit, ResponseBytes: reportedBytesLimit, }) @@ -1441,9 +1518,14 @@ func TestReportWritesTheLastLineWithTheDecisionsAndTheReportsFields( req.Header.Set("X-Forwarded-Proto", "https") id := req.Header.Get(requestIDHeader) - wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow) + wantDisposition( + t, + do(t, req), + http.StatusOK, + wafprotocol.DispositionAllow, + ) - got := s.report(waf.Report{ + got := s.report(wafprotocol.Report{ RequestID: id, Status: http.StatusCreated, RequestBytes: bodyBytes, ResponseBytes: answerBytes, DurationTotalMS: 12.5, DurationUpstreamTotalMS: 10.25, @@ -1490,7 +1572,7 @@ func TestEveryDeniedRequestsReportGetsItsLastLine(t *testing.T) { "close", map[string]string{denyNets: client, banResponse: banResponseClose}, "/", 0, requestlog.ActionClientDenied, }, - {"own prefix", nil, waf.HealthPath, http.StatusNotFound, + {"own prefix", nil, wafprotocol.HealthPath, http.StatusNotFound, requestlog.ActionOwnEndpoint}, } { t.Run(tc.name, func(t *testing.T) { @@ -1502,7 +1584,7 @@ func TestEveryDeniedRequestsReportGetsItsLastLine(t *testing.T) { id := req.Header.Get(requestIDHeader) do(t, req) - got := s.report(waf.Report{ + got := s.report(wafprotocol.Report{ RequestID: id, Status: tc.status, ResponseBytes: 8, DurationTotalMS: 3, }) wantStatus(t, got, http.StatusNoContent) @@ -1542,13 +1624,13 @@ func TestOldestAnsweredRequestIsDroppedPastTheBound(t *testing.T) { t, readAnswer(res), http.StatusOK, - waf.DispositionAllow, + wafprotocol.DispositionAllow, ) } // The table is full again with next and those, without oldest. got := s.report( - waf.Report{RequestID: oldest, ResponseBytes: answerBytes}, + wafprotocol.Report{RequestID: oldest, ResponseBytes: answerBytes}, ) wantStatus(t, got, http.StatusNoContent) @@ -1559,7 +1641,9 @@ func TestOldestAnsweredRequestIsDroppedPastTheBound(t *testing.T) { ) } - got = s.report(waf.Report{RequestID: next, ResponseBytes: answerBytes}) + got = s.report( + wafprotocol.Report{RequestID: next, ResponseBytes: answerBytes}, + ) wantStatus(t, got, http.StatusNoContent) if counted := s.history().Requests; counted != 1 { @@ -1580,10 +1664,20 @@ func TestBanMadeThroughOneHostRefusesTheClientOnAnother(t *testing.T) { ) got := do(t, s.question(siteA, client, "/.env")) - wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny) + wantDisposition( + t, + got, + http.StatusForbidden, + wafprotocol.DispositionDeny, + ) got = do(t, s.question(siteB, client, "/")) - wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny) + wantDisposition( + t, + got, + http.StatusForbidden, + wafprotocol.DispositionDeny, + ) lines := s.out.requestLines(t, 2) if lines[0].Host != siteA || lines[1].Host != siteB { @@ -1621,7 +1715,12 @@ func TestRequestsOnEverySiteCountTowardOneSetOfLimits(t *testing.T) { s.allowed(client) got := do(t, s.question(siteB, client, "/")) - wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny) + wantDisposition( + t, + got, + http.StatusForbidden, + wafprotocol.DispositionDeny, + ) } func TestOwnEndpointsAnswerOnlyWithoutTheClientHeader(t *testing.T) { @@ -1636,13 +1735,13 @@ func TestOwnEndpointsAnswerOnlyWithoutTheClientHeader(t *testing.T) { path, authorization string want int }{ - {waf.HealthPath, "", http.StatusOK}, + {wafprotocol.HealthPath, "", http.StatusOK}, {waf.MetricsPath, bearer, http.StatusOK}, {waf.MetricsPath, "", http.StatusUnauthorized}, {waf.BansPath, adminBearer, http.StatusOK}, {waf.BansPath, "", http.StatusUnauthorized}, {waf.ClientsPath + client, adminBearer, http.StatusOK}, - {waf.ReportPath, "", http.StatusNotFound}, + {wafprotocol.ReportPath, "", http.StatusNotFound}, {"/_sww/other", "", http.StatusNotFound}, {"/", "", http.StatusNotFound}, } { @@ -1659,10 +1758,10 @@ func TestOwnEndpointsAnswerOnlyWithoutTheClientHeader(t *testing.T) { t, do(t, req), http.StatusNotFound, - waf.DispositionDeny, + wafprotocol.DispositionDeny, ) } else { - wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow) + wantDisposition(t, do(t, req), http.StatusOK, wafprotocol.DispositionAllow) } } } @@ -1672,11 +1771,11 @@ func TestOwnEndpointsLinesDoNotSayWhichOfTheTwoTheyAre(t *testing.T) { s := startWAF(t, "", nil) - wantStatus(t, get(t, s.addr, waf.HealthPath), http.StatusOK) + wantStatus(t, get(t, s.addr, wafprotocol.HealthPath), http.StatusOK) line := s.out.requestLine(t) if _, there := line.fields["waf_line"]; there || - line.Path != waf.HealthPath { + line.Path != wafprotocol.HealthPath { t.Errorf("the health check's log line is %+v", line.fields) } } @@ -1688,7 +1787,7 @@ func TestALargeBodyAnnouncedAndLongHeadersAreNoReasonToRefuse(t *testing.T) { req := s.questionWith(http.MethodPost, siteA, client, "/", strings.NewReader("a body")) - req.Header.Set(waf.ContentLengthHeader, "1073741824") + req.Header.Set(wafprotocol.ContentLengthHeader, "1073741824") req.Header.Set(forwardedFor, otherClient) // 40,000 bytes of headers, over rproxy's 32 KiB, in lines within the @@ -1698,7 +1797,7 @@ func TestALargeBodyAnnouncedAndLongHeadersAreNoReasonToRefuse(t *testing.T) { } got := do(t, req) - wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) + wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow) line := s.out.requestLine(t) if line.ClientIP != client || line.ContentLength != 1<<30 { @@ -1776,7 +1875,7 @@ carried header:Content-Length block ^6$ strings.NewReader("a body")) if tc.announced != "" { - req.Header.Set(waf.ContentLengthHeader, tc.announced) + req.Header.Set(wafprotocol.ContentLengthHeader, tc.announced) } wantStatus(t, do(t, req), tc.want) @@ -1794,19 +1893,24 @@ func TestCoreRuleSetReadsTheBodyAQuestionCarries(t *testing.T) { req := s.questionWith(http.MethodPost, siteA, client, "/", strings.NewReader(body)) req.Header.Set("Content-Type", contentType) - req.Header.Set(waf.ContentLengthHeader, "100000") + req.Header.Set(wafprotocol.ContentLengthHeader, "100000") return do(t, req) } got := ask(formData, "id=1'%20OR%20'1'='1") - wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny) + wantDisposition( + t, + got, + http.StatusForbidden, + wafprotocol.DispositionDeny, + ) // The first 8 KiB of a longer JSON body cannot be parsed, which is no // attack, so the Core Rule Set does not read it. cut := `{"text":"` + strings.Repeat("x", 8<<10) got = ask("application/json", cut) - wantDisposition(t, got, http.StatusOK, waf.DispositionAllow) + wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow) } func TestCoreRuleSetAlertNamesTheSite(t *testing.T) { @@ -1815,7 +1919,12 @@ func TestCoreRuleSetAlertNamesTheSite(t *testing.T) { s := startWAF(t, "", map[string]string{coreRuleSetMode: block}) got := s.ask(client, sqlInjection) - wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny) + wantDisposition( + t, + got, + http.StatusForbidden, + wafprotocol.DispositionDeny, + ) waiting := s.queue.Snapshot().Waiting[alerts.DestinationWebhook] if len(waiting) != 1 || @@ -1836,7 +1945,7 @@ func TestReputationHitAlertNamesTheSite(t *testing.T) { loadLists(t, s.server, map[string][]string{dropURL: {client}}) wantDisposition(t, s.ask(client, "/"), http.StatusForbidden, - waf.DispositionDeny) + wafprotocol.DispositionDeny) waiting := s.queue.Snapshot().Waiting[alerts.DestinationWebhook] if len(waiting) != 1 || waiting[0].Event != alerts.EventReputationHit || diff --git a/internal/wafprotocol/answer.go b/internal/wafprotocol/answer.go new file mode 100644 index 0000000..9f3221d --- /dev/null +++ b/internal/wafprotocol/answer.go @@ -0,0 +1,46 @@ +package wafprotocol + +import "net/http" + +// Answer is what waf's answer to a question tells rproxy in its headers. +// The status, the other headers and the body of an answer of deny are what +// the visitor gets. +type Answer struct { + // Disposition is what rproxy does with the request: DispositionAllow, + // DispositionDeny or DispositionClose. + Disposition string + // ASN and Country are the client's AS number and country, each empty + // when the answer does not give it. + ASN, Country string +} + +// SetAnswerHeaders sets the headers of waf's answer that tell rproxy answer, +// leaving out an ASN or Country that is empty. +func SetAnswerHeaders(header http.Header, answer Answer) { + header.Set(DispositionHeader, answer.Disposition) + + if answer.ASN != "" { + header.Set(ASNHeader, answer.ASN) + } + + if answer.Country != "" { + header.Set(CountryHeader, answer.Country) + } +} + +// ReadAnswer returns what the headers of waf's answer tell rproxy, and +// whether they give a valid disposition. +func ReadAnswer(header http.Header) (Answer, bool) { + disposition := header.Get(DispositionHeader) + + switch disposition { + case DispositionAllow, DispositionDeny, DispositionClose: + return Answer{ + Disposition: disposition, + ASN: header.Get(ASNHeader), + Country: header.Get(CountryHeader), + }, true + } + + return Answer{}, false +} diff --git a/internal/wafprotocol/question.go b/internal/wafprotocol/question.go new file mode 100644 index 0000000..4f7d2f4 --- /dev/null +++ b/internal/wafprotocol/question.go @@ -0,0 +1,189 @@ +package wafprotocol + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/netip" + "net/url" + "strconv" + "unicode/utf8" +) + +// maxRequestIDBytes is the longest id a question may give. +const maxRequestIDBytes = 128 + +// ErrBodyStoppedArriving is the error ReadQuestion wraps when the question's +// body stops arriving. Every other error it returns is why waf refuses the +// question. +var ErrBodyStoppedArriving = errors.New("the body stopped arriving") + +// Why waf refuses a question. Each is also the body of the refusal. +var ( + errClientNotAnAddress = errors.New( + ClientHeader + " is not an address without a zone", + ) + errRequestIDNotValid = errors.New(RequestIDHeader + " is not a valid id") + errBodyTooLarge = errors.New("the body is over 64 KiB") +) + +// QuestionParams are what NewQuestion needs. +type QuestionParams struct { + // Visitor is the visitor's request. + Visitor *http.Request + // PathAsSent is Visitor's path as the visitor sent it, which the question + // sends unchanged. + PathAsSent string + // Body is the part of Visitor's body the question carries, at most + // QuestionBodyMaxBytes. + Body []byte + // WAFBackend is waf's address. The question takes its scheme and host. + WAFBackend *url.URL + // Client is the visitor's address, and Peer that of rproxy's own TCP + // peer. + Client, Peer netip.Addr + // RequestID is the request's id. + RequestID string + // Scheme is how the visitor reached rproxy, or the trusted proxy in front + // of it. + Scheme string +} + +// Question is a question as waf reads it. +type Question struct { + // Client is the visitor's address. + Client netip.Addr + // Peer is the address of rproxy's own TCP peer. It is not valid when the + // question gives none that can be read. + Peer netip.Addr + // RequestID is the request's id. + RequestID string + // Visitor is the visitor's request, as asTheVisitorSentIt gives it. + Visitor *http.Request +} + +// NewQuestion returns the question about a visitor's request, as "The +// question" in SPEC.md gives it: the request as the visitor sent it, with the +// part of the body it carries and rproxy's headers. +func NewQuestion( + ctx context.Context, + params QuestionParams, +) (*http.Request, error) { + in := params.Visitor + + question, err := http.NewRequestWithContext( + ctx, + in.Method, + params.WAFBackend.String(), + bytes.NewReader(params.Body), + ) + if err != nil { + return nil, fmt.Errorf("make the question: %w", err) + } + + backend := question.URL + target := *in.URL + target.Scheme = backend.Scheme + target.Host = backend.Host + target.Opaque = params.PathAsSent + question.URL = &target + question.Host = in.Host + + // The length NewRequestWithContext set is that of the body carried. This + // line makes a body whose length the visitor did not announce go chunked, + // as "The question" in SPEC.md requires. + question.TransferEncoding = in.TransferEncoding + + question.Header = in.Header.Clone() + question.Header.Set(ClientHeader, params.Client.String()) + question.Header.Set(PeerHeader, params.Peer.String()) + question.Header.Del(ContentLengthHeader) + + announced := in.Header.Get("Content-Length") + if announced != "" { + question.Header.Set(ContentLengthHeader, announced) + } + + question.Header.Set(RequestIDHeader, params.RequestID) + question.Header.Set("X-Forwarded-Proto", params.Scheme) + + // Go's client adds a User-Agent of its own to a request without the key. + _, sent := question.Header["User-Agent"] + if !sent { + question.Header["User-Agent"] = nil + } + + return question, nil +} + +// ReadQuestion reads the question r, with the body it carries. An error +// wrapping ErrBodyStoppedArriving means the body stopped arriving. Any other +// error is why waf refuses the question, and the question's body is then not +// read. The body is read whole, since Go's server does not send an answer +// until it has read the body. +func ReadQuestion(w http.ResponseWriter, r *http.Request) (Question, error) { + client, err := netip.ParseAddr(r.Header.Get(ClientHeader)) + if err != nil || client.Zone() != "" { + return Question{}, errClientNotAnAddress + } + + id := r.Header.Get(RequestIDHeader) + if id == "" || len(id) > maxRequestIDBytes || !utf8.ValidString(id) { + return Question{}, errRequestIDNotValid + } + + body, err := io.ReadAll( + http.MaxBytesReader(w, r.Body, QuestionBodyMaxBytes), + ) + + if _, tooLarge := errors.AsType[*http.MaxBytesError](err); tooLarge { + return Question{}, errBodyTooLarge + } + + if err != nil { + return Question{}, fmt.Errorf("%w: %w", ErrBodyStoppedArriving, err) + } + + // One that cannot be read leaves the log line's peer_ip invalid. + peer, _ := netip.ParseAddr(r.Header.Get(PeerHeader)) + + return Question{ + Client: client.Unmap(), + Peer: peer.Unmap(), + RequestID: id, + Visitor: asTheVisitorSentIt(r, body), + }, nil +} + +// asTheVisitorSentIt returns a copy of r, a question, as the checks and the +// log line are to see it: without the headers rproxy adds for waf, +// with body, which has been read of r, and with the length the visitor +// announced, as -1 when it announced none. The question's own length covers +// only the part of the body it carries. +func asTheVisitorSentIt(r *http.Request, body []byte) *http.Request { + sent := r.Clone(r.Context()) + announced := sent.Header.Get(ContentLengthHeader) + + sent.Header.Del(ClientHeader) + sent.Header.Del(PeerHeader) + sent.Header.Del(ContentLengthHeader) + sent.Header.Del("Content-Length") + + sent.ContentLength = -1 + + length, err := strconv.ParseInt(announced, 10, 64) + if err == nil && length >= 0 { + sent.ContentLength = length + sent.Header.Set("Content-Length", announced) + } + + sent.Body = http.NoBody + if len(body) > 0 { + sent.Body = io.NopCloser(bytes.NewReader(body)) + } + + return sent +} diff --git a/internal/wafprotocol/report.go b/internal/wafprotocol/report.go new file mode 100644 index 0000000..56fdbc1 --- /dev/null +++ b/internal/wafprotocol/report.go @@ -0,0 +1,123 @@ +package wafprotocol + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" +) + +const ( + // reportBodyMaxBytes is the most of a report's body that is read. It is + // far above any report's. + reportBodyMaxBytes = 1 << 20 + // maxReportedBytes is the most bytes a report may give for each way, 1 + // PiB, so that adding them up cannot overflow. + maxReportedBytes = 1 << 50 + // maxReportedMilliseconds is the longest duration a report may give, + // about 31 years, so that it fits a time.Duration. + maxReportedMilliseconds = 1e12 +) + +var ( + errReportFieldMissing = errors.New("a field is missing") + errReportValue = errors.New("a value is out of range") +) + +// Report is the JSON object rproxy posts to ReportPath once the visitor's +// answer has ended, its fields named as the request log's of the same +// meaning. RequestID is the question's id. Status is what the visitor got, 0 +// if nothing. The durations are in milliseconds. +// +//nolint:tagliatelle // docs/request-log.md names the fields in snake_case +type Report struct { + RequestID string `json:"request_id"` + Status int `json:"status"` + RequestBytes int64 `json:"request_bytes"` + ResponseBytes int64 `json:"response_bytes"` + DurationTotalMS float64 `json:"duration_total_ms"` + DurationUpstreamTotalMS float64 `json:"duration_upstream_total_ms"` +} + +// NewReportRequest returns the request that posts report to reportURL, which +// is waf's ReportPath. +func NewReportRequest( + ctx context.Context, + reportURL string, + report *Report, +) (*http.Request, error) { + body, err := json.Marshal(report) + if err != nil { + return nil, fmt.Errorf("encode the report: %w", err) + } + + req, err := http.NewRequestWithContext( + ctx, http.MethodPost, reportURL, bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("make the request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + + return req, nil +} + +// ReadReport reads a report from r's body, which must be one JSON object with +// all of Report's fields, in the ranges SPEC.md gives. +func ReadReport(w http.ResponseWriter, r *http.Request) (Report, error) { + var report Report + + body, err := io.ReadAll( + http.MaxBytesReader(w, r.Body, reportBodyMaxBytes), + ) + if err != nil { + return report, fmt.Errorf("read the body: %w", err) + } + + var fields map[string]json.RawMessage + + err = json.Unmarshal(body, &fields) + if err != nil { + return report, fmt.Errorf("decode the body: %w", err) + } + + for _, name := range []string{ + "request_id", "status", "request_bytes", "response_bytes", + "duration_total_ms", "duration_upstream_total_ms", + } { + value, there := fields[name] + if !there || string(value) == "null" { + return report, fmt.Errorf("%w: %s", errReportFieldMissing, name) + } + } + + err = json.Unmarshal(body, &report) + if err != nil { + return report, fmt.Errorf("decode the body: %w", err) + } + + return report, report.checkRanges() +} + +// checkRanges returns an error for a value of the report that cannot be true. +func (report *Report) checkRanges() error { + switch { + case report.Status != 0 && + (report.Status < http.StatusContinue || report.Status > ReportStatusMax): + return fmt.Errorf("%w: status", errReportValue) + case report.RequestBytes < 0 || report.RequestBytes > maxReportedBytes: + return fmt.Errorf("%w: request_bytes", errReportValue) + case report.ResponseBytes < 0 || report.ResponseBytes > maxReportedBytes: + return fmt.Errorf("%w: response_bytes", errReportValue) + case report.DurationTotalMS < 0 || report.DurationTotalMS > maxReportedMilliseconds: + return fmt.Errorf("%w: duration_total_ms", errReportValue) + case report.DurationUpstreamTotalMS < 0 || + report.DurationUpstreamTotalMS > maxReportedMilliseconds: + return fmt.Errorf("%w: duration_upstream_total_ms", errReportValue) + } + + return nil +} diff --git a/internal/wafprotocol/wafprotocol.go b/internal/wafprotocol/wafprotocol.go new file mode 100644 index 0000000..06af148 --- /dev/null +++ b/internal/wafprotocol/wafprotocol.go @@ -0,0 +1,61 @@ +// Package wafprotocol holds the three messages rproxy and waf send each +// other, as "`waf`" in SPEC.md gives them. rproxy asks waf a question about +// each visitor's request, and waf answers it. Once the visitor's answer has +// ended, rproxy sends waf a report. This package writes and reads each +// message, and declares every name and limit both sides must agree on. +package wafprotocol + +// The headers of the question and of the answer. Go writes every header name +// in this form, as it sends it and as it receives it, so X-Client-ASN arrives +// as X-Client-Asn; header names are not case-sensitive. +const ( + // ClientHeader marks a request as rproxy's question, and gives the + // visitor's address. + ClientHeader = "X-Sww-Client" + // PeerHeader gives the address of rproxy's own TCP peer. + PeerHeader = "X-Sww-Peer" + // ContentLengthHeader gives the body length the visitor announced. + ContentLengthHeader = "X-Sww-Content-Length" + // RequestIDHeader gives the request's id, which rproxy makes. rproxy + // also sets it on the request it passes to the app. + RequestIDHeader = "X-Request-ID" + // DispositionHeader gives waf's answer. + DispositionHeader = "X-Sww-Disposition" + // ASNHeader and CountryHeader give the client's AS number and country in + // an answer of allow. rproxy passes them to the app. + ASNHeader = "X-Client-Asn" + CountryHeader = "X-Client-Country" +) + +// The values of DispositionHeader. +const ( + // DispositionAllow tells rproxy to pass the request to the app. + DispositionAllow = "allow" + // DispositionDeny tells rproxy to give the visitor the answer's status, + // headers and body. + DispositionDeny = "deny" + // DispositionClose tells rproxy to close the visitor's connection + // without an answer, which SWW_WAF_BAN_RESPONSE asks for when it is close. + // waf closing the question's connection instead would read as + // waf being down, and rproxy would then let the request through. + DispositionClose = "close" +) + +// OwnEndpointPrefix starts the path of every request for sww itself: waf's +// own endpoints and rproxy's health endpoint. The app never sees a path under +// it. +const OwnEndpointPrefix = "/_sww/" + +// HealthPath is the health endpoint of waf and of rproxy, which the +// container's health check asks. +const HealthPath = "/_sww/healthz" + +// ReportPath is where rproxy sends the report of each request waf +// answered. +const ReportPath = "/_sww/report" + +// QuestionBodyMaxBytes is the most of a visitor's body a question carries. +const QuestionBodyMaxBytes = 64 << 10 + +// ReportStatusMax is the highest status a report may give. +const ReportStatusMax = 599 diff --git a/internal/wafprotocol/wafprotocol_test.go b/internal/wafprotocol/wafprotocol_test.go new file mode 100644 index 0000000..643f64a --- /dev/null +++ b/internal/wafprotocol/wafprotocol_test.go @@ -0,0 +1,204 @@ +package wafprotocol_test + +import ( + "io" + "net/http" + "net/http/httptest" + "net/netip" + "net/url" + "strings" + "testing" + + "sneak.berlin/go/smallwebwaf/internal/wafprotocol" +) + +// requestID is the id the tests' questions and reports give. +const requestID = "an-id" + +// questionRead is what the server startQuestionReader starts read of a +// question: the question, and the body it carries. +type questionRead struct { + question wafprotocol.Question + body string +} + +// startQuestionReader starts a server that reads each question it gets with +// ReadQuestion. It returns the server's URL and the channel what it reads goes +// to. +func startQuestionReader(t *testing.T) (*url.URL, <-chan questionRead) { + t.Helper() + + read := make(chan questionRead, 1) + server := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + question, err := wafprotocol.ReadQuestion(w, r) + if err != nil { + t.Errorf("read the question: %v", err) + } + + var body []byte + if question.Visitor != nil { + body, _ = io.ReadAll(question.Visitor.Body) + } + + read <- questionRead{question, string(body)} + }, + )) + t.Cleanup(server.Close) + + serverURL, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("parse the server's URL: %v", err) + } + + return serverURL, read +} + +// TestQuestionReadsAsWritten sends a question NewQuestion writes over HTTP, +// and checks that ReadQuestion gives back the visitor's request and what +// rproxy added. +func TestQuestionReadsAsWritten(t *testing.T) { + t.Parallel() + + const ( + target = "/a%2Fb?q=1" + body = "name=value" + held = "name" + ) + + client := netip.MustParseAddr("2001:db8::1") + peer := netip.MustParseAddr("192.0.2.1") + backend, read := startQuestionReader(t) + + visitor := httptest.NewRequestWithContext( + t.Context(), + http.MethodPost, + "http://site.example.com"+target, + strings.NewReader(body), + ) + visitor.Header.Set("Content-Length", "10") + visitor.Header.Set("User-Agent", "a visitor") + visitor.Header.Set(wafprotocol.PeerHeader, "the visitor's own") + + question, err := wafprotocol.NewQuestion( + t.Context(), + wafprotocol.QuestionParams{ + Visitor: visitor, + PathAsSent: "/a%2Fb", + Body: []byte(held), + WAFBackend: backend, + Client: client, + Peer: peer, + RequestID: requestID, + Scheme: "https", + }, + ) + if err != nil { + t.Fatalf("write the question: %v", err) + } + + res, err := http.DefaultTransport.RoundTrip(question) + if err != nil { + t.Fatalf("send the question: %v", err) + } + + _ = res.Body.Close() + + got := <-read + if got.question.Visitor == nil { + t.FailNow() + } + + if got.question.Client != client || got.question.Peer != peer || + got.question.RequestID != requestID { + t.Errorf("read %+v", got.question) + } + + sent := got.question.Visitor + if sent.Host != "site.example.com" || sent.RequestURI != target || + sent.ContentLength != int64(len(body)) || got.body != held { + t.Errorf( + "read %s at %s, of length %d, carrying %q", + sent.RequestURI, sent.Host, sent.ContentLength, got.body, + ) + } + + for name, want := range map[string]string{ + "Content-Length": "10", + "User-Agent": "a visitor", + "X-Forwarded-Proto": "https", + wafprotocol.RequestIDHeader: requestID, + wafprotocol.ClientHeader: "", + wafprotocol.PeerHeader: "", + wafprotocol.ContentLengthHeader: "", + } { + if sent.Header.Get(name) != want { + t.Errorf("%s is %q, want %q", name, sent.Header.Get(name), want) + } + } +} + +// TestAnswerReadsAsWritten checks that ReadAnswer gives back what +// SetAnswerHeaders sets, and that headers without a valid disposition give +// nothing. +func TestAnswerReadsAsWritten(t *testing.T) { + t.Parallel() + + for _, want := range []wafprotocol.Answer{ + {Disposition: wafprotocol.DispositionAllow, ASN: "64496", Country: "DE"}, + {Disposition: wafprotocol.DispositionAllow, Country: "DE"}, + {Disposition: wafprotocol.DispositionDeny}, + {Disposition: wafprotocol.DispositionClose}, + } { + header := http.Header{} + wafprotocol.SetAnswerHeaders(header, want) + + got, valid := wafprotocol.ReadAnswer(header) + if !valid || got != want { + t.Errorf("read %+v (valid: %t), want %+v", got, valid, want) + } + } + + header := http.Header{} + header.Set(wafprotocol.DispositionHeader, "maybe") + header.Set(wafprotocol.ASNHeader, "64496") + + got, valid := wafprotocol.ReadAnswer(header) + if valid || got != (wafprotocol.Answer{}) { + t.Errorf("read %+v (valid: %t), want nothing", got, valid) + } +} + +// TestReportReadsAsWritten checks that ReadReport gives back the report +// NewReportRequest posts. +func TestReportReadsAsWritten(t *testing.T) { + t.Parallel() + + want := wafprotocol.Report{ + RequestID: requestID, + Status: http.StatusOK, + RequestBytes: 10, + ResponseBytes: 20, + DurationTotalMS: 1.5, + DurationUpstreamTotalMS: 0.5, + } + + req, err := wafprotocol.NewReportRequest( + t.Context(), + "http://waf.example.com"+wafprotocol.ReportPath, + &want, + ) + if err != nil { + t.Fatalf("write the report: %v", err) + } + + if req.Method != http.MethodPost || + req.URL.Path != wafprotocol.ReportPath { + t.Errorf("the report is %s %s", req.Method, req.URL.Path) + } + + got, err := wafprotocol.ReadReport(httptest.NewRecorder(), req) + if err != nil || got != want { + t.Errorf("read %+v (%v), want %+v", got, err, want) + } +} -- 2.54.0