Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9ff9f23a8f | ||
|
|
73e0137c07 | ||
|
|
98d7432077 | ||
|
|
11ac8aee50 | ||
|
|
e7cc0adbdb |
No files matched your search
@@ -902,6 +902,8 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
|
||||
- `internal/rproxy`: the reverse proxy in front of one app.
|
||||
- `internal/waf`: `waf`'s server, which runs the checks and answers its own
|
||||
endpoints.
|
||||
- `internal/wafprotocol`: the question, the answer and the report that `rproxy`
|
||||
and `waf` send each other.
|
||||
- `internal/ratelimit`, `internal/bans`, `internal/rules`,
|
||||
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
||||
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
||||
|
||||
@@ -73,13 +73,16 @@ never as a time.
|
||||
`go.uber.org/fx`. fx builds the parts of `waf` and `rproxy`, and starts
|
||||
and stops their background work.
|
||||
- `internal/config`: reads the settings, the one place they are read.
|
||||
- `internal/waf`: `waf`'s server. It runs the checks in `check`, and holds
|
||||
`waf`'s question, answer and report. It answers `waf`'s own endpoints in
|
||||
`answerOwnEndpoint`. It counts bytes, refusals and anomalies, and writes
|
||||
the log lines.
|
||||
- `internal/waf`: `waf`'s server. It runs the checks in `check`, and answers
|
||||
`waf`'s own endpoints in `answerOwnEndpoint`. It counts bytes, refusals
|
||||
and anomalies, and writes the log lines.
|
||||
- `internal/rproxy`: the reverse proxy in front of one app that asks `waf`
|
||||
about each request. It passes a request to the app with the standard
|
||||
library's `httputil.ReverseProxy`, within the time and size limits.
|
||||
- `internal/wafprotocol`: the question, the answer and the report that
|
||||
`rproxy` and `waf` send each other (see "`waf`"). It writes and reads each
|
||||
one, and declares every header, path and limit both sides use.
|
||||
`internal/rproxy` imports it, not `internal/waf`.
|
||||
- `internal/ratelimit`: the table of clients. It counts requests, bytes and
|
||||
refusals, and keeps each client's history.
|
||||
- `internal/bans`: the bans. It holds each netblock's bans with their notes,
|
||||
|
||||
+47
-14
@@ -99,8 +99,13 @@ type GeoJS struct {
|
||||
// waiting are the clients without an answer: those to ask GeoJS about,
|
||||
// and those it is being asked about.
|
||||
waiting map[netip.Prefix]*wait
|
||||
// asking is true while a request to GeoJS is under way.
|
||||
asking bool
|
||||
// asking is true while a request to GeoJS is under way, and endAsking
|
||||
// ends it. stopped is true once StopAsking has been called, and
|
||||
// askGoroutines waits for the goroutine that makes the requests.
|
||||
asking bool
|
||||
endAsking context.CancelFunc
|
||||
stopped bool
|
||||
askGoroutines sync.WaitGroup
|
||||
// retryDelay is how long GeoJS is left alone after its last failure,
|
||||
// zero after an answer; retryAt is when it may be asked again.
|
||||
retryDelay time.Duration
|
||||
@@ -210,6 +215,23 @@ func (g *GeoJS) LookUpWithoutWaiting(
|
||||
return answer
|
||||
}
|
||||
|
||||
// StopAsking ends the request to GeoJS under way, and returns once the
|
||||
// goroutine that made it has. A request it ends is no failure: Failed is not
|
||||
// called, and GeoJS is not left alone after it. After StopAsking, GeoJS is
|
||||
// asked nothing.
|
||||
func (g *GeoJS) StopAsking() {
|
||||
g.mu.Lock()
|
||||
|
||||
g.stopped = true
|
||||
if g.asking {
|
||||
g.endAsking()
|
||||
}
|
||||
|
||||
g.mu.Unlock()
|
||||
|
||||
g.askGoroutines.Wait()
|
||||
}
|
||||
|
||||
// Kept returns client's answer, if one is kept, without asking GeoJS.
|
||||
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
|
||||
g.mu.Lock()
|
||||
@@ -294,7 +316,8 @@ func (g *GeoJS) answerOrWait(
|
||||
}
|
||||
|
||||
if !g.asking {
|
||||
// GeoJS is left alone after a failure, so no answer can come.
|
||||
// GeoJS is left alone after a failure, or asked nothing after
|
||||
// StopAsking, so no answer can come.
|
||||
w.late = true
|
||||
}
|
||||
|
||||
@@ -323,22 +346,28 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
||||
}
|
||||
|
||||
// ask starts asking GeoJS about the waiting clients, unless a request to
|
||||
// it is under way or it is left alone after a failure. The requests to
|
||||
// GeoJS are for every client waiting, so they go on when the caller whose
|
||||
// ctx is given stops waiting.
|
||||
// it is under way, it is left alone after a failure, or StopAsking has
|
||||
// been called. The requests to GeoJS are for every client waiting, so they
|
||||
// go on when the caller whose ctx is given stops waiting, until StopAsking.
|
||||
func (g *GeoJS) ask(ctx context.Context) {
|
||||
if g.asking || g.now().Before(g.retryAt) {
|
||||
if g.asking || g.stopped || g.now().Before(g.retryAt) {
|
||||
return
|
||||
}
|
||||
|
||||
g.asking = true
|
||||
askCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||
g.asking, g.endAsking = true, end
|
||||
|
||||
go g.askAboutWaiting(context.WithoutCancel(ctx))
|
||||
g.askGoroutines.Go(func() {
|
||||
defer end()
|
||||
|
||||
g.askAboutWaiting(askCtx)
|
||||
})
|
||||
}
|
||||
|
||||
// askAboutWaiting asks GeoJS about the waiting clients, one request at a
|
||||
// time, until none is left or GeoJS fails. Each answer kept is given to
|
||||
// Answered, outside the lock, since Answered takes locks of its own.
|
||||
// time, until none is left, GeoJS fails, or StopAsking ends ctx. Each
|
||||
// answer kept is given to Answered, outside the lock, since Answered takes
|
||||
// locks of its own.
|
||||
func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
||||
for {
|
||||
clients := g.nextClients()
|
||||
@@ -347,6 +376,10 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
||||
}
|
||||
|
||||
given, err := g.request(ctx, clients)
|
||||
if err != nil && ctx.Err() != nil {
|
||||
continue // ended by StopAsking, which is no failure
|
||||
}
|
||||
|
||||
kept, answered := g.keep(clients, given, err)
|
||||
|
||||
if g.answered != nil {
|
||||
@@ -362,13 +395,13 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
||||
}
|
||||
|
||||
// nextClients returns up to maxPerRequest of the waiting clients. When
|
||||
// none is waiting, it returns none and notes that no request to GeoJS is
|
||||
// under way.
|
||||
// none is waiting, or StopAsking has been called, it returns none and
|
||||
// notes that no request to GeoJS is under way.
|
||||
func (g *GeoJS) nextClients() []netip.Prefix {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
|
||||
if len(g.waiting) == 0 {
|
||||
if len(g.waiting) == 0 || g.stopped {
|
||||
g.asking = false
|
||||
|
||||
return nil
|
||||
|
||||
@@ -318,6 +318,44 @@ func TestCallerWhoseContextEndsEarlyDoesNotStopTheRequest(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestStopAskingEndsTheRequestUnderWayAsNoFailureAndAsksNoMore(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
service, _, g, reports := startWithReports()
|
||||
clients := newClients()
|
||||
|
||||
service.set(hanging)
|
||||
g.LookUpWithoutWaiting(t.Context(), clients())
|
||||
waitForRequests(t, service, 1)
|
||||
|
||||
// The request is ended, not waited out, and is no failure.
|
||||
began := time.Now()
|
||||
|
||||
g.StopAsking()
|
||||
|
||||
if waited := time.Since(began); waited != 0 {
|
||||
t.Errorf("StopAsking returned after %s, want at once", waited)
|
||||
}
|
||||
|
||||
reports.wantFailures(t)
|
||||
|
||||
// A new client is then not asked about, and its lookup does not
|
||||
// wait.
|
||||
service.set(answering)
|
||||
wantCountry(t, g, clients(), "")
|
||||
|
||||
if waited := time.Since(began); waited != 0 {
|
||||
t.Errorf("the lookup returned after %s, want at once", waited)
|
||||
}
|
||||
|
||||
waitForRequests(t, service, 1)
|
||||
reports.wantFailures(t)
|
||||
})
|
||||
}
|
||||
|
||||
func TestRequestOver10SecondsIsAFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -110,8 +110,12 @@ type AbuseIPDB struct {
|
||||
// scores are by client. Each is added as it is fetched and never moved
|
||||
// up, so that the one fetched longest ago is the first dropped.
|
||||
scores *simplelru.LRU[netip.Prefix, Score]
|
||||
// checking are the clients whose check is under way.
|
||||
checking map[netip.Prefix]bool
|
||||
// checking are the clients whose check is under way, each with the
|
||||
// function that ends it. stopped is true once StopChecking has been
|
||||
// called, and checkGoroutines waits for the goroutine of each check.
|
||||
checking map[netip.Prefix]context.CancelFunc
|
||||
stopped bool
|
||||
checkGoroutines sync.WaitGroup
|
||||
// day is the day, in UTC, of the checks spent counts.
|
||||
day civil.Date
|
||||
spent int
|
||||
@@ -134,7 +138,7 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
|
||||
params: params,
|
||||
httpClient: &http.Client{},
|
||||
scores: scores,
|
||||
checking: map[netip.Prefix]bool{},
|
||||
checking: map[netip.Prefix]context.CancelFunc{},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -144,11 +148,11 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
|
||||
// addresses its request comes from. A client without one is checked in
|
||||
// the background, by addr, the address its request came from, if
|
||||
// offender, if it has committed an offence, unless its check is under
|
||||
// way, a check failed less than failureDelay ago, or the day's checks
|
||||
// have used up DailyBudget; Hit never waits for a check. The check that
|
||||
// uses the budget up is logged and raised as a source_failure alert. ctx
|
||||
// is the context of the client's request, and a check goes on after the
|
||||
// request ends.
|
||||
// way, a check failed less than failureDelay ago, the day's checks have
|
||||
// used up DailyBudget, or StopChecking has been called; Hit never waits
|
||||
// for a check. The check that uses the budget up is logged and raised as a
|
||||
// source_failure alert. ctx is the context of the client's request, and a
|
||||
// check goes on after the request ends, until StopChecking.
|
||||
func (a *AbuseIPDB) Hit(
|
||||
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
|
||||
) (int64, bool) {
|
||||
@@ -167,14 +171,21 @@ func (a *AbuseIPDB) Hit(
|
||||
a.day, a.spent = today, 0
|
||||
}
|
||||
|
||||
check := offender && !a.checking[client] && !now.Before(a.retryAt) &&
|
||||
a.spent < a.params.DailyBudget
|
||||
_, checking := a.checking[client]
|
||||
|
||||
check := offender && !checking && !a.stopped &&
|
||||
!now.Before(a.retryAt) && a.spent < a.params.DailyBudget
|
||||
if check {
|
||||
a.checking[client] = true
|
||||
checkCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||
a.checking[client] = end
|
||||
a.checks++
|
||||
a.spent++
|
||||
|
||||
go a.check(context.WithoutCancel(ctx), client, addr)
|
||||
a.checkGoroutines.Go(func() {
|
||||
defer end()
|
||||
|
||||
a.check(checkCtx, client, addr)
|
||||
})
|
||||
}
|
||||
|
||||
usedUp := check && a.spent == a.params.DailyBudget
|
||||
@@ -189,6 +200,24 @@ func (a *AbuseIPDB) Hit(
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// StopChecking ends the checks under way, and returns once their goroutines
|
||||
// have. A check it ends gives no score and is no failure: it is neither
|
||||
// counted, logged nor raised as an alert, and checks are not left alone
|
||||
// after it. The ended check stays spent from the day's budget. After
|
||||
// StopChecking, Hit checks no client.
|
||||
func (a *AbuseIPDB) StopChecking() {
|
||||
a.mu.Lock()
|
||||
|
||||
a.stopped = true
|
||||
for _, end := range a.checking {
|
||||
end()
|
||||
}
|
||||
|
||||
a.mu.Unlock()
|
||||
|
||||
a.checkGoroutines.Wait()
|
||||
}
|
||||
|
||||
// Checked returns how many checks were made.
|
||||
func (a *AbuseIPDB) Checked() int {
|
||||
a.mu.Lock()
|
||||
@@ -315,7 +344,8 @@ func (a *AbuseIPDB) RunBlacklist(ctx context.Context) {
|
||||
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
|
||||
// the score as client's, and notes the check as no longer under way. A
|
||||
// check that fails gives no score: it is counted, logged and raised as a
|
||||
// source_failure alert, and no client is checked for failureDelay.
|
||||
// source_failure alert, and no client is checked for failureDelay. A check
|
||||
// ended as ctx ends, by StopChecking, is no failure.
|
||||
func (a *AbuseIPDB) check(
|
||||
ctx context.Context,
|
||||
client netip.Prefix,
|
||||
@@ -323,24 +353,26 @@ func (a *AbuseIPDB) check(
|
||||
) {
|
||||
score, err := a.ask(ctx, addr)
|
||||
now := a.params.Now()
|
||||
isFailure := err != nil && ctx.Err() == nil
|
||||
|
||||
a.mu.Lock()
|
||||
|
||||
delete(a.checking, client)
|
||||
|
||||
if err == nil {
|
||||
switch {
|
||||
case err == nil:
|
||||
a.scores.Add(
|
||||
client,
|
||||
Score{Client: client, Score: score, Fetched: now},
|
||||
)
|
||||
} else {
|
||||
case isFailure:
|
||||
a.failures++
|
||||
a.retryAt = now.Add(failureDelay)
|
||||
}
|
||||
|
||||
a.mu.Unlock()
|
||||
|
||||
if err != nil {
|
||||
if isFailure {
|
||||
a.alert("checking a client with AbuseIPDB failed", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -394,6 +394,43 @@ func TestCheckNotAnsweredWithinTheTimeoutFailsAndHitNeverWaits(
|
||||
})
|
||||
}
|
||||
|
||||
func TestStopCheckingEndsTheCheckUnderWayAsNoFailureAndChecksNoMore(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
queue := newQueue()
|
||||
p := abuseIPDBParams()
|
||||
p.Alerts = queue
|
||||
abuseIPDB := &abuseIPDBStandIn{hanging: true}
|
||||
checker := newAbuseIPDB(abuseIPDB, p)
|
||||
|
||||
wantScore(t, checker, suspect, true, 0, false)
|
||||
synctest.Wait()
|
||||
|
||||
// The check is ended, not waited out, and is no failure.
|
||||
began := time.Now()
|
||||
|
||||
checker.StopChecking()
|
||||
|
||||
if waited := time.Since(began); waited != 0 {
|
||||
t.Errorf("StopChecking returned after %s, want at once", waited)
|
||||
}
|
||||
|
||||
wantFailures(t, checker, 0)
|
||||
|
||||
if got := waiting(queue); len(got) != 0 {
|
||||
t.Errorf("alerts waiting %+v, want none", got)
|
||||
}
|
||||
|
||||
// An offender without a score is then not checked.
|
||||
wantScore(t, checker, other, true, 0, false)
|
||||
synctest.Wait()
|
||||
wantChecked(t, abuseIPDB, suspect)
|
||||
})
|
||||
}
|
||||
|
||||
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -82,8 +82,12 @@ type DNSBL struct {
|
||||
// verdicts are by query. Each is added as it is fetched and never moved
|
||||
// up, so that the one fetched longest ago is the first dropped.
|
||||
verdicts *simplelru.LRU[query, Verdict]
|
||||
// asking are the queries under way.
|
||||
asking map[query]bool
|
||||
// asking are the queries under way, each with the function that ends
|
||||
// it. stopped is true once StopAsking has been called, and
|
||||
// queryGoroutines waits for the goroutine of each query.
|
||||
asking map[query]context.CancelFunc
|
||||
stopped bool
|
||||
queryGoroutines sync.WaitGroup
|
||||
// queries and failures count, by zone, the queries made and those that
|
||||
// failed, and retryAt is when a zone whose last query failed may be
|
||||
// asked again.
|
||||
@@ -120,7 +124,7 @@ func NewDNSBL(params DNSBLParams) *DNSBL {
|
||||
params: params,
|
||||
resolver: resolver,
|
||||
verdicts: verdicts,
|
||||
asking: map[query]bool{},
|
||||
asking: map[query]context.CancelFunc{},
|
||||
queries: map[string]int{},
|
||||
failures: map[string]int{},
|
||||
retryAt: map[string]time.Time{},
|
||||
@@ -138,9 +142,10 @@ func (d *DNSBL) Zones() []string {
|
||||
// log, the alerts and the metrics. A verdict is used until CacheTTL has
|
||||
// passed since it was fetched. Each zone without one is asked about addr
|
||||
// in the background, unless a query about addr to it is under way, the
|
||||
// zone is left alone after a failure, or maxQueries are under way;
|
||||
// ListedBy never waits for a query. ctx is the context of the client's
|
||||
// request, and a query goes on after the request ends.
|
||||
// zone is left alone after a failure, maxQueries are under way, or
|
||||
// StopAsking has been called; ListedBy never waits for a query. ctx is the
|
||||
// context of the client's request, and a query goes on after the request
|
||||
// ends, until StopAsking.
|
||||
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
@@ -153,23 +158,47 @@ func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
||||
q := query{zone: zone, client: addr}
|
||||
|
||||
kept, found := d.verdicts.Peek(q)
|
||||
_, asking := d.asking[q]
|
||||
|
||||
switch {
|
||||
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
|
||||
if kept.Listed {
|
||||
listedBy = append(listedBy, config.MaskZoneKey(zone))
|
||||
}
|
||||
case !d.asking[q] && !now.Before(d.retryAt[zone]) && len(d.asking) < maxQueries:
|
||||
d.asking[q] = true
|
||||
case !asking && !d.stopped && !now.Before(d.retryAt[zone]) &&
|
||||
len(d.asking) < maxQueries:
|
||||
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||
d.asking[q] = end
|
||||
d.queries[zone]++
|
||||
|
||||
go d.ask(context.WithoutCancel(ctx), q)
|
||||
d.queryGoroutines.Go(func() {
|
||||
defer end()
|
||||
|
||||
d.ask(queryCtx, q)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return listedBy
|
||||
}
|
||||
|
||||
// StopAsking ends the queries under way, and returns once their
|
||||
// goroutines have. A query it ends gives no verdict and is no failure: it
|
||||
// is neither counted, logged nor raised as an alert, and the zone is not
|
||||
// left alone after it. After StopAsking, ListedBy asks no zone.
|
||||
func (d *DNSBL) StopAsking() {
|
||||
d.mu.Lock()
|
||||
|
||||
d.stopped = true
|
||||
for _, end := range d.asking {
|
||||
end()
|
||||
}
|
||||
|
||||
d.mu.Unlock()
|
||||
|
||||
d.queryGoroutines.Wait()
|
||||
}
|
||||
|
||||
// Queries returns how many queries were made to zone.
|
||||
func (d *DNSBL) Queries(zone string) int {
|
||||
d.mu.Lock()
|
||||
@@ -238,27 +267,29 @@ func (d *DNSBL) Load(verdicts []Verdict) {
|
||||
// query as no longer under way. A query that fails gives no verdict: it
|
||||
// is counted, logged and raised as a source_failure alert, which show the
|
||||
// zone with its key masked, and the zone is not asked again for
|
||||
// failureDelay.
|
||||
// failureDelay. A query ended as ctx ends, by StopAsking, is no failure.
|
||||
func (d *DNSBL) ask(ctx context.Context, q query) {
|
||||
listed, err := d.lookUp(ctx, q)
|
||||
now := d.params.Now()
|
||||
isFailure := err != nil && ctx.Err() == nil
|
||||
|
||||
d.mu.Lock()
|
||||
|
||||
delete(d.asking, q)
|
||||
|
||||
if err == nil {
|
||||
switch {
|
||||
case err == nil:
|
||||
d.verdicts.Add(q, Verdict{
|
||||
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
|
||||
})
|
||||
} else {
|
||||
case isFailure:
|
||||
d.failures[q.zone]++
|
||||
d.retryAt[q.zone] = now.Add(failureDelay)
|
||||
}
|
||||
|
||||
d.mu.Unlock()
|
||||
|
||||
if err != nil {
|
||||
if isFailure {
|
||||
const failed = "asking a DNSBL zone failed"
|
||||
|
||||
shown := config.MaskZoneKey(q.zone)
|
||||
|
||||
@@ -210,6 +210,45 @@ func TestQueryNotAnsweredWithinTheTimeoutFails(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
//
|
||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
|
||||
t *testing.T,
|
||||
) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
queue := newQueue()
|
||||
p := dnsblParams(zone)
|
||||
p.Alerts = queue
|
||||
resolver := &resolverStandIn{hanging: true}
|
||||
dnsbl := newDNSBL(resolver, p)
|
||||
|
||||
wantZones(t, dnsbl, listed)
|
||||
synctest.Wait()
|
||||
|
||||
// The query is ended, not waited out, and is no failure.
|
||||
began := time.Now()
|
||||
|
||||
dnsbl.StopAsking()
|
||||
|
||||
if waited := time.Since(began); waited != 0 {
|
||||
t.Errorf("StopAsking returned after %s, want at once", waited)
|
||||
}
|
||||
|
||||
wantQueries(t, dnsbl, 1, 0)
|
||||
|
||||
if got := waiting(queue); len(got) != 0 {
|
||||
t.Errorf("alerts waiting %+v, want none", got)
|
||||
}
|
||||
|
||||
// A client without a verdict is then asked about in no zone.
|
||||
wantZones(t, dnsbl, unlisted)
|
||||
synctest.Wait()
|
||||
wantQueries(t, dnsbl, 1, 0)
|
||||
wantAsked(t, resolver, listedName)
|
||||
waitForTheResolver()
|
||||
})
|
||||
}
|
||||
|
||||
//
|
||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(
|
||||
|
||||
@@ -99,8 +99,12 @@ type Client struct {
|
||||
// answers are by client. Each is added as it comes and never moved up,
|
||||
// so that the one that came longest ago is the first dropped.
|
||||
answers *simplelru.LRU[netip.Prefix, Answer]
|
||||
// asking are the clients whose query is under way.
|
||||
asking map[netip.Prefix]bool
|
||||
// asking are the clients whose query is under way, each with the
|
||||
// function that ends it. stopped is true once StopAsking has been
|
||||
// called, and queryGoroutines waits for the goroutine of each query.
|
||||
asking map[netip.Prefix]context.CancelFunc
|
||||
stopped bool
|
||||
queryGoroutines sync.WaitGroup
|
||||
// retryAt is when routewatch may be asked again after a query failed.
|
||||
retryAt time.Time
|
||||
}
|
||||
@@ -116,7 +120,7 @@ func New(params Params) *Client {
|
||||
params: params,
|
||||
httpClient: &http.Client{},
|
||||
answers: answers,
|
||||
asking: map[netip.Prefix]bool{},
|
||||
asking: map[netip.Prefix]context.CancelFunc{},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -124,8 +128,9 @@ func New(params Params) *Client {
|
||||
// IPv6 group, or why it has no route. It never waits for a query. A client
|
||||
// without an answer is asked about in the background, by addr, the address its
|
||||
// request came from, unless its query is under way, a query failed less than
|
||||
// failureKept ago, or maxQueries are under way. ctx is the context of the
|
||||
// client's request, and a query goes on after the request ends.
|
||||
// failureKept ago, maxQueries are under way, or StopAsking has been called. ctx
|
||||
// is the context of the client's request, and a query goes on after the request
|
||||
// ends, until StopAsking.
|
||||
func (c *Client) Route(
|
||||
ctx context.Context, client netip.Prefix, addr netip.Addr,
|
||||
) (Route, Missing) {
|
||||
@@ -147,15 +152,38 @@ func (c *Client) Route(
|
||||
return Route{}, RequestFailed
|
||||
}
|
||||
|
||||
if !c.asking[client] && len(c.asking) < maxQueries {
|
||||
c.asking[client] = true
|
||||
_, asking := c.asking[client]
|
||||
if !asking && !c.stopped && len(c.asking) < maxQueries {
|
||||
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||
c.asking[client] = end
|
||||
|
||||
go c.ask(context.WithoutCancel(ctx), client, addr)
|
||||
c.queryGoroutines.Go(func() {
|
||||
defer end()
|
||||
|
||||
c.ask(queryCtx, client, addr)
|
||||
})
|
||||
}
|
||||
|
||||
return Route{}, NoAnswerYet
|
||||
}
|
||||
|
||||
// StopAsking ends the queries under way, and returns once their goroutines
|
||||
// have. A query it ends gives no answer and is no failure: it is not logged,
|
||||
// and routewatch is not left alone after it. After StopAsking, Route asks
|
||||
// routewatch nothing.
|
||||
func (c *Client) StopAsking() {
|
||||
c.mu.Lock()
|
||||
|
||||
c.stopped = true
|
||||
for _, end := range c.asking {
|
||||
end()
|
||||
}
|
||||
|
||||
c.mu.Unlock()
|
||||
|
||||
c.queryGoroutines.Wait()
|
||||
}
|
||||
|
||||
// Snapshot returns every answer still in use, sorted by client, as
|
||||
// routewatch.json lists them.
|
||||
func (c *Client) Snapshot() []Answer {
|
||||
@@ -205,7 +233,8 @@ func (c *Client) Load(answers []Answer) {
|
||||
// ask asks routewatch about addr, keeps the answer as client's, and notes the
|
||||
// query as no longer under way. A query that fails gives no answer, and
|
||||
// routewatch is not asked again for failureKept. Only the first failure of
|
||||
// those under way is logged.
|
||||
// those under way is logged. A query ended as ctx ends, by StopAsking, is no
|
||||
// failure.
|
||||
func (c *Client) ask(
|
||||
ctx context.Context,
|
||||
client netip.Prefix,
|
||||
@@ -213,18 +242,20 @@ func (c *Client) ask(
|
||||
) {
|
||||
route, err := c.query(ctx, addr)
|
||||
now := c.params.Now()
|
||||
isFailure := err != nil && ctx.Err() == nil
|
||||
|
||||
c.mu.Lock()
|
||||
|
||||
delete(c.asking, client)
|
||||
|
||||
firstFailure := err != nil && !now.Before(c.retryAt)
|
||||
firstFailure := isFailure && !now.Before(c.retryAt)
|
||||
|
||||
if err == nil {
|
||||
switch {
|
||||
case err == nil:
|
||||
c.answers.Add(client, Answer{
|
||||
Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
|
||||
})
|
||||
} else {
|
||||
case isFailure:
|
||||
c.retryAt = now.Add(failureKept)
|
||||
}
|
||||
|
||||
|
||||
@@ -361,6 +361,54 @@ func TestHangingRoutewatchNeverDelaysAndFailsTheQuery(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
// In a synctest bubble, as TestHangingRoutewatchNeverDelaysAndFailsTheQuery
|
||||
// says.
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
f := &fixture{t: t, log: &syncBuffer{}}
|
||||
f.client = routewatch.New(routewatch.Params{
|
||||
URL: silentURL, Now: time.Now,
|
||||
ProcessLog: slog.New(slog.NewTextHandler(f.log, nil)),
|
||||
})
|
||||
|
||||
f.route()
|
||||
synctest.Wait()
|
||||
|
||||
// The query is ended, not waited out.
|
||||
began := time.Now()
|
||||
|
||||
f.client.StopAsking()
|
||||
|
||||
if waited := time.Since(began); waited != 0 {
|
||||
t.Errorf("StopAsking returned after %s, want at once", waited)
|
||||
}
|
||||
|
||||
// A query started now would fail within waitLimit. None is, and the
|
||||
// one ended is no failure: nothing is logged, and routewatch is not
|
||||
// left alone.
|
||||
for range 2 {
|
||||
if _, missing := f.route(); missing != routewatch.NoAnswerYet {
|
||||
t.Errorf(
|
||||
"Route gave %q, want %q",
|
||||
missing,
|
||||
routewatch.NoAnswerYet,
|
||||
)
|
||||
}
|
||||
|
||||
time.Sleep(waitLimit)
|
||||
synctest.Wait()
|
||||
}
|
||||
|
||||
if f.log.String() != "" {
|
||||
t.Errorf("process log %q, want nothing", f.log.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -7,16 +7,6 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The headers rproxy sets on the requests it passes to the app. Go writes every
|
||||
// header name in this form, as it sends it and as it receives it, so
|
||||
// X-Client-ASN arrives as X-Client-Asn, and Del removes a visitor's own whatever
|
||||
// their case; header names are not case-sensitive.
|
||||
const (
|
||||
requestIDHeader = "X-Request-ID"
|
||||
asnHeader = "X-Client-Asn"
|
||||
countryHeader = "X-Client-Country"
|
||||
)
|
||||
|
||||
// peerAddress is the address of the request's TCP peer, normally traefik, without
|
||||
// any IPv6 zone, which waf refuses.
|
||||
func peerAddress(r *http.Request) netip.Addr {
|
||||
|
||||
@@ -20,7 +20,7 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||
)
|
||||
|
||||
// post is the start of a POST whose headers end with what follows it.
|
||||
@@ -492,7 +492,7 @@ func TestASettingOfOffSetsNoLimit(t *testing.T) {
|
||||
strings.NewReader("a body"))),
|
||||
)
|
||||
|
||||
if got := get(t, addr, waf.HealthPath); got.status != http.StatusOK {
|
||||
if got := get(t, addr, wafprotocol.HealthPath); got.status != http.StatusOK {
|
||||
t.Errorf("answered %+v, want rproxy's own 200", got)
|
||||
}
|
||||
})
|
||||
@@ -1471,7 +1471,7 @@ func TestAVisitorThatStopsALittleShortGetsAnAnswerRProxyGivesAtOnce(
|
||||
{"a deny", post, http.StatusForbidden},
|
||||
{
|
||||
"the health check",
|
||||
"GET " + waf.HealthPath + " HTTP/1.1\r\nHost: site.example\r\n",
|
||||
"GET " + wafprotocol.HealthPath + " HTTP/1.1\r\nHost: site.example\r\n",
|
||||
http.StatusOK,
|
||||
},
|
||||
} {
|
||||
|
||||
@@ -12,11 +12,11 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||
)
|
||||
|
||||
// questionBodyMaxBytes is the most of a body a question carries.
|
||||
const questionBodyMaxBytes = waf.QuestionBodyMaxBytes
|
||||
const questionBodyMaxBytes = wafprotocol.QuestionBodyMaxBytes
|
||||
|
||||
// chunked is the encoding of a body whose length the visitor did not announce.
|
||||
const chunked = "chunked"
|
||||
@@ -42,9 +42,9 @@ func TestQuestionIsTheVisitorsRequestWithRProxysHeaders(t *testing.T) {
|
||||
req.Header.Set(forwardedForHeader, "198.51.100.1")
|
||||
req.Header.Set(forwardedProtoHeader, "gopher")
|
||||
req.Header.Set(requestIDHeader, "the-visitor's")
|
||||
req.Header.Set(waf.ClientHeader, "198.51.100.2")
|
||||
req.Header.Set(waf.PeerHeader, "198.51.100.3")
|
||||
req.Header.Set(waf.ContentLengthHeader, "12345")
|
||||
req.Header.Set(wafprotocol.ClientHeader, "198.51.100.2")
|
||||
req.Header.Set(wafprotocol.PeerHeader, "198.51.100.3")
|
||||
req.Header.Set(wafprotocol.ContentLengthHeader, "12345")
|
||||
// Go's client then sends no User-Agent, so that one in the question would be
|
||||
// Go's own.
|
||||
req.Header["User-Agent"] = nil
|
||||
@@ -73,8 +73,8 @@ func TestQuestionIsTheVisitorsRequestWithRProxysHeaders(t *testing.T) {
|
||||
want.Set(forwardedForHeader, "198.51.100.1")
|
||||
want.Set(forwardedProtoHeader, "http")
|
||||
want.Set(requestIDHeader, asked.header.Get(requestIDHeader))
|
||||
want.Set(waf.ClientHeader, localhost)
|
||||
want.Set(waf.PeerHeader, localhost)
|
||||
want.Set(wafprotocol.ClientHeader, localhost)
|
||||
want.Set(wafprotocol.PeerHeader, localhost)
|
||||
|
||||
if !maps.EqualFunc(asked.header, want, slices.Equal) {
|
||||
t.Errorf(
|
||||
@@ -124,8 +124,8 @@ func TestQuestionNamesTheVisitorATrustedProxySpeaksFor(t *testing.T) {
|
||||
wantGreeting(t, visit(t, req))
|
||||
|
||||
asked := standIn.question(t)
|
||||
if asked.header.Get(waf.ClientHeader) != tc.client ||
|
||||
asked.header.Get(waf.PeerHeader) != localhost ||
|
||||
if asked.header.Get(wafprotocol.ClientHeader) != tc.client ||
|
||||
asked.header.Get(wafprotocol.PeerHeader) != localhost ||
|
||||
asked.header.Get(forwardedForHeader) != visitor ||
|
||||
asked.header.Get(forwardedProtoHeader) != tc.proto {
|
||||
t.Errorf("the question's headers are %v", asked.header)
|
||||
@@ -191,7 +191,7 @@ func TestQuestionNamesTheClientAsTheForwardedAddressesSay(t *testing.T) {
|
||||
req.Header[forwardedForHeader] = tc.header
|
||||
wantGreeting(t, visit(t, req))
|
||||
|
||||
if got := standIn.question(t).header.Get(waf.ClientHeader); got !=
|
||||
if got := standIn.question(t).header.Get(wafprotocol.ClientHeader); got !=
|
||||
tc.client {
|
||||
t.Errorf("the question names %s, want %s", got, tc.client)
|
||||
}
|
||||
@@ -282,7 +282,7 @@ func wantQuestionBody(
|
||||
)
|
||||
}
|
||||
|
||||
values := asked.header.Values(waf.ContentLengthHeader)
|
||||
values := asked.header.Values(wafprotocol.ContentLengthHeader)
|
||||
|
||||
if announced {
|
||||
if !slices.Equal(values, []string{strconv.Itoa(size)}) ||
|
||||
@@ -586,8 +586,8 @@ func TestAQuestionNamesNoZone(t *testing.T) {
|
||||
server.Handler.ServeHTTP(httptest.NewRecorder(), req)
|
||||
|
||||
asked := standIn.question(t)
|
||||
if asked.header.Get(waf.ClientHeader) != tc.client ||
|
||||
asked.header.Get(waf.PeerHeader) != tc.peer {
|
||||
if asked.header.Get(wafprotocol.ClientHeader) != tc.client ||
|
||||
asked.header.Get(wafprotocol.PeerHeader) != tc.peer {
|
||||
t.Errorf("the question's headers are %v", asked.header)
|
||||
}
|
||||
|
||||
|
||||
+11
-13
@@ -18,17 +18,13 @@ import (
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||
)
|
||||
|
||||
// flushAfterEachWrite has ReverseProxy pass on each part of the app's answer as
|
||||
// soon as it arrives.
|
||||
const flushAfterEachWrite time.Duration = -1
|
||||
|
||||
// reportStatusMax is the highest status waf takes in a report. An app can
|
||||
// send a higher one, up to 999, and the visitor gets that as it is.
|
||||
const reportStatusMax = 599
|
||||
|
||||
// plainText is the Content-Type of the answers rproxy gives itself.
|
||||
const plainText = "text/plain; charset=utf-8"
|
||||
|
||||
@@ -154,7 +150,7 @@ func (rq *request) readFirstPartOfBody() bool {
|
||||
|
||||
_ = rq.rc.SetReadDeadline(rq.requestDeadline())
|
||||
held, err := io.ReadAll(
|
||||
io.LimitReader(rq.body, waf.QuestionBodyMaxBytes),
|
||||
io.LimitReader(rq.body, wafprotocol.QuestionBodyMaxBytes),
|
||||
)
|
||||
// From here on, the request timer ends a read of the rest of the body, so that
|
||||
// the answer can name the side rproxy was waiting on.
|
||||
@@ -228,16 +224,17 @@ func (rq *request) rewrite(pr *httputil.ProxyRequest) {
|
||||
setPathAsSent(pr.Out.URL, pr.In)
|
||||
|
||||
setForwardedHeaders(pr.In, pr.Out, rq.peer, rq.peerTrusted)
|
||||
pr.Out.Header.Set(requestIDHeader, rq.id)
|
||||
pr.Out.Header.Del(asnHeader)
|
||||
pr.Out.Header.Del(countryHeader)
|
||||
pr.Out.Header.Set(wafprotocol.RequestIDHeader, rq.id)
|
||||
// Del removes a visitor's own, whatever their case.
|
||||
pr.Out.Header.Del(wafprotocol.ASNHeader)
|
||||
pr.Out.Header.Del(wafprotocol.CountryHeader)
|
||||
|
||||
if rq.asn != "" {
|
||||
pr.Out.Header.Set(asnHeader, rq.asn)
|
||||
pr.Out.Header.Set(wafprotocol.ASNHeader, rq.asn)
|
||||
}
|
||||
|
||||
if rq.country != "" {
|
||||
pr.Out.Header.Set(countryHeader, rq.country)
|
||||
pr.Out.Header.Set(wafprotocol.CountryHeader, rq.country)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -470,9 +467,10 @@ func (rq *request) finish(ctx context.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
report := &waf.Report{
|
||||
// An app can send a status up to 999, and the visitor gets it as it is.
|
||||
report := &wafprotocol.Report{
|
||||
RequestID: rq.id,
|
||||
Status: min(rq.out.status, reportStatusMax),
|
||||
Status: min(rq.out.status, wafprotocol.ReportStatusMax),
|
||||
RequestBytes: requestBytes,
|
||||
ResponseBytes: responseBytes,
|
||||
DurationTotalMS: requestlog.Milliseconds(end.Sub(rq.start)),
|
||||
|
||||
+10
-12
@@ -24,7 +24,7 @@ import (
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||
)
|
||||
|
||||
// How rproxy keeps connections to the app and to waf open between
|
||||
@@ -47,10 +47,6 @@ const (
|
||||
idleTimeout = 120 * time.Second
|
||||
)
|
||||
|
||||
// ownEndpointPrefix starts the path of every request for sww itself, which
|
||||
// never reaches the app.
|
||||
const ownEndpointPrefix = "/_sww/"
|
||||
|
||||
// allowedMethods lists the methods rproxy passes on, as an Allow header gives
|
||||
// them. It answers any other method 405 itself.
|
||||
// The WebDAV ones are for Nextcloud.
|
||||
@@ -92,7 +88,9 @@ func New(params Params) *Server {
|
||||
transport: newTransport(),
|
||||
wafTransport: newWAFTransport(),
|
||||
reportURL: (&url.URL{
|
||||
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath,
|
||||
Scheme: backend.Scheme,
|
||||
Host: backend.Host,
|
||||
Path: wafprotocol.ReportPath,
|
||||
}).String(),
|
||||
}
|
||||
|
||||
@@ -176,7 +174,7 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
defer rq.readRestOfBody()
|
||||
defer rq.finish(r.Context())
|
||||
|
||||
if r.Method == http.MethodGet && r.URL.Path == waf.HealthPath {
|
||||
if r.Method == http.MethodGet && r.URL.Path == wafprotocol.HealthPath {
|
||||
rq.answerHealth()
|
||||
|
||||
return
|
||||
@@ -209,9 +207,9 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
rq.noteAnswer(answer)
|
||||
|
||||
switch rq.disposition {
|
||||
case waf.DispositionAllow:
|
||||
case wafprotocol.DispositionAllow:
|
||||
rq.passToApp(r.Context())
|
||||
case waf.DispositionClose:
|
||||
case wafprotocol.DispositionClose:
|
||||
// This panic has Go's server close the connection without an answer,
|
||||
// and log nothing; the deferred finish still runs.
|
||||
panic(http.ErrAbortHandler)
|
||||
@@ -221,8 +219,8 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// passToApp passes the request on to the app, unless a limit or its path keeps
|
||||
// it from the app. waf answers a path under ownEndpointPrefix itself, so
|
||||
// such a request gets here only when
|
||||
// it from the app. waf answers a path under wafprotocol.OwnEndpointPrefix
|
||||
// itself, so such a request gets here only when
|
||||
// SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION allowed it.
|
||||
// ctx is the request's own context.
|
||||
func (rq *request) passToApp(ctx context.Context) {
|
||||
@@ -231,7 +229,7 @@ func (rq *request) passToApp(ctx context.Context) {
|
||||
switch {
|
||||
case rq.refused.Load() != 0 || (maxBytes > 0 && rq.in.ContentLength > maxBytes):
|
||||
rq.answer(http.StatusRequestEntityTooLarge)
|
||||
case strings.HasPrefix(rq.in.URL.Path, ownEndpointPrefix):
|
||||
case strings.HasPrefix(rq.in.URL.Path, wafprotocol.OwnEndpointPrefix):
|
||||
rq.answer(http.StatusNotFound)
|
||||
default:
|
||||
rq.forward(ctx)
|
||||
|
||||
@@ -23,7 +23,7 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -56,7 +56,7 @@ const (
|
||||
|
||||
// The headers the tests set and look at.
|
||||
const (
|
||||
dispositionHeader = waf.DispositionHeader
|
||||
dispositionHeader = wafprotocol.DispositionHeader
|
||||
forwardedForHeader = "X-Forwarded-For"
|
||||
forwardedProtoHeader = "X-Forwarded-Proto"
|
||||
requestIDHeader = "X-Request-ID"
|
||||
@@ -72,9 +72,9 @@ const (
|
||||
|
||||
// What the stand-in for waf answers.
|
||||
const (
|
||||
dispositionAllow = waf.DispositionAllow
|
||||
dispositionDeny = waf.DispositionDeny
|
||||
dispositionClose = waf.DispositionClose
|
||||
dispositionAllow = wafprotocol.DispositionAllow
|
||||
dispositionDeny = wafprotocol.DispositionDeny
|
||||
dispositionClose = wafprotocol.DispositionClose
|
||||
// deniedBody is the body of a deny.
|
||||
deniedBody = "refused by waf"
|
||||
)
|
||||
@@ -175,7 +175,7 @@ type WAF struct {
|
||||
*httptest.Server
|
||||
|
||||
questions chan question
|
||||
reports chan waf.Report
|
||||
reports chan wafprotocol.Report
|
||||
}
|
||||
|
||||
// startWAF starts a stand-in for waf.
|
||||
@@ -184,11 +184,11 @@ func startWAF(t *testing.T, answer http.HandlerFunc) *WAF {
|
||||
|
||||
s := &WAF{
|
||||
questions: make(chan question, 10),
|
||||
reports: make(chan waf.Report, 10),
|
||||
reports: make(chan wafprotocol.Report, 10),
|
||||
}
|
||||
s.Server = httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get(waf.ClientHeader) == "" {
|
||||
if r.Header.Get(wafprotocol.ClientHeader) == "" {
|
||||
s.takeReport(t, w, r)
|
||||
|
||||
return
|
||||
@@ -217,11 +217,11 @@ func (s *WAF) takeReport(
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
var report waf.Report
|
||||
var report wafprotocol.Report
|
||||
|
||||
err := json.NewDecoder(r.Body).Decode(&report)
|
||||
if err != nil || r.Method != http.MethodPost ||
|
||||
r.URL.Path != waf.ReportPath {
|
||||
r.URL.Path != wafprotocol.ReportPath {
|
||||
t.Errorf("%s %s is no report: %v", r.Method, r.URL.Path, err)
|
||||
}
|
||||
|
||||
@@ -242,7 +242,7 @@ func (s *WAF) wantNoReport(t *testing.T) {
|
||||
}
|
||||
|
||||
// report waits for a report.
|
||||
func (s *WAF) report(t *testing.T) waf.Report {
|
||||
func (s *WAF) report(t *testing.T) wafprotocol.Report {
|
||||
t.Helper()
|
||||
|
||||
select {
|
||||
@@ -251,7 +251,7 @@ func (s *WAF) report(t *testing.T) waf.Report {
|
||||
case <-time.After(waitLimit):
|
||||
t.Fatal("no report")
|
||||
|
||||
return waf.Report{}
|
||||
return wafprotocol.Report{}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -279,7 +279,7 @@ func startWAFAnsweringReports(
|
||||
|
||||
backend := httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get(waf.ClientHeader) != "" {
|
||||
if r.Header.Get(wafprotocol.ClientHeader) != "" {
|
||||
answer(w, r)
|
||||
|
||||
return
|
||||
@@ -811,7 +811,7 @@ func TestAnAnswerRProxyHoldsWholeIsSentBeforeItsLogLine(t *testing.T) {
|
||||
}{
|
||||
{"a deny", "GET /" + host, deniedBody, http.StatusForbidden},
|
||||
{"rproxy's own answer", "GET x:admin" + host, "Bad Request\n", http.StatusBadRequest},
|
||||
{"the health check", "GET " + waf.HealthPath + host, "ok\n", http.StatusOK},
|
||||
{"the health check", "GET " + wafprotocol.HealthPath + host, "ok\n", http.StatusOK},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
@@ -1576,19 +1576,19 @@ func TestHealthCheckIsAnsweredWhileWAFIsDown(t *testing.T) {
|
||||
app := startApp(t, greet)
|
||||
addr, out := startRProxy(t, nowhere, app.URL, nil)
|
||||
|
||||
got := get(t, addr, waf.HealthPath)
|
||||
got := get(t, addr, wafprotocol.HealthPath)
|
||||
if got.status != http.StatusOK || got.body != "ok\n" {
|
||||
t.Errorf("answered %+v, want ok", got)
|
||||
}
|
||||
|
||||
line := out.requestLine(t)
|
||||
if line["path"] != waf.HealthPath || line["disposition"] != nil {
|
||||
if line["path"] != wafprotocol.HealthPath || line["disposition"] != nil {
|
||||
t.Errorf("line %v, want one with no disposition", line)
|
||||
}
|
||||
|
||||
// With waf up too, it is not asked.
|
||||
addr, _ = startRProxy(t, standIn.URL, app.URL, nil)
|
||||
get(t, addr, waf.HealthPath)
|
||||
get(t, addr, wafprotocol.HealthPath)
|
||||
|
||||
if len(standIn.questions) != 0 {
|
||||
t.Error("waf was asked about the health check")
|
||||
@@ -1749,7 +1749,7 @@ func TestWaitForReportsWaitsForAReportBeingSent(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
backend := httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get(waf.ClientHeader) != "" {
|
||||
if r.Header.Get(wafprotocol.ClientHeader) != "" {
|
||||
allow(w, r)
|
||||
|
||||
return
|
||||
|
||||
+34
-82
@@ -1,9 +1,7 @@
|
||||
package rproxy
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -12,7 +10,8 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||
)
|
||||
|
||||
// answerBodyMaxBytes is the most of the body of waf's answer that is read.
|
||||
@@ -41,15 +40,12 @@ type wafAnswer struct {
|
||||
// notes nothing, and no report is due for it: waf holds no decision to report
|
||||
// on.
|
||||
func (rq *request) noteAnswer(answer *wafAnswer) {
|
||||
disposition := answer.header.Get(waf.DispositionHeader)
|
||||
|
||||
switch disposition {
|
||||
case waf.DispositionAllow, waf.DispositionDeny,
|
||||
waf.DispositionClose:
|
||||
rq.disposition = disposition
|
||||
read, valid := wafprotocol.ReadAnswer(answer.header)
|
||||
if valid {
|
||||
rq.disposition = read.Disposition
|
||||
rq.answered = true
|
||||
rq.asn = answer.header.Get(asnHeader)
|
||||
rq.country = answer.header.Get(countryHeader)
|
||||
rq.asn = read.ASN
|
||||
rq.country = read.Country
|
||||
}
|
||||
}
|
||||
|
||||
@@ -80,7 +76,7 @@ func (rq *request) handleNoUsableAnswer(ctx context.Context, err error) {
|
||||
rq.disposition = rq.h.config.WAFBackendNoAnswerDisposition
|
||||
rq.wafNoAnswer = true
|
||||
|
||||
if rq.disposition == waf.DispositionAllow {
|
||||
if rq.disposition == wafprotocol.DispositionAllow {
|
||||
rq.passToApp(ctx)
|
||||
} else {
|
||||
rq.answer(http.StatusServiceUnavailable)
|
||||
@@ -120,7 +116,21 @@ func (rq *request) ask(ctx context.Context) (*wafAnswer, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, rq.h.config.WAFBackendTimeout)
|
||||
defer cancel()
|
||||
|
||||
question, err := rq.newQuestion(ctx)
|
||||
var held []byte
|
||||
if rq.body != nil {
|
||||
held = rq.body.held
|
||||
}
|
||||
|
||||
question, err := wafprotocol.NewQuestion(ctx, wafprotocol.QuestionParams{
|
||||
Visitor: rq.in,
|
||||
PathAsSent: rules.Path(rq.in),
|
||||
Body: held,
|
||||
WAFBackend: rq.h.config.WAFBackend,
|
||||
Client: rq.client,
|
||||
Peer: rq.peer,
|
||||
RequestID: rq.id,
|
||||
Scheme: scheme(rq.in, rq.peerTrusted),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -177,69 +187,13 @@ func canBeAsked(r *http.Request) bool {
|
||||
len(r.Header.Values("User-Agent")) <= 1
|
||||
}
|
||||
|
||||
// newQuestion returns the question about the request, as "The question" in
|
||||
// SPEC.md gives it: the visitor's request as the visitor sent it, with the part
|
||||
// of the body it holds and rproxy's headers.
|
||||
func (rq *request) newQuestion(ctx context.Context) (*http.Request, error) {
|
||||
in := rq.in
|
||||
|
||||
var held []byte
|
||||
if rq.body != nil {
|
||||
held = rq.body.held
|
||||
}
|
||||
|
||||
question, err := http.NewRequestWithContext( //nolint:gosec // a setting's URL
|
||||
ctx,
|
||||
in.Method,
|
||||
rq.h.config.WAFBackend.String(),
|
||||
bytes.NewReader(held),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("make the question: %w", err)
|
||||
}
|
||||
|
||||
backend := question.URL
|
||||
target := *in.URL
|
||||
target.Scheme = backend.Scheme
|
||||
target.Host = backend.Host
|
||||
setPathAsSent(&target, in)
|
||||
question.URL = &target
|
||||
question.Host = in.Host
|
||||
|
||||
// The length NewRequestWithContext set is that of what is held. This line makes
|
||||
// a body whose length the visitor did not announce go chunked, as "The question"
|
||||
// in SPEC.md requires.
|
||||
question.TransferEncoding = in.TransferEncoding
|
||||
|
||||
question.Header = in.Header.Clone()
|
||||
question.Header.Set(waf.ClientHeader, rq.client.String())
|
||||
question.Header.Set(waf.PeerHeader, rq.peer.String())
|
||||
question.Header.Del(waf.ContentLengthHeader)
|
||||
|
||||
announced := in.Header.Get("Content-Length")
|
||||
if announced != "" {
|
||||
question.Header.Set(waf.ContentLengthHeader, announced)
|
||||
}
|
||||
|
||||
question.Header.Set(requestIDHeader, rq.id)
|
||||
question.Header.Set("X-Forwarded-Proto", scheme(in, rq.peerTrusted))
|
||||
|
||||
// Go's client adds a User-Agent of its own to a request without the key.
|
||||
_, sent := question.Header["User-Agent"]
|
||||
if !sent {
|
||||
question.Header["User-Agent"] = nil
|
||||
}
|
||||
|
||||
return question, nil
|
||||
}
|
||||
|
||||
// answerWith gives the visitor waf's answer, less its disposition.
|
||||
func (rq *request) answerWith(answer *wafAnswer) {
|
||||
rq.prepareOwnAnswer(rq.requestDeadline())
|
||||
|
||||
header := rq.out.Header()
|
||||
maps.Copy(header, answer.header)
|
||||
header.Del(waf.DispositionHeader)
|
||||
header.Del(wafprotocol.DispositionHeader)
|
||||
|
||||
// Go's server adds a Content-Type it guesses from the body to a response whose
|
||||
// headers have no Content-Type key. The key with no value sends none.
|
||||
@@ -253,7 +207,10 @@ func (rq *request) answerWith(answer *wafAnswer) {
|
||||
// sendReport sends waf report on a goroutine of its own, so that the
|
||||
// visitor does not wait for it. The report goes on after ctx, the request's own
|
||||
// context, has ended.
|
||||
func (h *handler) sendReport(ctx context.Context, report *waf.Report) {
|
||||
func (h *handler) sendReport(
|
||||
ctx context.Context,
|
||||
report *wafprotocol.Report,
|
||||
) {
|
||||
h.reportsMu.Lock()
|
||||
defer h.reportsMu.Unlock()
|
||||
|
||||
@@ -271,23 +228,18 @@ func (h *handler) sendReport(ctx context.Context, report *waf.Report) {
|
||||
|
||||
// postReport posts report to waf, and waits for its answer for up to
|
||||
// SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS.
|
||||
func (h *handler) postReport(ctx context.Context, report *waf.Report) error {
|
||||
body, err := json.Marshal(report)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode the report: %w", err)
|
||||
}
|
||||
|
||||
func (h *handler) postReport(
|
||||
ctx context.Context,
|
||||
report *wafprotocol.Report,
|
||||
) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, h.config.WAFBackendTimeout)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext( //nolint:gosec // a setting's URL
|
||||
ctx, http.MethodPost, h.reportURL, bytes.NewReader(body))
|
||||
req, err := wafprotocol.NewReportRequest(ctx, h.reportURL, report)
|
||||
if err != nil {
|
||||
return fmt.Errorf("make the request: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
res, err := h.wafTransport.RoundTrip(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("send the report: %w", err)
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||
)
|
||||
|
||||
// healthCheckTimeout bounds the whole health check.
|
||||
@@ -116,7 +116,7 @@ func askHealthEndpoint(ctx context.Context, listenAddr string) error {
|
||||
health := "http://" + net.JoinHostPort(
|
||||
"127.0.0.1",
|
||||
port,
|
||||
) + waf.HealthPath
|
||||
) + wafprotocol.HealthPath
|
||||
|
||||
req, err := http.NewRequestWithContext(
|
||||
ctx,
|
||||
|
||||
+17
-17
@@ -21,7 +21,7 @@ import (
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||
"sneak.berlin/go/smallwebwaf/internal/sww"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -164,7 +164,7 @@ func TestABanThroughOneSiteRefusesTheClientOnTheOther(t *testing.T) {
|
||||
|
||||
// One request's three lines carry one id, rproxy's with waf's answer.
|
||||
line := firstOut.line(t, "path", bannedProbe)
|
||||
if line["disposition"] != waf.DispositionDeny ||
|
||||
if line["disposition"] != wafprotocol.DispositionDeny ||
|
||||
line["client_ip"] != placed {
|
||||
t.Errorf("rproxy's line is %v", line)
|
||||
}
|
||||
@@ -249,7 +249,7 @@ func TestTheSiteAnswersWhileWAFIsStoppedAndStartedAgain(t *testing.T) {
|
||||
wantStatus(t, site+"/after-stop", placed, http.StatusOK)
|
||||
|
||||
line := out.line(t, "path", "/after-stop")
|
||||
if line["disposition"] != waf.DispositionAllow ||
|
||||
if line["disposition"] != wafprotocol.DispositionAllow ||
|
||||
line["waf_backend_no_answer"] != true {
|
||||
t.Errorf("rproxy's line is %v", line)
|
||||
}
|
||||
@@ -437,7 +437,7 @@ func TestAChunkedBodyStaysABodyAtWAF(t *testing.T) {
|
||||
const smuggledID = "smuggled-request"
|
||||
|
||||
smuggled := "GET " + bannedProbe + " HTTP/1.1\r\nHost: site.example\r\n" +
|
||||
waf.ClientHeader + ": " + otherClient + "\r\n" +
|
||||
wafprotocol.ClientHeader + ": " + otherClient + "\r\n" +
|
||||
"X-Request-ID: " + smuggledID + "\r\n\r\n"
|
||||
|
||||
status := sendRaw(
|
||||
@@ -605,10 +605,11 @@ func TestRProxyAsksAWAFThatSpeaksHTTP2OverHTTP1(t *testing.T) {
|
||||
protocols := make(chan string, 10)
|
||||
backend := httptest.NewUnstartedServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get(waf.ClientHeader) != "" {
|
||||
if r.Header.Get(wafprotocol.ClientHeader) != "" {
|
||||
protocols <- r.Proto
|
||||
|
||||
w.Header().Set(waf.DispositionHeader, waf.DispositionDeny)
|
||||
w.Header().
|
||||
Set(wafprotocol.DispositionHeader, wafprotocol.DispositionDeny)
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
}
|
||||
}))
|
||||
@@ -690,8 +691,9 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
backend := httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get(waf.ClientHeader) != "" {
|
||||
w.Header().Set(waf.DispositionHeader, waf.DispositionAllow)
|
||||
if r.Header.Get(wafprotocol.ClientHeader) != "" {
|
||||
w.Header().
|
||||
Set(wafprotocol.DispositionHeader, wafprotocol.DispositionAllow)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -785,10 +787,7 @@ func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
|
||||
func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const (
|
||||
cutOffAfter = 5 * time.Second
|
||||
upstreamSetting = "SWW_RPROXY_UPSTREAM_URL"
|
||||
)
|
||||
const cutOffAfter = 5 * time.Second
|
||||
|
||||
asked := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
@@ -802,7 +801,8 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
|
||||
|
||||
backend := httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set(waf.DispositionHeader, waf.DispositionAllow)
|
||||
w.Header().
|
||||
Set(wafprotocol.DispositionHeader, wafprotocol.DispositionAllow)
|
||||
}))
|
||||
t.Cleanup(backend.Close)
|
||||
|
||||
@@ -810,9 +810,9 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
|
||||
defer stop()
|
||||
|
||||
env := map[string]string{
|
||||
rproxyListenAddr: localhost + ":0",
|
||||
upstreamSetting: app.URL,
|
||||
wafBackend: backend.URL,
|
||||
rproxyListenAddr: localhost + ":0",
|
||||
rproxyUpstreamURL: app.URL,
|
||||
wafBackend: backend.URL,
|
||||
}
|
||||
out := &output{}
|
||||
exited := make(chan int, 1)
|
||||
@@ -935,7 +935,7 @@ func TestMainRunsRProxy(t *testing.T) {
|
||||
addr, stop := startRProxyProcess(t, wafBackend+"=http://127.0.0.1:9")
|
||||
|
||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet,
|
||||
"http://"+addr+waf.HealthPath, http.NoBody)
|
||||
"http://"+addr+wafprotocol.HealthPath, http.NoBody)
|
||||
if err != nil {
|
||||
t.Fatalf("new request: %v", err)
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
|
||||
"sneak.berlin/go/smallwebwaf/internal/sww"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -1813,7 +1813,7 @@ func statusFrom(t *testing.T, url, from string) int {
|
||||
}
|
||||
|
||||
req.Header.Set("X-Forwarded-For", from)
|
||||
req.Header.Set(waf.ClientHeader, from)
|
||||
req.Header.Set(wafprotocol.ClientHeader, from)
|
||||
req.Header.Set("X-Request-ID", rand.Text())
|
||||
|
||||
transport := &http.Transport{}
|
||||
|
||||
+36
-12
@@ -118,7 +118,8 @@ func RunWAF(ctx context.Context, params Params) int {
|
||||
"settings", cfg)
|
||||
|
||||
// The last write follows the stop of every hook, and so the end of every
|
||||
// loop that changes the state or raises an alert. A hook of the state
|
||||
// loop that changes the state or raises an alert, and of every ask of
|
||||
// GeoJS, the DNSBL zones, AbuseIPDB and routewatch. A hook of the state
|
||||
// files would stop before the loops of the parts built ahead of them.
|
||||
return run(ctx, app, failed, processLog, func() error {
|
||||
err := files.WriteAll()
|
||||
@@ -297,11 +298,13 @@ func newLists(
|
||||
return lists
|
||||
}
|
||||
|
||||
// newDNSBL returns what asks the DNSBL zones, none asked yet.
|
||||
// newDNSBL returns what asks the DNSBL zones, none asked yet. At the stop it
|
||||
// ends the queries under way, as reputation.DNSBL.StopAsking says.
|
||||
func newDNSBL(
|
||||
cfg *config.WAF, processLog *slog.Logger, alertQueue *alerts.Queue,
|
||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||
alertQueue *alerts.Queue,
|
||||
) *reputation.DNSBL {
|
||||
return reputation.NewDNSBL(reputation.DNSBLParams{
|
||||
dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{
|
||||
Zones: cfg.DNSBLZones,
|
||||
Resolver: cfg.DNSBLResolver,
|
||||
CacheTTL: cfg.IPReputationCacheTTL,
|
||||
@@ -310,12 +313,17 @@ func newDNSBL(
|
||||
ProcessLog: processLog,
|
||||
Alerts: alertQueue,
|
||||
})
|
||||
|
||||
lc.Append(fx.StopHook(dnsbl.StopAsking))
|
||||
|
||||
return dnsbl
|
||||
}
|
||||
|
||||
// newAbuseIPDB returns what checks clients with AbuseIPDB while
|
||||
// SWW_WAF_ABUSEIPDB_KEY is set, none checked yet. Until waf stops, it
|
||||
// fetches AbuseIPDB's blacklist as it is due, and bans the clients on it in
|
||||
// ledger, as waf.BanAbuseIPDBBlacklist says.
|
||||
// ledger, as waf.BanAbuseIPDBBlacklist says. At the stop it ends the checks
|
||||
// under way, as reputation.AbuseIPDB.StopChecking says.
|
||||
func newAbuseIPDB(
|
||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||
alertQueue *alerts.Queue, ledger *bans.Ledger,
|
||||
@@ -341,24 +349,30 @@ func newAbuseIPDB(
|
||||
})
|
||||
|
||||
lc.Append(background(abuseIPDB.RunBlacklist))
|
||||
lc.Append(fx.StopHook(abuseIPDB.StopChecking))
|
||||
|
||||
return abuseIPDB
|
||||
}
|
||||
|
||||
// newRouteWatch returns what asks routewatch about clients, or nil while
|
||||
// SWW_WAF_ROUTEWATCH_URL is unset.
|
||||
// SWW_WAF_ROUTEWATCH_URL is unset. At the stop it ends the queries under
|
||||
// way, as routewatch.Client.StopAsking says.
|
||||
func newRouteWatch(
|
||||
cfg *config.WAF, processLog *slog.Logger,
|
||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||
) *routewatch.Client {
|
||||
if cfg.RouteWatchURL == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
return routewatch.New(routewatch.Params{
|
||||
routeWatch := routewatch.New(routewatch.Params{
|
||||
URL: cfg.RouteWatchURL,
|
||||
Now: utcNow,
|
||||
ProcessLog: processLog,
|
||||
})
|
||||
|
||||
lc.Append(fx.StopHook(routeWatch.StopAsking))
|
||||
|
||||
return routeWatch
|
||||
}
|
||||
|
||||
// newMetrics returns the metrics, with those read from the other parts as
|
||||
@@ -396,12 +410,18 @@ func newMetrics(
|
||||
|
||||
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
|
||||
// answer goes to the limiter and the ledger, as waf.AddLookup says, and each
|
||||
// failure is handled as waf.GeoJSFailed says.
|
||||
// failure is handled as waf.GeoJSFailed says. At the stop it ends the request
|
||||
// under way, as geojs.GeoJS.StopAsking says.
|
||||
func newGeoJS(
|
||||
cfg *config.WAF, m *metrics.Metrics, limiter *ratelimit.Limiter,
|
||||
ledger *bans.Ledger, alertQueue *alerts.Queue, processLog *slog.Logger,
|
||||
lc fx.Lifecycle,
|
||||
cfg *config.WAF,
|
||||
m *metrics.Metrics,
|
||||
limiter *ratelimit.Limiter,
|
||||
ledger *bans.Ledger,
|
||||
alertQueue *alerts.Queue,
|
||||
processLog *slog.Logger,
|
||||
) *geojs.GeoJS {
|
||||
return geojs.New(geojs.Params{
|
||||
geoJS := geojs.New(geojs.Params{
|
||||
URL: geojs.URL,
|
||||
Answered: func(answer geojs.Answer) {
|
||||
waf.AddLookup(cfg, limiter, ledger, answer)
|
||||
@@ -413,6 +433,10 @@ func newGeoJS(
|
||||
},
|
||||
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
||||
})
|
||||
|
||||
lc.Append(fx.StopHook(geoJS.StopAsking))
|
||||
|
||||
return geoJS
|
||||
}
|
||||
|
||||
// newWAFServer returns the server waf runs, with its parts and the time
|
||||
|
||||
@@ -20,7 +20,7 @@ import (
|
||||
_ "time/tzdata"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/sww"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||
)
|
||||
|
||||
// commandForMain, while set in the environment, has this test binary run Main
|
||||
@@ -106,7 +106,7 @@ func TestMainRunsWAF(t *testing.T) {
|
||||
|
||||
starting := out.lineBeforeExit(t, "starting", exited)
|
||||
addr, _ := starting["address"].(string)
|
||||
ask(t, addr, "/", http.StatusOK, waf.DispositionAllow)
|
||||
ask(t, addr, "/", http.StatusOK, wafprotocol.DispositionAllow)
|
||||
|
||||
// The files are written as sww stops.
|
||||
err = cmd.Process.Signal(syscall.SIGTERM)
|
||||
@@ -217,7 +217,7 @@ func ask(t *testing.T, addr, path string, status int, disposition string) {
|
||||
t.Fatalf("new request: %v", err)
|
||||
}
|
||||
|
||||
req.Header.Set(waf.ClientHeader, placed)
|
||||
req.Header.Set(wafprotocol.ClientHeader, placed)
|
||||
req.Header.Set("X-Request-ID", rand.Text())
|
||||
|
||||
transport := &http.Transport{}
|
||||
@@ -232,11 +232,11 @@ func ask(t *testing.T, addr, path string, status int, disposition string) {
|
||||
_ = res.Body.Close()
|
||||
|
||||
if res.StatusCode != status ||
|
||||
res.Header.Get(waf.DispositionHeader) != disposition {
|
||||
res.Header.Get(wafprotocol.DispositionHeader) != disposition {
|
||||
t.Errorf(
|
||||
"answered %d with disposition %q, want %d with %q",
|
||||
res.StatusCode,
|
||||
res.Header.Get(waf.DispositionHeader),
|
||||
res.Header.Get(wafprotocol.DispositionHeader),
|
||||
status,
|
||||
disposition,
|
||||
)
|
||||
@@ -287,12 +287,18 @@ func TestWAFKeepsItsBansAcrossARestartAndNeedsNoApp(t *testing.T) {
|
||||
}
|
||||
|
||||
runWAF(t, env, func(addr string) {
|
||||
ask(t, addr, "/", http.StatusOK, waf.DispositionAllow)
|
||||
ask(t, addr, "/", http.StatusOK, wafprotocol.DispositionAllow)
|
||||
// The default rule file bans a probe for this.
|
||||
ask(t, addr, "/.env", http.StatusForbidden, waf.DispositionDeny)
|
||||
ask(
|
||||
t,
|
||||
addr,
|
||||
"/.env",
|
||||
http.StatusForbidden,
|
||||
wafprotocol.DispositionDeny,
|
||||
)
|
||||
})
|
||||
|
||||
runWAF(t, env, func(addr string) {
|
||||
ask(t, addr, "/", http.StatusForbidden, waf.DispositionDeny)
|
||||
ask(t, addr, "/", http.StatusForbidden, wafprotocol.DispositionDeny)
|
||||
})
|
||||
}
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -84,7 +85,7 @@ func TestSixteenThousandCookiesAreRefusedBeforeTheCoreRuleSetReadsThem(
|
||||
|
||||
got := do(t, req)
|
||||
|
||||
wantDisposition(t, got, tooLarge, waf.DispositionDeny)
|
||||
wantDisposition(t, got, tooLarge, wafprotocol.DispositionDeny)
|
||||
|
||||
line := s.out.requestLine(t)
|
||||
wantLine(t, line, tooLarge, requestlog.ActionRequestHeaderTooLarge)
|
||||
@@ -164,9 +165,14 @@ func TestARequestLineOverTheLimitIsRefusedWith414BeforeAnyOtherCheck(
|
||||
line := s.out.requestLine(t)
|
||||
|
||||
if tc.status == http.StatusOK {
|
||||
wantDisposition(t, got, tc.status, waf.DispositionAllow)
|
||||
wantDisposition(
|
||||
t,
|
||||
got,
|
||||
tc.status,
|
||||
wafprotocol.DispositionAllow,
|
||||
)
|
||||
} else {
|
||||
wantDisposition(t, got, tc.status, waf.DispositionDeny)
|
||||
wantDisposition(t, got, tc.status, wafprotocol.DispositionDeny)
|
||||
}
|
||||
|
||||
wantLine(t, line, tc.status, tc.action)
|
||||
@@ -219,9 +225,14 @@ func TestAHeaderLineOverTheLimitIsRefusedWith431(t *testing.T) {
|
||||
got := do(t, req)
|
||||
|
||||
if tc.refused {
|
||||
wantDisposition(t, got, tooLarge, waf.DispositionDeny)
|
||||
wantDisposition(
|
||||
t,
|
||||
got,
|
||||
tooLarge,
|
||||
wafprotocol.DispositionDeny,
|
||||
)
|
||||
} else {
|
||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -244,10 +255,7 @@ func TestRequestLineRefusalNamesItsHeuristicLengthAndLimit(t *testing.T) {
|
||||
func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
repeated := make([]string, headerFieldLimit)
|
||||
for i := range repeated {
|
||||
repeated[i] = "1"
|
||||
}
|
||||
repeated := slices.Repeat([]string{"1"}, headerFieldLimit)
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
@@ -294,7 +302,12 @@ func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) {
|
||||
line := s.out.requestLine(t)
|
||||
|
||||
if tc.refused {
|
||||
wantDisposition(t, got, tooLarge, waf.DispositionDeny)
|
||||
wantDisposition(
|
||||
t,
|
||||
got,
|
||||
tooLarge,
|
||||
wafprotocol.DispositionDeny,
|
||||
)
|
||||
wantLine(
|
||||
t,
|
||||
line,
|
||||
@@ -305,7 +318,12 @@ func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||
wantDisposition(
|
||||
t,
|
||||
got,
|
||||
http.StatusOK,
|
||||
wafprotocol.DispositionAllow,
|
||||
)
|
||||
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
|
||||
|
||||
if line.DurationCoreRuleSetMS == nil {
|
||||
@@ -332,7 +350,12 @@ func TestHeaderLimitsHoldWhateverTheCoreRuleSetSettingsSay(t *testing.T) {
|
||||
req := s.question(siteA, client, "/")
|
||||
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
||||
|
||||
wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny)
|
||||
wantDisposition(
|
||||
t,
|
||||
do(t, req),
|
||||
tooLarge,
|
||||
wafprotocol.DispositionDeny,
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -348,7 +371,12 @@ func TestClientInTheAllowNetsSkipsTheHeaderLimits(t *testing.T) {
|
||||
req := s.question(siteA, client, "/")
|
||||
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
||||
|
||||
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow)
|
||||
wantDisposition(
|
||||
t,
|
||||
do(t, req),
|
||||
http.StatusOK,
|
||||
wafprotocol.DispositionAllow,
|
||||
)
|
||||
}
|
||||
|
||||
func TestObserveModeAllowsARequestOverTheHeaderLimitsUnread(t *testing.T) {
|
||||
@@ -364,7 +392,7 @@ func TestObserveModeAllowsARequestOverTheHeaderLimitsUnread(t *testing.T) {
|
||||
|
||||
got := do(t, req)
|
||||
|
||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
||||
|
||||
line := s.out.requestLine(t)
|
||||
if line.WouldAction != requestlog.ActionRequestHeaderTooLarge ||
|
||||
@@ -386,14 +414,14 @@ func TestTheRateLimitsDoNotCountRequestsOverTheHeaderLimits(t *testing.T) {
|
||||
for range 2 {
|
||||
req := s.question(siteA, client, "/")
|
||||
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
||||
wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny)
|
||||
wantDisposition(t, do(t, req), tooLarge, wafprotocol.DispositionDeny)
|
||||
}
|
||||
|
||||
wantDisposition(
|
||||
t,
|
||||
s.ask(client, "/"),
|
||||
http.StatusOK,
|
||||
waf.DispositionAllow,
|
||||
wafprotocol.DispositionAllow,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -466,7 +494,12 @@ func TestTheHeaderLimitsComeBeforeEveryCheckThatRefuses(t *testing.T) {
|
||||
req := s.question(siteA, client, tc.path)
|
||||
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
||||
|
||||
wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny)
|
||||
wantDisposition(
|
||||
t,
|
||||
do(t, req),
|
||||
tooLarge,
|
||||
wafprotocol.DispositionDeny,
|
||||
)
|
||||
wantLine(
|
||||
t,
|
||||
s.out.requestLine(t),
|
||||
@@ -503,11 +536,16 @@ func TestARequestOverTheHeaderLimitsRunsNoOtherCheck(t *testing.T) {
|
||||
|
||||
over := s.question(siteA, client, "/")
|
||||
over.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
||||
wantDisposition(t, do(t, over), tooLarge, waf.DispositionDeny)
|
||||
wantDisposition(t, do(t, over), tooLarge, wafprotocol.DispositionDeny)
|
||||
|
||||
within := s.question(siteA, client, "/")
|
||||
within.Header.Set(cookieHeader, cookies(cookieLimit))
|
||||
wantDisposition(t, do(t, within), http.StatusOK, waf.DispositionAllow)
|
||||
wantDisposition(
|
||||
t,
|
||||
do(t, within),
|
||||
http.StatusOK,
|
||||
wafprotocol.DispositionAllow,
|
||||
)
|
||||
|
||||
lines := s.out.requestLines(t, 2)
|
||||
|
||||
@@ -564,7 +602,7 @@ func TestNotWaitingForGeoJSNeverLowersAClientsLimit(t *testing.T) {
|
||||
t,
|
||||
s.ask(fromDE, "/"),
|
||||
http.StatusForbidden,
|
||||
waf.DispositionDeny,
|
||||
wafprotocol.DispositionDeny,
|
||||
)
|
||||
|
||||
line := s.out.requestLine(t)
|
||||
@@ -601,7 +639,7 @@ func TestAClientGeoJSPlacedInNoCountryCountsAsUnknownWithoutWaiting(
|
||||
t,
|
||||
s.ask(client, "/"),
|
||||
http.StatusForbidden,
|
||||
waf.DispositionDeny,
|
||||
wafprotocol.DispositionDeny,
|
||||
)
|
||||
|
||||
line := s.out.requestLine(t)
|
||||
@@ -630,7 +668,7 @@ func TestObserveModeAsksGeoJSNothingForARequestOverTheHeaderLimits(
|
||||
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
||||
|
||||
got := do(t, req)
|
||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
||||
|
||||
if len(asked()) != 0 || got.header.Get("X-Client-Asn") != "" {
|
||||
t.Errorf(
|
||||
@@ -672,7 +710,7 @@ func askDirectly(
|
||||
http.NoBody,
|
||||
)
|
||||
req.RemoteAddr = net.JoinHostPort(rproxyPeer, "1234")
|
||||
req.Header.Set(waf.ClientHeader, client)
|
||||
req.Header.Set(wafprotocol.ClientHeader, client)
|
||||
req.Header.Set(requestIDHeader, "the-id")
|
||||
change(req)
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||
)
|
||||
|
||||
// The biased thresholds.
|
||||
@@ -589,7 +590,7 @@ func serveFromDE(
|
||||
t.Helper()
|
||||
|
||||
req := httptest.NewRequestWithContext(t.Context(), method, "/", body)
|
||||
req.Header.Set(waf.ClientHeader, fromDE)
|
||||
req.Header.Set(wafprotocol.ClientHeader, fromDE)
|
||||
req.Header.Set(requestIDHeader, "the-id")
|
||||
|
||||
server.Handler.ServeHTTP(httptest.NewRecorder(), req)
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"slices"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||
)
|
||||
|
||||
// peerAddress is the address of the request's TCP peer.
|
||||
@@ -19,14 +20,11 @@ func peerAddress(r *http.Request) netip.Addr {
|
||||
return addrPort.Addr().Unmap()
|
||||
}
|
||||
|
||||
// requestIDHeader carries the request's id, from rproxy.
|
||||
const requestIDHeader = "X-Request-ID"
|
||||
|
||||
// requestID is the request's id: the one a question gives, or for any other
|
||||
// request a new random one.
|
||||
func requestID(r *http.Request, isQuestion bool) string {
|
||||
if isQuestion {
|
||||
return r.Header.Get(requestIDHeader)
|
||||
return r.Header.Get(wafprotocol.RequestIDHeader)
|
||||
}
|
||||
|
||||
return rand.Text()
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||
)
|
||||
|
||||
// The Core Rule Set's settings the tests set, besides SWW_WAF_CORE_RULE_SET_MODE, and
|
||||
@@ -395,9 +395,9 @@ func TestGiteasFixesApplyOnlyToAHostnameThatServesGitea(t *testing.T) {
|
||||
})
|
||||
|
||||
wantDisposition(t, do(t, s.question(siteA, client, fileName)),
|
||||
http.StatusOK, waf.DispositionAllow)
|
||||
http.StatusOK, wafprotocol.DispositionAllow)
|
||||
wantDisposition(t, do(t, s.question(siteB, client, fileName)),
|
||||
http.StatusForbidden, waf.DispositionDeny)
|
||||
http.StatusForbidden, wafprotocol.DispositionDeny)
|
||||
|
||||
lines := s.out.requestLines(t, 2)
|
||||
wantLine(t, lines[0], http.StatusOK, requestlog.ActionForward)
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||
)
|
||||
|
||||
func TestHealthEndpointCountsTowardNoLimit(t *testing.T) {
|
||||
@@ -25,8 +25,13 @@ func TestHealthEndpointCountsTowardNoLimit(t *testing.T) {
|
||||
)
|
||||
|
||||
for range healthChecks {
|
||||
got := s.ownEndpointRequest("", http.MethodGet, waf.HealthPath, "",
|
||||
http.StatusOK)
|
||||
got := s.ownEndpointRequest(
|
||||
"",
|
||||
http.MethodGet,
|
||||
wafprotocol.HealthPath,
|
||||
"",
|
||||
http.StatusOK,
|
||||
)
|
||||
|
||||
if string(got.body) != "ok\n" ||
|
||||
got.header.Get("Content-Type") != contentType {
|
||||
|
||||
@@ -104,12 +104,6 @@ func onReport() []heuristic {
|
||||
}
|
||||
}
|
||||
|
||||
// runAfterAnswer runs the heuristics of afterAnswer on the request, at the time
|
||||
// it is called.
|
||||
func (rq *request) runAfterAnswer(ctx context.Context) {
|
||||
rq.runBanningHeuristics(ctx, afterAnswer())
|
||||
}
|
||||
|
||||
// runHeuristics runs list, one of those checkClient runs, on the request at now,
|
||||
// and returns the first verdict, or nil when none gives one. The log line's
|
||||
// heuristic and heuristic_figures name the heuristic that gave it, and give its
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||
)
|
||||
|
||||
func TestHistoryKeepsEachRequestOfTheClient(t *testing.T) {
|
||||
@@ -118,7 +119,7 @@ func TestOwnEndpointsAreNotInTheHistory(t *testing.T) {
|
||||
s.ownEndpointRequest(
|
||||
"",
|
||||
http.MethodGet,
|
||||
waf.HealthPath,
|
||||
wafprotocol.HealthPath,
|
||||
"",
|
||||
http.StatusOK,
|
||||
)
|
||||
|
||||
@@ -3,7 +3,6 @@ package waf
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -16,15 +15,6 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
)
|
||||
|
||||
// The headers in which an answer of allow gives rproxy the client's AS number
|
||||
// and country, for the app, while SWW_WAF_ADD_LOOKUP_HEADERS is true. Go writes
|
||||
// every header name in this form, as it sends it and as it receives it, so
|
||||
// X-Client-ASN arrives as X-Client-Asn; header names are not case-sensitive.
|
||||
const (
|
||||
asnHeader = "X-Client-Asn"
|
||||
countryHeader = "X-Client-Country"
|
||||
)
|
||||
|
||||
// lookupResult is what the lookup gave for a request: whether its client was
|
||||
// looked up, the answer, and whether the request skipped the wait for GeoJS.
|
||||
type lookupResult struct {
|
||||
@@ -167,18 +157,6 @@ func GeoJSFailed(
|
||||
m.GeoJSFailures.Inc()
|
||||
}
|
||||
|
||||
// setLookupHeaders sets the headers that give the client's AS number and
|
||||
// country, leaving out one that is unknown.
|
||||
func setLookupHeaders(header http.Header, asn, country string) {
|
||||
if asn != "" {
|
||||
header.Set(asnHeader, asn)
|
||||
}
|
||||
|
||||
if country != "" {
|
||||
header.Set(countryHeader, country)
|
||||
}
|
||||
}
|
||||
|
||||
// canBePlaced reports whether a lookup can place addr: private, loopback
|
||||
// and link-local addresses have no AS number or country.
|
||||
func canBePlaced(addr netip.Addr) bool {
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -66,7 +67,7 @@ func TestMetricsNeedTheTokenAndOtherPathsAreNotFound(t *testing.T) {
|
||||
{http.MethodPost, waf.MetricsPath, bearer, http.StatusNotFound},
|
||||
{http.MethodGet, waf.MetricsPath + "/", bearer, http.StatusNotFound},
|
||||
{http.MethodGet, "/_sww/bans", bearer, http.StatusNotFound},
|
||||
{http.MethodPost, waf.HealthPath, "", http.StatusNotFound},
|
||||
{http.MethodPost, wafprotocol.HealthPath, "", http.StatusNotFound},
|
||||
} {
|
||||
got := s.ownEndpointRequest(
|
||||
tc.authorization,
|
||||
|
||||
@@ -28,14 +28,6 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||
)
|
||||
|
||||
// ownEndpointPrefix starts the path of each of waf's own endpoints. The app
|
||||
// never sees a path under it.
|
||||
const ownEndpointPrefix = "/_sww/"
|
||||
|
||||
// HealthPath is the health endpoint of waf and of rproxy, which the
|
||||
// container's health check asks.
|
||||
const HealthPath = "/_sww/healthz"
|
||||
|
||||
// MetricsPath is where the metrics are, for a request that carries
|
||||
// SWW_WAF_METRICS_TOKEN.
|
||||
const MetricsPath = "/_sww/metrics"
|
||||
|
||||
@@ -345,6 +345,16 @@ func timing(start, end time.Time) *float64 {
|
||||
return new(requestlog.Milliseconds(end.Sub(start)))
|
||||
}
|
||||
|
||||
// banAndCount runs list, afterAnswer or onReport, whose heuristics only ban.
|
||||
// It then counts the request for the anomaly thresholds and adds it to its
|
||||
// client's history. list runs first, so that a refusal limit it finds broken
|
||||
// is in that history. ctx is the request's own context.
|
||||
func (rq *request) banAndCount(ctx context.Context, list []heuristic) {
|
||||
rq.runBanningHeuristics(ctx, list)
|
||||
rq.countAnomalies()
|
||||
rq.addToHistory()
|
||||
}
|
||||
|
||||
// addToHistory adds the request, which has ended, to its client's
|
||||
// history, and counts its offences in the metrics, and then the lookup's
|
||||
// answer about the client, as answerAtTheEnd gives it, to that history and
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -248,7 +248,7 @@ func TestTimingsAreInOrder(t *testing.T) {
|
||||
wantStatus(t, do(t, req), http.StatusForbidden)
|
||||
refused := out.endedLines(t, 2)[1]
|
||||
|
||||
wantStatus(t, get(t, server.Addr, waf.HealthPath), http.StatusOK)
|
||||
wantStatus(t, get(t, server.Addr, wafprotocol.HealthPath), http.StatusOK)
|
||||
health := out.endedLines(t, 3)[2]
|
||||
|
||||
// A request passed to the app has every timing; one refused, none of
|
||||
|
||||
+74
-253
@@ -1,75 +1,21 @@
|
||||
package waf
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||
)
|
||||
|
||||
// The headers of rproxy's question and of waf's answer, as
|
||||
// "waf" in SPEC.md gives them.
|
||||
const (
|
||||
// ClientHeader marks a request as rproxy's question, and gives the
|
||||
// visitor's address.
|
||||
ClientHeader = "X-Sww-Client"
|
||||
// PeerHeader gives the address of rproxy's own TCP peer.
|
||||
PeerHeader = "X-Sww-Peer"
|
||||
// ContentLengthHeader gives the body length the visitor announced.
|
||||
ContentLengthHeader = "X-Sww-Content-Length"
|
||||
// DispositionHeader gives waf's answer.
|
||||
DispositionHeader = "X-Sww-Disposition"
|
||||
)
|
||||
|
||||
// The values of DispositionHeader.
|
||||
const (
|
||||
// DispositionAllow tells rproxy to pass the request to the app.
|
||||
DispositionAllow = "allow"
|
||||
// DispositionDeny tells rproxy to give the visitor the answer's status,
|
||||
// headers and body.
|
||||
DispositionDeny = "deny"
|
||||
// DispositionClose tells rproxy to close the visitor's connection
|
||||
// without an answer, which SWW_WAF_BAN_RESPONSE asks for when it is close.
|
||||
// waf closing the question's connection instead would read as
|
||||
// waf being down, and rproxy would then let the request through.
|
||||
DispositionClose = "close"
|
||||
)
|
||||
|
||||
// ReportPath is where rproxy sends the report of each request waf
|
||||
// answered.
|
||||
const ReportPath = "/_sww/report"
|
||||
|
||||
// QuestionBodyMaxBytes is the most of a visitor's body a question carries.
|
||||
const QuestionBodyMaxBytes = 64 << 10
|
||||
|
||||
const (
|
||||
// maxAwaitingReport is the most answered requests held until their
|
||||
// reports come. Past it, the request answered longest ago is dropped.
|
||||
maxAwaitingReport = 10000
|
||||
// maxRequestIDBytes is the longest id a question may give.
|
||||
maxRequestIDBytes = 128
|
||||
// reportBodyMaxBytes is the most of a report's body that is read. It is
|
||||
// far above any report's.
|
||||
reportBodyMaxBytes = 1 << 20
|
||||
// maxHTTPStatus is the highest status a report may give.
|
||||
maxHTTPStatus = 599
|
||||
// maxReportedBytes is the most bytes a report may give for each way, 1
|
||||
// PiB, so that adding them up cannot overflow.
|
||||
maxReportedBytes = 1 << 50
|
||||
// maxReportedMilliseconds is the longest duration a report may give,
|
||||
// about 31 years, so that it fits a time.Duration.
|
||||
maxReportedMilliseconds = 1e12
|
||||
// wafTimeout bounds how long a request to waf takes to
|
||||
// arrive, and its answer to leave, and wafIdleTimeout how long a
|
||||
// kept-open connection waits for its next request.
|
||||
@@ -77,26 +23,6 @@ const (
|
||||
wafIdleTimeout = 120 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
errReportFieldMissing = errors.New("a field is missing")
|
||||
errReportValue = errors.New("a value is out of range")
|
||||
)
|
||||
|
||||
// Report is the JSON object rproxy posts to ReportPath once the visitor's
|
||||
// answer has ended, its fields named as the request log's of the same
|
||||
// meaning. RequestID is the question's id. Status is what the visitor got, 0
|
||||
// if nothing. The durations are in milliseconds.
|
||||
//
|
||||
//nolint:tagliatelle // docs/request-log.md names the fields in snake_case
|
||||
type Report struct {
|
||||
RequestID string `json:"request_id"`
|
||||
Status int `json:"status"`
|
||||
RequestBytes int64 `json:"request_bytes"`
|
||||
ResponseBytes int64 `json:"response_bytes"`
|
||||
DurationTotalMS float64 `json:"duration_total_ms"`
|
||||
DurationUpstreamTotalMS float64 `json:"duration_upstream_total_ms"`
|
||||
}
|
||||
|
||||
// New returns the server `sww waf` runs. It answers
|
||||
// rproxy's questions, takes rproxy's reports and answers its own endpoints. It
|
||||
// passes nothing to an app. rproxy applies the size and time limits to the
|
||||
@@ -132,10 +58,10 @@ func New(params Params) *Server {
|
||||
}
|
||||
|
||||
// ServeHTTP handles one request to waf: a request carrying
|
||||
// ClientHeader is rproxy's question, and any other is for waf
|
||||
// wafprotocol.ClientHeader is rproxy's question, and any other is for waf
|
||||
// itself.
|
||||
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if len(r.Header.Values(ClientHeader)) > 0 {
|
||||
if len(r.Header.Values(wafprotocol.ClientHeader)) > 0 {
|
||||
h.answerQuestion(w, r)
|
||||
|
||||
return
|
||||
@@ -162,104 +88,76 @@ func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
|
||||
// carries, and answers rproxy what to do with it. A denied request's counting
|
||||
// is finished here, and an allowed one's by its report.
|
||||
func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
|
||||
client, body, ok := h.readQuestion(w, r)
|
||||
question, ok := h.readQuestion(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
// One that cannot be read leaves the log line's peer_ip invalid.
|
||||
peer, _ := netip.ParseAddr(r.Header.Get(PeerHeader))
|
||||
|
||||
rq := h.startRequest(
|
||||
w,
|
||||
asTheVisitorSentIt(r, body),
|
||||
client.Unmap(),
|
||||
peer.Unmap(),
|
||||
question.Visitor,
|
||||
question.Client,
|
||||
question.Peer,
|
||||
true,
|
||||
)
|
||||
// The last deferred runs first, so that what is held has the finished log
|
||||
// line. Go's server sends the answer only once this function has
|
||||
// returned, so the report cannot come before it is held.
|
||||
defer rq.holdForReport()
|
||||
defer rq.finish()
|
||||
// These run even when a check panics. finish runs first, so that what is
|
||||
// held has the finished log line. Go's server sends the answer only once
|
||||
// this function has returned, so the report cannot come before it is held.
|
||||
defer func() {
|
||||
rq.finish()
|
||||
rq.holdForReport()
|
||||
}()
|
||||
|
||||
rq.site = r.Host
|
||||
rq.line.WAFLine = requestlog.WAFLineDecision
|
||||
refused := rq.check(r.Context())
|
||||
rq.checked = time.Now()
|
||||
underOwnPrefix := strings.HasPrefix(r.URL.Path, ownEndpointPrefix)
|
||||
underOwnPrefix := strings.HasPrefix(
|
||||
r.URL.Path,
|
||||
wafprotocol.OwnEndpointPrefix,
|
||||
)
|
||||
|
||||
if refused == nil && !underOwnPrefix {
|
||||
// In observe mode, what enforce mode would have refused counts here.
|
||||
defer rq.runAfterAnswer(r.Context())
|
||||
|
||||
rq.allow()
|
||||
// In observe mode, what enforce mode would have refused counts here.
|
||||
rq.runBanningHeuristics(r.Context(), afterAnswer())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// The last deferred runs first: runAfterAnswer before addToHistory, so that
|
||||
// a broken refusal limit is in the client's history.
|
||||
defer rq.addToHistory()
|
||||
defer rq.countAnomalies()
|
||||
defer rq.runAfterAnswer(r.Context())
|
||||
|
||||
if refused != nil {
|
||||
rq.deny(*refused)
|
||||
|
||||
return
|
||||
} else {
|
||||
// The app never sees this prefix. waf's own endpoints are at its
|
||||
// own address, so this answers as a path with no endpoint does.
|
||||
rq.line.Action = requestlog.ActionOwnEndpoint
|
||||
wafprotocol.SetAnswerHeaders(
|
||||
rq.out.Header(),
|
||||
wafprotocol.Answer{Disposition: wafprotocol.DispositionDeny},
|
||||
)
|
||||
http.Error(
|
||||
rq.out,
|
||||
http.StatusText(http.StatusNotFound),
|
||||
http.StatusNotFound,
|
||||
)
|
||||
}
|
||||
|
||||
// The app never sees this prefix. waf's own endpoints are at its
|
||||
// own address, so this answers as a path with no endpoint does.
|
||||
rq.line.Action = requestlog.ActionOwnEndpoint
|
||||
rq.out.Header().Set(DispositionHeader, DispositionDeny)
|
||||
http.Error(
|
||||
rq.out,
|
||||
http.StatusText(http.StatusNotFound),
|
||||
http.StatusNotFound,
|
||||
)
|
||||
rq.banAndCount(r.Context(), afterAnswer())
|
||||
}
|
||||
|
||||
// readQuestion returns the visitor's address and the body a question carries,
|
||||
// and whether waf takes the question. It answers a question it does not
|
||||
// take with 400 and no disposition. rproxy passes that 400 to the visitor,
|
||||
// and SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION does not apply. The id of
|
||||
// a question waf takes is held from here on, without a decision yet, so
|
||||
// that another question with that id is refused while this one is being
|
||||
// handled.
|
||||
// readQuestion reads a question, and reports whether waf takes it. It answers
|
||||
// a question it does not take with 400 and no disposition. rproxy passes that
|
||||
// 400 to the visitor, and SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION does
|
||||
// not apply. The id of a question waf takes is held from here on, without a
|
||||
// decision yet, so that another question with that id is refused while this
|
||||
// one is being handled.
|
||||
//
|
||||
// The body is read before any check, since Go's server does not send an answer
|
||||
// until it has read the body. A question whose body stops arriving gets no
|
||||
// answer, and is not looked at.
|
||||
// A question whose body stops arriving gets no answer, and is not looked at.
|
||||
func (h *handler) readQuestion(
|
||||
w http.ResponseWriter, r *http.Request,
|
||||
) (netip.Addr, []byte, bool) {
|
||||
client, err := netip.ParseAddr(r.Header.Get(ClientHeader))
|
||||
if err != nil || client.Zone() != "" {
|
||||
h.refuseQuestion(w, ClientHeader+" is not an address without a zone")
|
||||
|
||||
return netip.Addr{}, nil, false
|
||||
}
|
||||
|
||||
id := r.Header.Get(requestIDHeader)
|
||||
if id == "" || len(id) > maxRequestIDBytes || !utf8.ValidString(id) {
|
||||
h.refuseQuestion(w, requestIDHeader+" is not a valid id")
|
||||
|
||||
return netip.Addr{}, nil, false
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(
|
||||
http.MaxBytesReader(w, r.Body, QuestionBodyMaxBytes),
|
||||
)
|
||||
|
||||
if _, tooLarge := errors.AsType[*http.MaxBytesError](err); tooLarge {
|
||||
h.refuseQuestion(w, "the body is over 64 KiB")
|
||||
|
||||
return netip.Addr{}, nil, false
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
) (wafprotocol.Question, bool) {
|
||||
question, err := wafprotocol.ReadQuestion(w, r)
|
||||
if errors.Is(err, wafprotocol.ErrBodyStoppedArriving) {
|
||||
h.processLog.Warn(
|
||||
"a question's body stopped arriving",
|
||||
"error",
|
||||
@@ -270,17 +168,23 @@ func (h *handler) readQuestion(
|
||||
panic(http.ErrAbortHandler)
|
||||
}
|
||||
|
||||
held, _ := h.awaitingReport.ContainsOrAdd(id, nil)
|
||||
if err != nil {
|
||||
h.refuseQuestion(w, err.Error())
|
||||
|
||||
return question, false
|
||||
}
|
||||
|
||||
held, _ := h.awaitingReport.ContainsOrAdd(question.RequestID, nil)
|
||||
if held {
|
||||
h.refuseQuestion(
|
||||
w,
|
||||
requestIDHeader+" is the id of a request waf holds",
|
||||
wafprotocol.RequestIDHeader+" is the id of a request waf holds",
|
||||
)
|
||||
|
||||
return netip.Addr{}, nil, false
|
||||
return question, false
|
||||
}
|
||||
|
||||
return client, body, true
|
||||
return question, true
|
||||
}
|
||||
|
||||
// refuseQuestion answers a question waf does not take, for reason.
|
||||
@@ -289,48 +193,18 @@ func (h *handler) refuseQuestion(w http.ResponseWriter, reason string) {
|
||||
http.Error(w, reason, http.StatusBadRequest)
|
||||
}
|
||||
|
||||
// asTheVisitorSentIt returns a copy of r, a question, as the checks and the
|
||||
// log line are to see it: without the headers rproxy adds for waf,
|
||||
// with body, which has been read of r, and with the length the visitor
|
||||
// announced, as -1 when it announced none. The question's own length covers
|
||||
// only the part of the body it carries.
|
||||
func asTheVisitorSentIt(r *http.Request, body []byte) *http.Request {
|
||||
sent := r.Clone(r.Context())
|
||||
announced := sent.Header.Get(ContentLengthHeader)
|
||||
|
||||
sent.Header.Del(ClientHeader)
|
||||
sent.Header.Del(PeerHeader)
|
||||
sent.Header.Del(ContentLengthHeader)
|
||||
sent.Header.Del("Content-Length")
|
||||
|
||||
sent.ContentLength = -1
|
||||
|
||||
length, err := strconv.ParseInt(announced, 10, 64)
|
||||
if err == nil && length >= 0 {
|
||||
sent.ContentLength = length
|
||||
sent.Header.Set("Content-Length", announced)
|
||||
}
|
||||
|
||||
sent.Body = http.NoBody
|
||||
if len(body) > 0 {
|
||||
sent.Body = io.NopCloser(bytes.NewReader(body))
|
||||
}
|
||||
|
||||
return sent
|
||||
}
|
||||
|
||||
// allow answers a question with DispositionAllow and, while
|
||||
// allow answers a question with wafprotocol.DispositionAllow and, while
|
||||
// SWW_WAF_ADD_LOOKUP_HEADERS is true, the client's AS number and country.
|
||||
func (rq *request) allow() {
|
||||
rq.allowed = true
|
||||
|
||||
header := rq.out.Header()
|
||||
header.Set(DispositionHeader, DispositionAllow)
|
||||
|
||||
answer := wafprotocol.Answer{Disposition: wafprotocol.DispositionAllow}
|
||||
if rq.h.config.AddLookupHeaders {
|
||||
setLookupHeaders(header, rq.line.ASN, rq.line.Country)
|
||||
answer.ASN = rq.line.ASN
|
||||
answer.Country = rq.line.Country
|
||||
}
|
||||
|
||||
wafprotocol.SetAnswerHeaders(rq.out.Header(), answer)
|
||||
rq.out.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
@@ -348,13 +222,19 @@ func (rq *request) deny(r refusal) {
|
||||
if r.status == 0 {
|
||||
// SWW_WAF_BAN_RESPONSE is close. Written past rq.out, since the log line's
|
||||
// status stays 0: the visitor is sent nothing.
|
||||
rq.out.Header().Set(DispositionHeader, DispositionClose)
|
||||
wafprotocol.SetAnswerHeaders(
|
||||
rq.out.Header(),
|
||||
wafprotocol.Answer{Disposition: wafprotocol.DispositionClose},
|
||||
)
|
||||
rq.out.ResponseWriter.WriteHeader(http.StatusOK)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
rq.out.Header().Set(DispositionHeader, DispositionDeny)
|
||||
wafprotocol.SetAnswerHeaders(
|
||||
rq.out.Header(),
|
||||
wafprotocol.Answer{Disposition: wafprotocol.DispositionDeny},
|
||||
)
|
||||
http.Error(rq.out, http.StatusText(r.status), r.status)
|
||||
}
|
||||
|
||||
@@ -362,7 +242,7 @@ func (rq *request) deny(r refusal) {
|
||||
// runs on it: the address is on the private network, where the peers are
|
||||
// rproxy and the admin's tools, not visitors.
|
||||
func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodPost && r.URL.Path == ReportPath {
|
||||
if r.Method == http.MethodPost && r.URL.Path == wafprotocol.ReportPath {
|
||||
h.takeReport(w, r)
|
||||
|
||||
return
|
||||
@@ -373,7 +253,7 @@ func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) {
|
||||
rq := h.startRequest(w, r, peer, peer, false)
|
||||
defer rq.finish()
|
||||
|
||||
if r.Method == http.MethodGet && r.URL.Path == HealthPath {
|
||||
if r.Method == http.MethodGet && r.URL.Path == wafprotocol.HealthPath {
|
||||
rq.answerHealth()
|
||||
|
||||
return
|
||||
@@ -382,11 +262,11 @@ func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) {
|
||||
rq.answerAdmin()
|
||||
}
|
||||
|
||||
// takeReport answers POST ReportPath: it finishes the request the report
|
||||
// names, if it holds that request's decision, and ignores the report
|
||||
// takeReport answers POST wafprotocol.ReportPath: it finishes the request the
|
||||
// report names, if it holds that request's decision, and ignores the report
|
||||
// otherwise.
|
||||
func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) {
|
||||
report, err := readReport(w, r)
|
||||
report, err := wafprotocol.ReadReport(w, r)
|
||||
if err != nil {
|
||||
h.processLog.Warn("a report is refused", "error", err.Error())
|
||||
http.Error(w, "the body is not a report", http.StatusBadRequest)
|
||||
@@ -404,63 +284,6 @@ func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// readReport reads a report from r's body, which must be one JSON object with
|
||||
// all of Report's fields, in the ranges SPEC.md gives.
|
||||
func readReport(w http.ResponseWriter, r *http.Request) (Report, error) {
|
||||
var report Report
|
||||
|
||||
body, err := io.ReadAll(
|
||||
http.MaxBytesReader(w, r.Body, reportBodyMaxBytes),
|
||||
)
|
||||
if err != nil {
|
||||
return report, fmt.Errorf("read the body: %w", err)
|
||||
}
|
||||
|
||||
var fields map[string]json.RawMessage
|
||||
|
||||
err = json.Unmarshal(body, &fields)
|
||||
if err != nil {
|
||||
return report, fmt.Errorf("decode the body: %w", err)
|
||||
}
|
||||
|
||||
for _, name := range []string{
|
||||
"request_id", "status", "request_bytes", "response_bytes",
|
||||
"duration_total_ms", "duration_upstream_total_ms",
|
||||
} {
|
||||
value, there := fields[name]
|
||||
if !there || string(value) == "null" {
|
||||
return report, fmt.Errorf("%w: %s", errReportFieldMissing, name)
|
||||
}
|
||||
}
|
||||
|
||||
err = json.Unmarshal(body, &report)
|
||||
if err != nil {
|
||||
return report, fmt.Errorf("decode the body: %w", err)
|
||||
}
|
||||
|
||||
return report, report.checkRanges()
|
||||
}
|
||||
|
||||
// checkRanges returns an error for a value of the report that cannot be true.
|
||||
func (report *Report) checkRanges() error {
|
||||
switch {
|
||||
case report.Status != 0 &&
|
||||
(report.Status < http.StatusContinue || report.Status > maxHTTPStatus):
|
||||
return fmt.Errorf("%w: status", errReportValue)
|
||||
case report.RequestBytes < 0 || report.RequestBytes > maxReportedBytes:
|
||||
return fmt.Errorf("%w: request_bytes", errReportValue)
|
||||
case report.ResponseBytes < 0 || report.ResponseBytes > maxReportedBytes:
|
||||
return fmt.Errorf("%w: response_bytes", errReportValue)
|
||||
case report.DurationTotalMS < 0 || report.DurationTotalMS > maxReportedMilliseconds:
|
||||
return fmt.Errorf("%w: duration_total_ms", errReportValue)
|
||||
case report.DurationUpstreamTotalMS < 0 ||
|
||||
report.DurationUpstreamTotalMS > maxReportedMilliseconds:
|
||||
return fmt.Errorf("%w: duration_upstream_total_ms", errReportValue)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// finishWith finishes a request with what its report says, and writes the
|
||||
// request's last log line, which holds its decision's fields and the report's.
|
||||
// For an allowed request it first runs the heuristics of onReport, and adds the
|
||||
@@ -470,7 +293,7 @@ func (report *Report) checkRanges() error {
|
||||
func (h *handler) finishWith(
|
||||
ctx context.Context,
|
||||
held *decision,
|
||||
report Report,
|
||||
report wafprotocol.Report,
|
||||
) {
|
||||
rq := &request{
|
||||
decision: *held,
|
||||
@@ -491,9 +314,7 @@ func (h *handler) finishWith(
|
||||
if rq.allowed {
|
||||
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
|
||||
|
||||
rq.runBanningHeuristics(ctx, onReport())
|
||||
rq.countAnomalies()
|
||||
rq.addToHistory()
|
||||
rq.banAndCount(ctx, onReport())
|
||||
|
||||
rq.h.metrics.RequestReported(
|
||||
&rq.line,
|
||||
|
||||
+195
-86
@@ -22,6 +22,7 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -106,8 +107,8 @@ func (s *WAF) questionWith(
|
||||
|
||||
req := newRequest(s.t, method, s.addr, path, body)
|
||||
req.Host = host
|
||||
req.Header.Set(waf.ClientHeader, from)
|
||||
req.Header.Set(waf.PeerHeader, rproxyPeer)
|
||||
req.Header.Set(wafprotocol.ClientHeader, from)
|
||||
req.Header.Set(wafprotocol.PeerHeader, rproxyPeer)
|
||||
req.Header.Set(requestIDHeader, rand.Text())
|
||||
|
||||
return req
|
||||
@@ -134,13 +135,18 @@ func (s *WAF) allowed(from string) string {
|
||||
s.t.Helper()
|
||||
|
||||
req := s.question(siteA, from, "/")
|
||||
wantDisposition(s.t, do(s.t, req), http.StatusOK, waf.DispositionAllow)
|
||||
wantDisposition(
|
||||
s.t,
|
||||
do(s.t, req),
|
||||
http.StatusOK,
|
||||
wafprotocol.DispositionAllow,
|
||||
)
|
||||
|
||||
return req.Header.Get(requestIDHeader)
|
||||
}
|
||||
|
||||
// report sends rproxy's report and returns the answer.
|
||||
func (s *WAF) report(report waf.Report) answer {
|
||||
func (s *WAF) report(report wafprotocol.Report) answer {
|
||||
s.t.Helper()
|
||||
|
||||
body, err := json.Marshal(report)
|
||||
@@ -155,8 +161,11 @@ func (s *WAF) report(report waf.Report) answer {
|
||||
func (s *WAF) reportBody(body string) answer {
|
||||
s.t.Helper()
|
||||
|
||||
return do(s.t, newRequest(s.t, http.MethodPost, s.addr, waf.ReportPath,
|
||||
strings.NewReader(body)))
|
||||
return do(
|
||||
s.t,
|
||||
newRequest(s.t, http.MethodPost, s.addr, wafprotocol.ReportPath,
|
||||
strings.NewReader(body)),
|
||||
)
|
||||
}
|
||||
|
||||
// passedOn asks about a request from the visitor at from on siteA, and reports
|
||||
@@ -164,7 +173,7 @@ func (s *WAF) reportBody(body string) answer {
|
||||
func (s *WAF) passedOn(from string) {
|
||||
s.t.Helper()
|
||||
|
||||
got := s.report(waf.Report{
|
||||
got := s.report(wafprotocol.Report{
|
||||
RequestID: s.allowed(from),
|
||||
Status: http.StatusOK,
|
||||
RequestBytes: bodyBytes,
|
||||
@@ -210,7 +219,7 @@ func wantDisposition(
|
||||
t.Helper()
|
||||
wantStatus(t, got, status)
|
||||
|
||||
if answered := got.header.Get(waf.DispositionHeader); answered !=
|
||||
if answered := got.header.Get(wafprotocol.DispositionHeader); answered !=
|
||||
disposition {
|
||||
t.Errorf(
|
||||
"answered with disposition %q, want %q",
|
||||
@@ -272,10 +281,10 @@ func TestQuestionAllowedIsAnsweredAndLogged(t *testing.T) {
|
||||
req := s.question(siteA, client, "/page?q=1&token=visible")
|
||||
req.Header.Set(requestIDHeader, "the-id")
|
||||
req.Header.Set("X-Forwarded-Proto", "https")
|
||||
req.Header.Set(waf.ContentLengthHeader, "1234")
|
||||
req.Header.Set(wafprotocol.ContentLengthHeader, "1234")
|
||||
|
||||
got := do(t, req)
|
||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
||||
wantBody(t, got, "")
|
||||
|
||||
if id := got.header.Get(requestIDHeader); id != "" {
|
||||
@@ -308,7 +317,12 @@ func TestIPv6VisitorIsCountedInItsGroup(t *testing.T) {
|
||||
s.allowed("2001:db8::1")
|
||||
|
||||
got := s.ask("2001:db8::2", "/")
|
||||
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
||||
wantDisposition(
|
||||
t,
|
||||
got,
|
||||
http.StatusForbidden,
|
||||
wafprotocol.DispositionDeny,
|
||||
)
|
||||
}
|
||||
|
||||
func TestIPv4VisitorInIPv6FormIsTheIPv4Client(t *testing.T) {
|
||||
@@ -317,9 +331,14 @@ func TestIPv4VisitorInIPv6FormIsTheIPv4Client(t *testing.T) {
|
||||
s := startWAF(t, "", map[string]string{denyNets: client})
|
||||
|
||||
req := s.question(siteA, "::ffff:"+client, "/")
|
||||
req.Header.Set(waf.PeerHeader, "::ffff:"+rproxyPeer)
|
||||
req.Header.Set(wafprotocol.PeerHeader, "::ffff:"+rproxyPeer)
|
||||
|
||||
wantDisposition(t, do(t, req), http.StatusForbidden, waf.DispositionDeny)
|
||||
wantDisposition(
|
||||
t,
|
||||
do(t, req),
|
||||
http.StatusForbidden,
|
||||
wafprotocol.DispositionDeny,
|
||||
)
|
||||
|
||||
line := s.out.requestLine(t)
|
||||
if line.ClientIP != client || line.PeerIP != rproxyPeer {
|
||||
@@ -339,13 +358,13 @@ func TestMalformedQuestionGets400AndAWarningAndNoLine(t *testing.T) {
|
||||
change func(*http.Request)
|
||||
}{
|
||||
{"client not an address", func(req *http.Request) {
|
||||
req.Header.Set(waf.ClientHeader, "not an address")
|
||||
req.Header.Set(wafprotocol.ClientHeader, "not an address")
|
||||
}},
|
||||
{"client empty", func(req *http.Request) {
|
||||
req.Header.Set(waf.ClientHeader, "")
|
||||
req.Header.Set(wafprotocol.ClientHeader, "")
|
||||
}},
|
||||
{"client address with a zone", func(req *http.Request) {
|
||||
req.Header.Set(waf.ClientHeader, "2001:db8::1%eth0")
|
||||
req.Header.Set(wafprotocol.ClientHeader, "2001:db8::1%eth0")
|
||||
}},
|
||||
{"id absent", func(req *http.Request) {
|
||||
req.Header.Del(requestIDHeader)
|
||||
@@ -390,7 +409,12 @@ func TestLongestValidIDIsTaken(t *testing.T) {
|
||||
|
||||
req := s.question(siteA, client, "/")
|
||||
req.Header.Set(requestIDHeader, strings.Repeat("x", 128))
|
||||
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow)
|
||||
wantDisposition(
|
||||
t,
|
||||
do(t, req),
|
||||
http.StatusOK,
|
||||
wafprotocol.DispositionAllow,
|
||||
)
|
||||
}
|
||||
|
||||
func TestQuestionWithABodyOver64KiBGets400(t *testing.T) {
|
||||
@@ -399,16 +423,18 @@ func TestQuestionWithABodyOver64KiBGets400(t *testing.T) {
|
||||
s := startWAF(t, "", nil)
|
||||
|
||||
over := strings.NewReader(
|
||||
strings.Repeat("x", waf.QuestionBodyMaxBytes+1),
|
||||
strings.Repeat("x", wafprotocol.QuestionBodyMaxBytes+1),
|
||||
)
|
||||
got := do(t, s.questionWith(http.MethodPost, siteA, client, "/", over))
|
||||
wantDisposition(t, got, http.StatusBadRequest, "")
|
||||
wantProcessMessages(t, s, "a question is refused", 1)
|
||||
wantNoRequestLines(t, s)
|
||||
|
||||
whole := strings.NewReader(strings.Repeat("x", waf.QuestionBodyMaxBytes))
|
||||
whole := strings.NewReader(
|
||||
strings.Repeat("x", wafprotocol.QuestionBodyMaxBytes),
|
||||
)
|
||||
got = do(t, s.questionWith(http.MethodPost, siteA, client, "/", whole))
|
||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
||||
}
|
||||
|
||||
func TestQuestionWithAHeldIDIsRefused(t *testing.T) {
|
||||
@@ -424,12 +450,12 @@ func TestQuestionWithAHeldIDIsRefused(t *testing.T) {
|
||||
wantProcessMessages(t, s, "a question is refused", 1)
|
||||
|
||||
// A denied request's id is held as well.
|
||||
denied := s.question(siteA, otherClient, waf.HealthPath)
|
||||
denied := s.question(siteA, otherClient, wafprotocol.HealthPath)
|
||||
wantDisposition(
|
||||
t,
|
||||
do(t, denied),
|
||||
http.StatusNotFound,
|
||||
waf.DispositionDeny,
|
||||
wafprotocol.DispositionDeny,
|
||||
)
|
||||
|
||||
again = s.question(siteA, client, "/")
|
||||
@@ -437,7 +463,9 @@ func TestQuestionWithAHeldIDIsRefused(t *testing.T) {
|
||||
wantDisposition(t, do(t, again), http.StatusBadRequest, "")
|
||||
|
||||
// The report ends the holding, and the refused questions changed nothing.
|
||||
got := s.report(waf.Report{RequestID: id, ResponseBytes: answerBytes})
|
||||
got := s.report(
|
||||
wafprotocol.Report{RequestID: id, ResponseBytes: answerBytes},
|
||||
)
|
||||
wantStatus(t, got, http.StatusNoContent)
|
||||
|
||||
if history := s.history(); history.ResponseBytes != answerBytes {
|
||||
@@ -446,7 +474,12 @@ func TestQuestionWithAHeldIDIsRefused(t *testing.T) {
|
||||
|
||||
again = s.question(siteA, client, "/")
|
||||
again.Header.Set(requestIDHeader, id)
|
||||
wantDisposition(t, do(t, again), http.StatusOK, waf.DispositionAllow)
|
||||
wantDisposition(
|
||||
t,
|
||||
do(t, again),
|
||||
http.StatusOK,
|
||||
wafprotocol.DispositionAllow,
|
||||
)
|
||||
}
|
||||
|
||||
func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
|
||||
@@ -481,7 +514,7 @@ func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
|
||||
|
||||
// A report for a question not yet answered is ignored, and that question's
|
||||
// id is taken.
|
||||
got := s.report(waf.Report{RequestID: id, ResponseBytes: 1000})
|
||||
got := s.report(wafprotocol.Report{RequestID: id, ResponseBytes: 1000})
|
||||
wantStatus(t, got, http.StatusNoContent)
|
||||
|
||||
// This would wait on the same lookup were the id not taken.
|
||||
@@ -496,7 +529,7 @@ func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
|
||||
|
||||
select {
|
||||
case got := <-answered:
|
||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
||||
case <-time.After(waitLimit):
|
||||
t.Fatal("the question got no answer")
|
||||
}
|
||||
@@ -515,7 +548,9 @@ func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
|
||||
}
|
||||
|
||||
// The true report comes after, and is the one that counts.
|
||||
got = s.report(waf.Report{RequestID: id, ResponseBytes: answerBytes})
|
||||
got = s.report(
|
||||
wafprotocol.Report{RequestID: id, ResponseBytes: answerBytes},
|
||||
)
|
||||
wantStatus(t, got, http.StatusNoContent)
|
||||
|
||||
if history := s.history(); history.ResponseBytes != answerBytes {
|
||||
@@ -533,17 +568,17 @@ func TestPeerAbsentOrUnreadableIsAnsweredAndLoggedInvalid(t *testing.T) {
|
||||
s := startWAF(t, "", nil)
|
||||
|
||||
req := s.question(siteA, client, "/")
|
||||
req.Header.Del(waf.PeerHeader)
|
||||
req.Header.Del(wafprotocol.PeerHeader)
|
||||
|
||||
if peer != "" {
|
||||
req.Header.Set(waf.PeerHeader, peer)
|
||||
req.Header.Set(wafprotocol.PeerHeader, peer)
|
||||
}
|
||||
|
||||
wantDisposition(
|
||||
t,
|
||||
do(t, req),
|
||||
http.StatusOK,
|
||||
waf.DispositionAllow,
|
||||
wafprotocol.DispositionAllow,
|
||||
)
|
||||
|
||||
if line := s.out.requestLine(t); line.PeerIP != "invalid IP" {
|
||||
@@ -571,12 +606,12 @@ func TestContentLengthUnreadableOrNegativeCountsAsAbsent(t *testing.T) {
|
||||
})
|
||||
|
||||
req := s.question(siteA, client, "/")
|
||||
req.Header.Set(waf.ContentLengthHeader, announced)
|
||||
req.Header.Set(wafprotocol.ContentLengthHeader, announced)
|
||||
wantDisposition(
|
||||
t,
|
||||
do(t, req),
|
||||
http.StatusOK,
|
||||
waf.DispositionAllow,
|
||||
wafprotocol.DispositionAllow,
|
||||
)
|
||||
|
||||
if line := s.out.requestLine(t); line.ContentLength != 0 {
|
||||
@@ -595,13 +630,18 @@ func TestHeaderSentTwiceUsesItsFirstValue(t *testing.T) {
|
||||
s := startWAF(t, "", nil)
|
||||
|
||||
req := s.question(siteA, client, "/")
|
||||
req.Header.Add(waf.ClientHeader, otherClient)
|
||||
req.Header.Add(waf.PeerHeader, "172.18.0.3")
|
||||
req.Header.Add(wafprotocol.ClientHeader, otherClient)
|
||||
req.Header.Add(wafprotocol.PeerHeader, "172.18.0.3")
|
||||
req.Header.Set(requestIDHeader, "first")
|
||||
req.Header.Add(requestIDHeader, "second")
|
||||
req.Header.Set(waf.ContentLengthHeader, "1234")
|
||||
req.Header.Add(waf.ContentLengthHeader, "5678")
|
||||
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow)
|
||||
req.Header.Set(wafprotocol.ContentLengthHeader, "1234")
|
||||
req.Header.Add(wafprotocol.ContentLengthHeader, "5678")
|
||||
wantDisposition(
|
||||
t,
|
||||
do(t, req),
|
||||
http.StatusOK,
|
||||
wafprotocol.DispositionAllow,
|
||||
)
|
||||
|
||||
line := s.out.requestLine(t)
|
||||
if line.ClientIP != client || line.PeerIP != rproxyPeer ||
|
||||
@@ -630,7 +670,7 @@ func sendQuestionWithPartOfItsBody(t *testing.T, conn net.Conn) {
|
||||
_, err := fmt.Fprintf(conn,
|
||||
"POST / HTTP/1.1\r\nHost: %s\r\n%s: %s\r\n%s: an-id\r\n"+
|
||||
"Content-Length: 100\r\n\r\npart",
|
||||
siteA, waf.ClientHeader, client, requestIDHeader)
|
||||
siteA, wafprotocol.ClientHeader, client, requestIDHeader)
|
||||
if err != nil {
|
||||
t.Fatalf("send the question: %v", err)
|
||||
}
|
||||
@@ -699,7 +739,7 @@ func TestQuestionWhoseBodyStopsArrivingGetsNoAnswer(t *testing.T) {
|
||||
t,
|
||||
do(t, again),
|
||||
http.StatusOK,
|
||||
waf.DispositionAllow,
|
||||
wafprotocol.DispositionAllow,
|
||||
)
|
||||
})
|
||||
}
|
||||
@@ -717,7 +757,7 @@ func TestQuestionDeniedGetsTheBanResponse(t *testing.T) {
|
||||
})
|
||||
|
||||
got := s.ask(client, "/")
|
||||
wantDisposition(t, got, status, waf.DispositionDeny)
|
||||
wantDisposition(t, got, status, wafprotocol.DispositionDeny)
|
||||
wantBody(t, got, http.StatusText(status)+"\n")
|
||||
wantLine(
|
||||
t,
|
||||
@@ -737,7 +777,7 @@ func TestBanResponseCloseIsAnAnswerOfItsOwn(t *testing.T) {
|
||||
})
|
||||
|
||||
got := s.ask(client, "/")
|
||||
wantDisposition(t, got, http.StatusOK, waf.DispositionClose)
|
||||
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionClose)
|
||||
wantBody(t, got, "")
|
||||
// The visitor is sent nothing.
|
||||
wantLine(t, s.out.requestLine(t), 0, requestlog.ActionClientDenied)
|
||||
@@ -761,12 +801,22 @@ func TestBlockRuleIsDeniedWith403AndABanRuleWithTheBanResponse(
|
||||
})
|
||||
|
||||
got := s.ask(client, "/blocked")
|
||||
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
||||
wantDisposition(
|
||||
t,
|
||||
got,
|
||||
http.StatusForbidden,
|
||||
wafprotocol.DispositionDeny,
|
||||
)
|
||||
wantLine(t, s.out.requestLine(t), http.StatusForbidden,
|
||||
requestlog.ActionBlockedByBlockRule)
|
||||
|
||||
got = s.ask(client, "/.env")
|
||||
wantDisposition(t, got, http.StatusTooManyRequests, waf.DispositionDeny)
|
||||
wantDisposition(
|
||||
t,
|
||||
got,
|
||||
http.StatusTooManyRequests,
|
||||
wafprotocol.DispositionDeny,
|
||||
)
|
||||
wantLine(t, s.out.requestLines(t, 2)[1], http.StatusTooManyRequests,
|
||||
requestlog.ActionTemporarilyBanned)
|
||||
}
|
||||
@@ -781,10 +831,15 @@ func TestQuestionUnderTheOwnPrefixIsDeniedWith404AfterTheChecks(
|
||||
})
|
||||
|
||||
for _, path := range []string{
|
||||
waf.HealthPath, waf.MetricsPath, waf.ReportPath,
|
||||
wafprotocol.HealthPath, waf.MetricsPath, wafprotocol.ReportPath,
|
||||
} {
|
||||
got := s.ask(client, path)
|
||||
wantDisposition(t, got, http.StatusNotFound, waf.DispositionDeny)
|
||||
wantDisposition(
|
||||
t,
|
||||
got,
|
||||
http.StatusNotFound,
|
||||
wafprotocol.DispositionDeny,
|
||||
)
|
||||
}
|
||||
|
||||
wantLine(
|
||||
@@ -795,7 +850,12 @@ func TestQuestionUnderTheOwnPrefixIsDeniedWith404AfterTheChecks(
|
||||
)
|
||||
|
||||
got := s.ask(otherClient, waf.MetricsPath)
|
||||
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
||||
wantDisposition(
|
||||
t,
|
||||
got,
|
||||
http.StatusForbidden,
|
||||
wafprotocol.DispositionDeny,
|
||||
)
|
||||
}
|
||||
|
||||
func TestObserveModeAllowsAndLogsWhatEnforceModeWouldHaveDone(t *testing.T) {
|
||||
@@ -804,7 +864,7 @@ func TestObserveModeAllowsAndLogsWhatEnforceModeWouldHaveDone(t *testing.T) {
|
||||
s := startWAF(t, "", map[string]string{mode: observe, denyNets: client})
|
||||
|
||||
got := s.ask(client, "/")
|
||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
||||
|
||||
line := s.out.requestLine(t)
|
||||
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
|
||||
@@ -822,7 +882,7 @@ func TestObserveModeRefusalAnsweredAllowCountsForTheRefusalLimit(
|
||||
|
||||
for range 2 {
|
||||
got := s.ask(client, "/blocked")
|
||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
||||
}
|
||||
|
||||
lines := s.out.requestLines(t, 2)
|
||||
@@ -846,7 +906,12 @@ func TestDeniedQuestionCountsForTheRefusalLimit(t *testing.T) {
|
||||
|
||||
for range 2 {
|
||||
got := s.ask(client, "/blocked")
|
||||
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
||||
wantDisposition(
|
||||
t,
|
||||
got,
|
||||
http.StatusForbidden,
|
||||
wafprotocol.DispositionDeny,
|
||||
)
|
||||
}
|
||||
|
||||
lines := s.out.requestLines(t, 2)
|
||||
@@ -882,7 +947,12 @@ func TestAllowCarriesTheLookupHeadersWhileTheSettingIsSet(t *testing.T) {
|
||||
})
|
||||
|
||||
got := s.ask(fromDE, "/")
|
||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||
wantDisposition(
|
||||
t,
|
||||
got,
|
||||
http.StatusOK,
|
||||
wafprotocol.DispositionAllow,
|
||||
)
|
||||
|
||||
asn, country := got.header.Get(
|
||||
"X-Client-Asn",
|
||||
@@ -916,7 +986,7 @@ func TestReportCountsTowardAByteLimitWhichThenBans(t *testing.T) {
|
||||
}
|
||||
|
||||
// 100 bytes are over the limit of 99.
|
||||
got := s.report(waf.Report{
|
||||
got := s.report(wafprotocol.Report{
|
||||
RequestID: id, Status: http.StatusBadGateway,
|
||||
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
|
||||
})
|
||||
@@ -940,7 +1010,12 @@ func TestReportCountsTowardAByteLimitWhichThenBans(t *testing.T) {
|
||||
}
|
||||
|
||||
got = s.ask(client, "/")
|
||||
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
||||
wantDisposition(
|
||||
t,
|
||||
got,
|
||||
http.StatusForbidden,
|
||||
wafprotocol.DispositionDeny,
|
||||
)
|
||||
}
|
||||
|
||||
func TestReportCountsAtThePercentageItsQuestionWasGiven(t *testing.T) {
|
||||
@@ -958,12 +1033,12 @@ func TestReportCountsAtThePercentageItsQuestionWasGiven(t *testing.T) {
|
||||
t,
|
||||
s.ask(fromKP, "/"),
|
||||
http.StatusOK,
|
||||
waf.DispositionAllow,
|
||||
wafprotocol.DispositionAllow,
|
||||
)
|
||||
|
||||
s.passedOn(fromDE)
|
||||
wantDisposition(t, s.ask(fromDE, "/"), http.StatusForbidden,
|
||||
waf.DispositionDeny)
|
||||
wafprotocol.DispositionDeny)
|
||||
}
|
||||
|
||||
func TestReportAddsTheRequestToItsClientsHistoryAndTheAnomalyCounters(
|
||||
@@ -989,7 +1064,7 @@ func TestReportAddsTheRequestToItsClientsHistoryAndTheAnomalyCounters(
|
||||
)
|
||||
}
|
||||
|
||||
got := s.report(waf.Report{
|
||||
got := s.report(wafprotocol.Report{
|
||||
RequestID: id, Status: http.StatusBadGateway,
|
||||
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
|
||||
})
|
||||
@@ -1021,7 +1096,7 @@ func TestDeniedRequestIsFinishedAtItsDecision(t *testing.T) {
|
||||
t,
|
||||
do(t, denied),
|
||||
http.StatusForbidden,
|
||||
waf.DispositionDeny,
|
||||
wafprotocol.DispositionDeny,
|
||||
)
|
||||
|
||||
counters := s.server.Anomalies.Snapshot()
|
||||
@@ -1039,7 +1114,7 @@ func TestDeniedRequestIsFinishedAtItsDecision(t *testing.T) {
|
||||
|
||||
wantMetric(t, s.metrics(), series, 1)
|
||||
|
||||
got := s.report(waf.Report{
|
||||
got := s.report(wafprotocol.Report{
|
||||
RequestID: denied.Header.Get(
|
||||
requestIDHeader,
|
||||
), Status: http.StatusForbidden,
|
||||
@@ -1082,7 +1157,7 @@ func TestMetricsCountAnAllowedRequestFromItsReport(t *testing.T) {
|
||||
1,
|
||||
) // the scrape itself
|
||||
|
||||
got := s.report(waf.Report{
|
||||
got := s.report(wafprotocol.Report{
|
||||
RequestID: id, Status: http.StatusBadGateway,
|
||||
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
|
||||
DurationTotalMS: 5, DurationUpstreamTotalMS: 4,
|
||||
@@ -1100,7 +1175,7 @@ func TestMetricsCountAnAllowedRequestFromItsReport(t *testing.T) {
|
||||
// check needs a server that has not been scraped yet.
|
||||
unscraped := startWAF(t, "", map[string]string{metricsToken: token})
|
||||
|
||||
got = unscraped.report(waf.Report{
|
||||
got = unscraped.report(wafprotocol.Report{
|
||||
RequestID: unscraped.allowed(client), Status: http.StatusBadGateway,
|
||||
DurationTotalMS: 5,
|
||||
})
|
||||
@@ -1118,7 +1193,7 @@ func TestReportForARequestNotHeldIsIgnored(t *testing.T) {
|
||||
|
||||
s := startWAF(t, "", map[string]string{bytesLimitPerMinute: byteLimit})
|
||||
|
||||
got := s.report(waf.Report{
|
||||
got := s.report(wafprotocol.Report{
|
||||
RequestID: "no-such-request", ResponseBytes: 1000,
|
||||
})
|
||||
wantStatus(t, got, http.StatusNoContent)
|
||||
@@ -1137,7 +1212,9 @@ func TestReportForARequestNotHeldIsIgnored(t *testing.T) {
|
||||
// A second report of the same request does not count it twice.
|
||||
id := s.allowed(client)
|
||||
for range 2 {
|
||||
got = s.report(waf.Report{RequestID: id, ResponseBytes: answerBytes})
|
||||
got = s.report(
|
||||
wafprotocol.Report{RequestID: id, ResponseBytes: answerBytes},
|
||||
)
|
||||
wantStatus(t, got, http.StatusNoContent)
|
||||
}
|
||||
|
||||
@@ -1347,7 +1424,7 @@ func TestBytesAtTheirMostEachWayBanRatherThanOverflow(t *testing.T) {
|
||||
|
||||
s := startWAF(t, "", map[string]string{bytesLimitPerMinute: byteLimit})
|
||||
|
||||
got := s.report(waf.Report{
|
||||
got := s.report(wafprotocol.Report{
|
||||
RequestID: s.allowed(client), Status: http.StatusOK,
|
||||
RequestBytes: reportedBytesLimit, ResponseBytes: reportedBytesLimit,
|
||||
})
|
||||
@@ -1441,9 +1518,14 @@ func TestReportWritesTheLastLineWithTheDecisionsAndTheReportsFields(
|
||||
req.Header.Set("X-Forwarded-Proto", "https")
|
||||
id := req.Header.Get(requestIDHeader)
|
||||
|
||||
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow)
|
||||
wantDisposition(
|
||||
t,
|
||||
do(t, req),
|
||||
http.StatusOK,
|
||||
wafprotocol.DispositionAllow,
|
||||
)
|
||||
|
||||
got := s.report(waf.Report{
|
||||
got := s.report(wafprotocol.Report{
|
||||
RequestID: id, Status: http.StatusCreated,
|
||||
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
|
||||
DurationTotalMS: 12.5, DurationUpstreamTotalMS: 10.25,
|
||||
@@ -1490,7 +1572,7 @@ func TestEveryDeniedRequestsReportGetsItsLastLine(t *testing.T) {
|
||||
"close", map[string]string{denyNets: client, banResponse: banResponseClose}, "/",
|
||||
0, requestlog.ActionClientDenied,
|
||||
},
|
||||
{"own prefix", nil, waf.HealthPath, http.StatusNotFound,
|
||||
{"own prefix", nil, wafprotocol.HealthPath, http.StatusNotFound,
|
||||
requestlog.ActionOwnEndpoint},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
@@ -1502,7 +1584,7 @@ func TestEveryDeniedRequestsReportGetsItsLastLine(t *testing.T) {
|
||||
id := req.Header.Get(requestIDHeader)
|
||||
do(t, req)
|
||||
|
||||
got := s.report(waf.Report{
|
||||
got := s.report(wafprotocol.Report{
|
||||
RequestID: id, Status: tc.status, ResponseBytes: 8, DurationTotalMS: 3,
|
||||
})
|
||||
wantStatus(t, got, http.StatusNoContent)
|
||||
@@ -1542,13 +1624,13 @@ func TestOldestAnsweredRequestIsDroppedPastTheBound(t *testing.T) {
|
||||
t,
|
||||
readAnswer(res),
|
||||
http.StatusOK,
|
||||
waf.DispositionAllow,
|
||||
wafprotocol.DispositionAllow,
|
||||
)
|
||||
}
|
||||
|
||||
// The table is full again with next and those, without oldest.
|
||||
got := s.report(
|
||||
waf.Report{RequestID: oldest, ResponseBytes: answerBytes},
|
||||
wafprotocol.Report{RequestID: oldest, ResponseBytes: answerBytes},
|
||||
)
|
||||
wantStatus(t, got, http.StatusNoContent)
|
||||
|
||||
@@ -1559,7 +1641,9 @@ func TestOldestAnsweredRequestIsDroppedPastTheBound(t *testing.T) {
|
||||
)
|
||||
}
|
||||
|
||||
got = s.report(waf.Report{RequestID: next, ResponseBytes: answerBytes})
|
||||
got = s.report(
|
||||
wafprotocol.Report{RequestID: next, ResponseBytes: answerBytes},
|
||||
)
|
||||
wantStatus(t, got, http.StatusNoContent)
|
||||
|
||||
if counted := s.history().Requests; counted != 1 {
|
||||
@@ -1580,10 +1664,20 @@ func TestBanMadeThroughOneHostRefusesTheClientOnAnother(t *testing.T) {
|
||||
)
|
||||
|
||||
got := do(t, s.question(siteA, client, "/.env"))
|
||||
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
||||
wantDisposition(
|
||||
t,
|
||||
got,
|
||||
http.StatusForbidden,
|
||||
wafprotocol.DispositionDeny,
|
||||
)
|
||||
|
||||
got = do(t, s.question(siteB, client, "/"))
|
||||
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
||||
wantDisposition(
|
||||
t,
|
||||
got,
|
||||
http.StatusForbidden,
|
||||
wafprotocol.DispositionDeny,
|
||||
)
|
||||
|
||||
lines := s.out.requestLines(t, 2)
|
||||
if lines[0].Host != siteA || lines[1].Host != siteB {
|
||||
@@ -1621,7 +1715,12 @@ func TestRequestsOnEverySiteCountTowardOneSetOfLimits(t *testing.T) {
|
||||
s.allowed(client)
|
||||
|
||||
got := do(t, s.question(siteB, client, "/"))
|
||||
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
||||
wantDisposition(
|
||||
t,
|
||||
got,
|
||||
http.StatusForbidden,
|
||||
wafprotocol.DispositionDeny,
|
||||
)
|
||||
}
|
||||
|
||||
func TestOwnEndpointsAnswerOnlyWithoutTheClientHeader(t *testing.T) {
|
||||
@@ -1636,13 +1735,13 @@ func TestOwnEndpointsAnswerOnlyWithoutTheClientHeader(t *testing.T) {
|
||||
path, authorization string
|
||||
want int
|
||||
}{
|
||||
{waf.HealthPath, "", http.StatusOK},
|
||||
{wafprotocol.HealthPath, "", http.StatusOK},
|
||||
{waf.MetricsPath, bearer, http.StatusOK},
|
||||
{waf.MetricsPath, "", http.StatusUnauthorized},
|
||||
{waf.BansPath, adminBearer, http.StatusOK},
|
||||
{waf.BansPath, "", http.StatusUnauthorized},
|
||||
{waf.ClientsPath + client, adminBearer, http.StatusOK},
|
||||
{waf.ReportPath, "", http.StatusNotFound},
|
||||
{wafprotocol.ReportPath, "", http.StatusNotFound},
|
||||
{"/_sww/other", "", http.StatusNotFound},
|
||||
{"/", "", http.StatusNotFound},
|
||||
} {
|
||||
@@ -1659,10 +1758,10 @@ func TestOwnEndpointsAnswerOnlyWithoutTheClientHeader(t *testing.T) {
|
||||
t,
|
||||
do(t, req),
|
||||
http.StatusNotFound,
|
||||
waf.DispositionDeny,
|
||||
wafprotocol.DispositionDeny,
|
||||
)
|
||||
} else {
|
||||
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow)
|
||||
wantDisposition(t, do(t, req), http.StatusOK, wafprotocol.DispositionAllow)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1672,11 +1771,11 @@ func TestOwnEndpointsLinesDoNotSayWhichOfTheTwoTheyAre(t *testing.T) {
|
||||
|
||||
s := startWAF(t, "", nil)
|
||||
|
||||
wantStatus(t, get(t, s.addr, waf.HealthPath), http.StatusOK)
|
||||
wantStatus(t, get(t, s.addr, wafprotocol.HealthPath), http.StatusOK)
|
||||
|
||||
line := s.out.requestLine(t)
|
||||
if _, there := line.fields["waf_line"]; there ||
|
||||
line.Path != waf.HealthPath {
|
||||
line.Path != wafprotocol.HealthPath {
|
||||
t.Errorf("the health check's log line is %+v", line.fields)
|
||||
}
|
||||
}
|
||||
@@ -1688,7 +1787,7 @@ func TestALargeBodyAnnouncedAndLongHeadersAreNoReasonToRefuse(t *testing.T) {
|
||||
|
||||
req := s.questionWith(http.MethodPost, siteA, client, "/",
|
||||
strings.NewReader("a body"))
|
||||
req.Header.Set(waf.ContentLengthHeader, "1073741824")
|
||||
req.Header.Set(wafprotocol.ContentLengthHeader, "1073741824")
|
||||
req.Header.Set(forwardedFor, otherClient)
|
||||
|
||||
// 40,000 bytes of headers, over rproxy's 32 KiB, in lines within the
|
||||
@@ -1698,7 +1797,7 @@ func TestALargeBodyAnnouncedAndLongHeadersAreNoReasonToRefuse(t *testing.T) {
|
||||
}
|
||||
|
||||
got := do(t, req)
|
||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
||||
|
||||
line := s.out.requestLine(t)
|
||||
if line.ClientIP != client || line.ContentLength != 1<<30 {
|
||||
@@ -1776,7 +1875,7 @@ carried header:Content-Length block ^6$
|
||||
strings.NewReader("a body"))
|
||||
|
||||
if tc.announced != "" {
|
||||
req.Header.Set(waf.ContentLengthHeader, tc.announced)
|
||||
req.Header.Set(wafprotocol.ContentLengthHeader, tc.announced)
|
||||
}
|
||||
|
||||
wantStatus(t, do(t, req), tc.want)
|
||||
@@ -1794,19 +1893,24 @@ func TestCoreRuleSetReadsTheBodyAQuestionCarries(t *testing.T) {
|
||||
req := s.questionWith(http.MethodPost, siteA, client, "/",
|
||||
strings.NewReader(body))
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
req.Header.Set(waf.ContentLengthHeader, "100000")
|
||||
req.Header.Set(wafprotocol.ContentLengthHeader, "100000")
|
||||
|
||||
return do(t, req)
|
||||
}
|
||||
|
||||
got := ask(formData, "id=1'%20OR%20'1'='1")
|
||||
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
||||
wantDisposition(
|
||||
t,
|
||||
got,
|
||||
http.StatusForbidden,
|
||||
wafprotocol.DispositionDeny,
|
||||
)
|
||||
|
||||
// The first 8 KiB of a longer JSON body cannot be parsed, which is no
|
||||
// attack, so the Core Rule Set does not read it.
|
||||
cut := `{"text":"` + strings.Repeat("x", 8<<10)
|
||||
got = ask("application/json", cut)
|
||||
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
||||
}
|
||||
|
||||
func TestCoreRuleSetAlertNamesTheSite(t *testing.T) {
|
||||
@@ -1815,7 +1919,12 @@ func TestCoreRuleSetAlertNamesTheSite(t *testing.T) {
|
||||
s := startWAF(t, "", map[string]string{coreRuleSetMode: block})
|
||||
|
||||
got := s.ask(client, sqlInjection)
|
||||
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
||||
wantDisposition(
|
||||
t,
|
||||
got,
|
||||
http.StatusForbidden,
|
||||
wafprotocol.DispositionDeny,
|
||||
)
|
||||
|
||||
waiting := s.queue.Snapshot().Waiting[alerts.DestinationWebhook]
|
||||
if len(waiting) != 1 ||
|
||||
@@ -1836,7 +1945,7 @@ func TestReputationHitAlertNamesTheSite(t *testing.T) {
|
||||
loadLists(t, s.server, map[string][]string{dropURL: {client}})
|
||||
|
||||
wantDisposition(t, s.ask(client, "/"), http.StatusForbidden,
|
||||
waf.DispositionDeny)
|
||||
wafprotocol.DispositionDeny)
|
||||
|
||||
waiting := s.queue.Snapshot().Waiting[alerts.DestinationWebhook]
|
||||
if len(waiting) != 1 || waiting[0].Event != alerts.EventReputationHit ||
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package wafprotocol
|
||||
|
||||
import "net/http"
|
||||
|
||||
// Answer is what waf's answer to a question tells rproxy in its headers.
|
||||
// The status, the other headers and the body of an answer of deny are what
|
||||
// the visitor gets.
|
||||
type Answer struct {
|
||||
// Disposition is what rproxy does with the request: DispositionAllow,
|
||||
// DispositionDeny or DispositionClose.
|
||||
Disposition string
|
||||
// ASN and Country are the client's AS number and country, each empty
|
||||
// when the answer does not give it.
|
||||
ASN, Country string
|
||||
}
|
||||
|
||||
// SetAnswerHeaders sets the headers of waf's answer that tell rproxy answer,
|
||||
// leaving out an ASN or Country that is empty.
|
||||
func SetAnswerHeaders(header http.Header, answer Answer) {
|
||||
header.Set(DispositionHeader, answer.Disposition)
|
||||
|
||||
if answer.ASN != "" {
|
||||
header.Set(ASNHeader, answer.ASN)
|
||||
}
|
||||
|
||||
if answer.Country != "" {
|
||||
header.Set(CountryHeader, answer.Country)
|
||||
}
|
||||
}
|
||||
|
||||
// ReadAnswer returns what the headers of waf's answer tell rproxy, and
|
||||
// whether they give a valid disposition.
|
||||
func ReadAnswer(header http.Header) (Answer, bool) {
|
||||
disposition := header.Get(DispositionHeader)
|
||||
|
||||
switch disposition {
|
||||
case DispositionAllow, DispositionDeny, DispositionClose:
|
||||
return Answer{
|
||||
Disposition: disposition,
|
||||
ASN: header.Get(ASNHeader),
|
||||
Country: header.Get(CountryHeader),
|
||||
}, true
|
||||
}
|
||||
|
||||
return Answer{}, false
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
package wafprotocol
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// maxRequestIDBytes is the longest id a question may give.
|
||||
const maxRequestIDBytes = 128
|
||||
|
||||
// ErrBodyStoppedArriving is the error ReadQuestion wraps when the question's
|
||||
// body stops arriving. Every other error it returns is why waf refuses the
|
||||
// question.
|
||||
var ErrBodyStoppedArriving = errors.New("the body stopped arriving")
|
||||
|
||||
// Why waf refuses a question. Each is also the body of the refusal.
|
||||
var (
|
||||
errClientNotAnAddress = errors.New(
|
||||
ClientHeader + " is not an address without a zone",
|
||||
)
|
||||
errRequestIDNotValid = errors.New(RequestIDHeader + " is not a valid id")
|
||||
errBodyTooLarge = errors.New("the body is over 64 KiB")
|
||||
)
|
||||
|
||||
// QuestionParams are what NewQuestion needs.
|
||||
type QuestionParams struct {
|
||||
// Visitor is the visitor's request.
|
||||
Visitor *http.Request
|
||||
// PathAsSent is Visitor's path as the visitor sent it, which the question
|
||||
// sends unchanged.
|
||||
PathAsSent string
|
||||
// Body is the part of Visitor's body the question carries, at most
|
||||
// QuestionBodyMaxBytes.
|
||||
Body []byte
|
||||
// WAFBackend is waf's address. The question takes its scheme and host.
|
||||
WAFBackend *url.URL
|
||||
// Client is the visitor's address, and Peer that of rproxy's own TCP
|
||||
// peer.
|
||||
Client, Peer netip.Addr
|
||||
// RequestID is the request's id.
|
||||
RequestID string
|
||||
// Scheme is how the visitor reached rproxy, or the trusted proxy in front
|
||||
// of it.
|
||||
Scheme string
|
||||
}
|
||||
|
||||
// Question is a question as waf reads it.
|
||||
type Question struct {
|
||||
// Client is the visitor's address.
|
||||
Client netip.Addr
|
||||
// Peer is the address of rproxy's own TCP peer. It is not valid when the
|
||||
// question gives none that can be read.
|
||||
Peer netip.Addr
|
||||
// RequestID is the request's id.
|
||||
RequestID string
|
||||
// Visitor is the visitor's request, as asTheVisitorSentIt gives it.
|
||||
Visitor *http.Request
|
||||
}
|
||||
|
||||
// NewQuestion returns the question about a visitor's request, as "The
|
||||
// question" in SPEC.md gives it: the request as the visitor sent it, with the
|
||||
// part of the body it carries and rproxy's headers.
|
||||
func NewQuestion(
|
||||
ctx context.Context,
|
||||
params QuestionParams,
|
||||
) (*http.Request, error) {
|
||||
in := params.Visitor
|
||||
|
||||
question, err := http.NewRequestWithContext(
|
||||
ctx,
|
||||
in.Method,
|
||||
params.WAFBackend.String(),
|
||||
bytes.NewReader(params.Body),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("make the question: %w", err)
|
||||
}
|
||||
|
||||
backend := question.URL
|
||||
target := *in.URL
|
||||
target.Scheme = backend.Scheme
|
||||
target.Host = backend.Host
|
||||
target.Opaque = params.PathAsSent
|
||||
question.URL = &target
|
||||
question.Host = in.Host
|
||||
|
||||
// The length NewRequestWithContext set is that of the body carried. This
|
||||
// line makes a body whose length the visitor did not announce go chunked,
|
||||
// as "The question" in SPEC.md requires.
|
||||
question.TransferEncoding = in.TransferEncoding
|
||||
|
||||
question.Header = in.Header.Clone()
|
||||
question.Header.Set(ClientHeader, params.Client.String())
|
||||
question.Header.Set(PeerHeader, params.Peer.String())
|
||||
question.Header.Del(ContentLengthHeader)
|
||||
|
||||
announced := in.Header.Get("Content-Length")
|
||||
if announced != "" {
|
||||
question.Header.Set(ContentLengthHeader, announced)
|
||||
}
|
||||
|
||||
question.Header.Set(RequestIDHeader, params.RequestID)
|
||||
question.Header.Set("X-Forwarded-Proto", params.Scheme)
|
||||
|
||||
// Go's client adds a User-Agent of its own to a request without the key.
|
||||
_, sent := question.Header["User-Agent"]
|
||||
if !sent {
|
||||
question.Header["User-Agent"] = nil
|
||||
}
|
||||
|
||||
return question, nil
|
||||
}
|
||||
|
||||
// ReadQuestion reads the question r, with the body it carries. An error
|
||||
// wrapping ErrBodyStoppedArriving means the body stopped arriving. Any other
|
||||
// error is why waf refuses the question, and the question's body is then not
|
||||
// read. The body is read whole, since Go's server does not send an answer
|
||||
// until it has read the body.
|
||||
func ReadQuestion(w http.ResponseWriter, r *http.Request) (Question, error) {
|
||||
client, err := netip.ParseAddr(r.Header.Get(ClientHeader))
|
||||
if err != nil || client.Zone() != "" {
|
||||
return Question{}, errClientNotAnAddress
|
||||
}
|
||||
|
||||
id := r.Header.Get(RequestIDHeader)
|
||||
if id == "" || len(id) > maxRequestIDBytes || !utf8.ValidString(id) {
|
||||
return Question{}, errRequestIDNotValid
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(
|
||||
http.MaxBytesReader(w, r.Body, QuestionBodyMaxBytes),
|
||||
)
|
||||
|
||||
if _, tooLarge := errors.AsType[*http.MaxBytesError](err); tooLarge {
|
||||
return Question{}, errBodyTooLarge
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return Question{}, fmt.Errorf("%w: %w", ErrBodyStoppedArriving, err)
|
||||
}
|
||||
|
||||
// One that cannot be read leaves the log line's peer_ip invalid.
|
||||
peer, _ := netip.ParseAddr(r.Header.Get(PeerHeader))
|
||||
|
||||
return Question{
|
||||
Client: client.Unmap(),
|
||||
Peer: peer.Unmap(),
|
||||
RequestID: id,
|
||||
Visitor: asTheVisitorSentIt(r, body),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// asTheVisitorSentIt returns a copy of r, a question, as the checks and the
|
||||
// log line are to see it: without the headers rproxy adds for waf,
|
||||
// with body, which has been read of r, and with the length the visitor
|
||||
// announced, as -1 when it announced none. The question's own length covers
|
||||
// only the part of the body it carries.
|
||||
func asTheVisitorSentIt(r *http.Request, body []byte) *http.Request {
|
||||
sent := r.Clone(r.Context())
|
||||
announced := sent.Header.Get(ContentLengthHeader)
|
||||
|
||||
sent.Header.Del(ClientHeader)
|
||||
sent.Header.Del(PeerHeader)
|
||||
sent.Header.Del(ContentLengthHeader)
|
||||
sent.Header.Del("Content-Length")
|
||||
|
||||
sent.ContentLength = -1
|
||||
|
||||
length, err := strconv.ParseInt(announced, 10, 64)
|
||||
if err == nil && length >= 0 {
|
||||
sent.ContentLength = length
|
||||
sent.Header.Set("Content-Length", announced)
|
||||
}
|
||||
|
||||
sent.Body = http.NoBody
|
||||
if len(body) > 0 {
|
||||
sent.Body = io.NopCloser(bytes.NewReader(body))
|
||||
}
|
||||
|
||||
return sent
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package wafprotocol
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
const (
|
||||
// reportBodyMaxBytes is the most of a report's body that is read. It is
|
||||
// far above any report's.
|
||||
reportBodyMaxBytes = 1 << 20
|
||||
// maxReportedBytes is the most bytes a report may give for each way, 1
|
||||
// PiB, so that adding them up cannot overflow.
|
||||
maxReportedBytes = 1 << 50
|
||||
// maxReportedMilliseconds is the longest duration a report may give,
|
||||
// about 31 years, so that it fits a time.Duration.
|
||||
maxReportedMilliseconds = 1e12
|
||||
)
|
||||
|
||||
var (
|
||||
errReportFieldMissing = errors.New("a field is missing")
|
||||
errReportValue = errors.New("a value is out of range")
|
||||
)
|
||||
|
||||
// Report is the JSON object rproxy posts to ReportPath once the visitor's
|
||||
// answer has ended, its fields named as the request log's of the same
|
||||
// meaning. RequestID is the question's id. Status is what the visitor got, 0
|
||||
// if nothing. The durations are in milliseconds.
|
||||
//
|
||||
//nolint:tagliatelle // docs/request-log.md names the fields in snake_case
|
||||
type Report struct {
|
||||
RequestID string `json:"request_id"`
|
||||
Status int `json:"status"`
|
||||
RequestBytes int64 `json:"request_bytes"`
|
||||
ResponseBytes int64 `json:"response_bytes"`
|
||||
DurationTotalMS float64 `json:"duration_total_ms"`
|
||||
DurationUpstreamTotalMS float64 `json:"duration_upstream_total_ms"`
|
||||
}
|
||||
|
||||
// NewReportRequest returns the request that posts report to reportURL, which
|
||||
// is waf's ReportPath.
|
||||
func NewReportRequest(
|
||||
ctx context.Context,
|
||||
reportURL string,
|
||||
report *Report,
|
||||
) (*http.Request, error) {
|
||||
body, err := json.Marshal(report)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode the report: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(
|
||||
ctx, http.MethodPost, reportURL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("make the request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// ReadReport reads a report from r's body, which must be one JSON object with
|
||||
// all of Report's fields, in the ranges SPEC.md gives.
|
||||
func ReadReport(w http.ResponseWriter, r *http.Request) (Report, error) {
|
||||
var report Report
|
||||
|
||||
body, err := io.ReadAll(
|
||||
http.MaxBytesReader(w, r.Body, reportBodyMaxBytes),
|
||||
)
|
||||
if err != nil {
|
||||
return report, fmt.Errorf("read the body: %w", err)
|
||||
}
|
||||
|
||||
var fields map[string]json.RawMessage
|
||||
|
||||
err = json.Unmarshal(body, &fields)
|
||||
if err != nil {
|
||||
return report, fmt.Errorf("decode the body: %w", err)
|
||||
}
|
||||
|
||||
for _, name := range []string{
|
||||
"request_id", "status", "request_bytes", "response_bytes",
|
||||
"duration_total_ms", "duration_upstream_total_ms",
|
||||
} {
|
||||
value, there := fields[name]
|
||||
if !there || string(value) == "null" {
|
||||
return report, fmt.Errorf("%w: %s", errReportFieldMissing, name)
|
||||
}
|
||||
}
|
||||
|
||||
err = json.Unmarshal(body, &report)
|
||||
if err != nil {
|
||||
return report, fmt.Errorf("decode the body: %w", err)
|
||||
}
|
||||
|
||||
return report, report.checkRanges()
|
||||
}
|
||||
|
||||
// checkRanges returns an error for a value of the report that cannot be true.
|
||||
func (report *Report) checkRanges() error {
|
||||
switch {
|
||||
case report.Status != 0 &&
|
||||
(report.Status < http.StatusContinue || report.Status > ReportStatusMax):
|
||||
return fmt.Errorf("%w: status", errReportValue)
|
||||
case report.RequestBytes < 0 || report.RequestBytes > maxReportedBytes:
|
||||
return fmt.Errorf("%w: request_bytes", errReportValue)
|
||||
case report.ResponseBytes < 0 || report.ResponseBytes > maxReportedBytes:
|
||||
return fmt.Errorf("%w: response_bytes", errReportValue)
|
||||
case report.DurationTotalMS < 0 || report.DurationTotalMS > maxReportedMilliseconds:
|
||||
return fmt.Errorf("%w: duration_total_ms", errReportValue)
|
||||
case report.DurationUpstreamTotalMS < 0 ||
|
||||
report.DurationUpstreamTotalMS > maxReportedMilliseconds:
|
||||
return fmt.Errorf("%w: duration_upstream_total_ms", errReportValue)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
// Package wafprotocol holds the three messages rproxy and waf send each
|
||||
// other, as "`waf`" in SPEC.md gives them. rproxy asks waf a question about
|
||||
// each visitor's request, and waf answers it. Once the visitor's answer has
|
||||
// ended, rproxy sends waf a report. This package writes and reads each
|
||||
// message, and declares every name and limit both sides must agree on.
|
||||
package wafprotocol
|
||||
|
||||
// The headers of the question and of the answer. Go writes every header name
|
||||
// in this form, as it sends it and as it receives it, so X-Client-ASN arrives
|
||||
// as X-Client-Asn; header names are not case-sensitive.
|
||||
const (
|
||||
// ClientHeader marks a request as rproxy's question, and gives the
|
||||
// visitor's address.
|
||||
ClientHeader = "X-Sww-Client"
|
||||
// PeerHeader gives the address of rproxy's own TCP peer.
|
||||
PeerHeader = "X-Sww-Peer"
|
||||
// ContentLengthHeader gives the body length the visitor announced.
|
||||
ContentLengthHeader = "X-Sww-Content-Length"
|
||||
// RequestIDHeader gives the request's id, which rproxy makes. rproxy
|
||||
// also sets it on the request it passes to the app.
|
||||
RequestIDHeader = "X-Request-ID"
|
||||
// DispositionHeader gives waf's answer.
|
||||
DispositionHeader = "X-Sww-Disposition"
|
||||
// ASNHeader and CountryHeader give the client's AS number and country in
|
||||
// an answer of allow. rproxy passes them to the app.
|
||||
ASNHeader = "X-Client-Asn"
|
||||
CountryHeader = "X-Client-Country"
|
||||
)
|
||||
|
||||
// The values of DispositionHeader.
|
||||
const (
|
||||
// DispositionAllow tells rproxy to pass the request to the app.
|
||||
DispositionAllow = "allow"
|
||||
// DispositionDeny tells rproxy to give the visitor the answer's status,
|
||||
// headers and body.
|
||||
DispositionDeny = "deny"
|
||||
// DispositionClose tells rproxy to close the visitor's connection
|
||||
// without an answer, which SWW_WAF_BAN_RESPONSE asks for when it is close.
|
||||
// waf closing the question's connection instead would read as
|
||||
// waf being down, and rproxy would then let the request through.
|
||||
DispositionClose = "close"
|
||||
)
|
||||
|
||||
// OwnEndpointPrefix starts the path of every request for sww itself: waf's
|
||||
// own endpoints and rproxy's health endpoint. The app never sees a path under
|
||||
// it.
|
||||
const OwnEndpointPrefix = "/_sww/"
|
||||
|
||||
// HealthPath is the health endpoint of waf and of rproxy, which the
|
||||
// container's health check asks.
|
||||
const HealthPath = "/_sww/healthz"
|
||||
|
||||
// ReportPath is where rproxy sends the report of each request waf
|
||||
// answered.
|
||||
const ReportPath = "/_sww/report"
|
||||
|
||||
// QuestionBodyMaxBytes is the most of a visitor's body a question carries.
|
||||
const QuestionBodyMaxBytes = 64 << 10
|
||||
|
||||
// ReportStatusMax is the highest status a report may give.
|
||||
const ReportStatusMax = 599
|
||||
@@ -0,0 +1,204 @@
|
||||
package wafprotocol_test
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
||||
)
|
||||
|
||||
// requestID is the id the tests' questions and reports give.
|
||||
const requestID = "an-id"
|
||||
|
||||
// questionRead is what the server startQuestionReader starts read of a
|
||||
// question: the question, and the body it carries.
|
||||
type questionRead struct {
|
||||
question wafprotocol.Question
|
||||
body string
|
||||
}
|
||||
|
||||
// startQuestionReader starts a server that reads each question it gets with
|
||||
// ReadQuestion. It returns the server's URL and the channel what it reads goes
|
||||
// to.
|
||||
func startQuestionReader(t *testing.T) (*url.URL, <-chan questionRead) {
|
||||
t.Helper()
|
||||
|
||||
read := make(chan questionRead, 1)
|
||||
server := httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
question, err := wafprotocol.ReadQuestion(w, r)
|
||||
if err != nil {
|
||||
t.Errorf("read the question: %v", err)
|
||||
}
|
||||
|
||||
var body []byte
|
||||
if question.Visitor != nil {
|
||||
body, _ = io.ReadAll(question.Visitor.Body)
|
||||
}
|
||||
|
||||
read <- questionRead{question, string(body)}
|
||||
},
|
||||
))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
serverURL, err := url.Parse(server.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("parse the server's URL: %v", err)
|
||||
}
|
||||
|
||||
return serverURL, read
|
||||
}
|
||||
|
||||
// TestQuestionReadsAsWritten sends a question NewQuestion writes over HTTP,
|
||||
// and checks that ReadQuestion gives back the visitor's request and what
|
||||
// rproxy added.
|
||||
func TestQuestionReadsAsWritten(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const (
|
||||
target = "/a%2Fb?q=1"
|
||||
body = "name=value"
|
||||
held = "name"
|
||||
)
|
||||
|
||||
client := netip.MustParseAddr("2001:db8::1")
|
||||
peer := netip.MustParseAddr("192.0.2.1")
|
||||
backend, read := startQuestionReader(t)
|
||||
|
||||
visitor := httptest.NewRequestWithContext(
|
||||
t.Context(),
|
||||
http.MethodPost,
|
||||
"http://site.example.com"+target,
|
||||
strings.NewReader(body),
|
||||
)
|
||||
visitor.Header.Set("Content-Length", "10")
|
||||
visitor.Header.Set("User-Agent", "a visitor")
|
||||
visitor.Header.Set(wafprotocol.PeerHeader, "the visitor's own")
|
||||
|
||||
question, err := wafprotocol.NewQuestion(
|
||||
t.Context(),
|
||||
wafprotocol.QuestionParams{
|
||||
Visitor: visitor,
|
||||
PathAsSent: "/a%2Fb",
|
||||
Body: []byte(held),
|
||||
WAFBackend: backend,
|
||||
Client: client,
|
||||
Peer: peer,
|
||||
RequestID: requestID,
|
||||
Scheme: "https",
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("write the question: %v", err)
|
||||
}
|
||||
|
||||
res, err := http.DefaultTransport.RoundTrip(question)
|
||||
if err != nil {
|
||||
t.Fatalf("send the question: %v", err)
|
||||
}
|
||||
|
||||
_ = res.Body.Close()
|
||||
|
||||
got := <-read
|
||||
if got.question.Visitor == nil {
|
||||
t.FailNow()
|
||||
}
|
||||
|
||||
if got.question.Client != client || got.question.Peer != peer ||
|
||||
got.question.RequestID != requestID {
|
||||
t.Errorf("read %+v", got.question)
|
||||
}
|
||||
|
||||
sent := got.question.Visitor
|
||||
if sent.Host != "site.example.com" || sent.RequestURI != target ||
|
||||
sent.ContentLength != int64(len(body)) || got.body != held {
|
||||
t.Errorf(
|
||||
"read %s at %s, of length %d, carrying %q",
|
||||
sent.RequestURI, sent.Host, sent.ContentLength, got.body,
|
||||
)
|
||||
}
|
||||
|
||||
for name, want := range map[string]string{
|
||||
"Content-Length": "10",
|
||||
"User-Agent": "a visitor",
|
||||
"X-Forwarded-Proto": "https",
|
||||
wafprotocol.RequestIDHeader: requestID,
|
||||
wafprotocol.ClientHeader: "",
|
||||
wafprotocol.PeerHeader: "",
|
||||
wafprotocol.ContentLengthHeader: "",
|
||||
} {
|
||||
if sent.Header.Get(name) != want {
|
||||
t.Errorf("%s is %q, want %q", name, sent.Header.Get(name), want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnswerReadsAsWritten checks that ReadAnswer gives back what
|
||||
// SetAnswerHeaders sets, and that headers without a valid disposition give
|
||||
// nothing.
|
||||
func TestAnswerReadsAsWritten(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, want := range []wafprotocol.Answer{
|
||||
{Disposition: wafprotocol.DispositionAllow, ASN: "64496", Country: "DE"},
|
||||
{Disposition: wafprotocol.DispositionAllow, Country: "DE"},
|
||||
{Disposition: wafprotocol.DispositionDeny},
|
||||
{Disposition: wafprotocol.DispositionClose},
|
||||
} {
|
||||
header := http.Header{}
|
||||
wafprotocol.SetAnswerHeaders(header, want)
|
||||
|
||||
got, valid := wafprotocol.ReadAnswer(header)
|
||||
if !valid || got != want {
|
||||
t.Errorf("read %+v (valid: %t), want %+v", got, valid, want)
|
||||
}
|
||||
}
|
||||
|
||||
header := http.Header{}
|
||||
header.Set(wafprotocol.DispositionHeader, "maybe")
|
||||
header.Set(wafprotocol.ASNHeader, "64496")
|
||||
|
||||
got, valid := wafprotocol.ReadAnswer(header)
|
||||
if valid || got != (wafprotocol.Answer{}) {
|
||||
t.Errorf("read %+v (valid: %t), want nothing", got, valid)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReportReadsAsWritten checks that ReadReport gives back the report
|
||||
// NewReportRequest posts.
|
||||
func TestReportReadsAsWritten(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
want := wafprotocol.Report{
|
||||
RequestID: requestID,
|
||||
Status: http.StatusOK,
|
||||
RequestBytes: 10,
|
||||
ResponseBytes: 20,
|
||||
DurationTotalMS: 1.5,
|
||||
DurationUpstreamTotalMS: 0.5,
|
||||
}
|
||||
|
||||
req, err := wafprotocol.NewReportRequest(
|
||||
t.Context(),
|
||||
"http://waf.example.com"+wafprotocol.ReportPath,
|
||||
&want,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("write the report: %v", err)
|
||||
}
|
||||
|
||||
if req.Method != http.MethodPost ||
|
||||
req.URL.Path != wafprotocol.ReportPath {
|
||||
t.Errorf("the report is %s %s", req.Method, req.URL.Path)
|
||||
}
|
||||
|
||||
got, err := wafprotocol.ReadReport(httptest.NewRecorder(), req)
|
||||
if err != nil || got != want {
|
||||
t.Errorf("read %+v (%v), want %+v", got, err, want)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user