Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e52edb9f62 | ||
|
|
5667661dd8 | ||
|
|
5ce35505bc |
No files matched your search
@@ -902,15 +902,15 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
|
|||||||
- `internal/rproxy`: the reverse proxy in front of one app.
|
- `internal/rproxy`: the reverse proxy in front of one app.
|
||||||
- `internal/waf`: `waf`'s server, which runs the checks and answers its own
|
- `internal/waf`: `waf`'s server, which runs the checks and answers its own
|
||||||
endpoints.
|
endpoints.
|
||||||
- `internal/wafprotocol`: the question, the answer and the report that `rproxy`
|
|
||||||
and `waf` send each other.
|
|
||||||
- `internal/ratelimit`, `internal/bans`, `internal/rules`,
|
- `internal/ratelimit`, `internal/bans`, `internal/rules`,
|
||||||
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
`internal/coreruleset`, `internal/geojs`, `internal/lookup`,
|
||||||
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and
|
||||||
`internal/alerts`: one check or source each.
|
`internal/alerts`: one check or source each.
|
||||||
- `internal/state`, `internal/requestlog`, `internal/remotelog` and
|
- `internal/state`: the state and cache files.
|
||||||
`internal/metrics`: the state and cache files, the log lines, the syslog
|
- `internal/requestlog`: the request log.
|
||||||
sender and the metrics.
|
- `internal/processlog`: the process's own messages.
|
||||||
|
- `internal/remotelog`: the syslog sender.
|
||||||
|
- `internal/metrics`: the metrics.
|
||||||
|
|
||||||
## Rationale
|
## Rationale
|
||||||
|
|
||||||
|
|||||||
@@ -73,16 +73,13 @@ never as a time.
|
|||||||
`go.uber.org/fx`. fx builds the parts of `waf` and `rproxy`, and starts
|
`go.uber.org/fx`. fx builds the parts of `waf` and `rproxy`, and starts
|
||||||
and stops their background work.
|
and stops their background work.
|
||||||
- `internal/config`: reads the settings, the one place they are read.
|
- `internal/config`: reads the settings, the one place they are read.
|
||||||
- `internal/waf`: `waf`'s server. It runs the checks in `check`, and answers
|
- `internal/waf`: `waf`'s server. It runs the checks in `check`, and holds
|
||||||
`waf`'s own endpoints in `answerOwnEndpoint`. It counts bytes, refusals
|
`waf`'s question, answer and report. It answers `waf`'s own endpoints in
|
||||||
and anomalies, and writes the log lines.
|
`answerOwnEndpoint`. It counts bytes, refusals and anomalies, and writes
|
||||||
|
the log lines.
|
||||||
- `internal/rproxy`: the reverse proxy in front of one app that asks `waf`
|
- `internal/rproxy`: the reverse proxy in front of one app that asks `waf`
|
||||||
about each request. It passes a request to the app with the standard
|
about each request. It passes a request to the app with the standard
|
||||||
library's `httputil.ReverseProxy`, within the time and size limits.
|
library's `httputil.ReverseProxy`, within the time and size limits.
|
||||||
- `internal/wafprotocol`: the question, the answer and the report that
|
|
||||||
`rproxy` and `waf` send each other (see "`waf`"). It writes and reads each
|
|
||||||
one, and declares every header, path and limit both sides use.
|
|
||||||
`internal/rproxy` imports it, not `internal/waf`.
|
|
||||||
- `internal/ratelimit`: the table of clients. It counts requests, bytes and
|
- `internal/ratelimit`: the table of clients. It counts requests, bytes and
|
||||||
refusals, and keeps each client's history.
|
refusals, and keeps each client's history.
|
||||||
- `internal/bans`: the bans. It holds each netblock's bans with their notes,
|
- `internal/bans`: the bans. It holds each netblock's bans with their notes,
|
||||||
@@ -107,8 +104,10 @@ never as a time.
|
|||||||
its own.
|
its own.
|
||||||
- `internal/state`: reads the state and cache files at start, takes in edits
|
- `internal/state`: reads the state and cache files at start, takes in edits
|
||||||
while running, and writes the files when due and at the stop.
|
while running, and writes the files when due and at the stop.
|
||||||
- `internal/requestlog` and `internal/remotelog`: the lines on stdout, and
|
- `internal/requestlog`: the request log, as JSON lines on stdout.
|
||||||
sending them to the syslog server.
|
- `internal/processlog`: the process lines on stdout, which hold the
|
||||||
|
process's own messages.
|
||||||
|
- `internal/remotelog`: sends the lines on stdout to the syslog server.
|
||||||
- `internal/metrics`: the metrics, in the Prometheus text format.
|
- `internal/metrics`: the metrics, in the Prometheus text format.
|
||||||
- `Dockerfile`: the lint and test phases, then the image.
|
- `Dockerfile`: the lint and test phases, then the image.
|
||||||
`deploy/example-app` is an app built on the image, with `rproxy` in its
|
`deploy/example-app` is an app built on the image, with `rproxy` in its
|
||||||
|
|||||||
+17
-30
@@ -27,19 +27,6 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The names of the settings that waf writes into log lines and ban notes,
|
|
||||||
// as the setting that refused a request or lowered a client's limits.
|
|
||||||
const (
|
|
||||||
ExclusivelyAllowedCountriesSetting = "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES"
|
|
||||||
TagLimitPercentSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
|
|
||||||
ASNBytesLimitPercentSetting = "SWW_WAF_ASN_BYTES_LIMIT_PERCENT"
|
|
||||||
CountryBytesLimitPercentSetting = "SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT"
|
|
||||||
UnknownCountryLimitPercentSetting = "SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT"
|
|
||||||
ASNLimitPercentURLSetting = "SWW_WAF_ASN_LIMIT_PERCENT_URL"
|
|
||||||
BlocklistActionSetting = "SWW_WAF_BLOCKLIST_ACTION"
|
|
||||||
DNSBLAbuseIPDBActionSetting = "SWW_WAF_DNSBL_ABUSEIPDB_ACTION"
|
|
||||||
)
|
|
||||||
|
|
||||||
// WAF is the settings of `sww waf`. A time or size limit of zero is off.
|
// WAF is the settings of `sww waf`. A time or size limit of zero is off.
|
||||||
type WAF struct {
|
type WAF struct {
|
||||||
// ListenAddr is where waf listens (SWW_WAF_LISTEN_ADDR).
|
// ListenAddr is where waf listens (SWW_WAF_LISTEN_ADDR).
|
||||||
@@ -631,24 +618,24 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
|||||||
"800",
|
"800",
|
||||||
),
|
),
|
||||||
ExclusivelyAllowedCountries: env.countries(
|
ExclusivelyAllowedCountries: env.countries(
|
||||||
ExclusivelyAllowedCountriesSetting, ""),
|
"SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""),
|
||||||
TagLists: env.tagLists("SWW_WAF_TAG_LISTS"),
|
TagLists: env.tagLists("SWW_WAF_TAG_LISTS"),
|
||||||
TagDeny: env.tags("SWW_WAF_TAG_DENY"),
|
TagDeny: env.tags("SWW_WAF_TAG_DENY"),
|
||||||
TagLimitPercent: env.tagPercents(TagLimitPercentSetting),
|
TagLimitPercent: env.tagPercents("SWW_WAF_TAG_LIMIT_PERCENT"),
|
||||||
RouteWatchURL: env.routeWatchURL("SWW_WAF_ROUTEWATCH_URL"),
|
RouteWatchURL: env.routeWatchURL("SWW_WAF_ROUTEWATCH_URL"),
|
||||||
ASNBytesLimitPercent: env.percents(
|
ASNBytesLimitPercent: env.percents(
|
||||||
ASNBytesLimitPercentSetting,
|
"SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
|
||||||
ParseASN,
|
ParseASN,
|
||||||
),
|
),
|
||||||
CountryBytesLimitPercent: env.percents(
|
CountryBytesLimitPercent: env.percents(
|
||||||
CountryBytesLimitPercentSetting,
|
"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
|
||||||
parseCountry,
|
parseCountry,
|
||||||
),
|
),
|
||||||
UnknownCountryLimitPercent: env.percent(
|
UnknownCountryLimitPercent: env.percent(
|
||||||
UnknownCountryLimitPercentSetting,
|
"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
|
||||||
"100",
|
"100",
|
||||||
),
|
),
|
||||||
ASNLimitPercentURL: env.listURL(ASNLimitPercentURLSetting),
|
ASNLimitPercentURL: env.listURL("SWW_WAF_ASN_LIMIT_PERCENT_URL"),
|
||||||
BlocklistURLs: env.listURLs("SWW_WAF_BLOCKLIST_URLS"),
|
BlocklistURLs: env.listURLs("SWW_WAF_BLOCKLIST_URLS"),
|
||||||
BlocklistRefresh: env.refresh("SWW_WAF_BLOCKLIST_REFRESH", "24h"),
|
BlocklistRefresh: env.refresh("SWW_WAF_BLOCKLIST_REFRESH", "24h"),
|
||||||
DNSBLZones: env.zones("SWW_WAF_DNSBL_ZONES"),
|
DNSBLZones: env.zones("SWW_WAF_DNSBL_ZONES"),
|
||||||
@@ -822,9 +809,9 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
|||||||
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
|
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
|
||||||
strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
|
strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
|
||||||
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
|
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
|
||||||
BlocklistActionSetting, string(ReputationActionDeny))
|
"SWW_WAF_BLOCKLIST_ACTION", string(ReputationActionDeny))
|
||||||
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
|
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
|
||||||
DNSBLAbuseIPDBActionSetting, "limit:25")
|
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "limit:25")
|
||||||
|
|
||||||
env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
|
env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
|
||||||
env.checkLookupDBPath(cfg)
|
env.checkLookupDBPath(cfg)
|
||||||
@@ -1233,7 +1220,7 @@ func (e *environment) checkLookupDBPath(cfg *WAF) {
|
|||||||
func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
||||||
for _, country := range cfg.ExclusivelyAllowedCountries {
|
for _, country := range cfg.ExclusivelyAllowedCountries {
|
||||||
if slices.Contains(cfg.TagDeny, CountryTagPrefix+country) {
|
if slices.Contains(cfg.TagDeny, CountryTagPrefix+country) {
|
||||||
e.check(ExclusivelyAllowedCountriesSetting,
|
e.check("SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
|
||||||
fmt.Errorf("%q %w", country, errOnBothLists))
|
fmt.Errorf("%q %w", country, errOnBothLists))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1252,15 +1239,15 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
|||||||
name string
|
name string
|
||||||
set bool
|
set bool
|
||||||
}{
|
}{
|
||||||
{ExclusivelyAllowedCountriesSetting, len(cfg.ExclusivelyAllowedCountries) > 0},
|
{"SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", len(cfg.ExclusivelyAllowedCountries) > 0},
|
||||||
{"SWW_WAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders},
|
{"SWW_WAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders},
|
||||||
{"SWW_WAF_TAG_DENY", slices.ContainsFunc(cfg.TagDeny, onlyLookupGives)},
|
{"SWW_WAF_TAG_DENY", slices.ContainsFunc(cfg.TagDeny, onlyLookupGives)},
|
||||||
{TagLimitPercentSetting, slices.ContainsFunc(
|
{"SWW_WAF_TAG_LIMIT_PERCENT", slices.ContainsFunc(
|
||||||
slices.Collect(maps.Keys(cfg.TagLimitPercent)), onlyLookupGives)},
|
slices.Collect(maps.Keys(cfg.TagLimitPercent)), onlyLookupGives)},
|
||||||
{ASNBytesLimitPercentSetting, len(cfg.ASNBytesLimitPercent) > 0},
|
{"SWW_WAF_ASN_BYTES_LIMIT_PERCENT", len(cfg.ASNBytesLimitPercent) > 0},
|
||||||
{CountryBytesLimitPercentSetting, len(cfg.CountryBytesLimitPercent) > 0},
|
{"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT", len(cfg.CountryBytesLimitPercent) > 0},
|
||||||
{UnknownCountryLimitPercentSetting, cfg.UnknownCountryLimitPercent < 100},
|
{"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT", cfg.UnknownCountryLimitPercent < 100},
|
||||||
{ASNLimitPercentURLSetting, cfg.ASNLimitPercentURL != ""},
|
{"SWW_WAF_ASN_LIMIT_PERCENT_URL", cfg.ASNLimitPercentURL != ""},
|
||||||
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0},
|
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0},
|
||||||
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0},
|
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0},
|
||||||
{"SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0},
|
{"SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0},
|
||||||
@@ -1280,7 +1267,7 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
|||||||
// blocklist too: the file at a URL is fetched as one list or the other.
|
// blocklist too: the file at a URL is fetched as one list or the other.
|
||||||
func (e *environment) checkASNLimitPercentURL(cfg *WAF) {
|
func (e *environment) checkASNLimitPercentURL(cfg *WAF) {
|
||||||
if slices.Contains(cfg.BlocklistURLs, cfg.ASNLimitPercentURL) {
|
if slices.Contains(cfg.BlocklistURLs, cfg.ASNLimitPercentURL) {
|
||||||
e.check(ASNLimitPercentURLSetting,
|
e.check("SWW_WAF_ASN_LIMIT_PERCENT_URL",
|
||||||
fmt.Errorf("%q %w", cfg.ASNLimitPercentURL, errInBlocklistURLs))
|
fmt.Errorf("%q %w", cfg.ASNLimitPercentURL, errInBlocklistURLs))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1307,7 +1294,7 @@ func (e *environment) checkTags(cfg *WAF) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for tag := range cfg.TagLimitPercent {
|
for tag := range cfg.TagLimitPercent {
|
||||||
e.check(TagLimitPercentSetting, checkTag(cfg, tag))
|
e.check("SWW_WAF_TAG_LIMIT_PERCENT", checkTag(cfg, tag))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+14
-24
@@ -84,12 +84,7 @@ type Params struct {
|
|||||||
// one request to GeoJS is under way at a time, and it asks about every
|
// one request to GeoJS is under way at a time, and it asks about every
|
||||||
// client waiting, up to maxPerRequest. It is safe for concurrent use.
|
// client waiting, up to maxPerRequest. It is safe for concurrent use.
|
||||||
type GeoJS struct {
|
type GeoJS struct {
|
||||||
url string
|
params Params
|
||||||
answered func(Answer)
|
|
||||||
now func() time.Time
|
|
||||||
asked func()
|
|
||||||
failed func(err error, askingAgainIn time.Duration)
|
|
||||||
wentWithoutAnswer func()
|
|
||||||
// httpClient follows no redirect, so that visitors' addresses go to
|
// httpClient follows no redirect, so that visitors' addresses go to
|
||||||
// GeoJS alone: a redirect is a failure.
|
// GeoJS alone: a redirect is a failure.
|
||||||
httpClient *http.Client
|
httpClient *http.Client
|
||||||
@@ -145,12 +140,7 @@ func New(params Params) *GeoJS {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &GeoJS{
|
return &GeoJS{
|
||||||
url: params.URL,
|
params: params,
|
||||||
answered: params.Answered,
|
|
||||||
now: params.Now,
|
|
||||||
asked: params.Asked,
|
|
||||||
failed: params.Failed,
|
|
||||||
wentWithoutAnswer: params.WentWithoutAnswer,
|
|
||||||
httpClient: &http.Client{
|
httpClient: &http.Client{
|
||||||
CheckRedirect: func(*http.Request, []*http.Request) error {
|
CheckRedirect: func(*http.Request, []*http.Request) error {
|
||||||
return http.ErrUseLastResponse
|
return http.ErrUseLastResponse
|
||||||
@@ -193,7 +183,7 @@ func (g *GeoJS) LookUp(
|
|||||||
|
|
||||||
answer, found := g.kept(client)
|
answer, found := g.kept(client)
|
||||||
if !found {
|
if !found {
|
||||||
g.wentWithoutAnswer()
|
g.params.WentWithoutAnswer()
|
||||||
}
|
}
|
||||||
|
|
||||||
w, waiting := g.waiting[client]
|
w, waiting := g.waiting[client]
|
||||||
@@ -273,7 +263,7 @@ func (g *GeoJS) Load(answers []Answer) {
|
|||||||
|
|
||||||
g.answers.Purge()
|
g.answers.Purge()
|
||||||
|
|
||||||
now := g.now()
|
now := g.params.Now()
|
||||||
|
|
||||||
for _, answer := range answers {
|
for _, answer := range answers {
|
||||||
if now.Sub(answer.Answered) < keepFor {
|
if now.Sub(answer.Answered) < keepFor {
|
||||||
@@ -310,7 +300,7 @@ func (g *GeoJS) answerOrWait(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if w == nil {
|
if w == nil {
|
||||||
g.wentWithoutAnswer()
|
g.params.WentWithoutAnswer()
|
||||||
|
|
||||||
return Answer{}, nil // too many clients wait already
|
return Answer{}, nil // too many clients wait already
|
||||||
}
|
}
|
||||||
@@ -322,7 +312,7 @@ func (g *GeoJS) answerOrWait(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if w.late {
|
if w.late {
|
||||||
g.wentWithoutAnswer()
|
g.params.WentWithoutAnswer()
|
||||||
|
|
||||||
return Answer{}, nil
|
return Answer{}, nil
|
||||||
}
|
}
|
||||||
@@ -333,7 +323,7 @@ func (g *GeoJS) answerOrWait(
|
|||||||
// kept returns client's answer, if GeoJS gave it less than keepFor ago,
|
// kept returns client's answer, if GeoJS gave it less than keepFor ago,
|
||||||
// and notes that it was used.
|
// and notes that it was used.
|
||||||
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
||||||
now := g.now()
|
now := g.params.Now()
|
||||||
|
|
||||||
kept, found := g.answers.Get(client)
|
kept, found := g.answers.Get(client)
|
||||||
if !found || now.Sub(kept.Answered) >= keepFor {
|
if !found || now.Sub(kept.Answered) >= keepFor {
|
||||||
@@ -350,7 +340,7 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
|||||||
// been called. The requests to GeoJS are for every client waiting, so they
|
// been called. The requests to GeoJS are for every client waiting, so they
|
||||||
// go on when the caller whose ctx is given stops waiting, until StopAsking.
|
// go on when the caller whose ctx is given stops waiting, until StopAsking.
|
||||||
func (g *GeoJS) ask(ctx context.Context) {
|
func (g *GeoJS) ask(ctx context.Context) {
|
||||||
if g.asking || g.stopped || g.now().Before(g.retryAt) {
|
if g.asking || g.stopped || g.params.Now().Before(g.retryAt) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -382,9 +372,9 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
|||||||
|
|
||||||
kept, answered := g.keep(clients, given, err)
|
kept, answered := g.keep(clients, given, err)
|
||||||
|
|
||||||
if g.answered != nil {
|
if g.params.Answered != nil {
|
||||||
for _, answer := range kept {
|
for _, answer := range kept {
|
||||||
g.answered(answer)
|
g.params.Answered(answer)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -433,7 +423,7 @@ func (g *GeoJS) keep(
|
|||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
defer g.mu.Unlock()
|
defer g.mu.Unlock()
|
||||||
|
|
||||||
now := g.now()
|
now := g.params.Now()
|
||||||
kept := make([]Answer, 0, len(clients))
|
kept := make([]Answer, 0, len(clients))
|
||||||
leftOut := 0
|
leftOut := 0
|
||||||
|
|
||||||
@@ -477,7 +467,7 @@ func (g *GeoJS) keep(
|
|||||||
w.late = true
|
w.late = true
|
||||||
}
|
}
|
||||||
|
|
||||||
g.failed(err, g.retryDelay)
|
g.params.Failed(err, g.retryDelay)
|
||||||
|
|
||||||
return kept, false
|
return kept, false
|
||||||
}
|
}
|
||||||
@@ -507,7 +497,7 @@ func (g *GeoJS) request(
|
|||||||
req, err := http.NewRequestWithContext(
|
req, err := http.NewRequestWithContext(
|
||||||
ctx,
|
ctx,
|
||||||
http.MethodGet,
|
http.MethodGet,
|
||||||
g.url,
|
g.params.URL,
|
||||||
http.NoBody,
|
http.NoBody,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -516,7 +506,7 @@ func (g *GeoJS) request(
|
|||||||
|
|
||||||
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
|
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
|
||||||
|
|
||||||
g.asked()
|
g.params.Asked()
|
||||||
|
|
||||||
res, err := g.httpClient.Do(req)
|
res, err := g.httpClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -0,0 +1,44 @@
|
|||||||
|
// Package processlog builds the logger for the process's own messages,
|
||||||
|
// which it writes as JSON lines marked "type":"process". The request log
|
||||||
|
// is package requestlog's.
|
||||||
|
package processlog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"log/slog"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Params are what New needs.
|
||||||
|
type Params struct {
|
||||||
|
// Output is where the lines are written.
|
||||||
|
Output io.Writer
|
||||||
|
// InstanceName, SWW_WAF_INSTANCE_NAME, is each line's instance.
|
||||||
|
InstanceName string
|
||||||
|
// Level, SWW_WAF_LOG_LEVEL, is the least severe level written.
|
||||||
|
Level slog.Level
|
||||||
|
}
|
||||||
|
|
||||||
|
// New returns the logger for the process's own messages: JSON lines on
|
||||||
|
// params.Output, marked "type":"process", with the time in the same form
|
||||||
|
// as a request line's. It writes only the messages at params.Level or more
|
||||||
|
// severe; the request lines requestlog.Write writes are never held back.
|
||||||
|
func New(params Params) *slog.Logger {
|
||||||
|
handler := slog.NewJSONHandler(params.Output, &slog.HandlerOptions{
|
||||||
|
Level: params.Level,
|
||||||
|
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
|
||||||
|
if attr.Key == slog.TimeKey && len(groups) == 0 {
|
||||||
|
return slog.String(
|
||||||
|
slog.TimeKey,
|
||||||
|
requestlog.FormatTime(attr.Value.Time()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return attr
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
return slog.New(handler).
|
||||||
|
With("type", "process", "instance", params.InstanceName)
|
||||||
|
}
|
||||||
@@ -0,0 +1,105 @@
|
|||||||
|
package processlog_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"log/slog"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
|
)
|
||||||
|
|
||||||
|
// instanceName is the instance the tests' loggers are built with.
|
||||||
|
const instanceName = "host1/blog"
|
||||||
|
|
||||||
|
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
|
||||||
|
processlog.New(processlog.Params{
|
||||||
|
Output: &out,
|
||||||
|
InstanceName: instanceName,
|
||||||
|
Level: slog.LevelInfo,
|
||||||
|
}).
|
||||||
|
Info("starting",
|
||||||
|
"version", "v1")
|
||||||
|
|
||||||
|
var fields map[string]any
|
||||||
|
|
||||||
|
err := json.Unmarshal(out.Bytes(), &fields)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decode %q: %v", out.String(), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if fields["type"] != "process" ||
|
||||||
|
fields["instance"] != "host1/blog" ||
|
||||||
|
fields["msg"] != "starting" ||
|
||||||
|
fields["level"] != "INFO" ||
|
||||||
|
fields["version"] != "v1" {
|
||||||
|
t.Errorf("process line %v", fields)
|
||||||
|
}
|
||||||
|
|
||||||
|
timeText, _ := fields["time"].(string)
|
||||||
|
|
||||||
|
logged, err := time.Parse(time.RFC3339, timeText)
|
||||||
|
if err != nil || !strings.HasSuffix(timeText, "Z") ||
|
||||||
|
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
|
||||||
|
time.Since(logged) > time.Minute {
|
||||||
|
t.Errorf(
|
||||||
|
"process line time %q, want now in UTC with milliseconds",
|
||||||
|
timeText,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
levels := []slog.Level{
|
||||||
|
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, level := range levels {
|
||||||
|
t.Run(level.String(), func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
|
||||||
|
processLog := processlog.New(processlog.Params{
|
||||||
|
Output: &out,
|
||||||
|
InstanceName: instanceName,
|
||||||
|
Level: level,
|
||||||
|
})
|
||||||
|
for _, at := range levels {
|
||||||
|
processLog.Log(t.Context(), at, "message")
|
||||||
|
}
|
||||||
|
|
||||||
|
var got, want []string
|
||||||
|
|
||||||
|
for line := range strings.Lines(out.String()) {
|
||||||
|
var fields struct {
|
||||||
|
Level string `json:"level"`
|
||||||
|
}
|
||||||
|
|
||||||
|
err := json.Unmarshal([]byte(line), &fields)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decode %q: %v", line, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
got = append(got, fields.Level)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, written := range levels[i:] {
|
||||||
|
want = append(want, written.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
if !slices.Equal(got, want) {
|
||||||
|
t.Errorf("lines at %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
func TestHistoryKeepsEveryRequest(t *testing.T) {
|
func TestHistoryKeepsEveryRequest(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -57,7 +57,7 @@ func TestHistoryKeepsEveryRequest(t *testing.T) {
|
|||||||
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
other := netip.MustParsePrefix("198.51.100.7/32")
|
other := netip.MustParsePrefix("198.51.100.7/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -100,7 +100,10 @@ func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
|
|||||||
func TestResetKeepsTheHistory(t *testing.T) {
|
func TestResetKeepsTheHistory(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerMinute: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -123,7 +126,7 @@ func TestResetKeepsTheHistory(t *testing.T) {
|
|||||||
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
|
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
|
|
||||||
for client, requests := range map[string]int{
|
for client, requests := range map[string]int{
|
||||||
"198.51.100.9/32": 2,
|
"198.51.100.9/32": 2,
|
||||||
|
|||||||
@@ -156,16 +156,28 @@ type Request struct {
|
|||||||
BlockedByCoreRuleSet bool
|
BlockedByCoreRuleSet bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// New returns a Limiter for limits, with no client counted yet, whose
|
// Params are what New needs.
|
||||||
// table holds at most maxClients clients (SWW_WAF_MAX_TRACKED_CLIENTS). Past
|
type Params struct {
|
||||||
// it, the least recently seen client is dropped, with its history, and
|
// Limits are what each client's requests and bytes are counted against.
|
||||||
// starts afresh if it comes back.
|
Limits Limits
|
||||||
func New(limits Limits, maxClients int) *Limiter {
|
// MaxClients is the most clients the table holds,
|
||||||
clients, err := simplelru.NewLRU[netip.Prefix, *Client](maxClients, nil)
|
// SWW_WAF_MAX_TRACKED_CLIENTS. Past it, the least recently seen client
|
||||||
|
// is dropped, with its history, and starts afresh if it comes back.
|
||||||
|
MaxClients int
|
||||||
|
}
|
||||||
|
|
||||||
|
// New returns a Limiter with no client counted yet.
|
||||||
|
func New(params Params) *Limiter {
|
||||||
|
clients, err := simplelru.NewLRU[netip.Prefix, *Client](
|
||||||
|
params.MaxClients,
|
||||||
|
nil,
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err) // NewLRU fails only for a size below one
|
panic(err) // NewLRU fails only for a size below one
|
||||||
}
|
}
|
||||||
|
|
||||||
|
limits := params.Limits
|
||||||
|
|
||||||
return &Limiter{
|
return &Limiter{
|
||||||
windows: [3]window{
|
windows: [3]window{
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -33,10 +33,10 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
|
|||||||
|
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
now := midnight()
|
now := midnight()
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
|
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
|
|
||||||
if limiter.OverARateLimit(client, now) {
|
if limiter.OverARateLimit(client, now) {
|
||||||
t.Error("a client never seen is over a rate limit")
|
t.Error("a client never seen is over a rate limit")
|
||||||
@@ -72,7 +72,7 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
|
|||||||
wantCount(t, limiter, client, now, minute)
|
wantCount(t, limiter, client, now, minute)
|
||||||
|
|
||||||
// A limit that is off is never gone over.
|
// A limit that is off is never gone over.
|
||||||
off := ratelimit.New(ratelimit.Limits{}, tableSize)
|
off := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
off.Count(client, now, whole)
|
off.Count(client, now, whole)
|
||||||
|
|
||||||
if off.OverARateLimit(client, now) {
|
if off.OverARateLimit(client, now) {
|
||||||
@@ -95,7 +95,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
|
|||||||
t.Run(tc.window, func(t *testing.T) {
|
t.Run(tc.window, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(tc.limits, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: tc.limits,
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
quarter := tc.length / 4
|
quarter := tc.length / 4
|
||||||
@@ -126,10 +129,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
|
|||||||
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: limit, PerHour: limit},
|
Limits: ratelimit.Limits{PerMinute: limit, PerHour: limit},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -159,10 +162,13 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
|
|||||||
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
|
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: 5, BytesPerDay: math.MaxInt64},
|
Limits: ratelimit.Limits{
|
||||||
tableSize,
|
PerMinute: 5,
|
||||||
)
|
BytesPerDay: math.MaxInt64,
|
||||||
|
},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -203,10 +209,10 @@ func TestZeroPercentIsAZeroAllowanceAndALimitOffStaysOff(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// Only the hour has limits: the minute's and the day's are off.
|
// Only the hour has limits: the minute's and the day's are off.
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
|
Limits: ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -253,7 +259,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
|
|||||||
t.Run(tc.window, func(t *testing.T) {
|
t.Run(tc.window, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(tc.limits, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: tc.limits,
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
|
|
||||||
// 600 bytes are within the limit, 600 more over it.
|
// 600 bytes are within the limit, 600 more over it.
|
||||||
@@ -287,10 +296,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
|
|||||||
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
|
Limits: ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
other := netip.MustParsePrefix("203.0.113.10/32")
|
other := netip.MustParsePrefix("203.0.113.10/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -314,7 +323,7 @@ func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
|
|||||||
func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -340,7 +349,10 @@ func TestCountGivesTheBytesInEachWindow(t *testing.T) {
|
|||||||
func TestResetSetsTheBytesBackToZero(t *testing.T) {
|
func TestResetSetsTheBytesBackToZero(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{BytesPerDay: 1000}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{BytesPerDay: 1000},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -356,7 +368,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -411,7 +423,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
|
|||||||
func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -438,10 +450,10 @@ func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
|
|||||||
func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: limit, PerDay: limit},
|
Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -463,7 +475,10 @@ func TestResetSetsTheCountsBackToZero(t *testing.T) {
|
|||||||
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerHour: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -482,10 +497,10 @@ func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
|
|||||||
func TestRefusedRequestsCount(t *testing.T) {
|
func TestRefusedRequestsCount(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
|
Limits: ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
|
||||||
tableSize,
|
MaxClients: tableSize,
|
||||||
)
|
})
|
||||||
refused := netip.MustParsePrefix("203.0.113.9/32")
|
refused := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
within := netip.MustParsePrefix("203.0.113.10/32")
|
within := netip.MustParsePrefix("203.0.113.10/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
@@ -518,7 +533,10 @@ func TestRefusedRequestsCount(t *testing.T) {
|
|||||||
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerMinute: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -534,7 +552,10 @@ func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
|
|||||||
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
|
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerHour: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
@@ -562,7 +583,10 @@ func TestKeepsAtMostMaxClientsDroppingTheLeastRecentlySeen(t *testing.T) {
|
|||||||
|
|
||||||
const maxClients = 3
|
const maxClients = 3
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{PerMinute: 1}, maxClients)
|
limiter := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerMinute: 1},
|
||||||
|
MaxClients: maxClients,
|
||||||
|
})
|
||||||
now := midnight()
|
now := midnight()
|
||||||
|
|
||||||
clients := make([]netip.Prefix, maxClients+1)
|
clients := make([]netip.Prefix, maxClients+1)
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ func TestSnapshotListsTheClientsByAddress(t *testing.T) {
|
|||||||
"2001:db8::/64",
|
"2001:db8::/64",
|
||||||
}
|
}
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
for _, i := range []int{2, 3, 0, 1} {
|
for _, i := range []int{2, 3, 0, 1} {
|
||||||
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
|
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
|
||||||
}
|
}
|
||||||
@@ -52,7 +52,10 @@ func TestLoadedCountsCarryOn(t *testing.T) {
|
|||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
before := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
before := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerHour: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
for range limit {
|
for range limit {
|
||||||
wantCount(t, before, client, start, "")
|
wantCount(t, before, client, start, "")
|
||||||
}
|
}
|
||||||
@@ -60,7 +63,10 @@ func TestLoadedCountsCarryOn(t *testing.T) {
|
|||||||
// Loaded into a new limiter, as across a restart, the client has no
|
// Loaded into a new limiter, as across a restart, the client has no
|
||||||
// fresh allowance.
|
// fresh allowance.
|
||||||
later := start.Add(time.Minute)
|
later := start.Add(time.Minute)
|
||||||
after := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize)
|
after := ratelimit.New(ratelimit.Params{
|
||||||
|
Limits: ratelimit.Limits{PerHour: limit},
|
||||||
|
MaxClients: tableSize,
|
||||||
|
})
|
||||||
after.Load(before.Snapshot(), later)
|
after.Load(before.Snapshot(), later)
|
||||||
wantCount(t, after, client, later, hour)
|
wantCount(t, after, client, later, hour)
|
||||||
}
|
}
|
||||||
@@ -71,7 +77,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
|
|||||||
client := netip.MustParsePrefix("203.0.113.9/32")
|
client := netip.MustParsePrefix("203.0.113.9/32")
|
||||||
start := midnight()
|
start := midnight()
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, tableSize)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
limiter.Count(client, start, whole)
|
limiter.Count(client, start, whole)
|
||||||
limiter.CountBytes(client, start, 5, whole)
|
limiter.CountBytes(client, start, 5, whole)
|
||||||
limiter.CountRefusal(client, start, limit)
|
limiter.CountRefusal(client, start, limit)
|
||||||
@@ -80,7 +86,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
|
|||||||
loaded := func(now time.Time) ratelimit.Client {
|
loaded := func(now time.Time) ratelimit.Client {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
after := ratelimit.New(ratelimit.Limits{}, tableSize)
|
after := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
|
||||||
after.Load(limiter.Snapshot(), now)
|
after.Load(limiter.Snapshot(), now)
|
||||||
|
|
||||||
return after.Snapshot()[0]
|
return after.Snapshot()[0]
|
||||||
@@ -142,7 +148,7 @@ func TestLoadDropsTheLeastRecentlySeenFirst(t *testing.T) {
|
|||||||
addr = addr.Next()
|
addr = addr.Next()
|
||||||
}
|
}
|
||||||
|
|
||||||
limiter := ratelimit.New(ratelimit.Limits{}, maxClients)
|
limiter := ratelimit.New(ratelimit.Params{MaxClients: maxClients})
|
||||||
limiter.Load(clients, midnight())
|
limiter.Load(clients, midnight())
|
||||||
|
|
||||||
got := limiter.Snapshot()
|
got := limiter.Snapshot()
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
// Package requestlog writes the lines sww prints on stdout: one
|
// Package requestlog writes the request log sww prints on stdout: JSON
|
||||||
// JSON object per request, two from waf and a short one from rproxy,
|
// lines marked "type":"request", at most two from waf and a short one
|
||||||
// marked "type":"request", and the process's own messages as JSON lines marked
|
// from rproxy for each request. Package processlog writes the process's
|
||||||
// "type":"process".
|
// own messages.
|
||||||
package requestlog
|
package requestlog
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -9,7 +9,6 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
@@ -300,31 +299,3 @@ func Milliseconds(d time.Duration) float64 {
|
|||||||
time.Millisecond/time.Microsecond,
|
time.Millisecond/time.Microsecond,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewProcessLogger returns the logger for the process's own messages:
|
|
||||||
// JSON lines on w, marked "type":"process", with the time in the same form
|
|
||||||
// as a request line's, and instanceName, SWW_WAF_INSTANCE_NAME, as instance.
|
|
||||||
// It writes only the messages at level, SWW_WAF_LOG_LEVEL, or more severe;
|
|
||||||
// the request lines Write writes are never held back.
|
|
||||||
func NewProcessLogger(
|
|
||||||
w io.Writer,
|
|
||||||
instanceName string,
|
|
||||||
level slog.Level,
|
|
||||||
) *slog.Logger {
|
|
||||||
handler := slog.NewJSONHandler(w, &slog.HandlerOptions{
|
|
||||||
Level: level,
|
|
||||||
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
|
|
||||||
if attr.Key == slog.TimeKey && len(groups) == 0 {
|
|
||||||
return slog.String(
|
|
||||||
slog.TimeKey,
|
|
||||||
FormatTime(attr.Value.Time()),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return attr
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
return slog.New(handler).
|
|
||||||
With("type", "process", "instance", instanceName)
|
|
||||||
}
|
|
||||||
@@ -3,8 +3,6 @@ package requestlog_test
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"log/slog"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -134,88 +132,3 @@ func TestWriteRProxyLineWritesOneJSONLineWithOnlyItsFields(t *testing.T) {
|
|||||||
t.Errorf("wrote %s, want %s", out.String(), want)
|
t.Errorf("wrote %s, want %s", out.String(), want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
requestlog.NewProcessLogger(&out, "host1/blog", slog.LevelInfo).
|
|
||||||
Info("starting",
|
|
||||||
"version", "v1")
|
|
||||||
|
|
||||||
var fields map[string]any
|
|
||||||
|
|
||||||
err := json.Unmarshal(out.Bytes(), &fields)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("decode %q: %v", out.String(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if fields["type"] != "process" ||
|
|
||||||
fields["instance"] != "host1/blog" ||
|
|
||||||
fields["msg"] != "starting" ||
|
|
||||||
fields["level"] != "INFO" ||
|
|
||||||
fields["version"] != "v1" {
|
|
||||||
t.Errorf("process line %v", fields)
|
|
||||||
}
|
|
||||||
|
|
||||||
timeText, _ := fields["time"].(string)
|
|
||||||
|
|
||||||
logged, err := time.Parse(time.RFC3339, timeText)
|
|
||||||
if err != nil || !strings.HasSuffix(timeText, "Z") ||
|
|
||||||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
|
|
||||||
time.Since(logged) > time.Minute {
|
|
||||||
t.Errorf(
|
|
||||||
"process line time %q, want now in UTC with milliseconds",
|
|
||||||
timeText,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
levels := []slog.Level{
|
|
||||||
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, level := range levels {
|
|
||||||
t.Run(level.String(), func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
processLog := requestlog.NewProcessLogger(
|
|
||||||
&out,
|
|
||||||
"host1/blog",
|
|
||||||
level,
|
|
||||||
)
|
|
||||||
for _, at := range levels {
|
|
||||||
processLog.Log(t.Context(), at, "message")
|
|
||||||
}
|
|
||||||
|
|
||||||
var got, want []string
|
|
||||||
|
|
||||||
for line := range strings.Lines(out.String()) {
|
|
||||||
var fields struct {
|
|
||||||
Level string `json:"level"`
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(line), &fields)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("decode %q: %v", line, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
got = append(got, fields.Level)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, written := range levels[i:] {
|
|
||||||
want = append(want, written.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(got, want) {
|
|
||||||
t.Errorf("lines at %v, want %v", got, want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -7,6 +7,16 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// The headers rproxy sets on the requests it passes to the app. Go writes every
|
||||||
|
// header name in this form, as it sends it and as it receives it, so
|
||||||
|
// X-Client-ASN arrives as X-Client-Asn, and Del removes a visitor's own whatever
|
||||||
|
// their case; header names are not case-sensitive.
|
||||||
|
const (
|
||||||
|
requestIDHeader = "X-Request-ID"
|
||||||
|
asnHeader = "X-Client-Asn"
|
||||||
|
countryHeader = "X-Client-Country"
|
||||||
|
)
|
||||||
|
|
||||||
// peerAddress is the address of the request's TCP peer, normally traefik, without
|
// peerAddress is the address of the request's TCP peer, normally traefik, without
|
||||||
// any IPv6 zone, which waf refuses.
|
// any IPv6 zone, which waf refuses.
|
||||||
func peerAddress(r *http.Request) netip.Addr {
|
func peerAddress(r *http.Request) netip.Addr {
|
||||||
|
|||||||
@@ -18,9 +18,9 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
|
|
||||||
// post is the start of a POST whose headers end with what follows it.
|
// post is the start of a POST whose headers end with what follows it.
|
||||||
@@ -51,11 +51,11 @@ func TestServerHasTheFixedLimits(t *testing.T) {
|
|||||||
server := rproxy.New(rproxy.Params{
|
server := rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: io.Discard,
|
RequestLog: io.Discard,
|
||||||
ProcessLog: requestlog.NewProcessLogger(
|
ProcessLog: processlog.New(processlog.Params{
|
||||||
io.Discard,
|
Output: io.Discard,
|
||||||
"test",
|
InstanceName: "test",
|
||||||
slog.LevelInfo,
|
Level: slog.LevelInfo,
|
||||||
),
|
}),
|
||||||
})
|
})
|
||||||
|
|
||||||
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
|
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
|
||||||
@@ -492,7 +492,7 @@ func TestASettingOfOffSetsNoLimit(t *testing.T) {
|
|||||||
strings.NewReader("a body"))),
|
strings.NewReader("a body"))),
|
||||||
)
|
)
|
||||||
|
|
||||||
if got := get(t, addr, wafprotocol.HealthPath); got.status != http.StatusOK {
|
if got := get(t, addr, waf.HealthPath); got.status != http.StatusOK {
|
||||||
t.Errorf("answered %+v, want rproxy's own 200", got)
|
t.Errorf("answered %+v, want rproxy's own 200", got)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -1471,7 +1471,7 @@ func TestAVisitorThatStopsALittleShortGetsAnAnswerRProxyGivesAtOnce(
|
|||||||
{"a deny", post, http.StatusForbidden},
|
{"a deny", post, http.StatusForbidden},
|
||||||
{
|
{
|
||||||
"the health check",
|
"the health check",
|
||||||
"GET " + wafprotocol.HealthPath + " HTTP/1.1\r\nHost: site.example\r\n",
|
"GET " + waf.HealthPath + " HTTP/1.1\r\nHost: site.example\r\n",
|
||||||
http.StatusOK,
|
http.StatusOK,
|
||||||
},
|
},
|
||||||
} {
|
} {
|
||||||
|
|||||||
@@ -12,11 +12,11 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
|
|
||||||
// questionBodyMaxBytes is the most of a body a question carries.
|
// questionBodyMaxBytes is the most of a body a question carries.
|
||||||
const questionBodyMaxBytes = wafprotocol.QuestionBodyMaxBytes
|
const questionBodyMaxBytes = waf.QuestionBodyMaxBytes
|
||||||
|
|
||||||
// chunked is the encoding of a body whose length the visitor did not announce.
|
// chunked is the encoding of a body whose length the visitor did not announce.
|
||||||
const chunked = "chunked"
|
const chunked = "chunked"
|
||||||
@@ -42,9 +42,9 @@ func TestQuestionIsTheVisitorsRequestWithRProxysHeaders(t *testing.T) {
|
|||||||
req.Header.Set(forwardedForHeader, "198.51.100.1")
|
req.Header.Set(forwardedForHeader, "198.51.100.1")
|
||||||
req.Header.Set(forwardedProtoHeader, "gopher")
|
req.Header.Set(forwardedProtoHeader, "gopher")
|
||||||
req.Header.Set(requestIDHeader, "the-visitor's")
|
req.Header.Set(requestIDHeader, "the-visitor's")
|
||||||
req.Header.Set(wafprotocol.ClientHeader, "198.51.100.2")
|
req.Header.Set(waf.ClientHeader, "198.51.100.2")
|
||||||
req.Header.Set(wafprotocol.PeerHeader, "198.51.100.3")
|
req.Header.Set(waf.PeerHeader, "198.51.100.3")
|
||||||
req.Header.Set(wafprotocol.ContentLengthHeader, "12345")
|
req.Header.Set(waf.ContentLengthHeader, "12345")
|
||||||
// Go's client then sends no User-Agent, so that one in the question would be
|
// Go's client then sends no User-Agent, so that one in the question would be
|
||||||
// Go's own.
|
// Go's own.
|
||||||
req.Header["User-Agent"] = nil
|
req.Header["User-Agent"] = nil
|
||||||
@@ -73,8 +73,8 @@ func TestQuestionIsTheVisitorsRequestWithRProxysHeaders(t *testing.T) {
|
|||||||
want.Set(forwardedForHeader, "198.51.100.1")
|
want.Set(forwardedForHeader, "198.51.100.1")
|
||||||
want.Set(forwardedProtoHeader, "http")
|
want.Set(forwardedProtoHeader, "http")
|
||||||
want.Set(requestIDHeader, asked.header.Get(requestIDHeader))
|
want.Set(requestIDHeader, asked.header.Get(requestIDHeader))
|
||||||
want.Set(wafprotocol.ClientHeader, localhost)
|
want.Set(waf.ClientHeader, localhost)
|
||||||
want.Set(wafprotocol.PeerHeader, localhost)
|
want.Set(waf.PeerHeader, localhost)
|
||||||
|
|
||||||
if !maps.EqualFunc(asked.header, want, slices.Equal) {
|
if !maps.EqualFunc(asked.header, want, slices.Equal) {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
@@ -124,8 +124,8 @@ func TestQuestionNamesTheVisitorATrustedProxySpeaksFor(t *testing.T) {
|
|||||||
wantGreeting(t, visit(t, req))
|
wantGreeting(t, visit(t, req))
|
||||||
|
|
||||||
asked := standIn.question(t)
|
asked := standIn.question(t)
|
||||||
if asked.header.Get(wafprotocol.ClientHeader) != tc.client ||
|
if asked.header.Get(waf.ClientHeader) != tc.client ||
|
||||||
asked.header.Get(wafprotocol.PeerHeader) != localhost ||
|
asked.header.Get(waf.PeerHeader) != localhost ||
|
||||||
asked.header.Get(forwardedForHeader) != visitor ||
|
asked.header.Get(forwardedForHeader) != visitor ||
|
||||||
asked.header.Get(forwardedProtoHeader) != tc.proto {
|
asked.header.Get(forwardedProtoHeader) != tc.proto {
|
||||||
t.Errorf("the question's headers are %v", asked.header)
|
t.Errorf("the question's headers are %v", asked.header)
|
||||||
@@ -191,7 +191,7 @@ func TestQuestionNamesTheClientAsTheForwardedAddressesSay(t *testing.T) {
|
|||||||
req.Header[forwardedForHeader] = tc.header
|
req.Header[forwardedForHeader] = tc.header
|
||||||
wantGreeting(t, visit(t, req))
|
wantGreeting(t, visit(t, req))
|
||||||
|
|
||||||
if got := standIn.question(t).header.Get(wafprotocol.ClientHeader); got !=
|
if got := standIn.question(t).header.Get(waf.ClientHeader); got !=
|
||||||
tc.client {
|
tc.client {
|
||||||
t.Errorf("the question names %s, want %s", got, tc.client)
|
t.Errorf("the question names %s, want %s", got, tc.client)
|
||||||
}
|
}
|
||||||
@@ -282,7 +282,7 @@ func wantQuestionBody(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
values := asked.header.Values(wafprotocol.ContentLengthHeader)
|
values := asked.header.Values(waf.ContentLengthHeader)
|
||||||
|
|
||||||
if announced {
|
if announced {
|
||||||
if !slices.Equal(values, []string{strconv.Itoa(size)}) ||
|
if !slices.Equal(values, []string{strconv.Itoa(size)}) ||
|
||||||
@@ -586,8 +586,8 @@ func TestAQuestionNamesNoZone(t *testing.T) {
|
|||||||
server.Handler.ServeHTTP(httptest.NewRecorder(), req)
|
server.Handler.ServeHTTP(httptest.NewRecorder(), req)
|
||||||
|
|
||||||
asked := standIn.question(t)
|
asked := standIn.question(t)
|
||||||
if asked.header.Get(wafprotocol.ClientHeader) != tc.client ||
|
if asked.header.Get(waf.ClientHeader) != tc.client ||
|
||||||
asked.header.Get(wafprotocol.PeerHeader) != tc.peer {
|
asked.header.Get(waf.PeerHeader) != tc.peer {
|
||||||
t.Errorf("the question's headers are %v", asked.header)
|
t.Errorf("the question's headers are %v", asked.header)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+13
-11
@@ -18,13 +18,17 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
|
|
||||||
// flushAfterEachWrite has ReverseProxy pass on each part of the app's answer as
|
// flushAfterEachWrite has ReverseProxy pass on each part of the app's answer as
|
||||||
// soon as it arrives.
|
// soon as it arrives.
|
||||||
const flushAfterEachWrite time.Duration = -1
|
const flushAfterEachWrite time.Duration = -1
|
||||||
|
|
||||||
|
// reportStatusMax is the highest status waf takes in a report. An app can
|
||||||
|
// send a higher one, up to 999, and the visitor gets that as it is.
|
||||||
|
const reportStatusMax = 599
|
||||||
|
|
||||||
// plainText is the Content-Type of the answers rproxy gives itself.
|
// plainText is the Content-Type of the answers rproxy gives itself.
|
||||||
const plainText = "text/plain; charset=utf-8"
|
const plainText = "text/plain; charset=utf-8"
|
||||||
|
|
||||||
@@ -150,7 +154,7 @@ func (rq *request) readFirstPartOfBody() bool {
|
|||||||
|
|
||||||
_ = rq.rc.SetReadDeadline(rq.requestDeadline())
|
_ = rq.rc.SetReadDeadline(rq.requestDeadline())
|
||||||
held, err := io.ReadAll(
|
held, err := io.ReadAll(
|
||||||
io.LimitReader(rq.body, wafprotocol.QuestionBodyMaxBytes),
|
io.LimitReader(rq.body, waf.QuestionBodyMaxBytes),
|
||||||
)
|
)
|
||||||
// From here on, the request timer ends a read of the rest of the body, so that
|
// From here on, the request timer ends a read of the rest of the body, so that
|
||||||
// the answer can name the side rproxy was waiting on.
|
// the answer can name the side rproxy was waiting on.
|
||||||
@@ -224,17 +228,16 @@ func (rq *request) rewrite(pr *httputil.ProxyRequest) {
|
|||||||
setPathAsSent(pr.Out.URL, pr.In)
|
setPathAsSent(pr.Out.URL, pr.In)
|
||||||
|
|
||||||
setForwardedHeaders(pr.In, pr.Out, rq.peer, rq.peerTrusted)
|
setForwardedHeaders(pr.In, pr.Out, rq.peer, rq.peerTrusted)
|
||||||
pr.Out.Header.Set(wafprotocol.RequestIDHeader, rq.id)
|
pr.Out.Header.Set(requestIDHeader, rq.id)
|
||||||
// Del removes a visitor's own, whatever their case.
|
pr.Out.Header.Del(asnHeader)
|
||||||
pr.Out.Header.Del(wafprotocol.ASNHeader)
|
pr.Out.Header.Del(countryHeader)
|
||||||
pr.Out.Header.Del(wafprotocol.CountryHeader)
|
|
||||||
|
|
||||||
if rq.asn != "" {
|
if rq.asn != "" {
|
||||||
pr.Out.Header.Set(wafprotocol.ASNHeader, rq.asn)
|
pr.Out.Header.Set(asnHeader, rq.asn)
|
||||||
}
|
}
|
||||||
|
|
||||||
if rq.country != "" {
|
if rq.country != "" {
|
||||||
pr.Out.Header.Set(wafprotocol.CountryHeader, rq.country)
|
pr.Out.Header.Set(countryHeader, rq.country)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -467,10 +470,9 @@ func (rq *request) finish(ctx context.Context) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// An app can send a status up to 999, and the visitor gets it as it is.
|
report := &waf.Report{
|
||||||
report := &wafprotocol.Report{
|
|
||||||
RequestID: rq.id,
|
RequestID: rq.id,
|
||||||
Status: min(rq.out.status, wafprotocol.ReportStatusMax),
|
Status: min(rq.out.status, reportStatusMax),
|
||||||
RequestBytes: requestBytes,
|
RequestBytes: requestBytes,
|
||||||
ResponseBytes: responseBytes,
|
ResponseBytes: responseBytes,
|
||||||
DurationTotalMS: requestlog.Milliseconds(end.Sub(rq.start)),
|
DurationTotalMS: requestlog.Milliseconds(end.Sub(rq.start)),
|
||||||
|
|||||||
+12
-10
@@ -24,7 +24,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
|
|
||||||
// How rproxy keeps connections to the app and to waf open between
|
// How rproxy keeps connections to the app and to waf open between
|
||||||
@@ -47,6 +47,10 @@ const (
|
|||||||
idleTimeout = 120 * time.Second
|
idleTimeout = 120 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ownEndpointPrefix starts the path of every request for sww itself, which
|
||||||
|
// never reaches the app.
|
||||||
|
const ownEndpointPrefix = "/_sww/"
|
||||||
|
|
||||||
// allowedMethods lists the methods rproxy passes on, as an Allow header gives
|
// allowedMethods lists the methods rproxy passes on, as an Allow header gives
|
||||||
// them. It answers any other method 405 itself.
|
// them. It answers any other method 405 itself.
|
||||||
// The WebDAV ones are for Nextcloud.
|
// The WebDAV ones are for Nextcloud.
|
||||||
@@ -88,9 +92,7 @@ func New(params Params) *Server {
|
|||||||
transport: newTransport(),
|
transport: newTransport(),
|
||||||
wafTransport: newWAFTransport(),
|
wafTransport: newWAFTransport(),
|
||||||
reportURL: (&url.URL{
|
reportURL: (&url.URL{
|
||||||
Scheme: backend.Scheme,
|
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath,
|
||||||
Host: backend.Host,
|
|
||||||
Path: wafprotocol.ReportPath,
|
|
||||||
}).String(),
|
}).String(),
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -174,7 +176,7 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
defer rq.readRestOfBody()
|
defer rq.readRestOfBody()
|
||||||
defer rq.finish(r.Context())
|
defer rq.finish(r.Context())
|
||||||
|
|
||||||
if r.Method == http.MethodGet && r.URL.Path == wafprotocol.HealthPath {
|
if r.Method == http.MethodGet && r.URL.Path == waf.HealthPath {
|
||||||
rq.answerHealth()
|
rq.answerHealth()
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -207,9 +209,9 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
rq.noteAnswer(answer)
|
rq.noteAnswer(answer)
|
||||||
|
|
||||||
switch rq.disposition {
|
switch rq.disposition {
|
||||||
case wafprotocol.DispositionAllow:
|
case waf.DispositionAllow:
|
||||||
rq.passToApp(r.Context())
|
rq.passToApp(r.Context())
|
||||||
case wafprotocol.DispositionClose:
|
case waf.DispositionClose:
|
||||||
// This panic has Go's server close the connection without an answer,
|
// This panic has Go's server close the connection without an answer,
|
||||||
// and log nothing; the deferred finish still runs.
|
// and log nothing; the deferred finish still runs.
|
||||||
panic(http.ErrAbortHandler)
|
panic(http.ErrAbortHandler)
|
||||||
@@ -219,8 +221,8 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// passToApp passes the request on to the app, unless a limit or its path keeps
|
// passToApp passes the request on to the app, unless a limit or its path keeps
|
||||||
// it from the app. waf answers a path under wafprotocol.OwnEndpointPrefix
|
// it from the app. waf answers a path under ownEndpointPrefix itself, so
|
||||||
// itself, so such a request gets here only when
|
// such a request gets here only when
|
||||||
// SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION allowed it.
|
// SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION allowed it.
|
||||||
// ctx is the request's own context.
|
// ctx is the request's own context.
|
||||||
func (rq *request) passToApp(ctx context.Context) {
|
func (rq *request) passToApp(ctx context.Context) {
|
||||||
@@ -229,7 +231,7 @@ func (rq *request) passToApp(ctx context.Context) {
|
|||||||
switch {
|
switch {
|
||||||
case rq.refused.Load() != 0 || (maxBytes > 0 && rq.in.ContentLength > maxBytes):
|
case rq.refused.Load() != 0 || (maxBytes > 0 && rq.in.ContentLength > maxBytes):
|
||||||
rq.answer(http.StatusRequestEntityTooLarge)
|
rq.answer(http.StatusRequestEntityTooLarge)
|
||||||
case strings.HasPrefix(rq.in.URL.Path, wafprotocol.OwnEndpointPrefix):
|
case strings.HasPrefix(rq.in.URL.Path, ownEndpointPrefix):
|
||||||
rq.answer(http.StatusNotFound)
|
rq.answer(http.StatusNotFound)
|
||||||
default:
|
default:
|
||||||
rq.forward(ctx)
|
rq.forward(ctx)
|
||||||
|
|||||||
@@ -21,9 +21,9 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -56,7 +56,7 @@ const (
|
|||||||
|
|
||||||
// The headers the tests set and look at.
|
// The headers the tests set and look at.
|
||||||
const (
|
const (
|
||||||
dispositionHeader = wafprotocol.DispositionHeader
|
dispositionHeader = waf.DispositionHeader
|
||||||
forwardedForHeader = "X-Forwarded-For"
|
forwardedForHeader = "X-Forwarded-For"
|
||||||
forwardedProtoHeader = "X-Forwarded-Proto"
|
forwardedProtoHeader = "X-Forwarded-Proto"
|
||||||
requestIDHeader = "X-Request-ID"
|
requestIDHeader = "X-Request-ID"
|
||||||
@@ -72,9 +72,9 @@ const (
|
|||||||
|
|
||||||
// What the stand-in for waf answers.
|
// What the stand-in for waf answers.
|
||||||
const (
|
const (
|
||||||
dispositionAllow = wafprotocol.DispositionAllow
|
dispositionAllow = waf.DispositionAllow
|
||||||
dispositionDeny = wafprotocol.DispositionDeny
|
dispositionDeny = waf.DispositionDeny
|
||||||
dispositionClose = wafprotocol.DispositionClose
|
dispositionClose = waf.DispositionClose
|
||||||
// deniedBody is the body of a deny.
|
// deniedBody is the body of a deny.
|
||||||
deniedBody = "refused by waf"
|
deniedBody = "refused by waf"
|
||||||
)
|
)
|
||||||
@@ -175,7 +175,7 @@ type WAF struct {
|
|||||||
*httptest.Server
|
*httptest.Server
|
||||||
|
|
||||||
questions chan question
|
questions chan question
|
||||||
reports chan wafprotocol.Report
|
reports chan waf.Report
|
||||||
}
|
}
|
||||||
|
|
||||||
// startWAF starts a stand-in for waf.
|
// startWAF starts a stand-in for waf.
|
||||||
@@ -184,11 +184,11 @@ func startWAF(t *testing.T, answer http.HandlerFunc) *WAF {
|
|||||||
|
|
||||||
s := &WAF{
|
s := &WAF{
|
||||||
questions: make(chan question, 10),
|
questions: make(chan question, 10),
|
||||||
reports: make(chan wafprotocol.Report, 10),
|
reports: make(chan waf.Report, 10),
|
||||||
}
|
}
|
||||||
s.Server = httptest.NewServer(http.HandlerFunc(
|
s.Server = httptest.NewServer(http.HandlerFunc(
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.Header.Get(wafprotocol.ClientHeader) == "" {
|
if r.Header.Get(waf.ClientHeader) == "" {
|
||||||
s.takeReport(t, w, r)
|
s.takeReport(t, w, r)
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -217,11 +217,11 @@ func (s *WAF) takeReport(
|
|||||||
) {
|
) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
var report wafprotocol.Report
|
var report waf.Report
|
||||||
|
|
||||||
err := json.NewDecoder(r.Body).Decode(&report)
|
err := json.NewDecoder(r.Body).Decode(&report)
|
||||||
if err != nil || r.Method != http.MethodPost ||
|
if err != nil || r.Method != http.MethodPost ||
|
||||||
r.URL.Path != wafprotocol.ReportPath {
|
r.URL.Path != waf.ReportPath {
|
||||||
t.Errorf("%s %s is no report: %v", r.Method, r.URL.Path, err)
|
t.Errorf("%s %s is no report: %v", r.Method, r.URL.Path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -242,7 +242,7 @@ func (s *WAF) wantNoReport(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// report waits for a report.
|
// report waits for a report.
|
||||||
func (s *WAF) report(t *testing.T) wafprotocol.Report {
|
func (s *WAF) report(t *testing.T) waf.Report {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
select {
|
select {
|
||||||
@@ -251,7 +251,7 @@ func (s *WAF) report(t *testing.T) wafprotocol.Report {
|
|||||||
case <-time.After(waitLimit):
|
case <-time.After(waitLimit):
|
||||||
t.Fatal("no report")
|
t.Fatal("no report")
|
||||||
|
|
||||||
return wafprotocol.Report{}
|
return waf.Report{}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -279,7 +279,7 @@ func startWAFAnsweringReports(
|
|||||||
|
|
||||||
backend := httptest.NewServer(http.HandlerFunc(
|
backend := httptest.NewServer(http.HandlerFunc(
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.Header.Get(wafprotocol.ClientHeader) != "" {
|
if r.Header.Get(waf.ClientHeader) != "" {
|
||||||
answer(w, r)
|
answer(w, r)
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -370,11 +370,11 @@ func startServerLogging(
|
|||||||
server := rproxy.New(rproxy.Params{
|
server := rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: requestLog,
|
RequestLog: requestLog,
|
||||||
ProcessLog: requestlog.NewProcessLogger(
|
ProcessLog: processlog.New(processlog.Params{
|
||||||
processLog,
|
Output: processLog,
|
||||||
"test",
|
InstanceName: "test",
|
||||||
slog.LevelInfo,
|
Level: slog.LevelInfo,
|
||||||
),
|
}),
|
||||||
})
|
})
|
||||||
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(
|
listener, err := (&net.ListenConfig{}).Listen(
|
||||||
@@ -811,7 +811,7 @@ func TestAnAnswerRProxyHoldsWholeIsSentBeforeItsLogLine(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{"a deny", "GET /" + host, deniedBody, http.StatusForbidden},
|
{"a deny", "GET /" + host, deniedBody, http.StatusForbidden},
|
||||||
{"rproxy's own answer", "GET x:admin" + host, "Bad Request\n", http.StatusBadRequest},
|
{"rproxy's own answer", "GET x:admin" + host, "Bad Request\n", http.StatusBadRequest},
|
||||||
{"the health check", "GET " + wafprotocol.HealthPath + host, "ok\n", http.StatusOK},
|
{"the health check", "GET " + waf.HealthPath + host, "ok\n", http.StatusOK},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
@@ -1576,19 +1576,19 @@ func TestHealthCheckIsAnsweredWhileWAFIsDown(t *testing.T) {
|
|||||||
app := startApp(t, greet)
|
app := startApp(t, greet)
|
||||||
addr, out := startRProxy(t, nowhere, app.URL, nil)
|
addr, out := startRProxy(t, nowhere, app.URL, nil)
|
||||||
|
|
||||||
got := get(t, addr, wafprotocol.HealthPath)
|
got := get(t, addr, waf.HealthPath)
|
||||||
if got.status != http.StatusOK || got.body != "ok\n" {
|
if got.status != http.StatusOK || got.body != "ok\n" {
|
||||||
t.Errorf("answered %+v, want ok", got)
|
t.Errorf("answered %+v, want ok", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
line := out.requestLine(t)
|
line := out.requestLine(t)
|
||||||
if line["path"] != wafprotocol.HealthPath || line["disposition"] != nil {
|
if line["path"] != waf.HealthPath || line["disposition"] != nil {
|
||||||
t.Errorf("line %v, want one with no disposition", line)
|
t.Errorf("line %v, want one with no disposition", line)
|
||||||
}
|
}
|
||||||
|
|
||||||
// With waf up too, it is not asked.
|
// With waf up too, it is not asked.
|
||||||
addr, _ = startRProxy(t, standIn.URL, app.URL, nil)
|
addr, _ = startRProxy(t, standIn.URL, app.URL, nil)
|
||||||
get(t, addr, wafprotocol.HealthPath)
|
get(t, addr, waf.HealthPath)
|
||||||
|
|
||||||
if len(standIn.questions) != 0 {
|
if len(standIn.questions) != 0 {
|
||||||
t.Error("waf was asked about the health check")
|
t.Error("waf was asked about the health check")
|
||||||
@@ -1749,7 +1749,7 @@ func TestWaitForReportsWaitsForAReportBeingSent(t *testing.T) {
|
|||||||
release := make(chan struct{})
|
release := make(chan struct{})
|
||||||
backend := httptest.NewServer(http.HandlerFunc(
|
backend := httptest.NewServer(http.HandlerFunc(
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.Header.Get(wafprotocol.ClientHeader) != "" {
|
if r.Header.Get(waf.ClientHeader) != "" {
|
||||||
allow(w, r)
|
allow(w, r)
|
||||||
|
|
||||||
return
|
return
|
||||||
|
|||||||
+82
-34
@@ -1,7 +1,9 @@
|
|||||||
package rproxy
|
package rproxy
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
@@ -10,8 +12,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// answerBodyMaxBytes is the most of the body of waf's answer that is read.
|
// answerBodyMaxBytes is the most of the body of waf's answer that is read.
|
||||||
@@ -40,12 +41,15 @@ type wafAnswer struct {
|
|||||||
// notes nothing, and no report is due for it: waf holds no decision to report
|
// notes nothing, and no report is due for it: waf holds no decision to report
|
||||||
// on.
|
// on.
|
||||||
func (rq *request) noteAnswer(answer *wafAnswer) {
|
func (rq *request) noteAnswer(answer *wafAnswer) {
|
||||||
read, valid := wafprotocol.ReadAnswer(answer.header)
|
disposition := answer.header.Get(waf.DispositionHeader)
|
||||||
if valid {
|
|
||||||
rq.disposition = read.Disposition
|
switch disposition {
|
||||||
|
case waf.DispositionAllow, waf.DispositionDeny,
|
||||||
|
waf.DispositionClose:
|
||||||
|
rq.disposition = disposition
|
||||||
rq.answered = true
|
rq.answered = true
|
||||||
rq.asn = read.ASN
|
rq.asn = answer.header.Get(asnHeader)
|
||||||
rq.country = read.Country
|
rq.country = answer.header.Get(countryHeader)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -76,7 +80,7 @@ func (rq *request) handleNoUsableAnswer(ctx context.Context, err error) {
|
|||||||
rq.disposition = rq.h.config.WAFBackendNoAnswerDisposition
|
rq.disposition = rq.h.config.WAFBackendNoAnswerDisposition
|
||||||
rq.wafNoAnswer = true
|
rq.wafNoAnswer = true
|
||||||
|
|
||||||
if rq.disposition == wafprotocol.DispositionAllow {
|
if rq.disposition == waf.DispositionAllow {
|
||||||
rq.passToApp(ctx)
|
rq.passToApp(ctx)
|
||||||
} else {
|
} else {
|
||||||
rq.answer(http.StatusServiceUnavailable)
|
rq.answer(http.StatusServiceUnavailable)
|
||||||
@@ -116,21 +120,7 @@ func (rq *request) ask(ctx context.Context) (*wafAnswer, error) {
|
|||||||
ctx, cancel := context.WithTimeout(ctx, rq.h.config.WAFBackendTimeout)
|
ctx, cancel := context.WithTimeout(ctx, rq.h.config.WAFBackendTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
var held []byte
|
question, err := rq.newQuestion(ctx)
|
||||||
if rq.body != nil {
|
|
||||||
held = rq.body.held
|
|
||||||
}
|
|
||||||
|
|
||||||
question, err := wafprotocol.NewQuestion(ctx, wafprotocol.QuestionParams{
|
|
||||||
Visitor: rq.in,
|
|
||||||
PathAsSent: rules.Path(rq.in),
|
|
||||||
Body: held,
|
|
||||||
WAFBackend: rq.h.config.WAFBackend,
|
|
||||||
Client: rq.client,
|
|
||||||
Peer: rq.peer,
|
|
||||||
RequestID: rq.id,
|
|
||||||
Scheme: scheme(rq.in, rq.peerTrusted),
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -187,13 +177,69 @@ func canBeAsked(r *http.Request) bool {
|
|||||||
len(r.Header.Values("User-Agent")) <= 1
|
len(r.Header.Values("User-Agent")) <= 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// newQuestion returns the question about the request, as "The question" in
|
||||||
|
// SPEC.md gives it: the visitor's request as the visitor sent it, with the part
|
||||||
|
// of the body it holds and rproxy's headers.
|
||||||
|
func (rq *request) newQuestion(ctx context.Context) (*http.Request, error) {
|
||||||
|
in := rq.in
|
||||||
|
|
||||||
|
var held []byte
|
||||||
|
if rq.body != nil {
|
||||||
|
held = rq.body.held
|
||||||
|
}
|
||||||
|
|
||||||
|
question, err := http.NewRequestWithContext( //nolint:gosec // a setting's URL
|
||||||
|
ctx,
|
||||||
|
in.Method,
|
||||||
|
rq.h.config.WAFBackend.String(),
|
||||||
|
bytes.NewReader(held),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("make the question: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
backend := question.URL
|
||||||
|
target := *in.URL
|
||||||
|
target.Scheme = backend.Scheme
|
||||||
|
target.Host = backend.Host
|
||||||
|
setPathAsSent(&target, in)
|
||||||
|
question.URL = &target
|
||||||
|
question.Host = in.Host
|
||||||
|
|
||||||
|
// The length NewRequestWithContext set is that of what is held. This line makes
|
||||||
|
// a body whose length the visitor did not announce go chunked, as "The question"
|
||||||
|
// in SPEC.md requires.
|
||||||
|
question.TransferEncoding = in.TransferEncoding
|
||||||
|
|
||||||
|
question.Header = in.Header.Clone()
|
||||||
|
question.Header.Set(waf.ClientHeader, rq.client.String())
|
||||||
|
question.Header.Set(waf.PeerHeader, rq.peer.String())
|
||||||
|
question.Header.Del(waf.ContentLengthHeader)
|
||||||
|
|
||||||
|
announced := in.Header.Get("Content-Length")
|
||||||
|
if announced != "" {
|
||||||
|
question.Header.Set(waf.ContentLengthHeader, announced)
|
||||||
|
}
|
||||||
|
|
||||||
|
question.Header.Set(requestIDHeader, rq.id)
|
||||||
|
question.Header.Set("X-Forwarded-Proto", scheme(in, rq.peerTrusted))
|
||||||
|
|
||||||
|
// Go's client adds a User-Agent of its own to a request without the key.
|
||||||
|
_, sent := question.Header["User-Agent"]
|
||||||
|
if !sent {
|
||||||
|
question.Header["User-Agent"] = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return question, nil
|
||||||
|
}
|
||||||
|
|
||||||
// answerWith gives the visitor waf's answer, less its disposition.
|
// answerWith gives the visitor waf's answer, less its disposition.
|
||||||
func (rq *request) answerWith(answer *wafAnswer) {
|
func (rq *request) answerWith(answer *wafAnswer) {
|
||||||
rq.prepareOwnAnswer(rq.requestDeadline())
|
rq.prepareOwnAnswer(rq.requestDeadline())
|
||||||
|
|
||||||
header := rq.out.Header()
|
header := rq.out.Header()
|
||||||
maps.Copy(header, answer.header)
|
maps.Copy(header, answer.header)
|
||||||
header.Del(wafprotocol.DispositionHeader)
|
header.Del(waf.DispositionHeader)
|
||||||
|
|
||||||
// Go's server adds a Content-Type it guesses from the body to a response whose
|
// Go's server adds a Content-Type it guesses from the body to a response whose
|
||||||
// headers have no Content-Type key. The key with no value sends none.
|
// headers have no Content-Type key. The key with no value sends none.
|
||||||
@@ -207,10 +253,7 @@ func (rq *request) answerWith(answer *wafAnswer) {
|
|||||||
// sendReport sends waf report on a goroutine of its own, so that the
|
// sendReport sends waf report on a goroutine of its own, so that the
|
||||||
// visitor does not wait for it. The report goes on after ctx, the request's own
|
// visitor does not wait for it. The report goes on after ctx, the request's own
|
||||||
// context, has ended.
|
// context, has ended.
|
||||||
func (h *handler) sendReport(
|
func (h *handler) sendReport(ctx context.Context, report *waf.Report) {
|
||||||
ctx context.Context,
|
|
||||||
report *wafprotocol.Report,
|
|
||||||
) {
|
|
||||||
h.reportsMu.Lock()
|
h.reportsMu.Lock()
|
||||||
defer h.reportsMu.Unlock()
|
defer h.reportsMu.Unlock()
|
||||||
|
|
||||||
@@ -228,18 +271,23 @@ func (h *handler) sendReport(
|
|||||||
|
|
||||||
// postReport posts report to waf, and waits for its answer for up to
|
// postReport posts report to waf, and waits for its answer for up to
|
||||||
// SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS.
|
// SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS.
|
||||||
func (h *handler) postReport(
|
func (h *handler) postReport(ctx context.Context, report *waf.Report) error {
|
||||||
ctx context.Context,
|
body, err := json.Marshal(report)
|
||||||
report *wafprotocol.Report,
|
if err != nil {
|
||||||
) error {
|
return fmt.Errorf("encode the report: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, h.config.WAFBackendTimeout)
|
ctx, cancel := context.WithTimeout(ctx, h.config.WAFBackendTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
req, err := wafprotocol.NewReportRequest(ctx, h.reportURL, report)
|
req, err := http.NewRequestWithContext( //nolint:gosec // a setting's URL
|
||||||
|
ctx, http.MethodPost, h.reportURL, bytes.NewReader(body))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return fmt.Errorf("make the request: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
res, err := h.wafTransport.RoundTrip(req)
|
res, err := h.wafTransport.RoundTrip(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("send the report: %w", err)
|
return fmt.Errorf("send the report: %w", err)
|
||||||
|
|||||||
@@ -2448,7 +2448,7 @@ func newParams(dir string) state.Params {
|
|||||||
AttackBanDuration: 7 * 24 * time.Hour,
|
AttackBanDuration: 7 * 24 * time.Hour,
|
||||||
MaxBans: 5000,
|
MaxBans: 5000,
|
||||||
}),
|
}),
|
||||||
Limiter: ratelimit.New(ratelimit.Limits{}, 20000),
|
Limiter: ratelimit.New(ratelimit.Params{MaxClients: 20000}),
|
||||||
GeoJS: geojs.New(geojs.Params{Now: midnight}),
|
GeoJS: geojs.New(geojs.Params{Now: midnight}),
|
||||||
Lists: reputation.New(reputation.Params{
|
Lists: reputation.New(reputation.Params{
|
||||||
BlocklistURLs: []string{
|
BlocklistURLs: []string{
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import (
|
|||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
|
|
||||||
// healthCheckTimeout bounds the whole health check.
|
// healthCheckTimeout bounds the whole health check.
|
||||||
@@ -116,7 +116,7 @@ func askHealthEndpoint(ctx context.Context, listenAddr string) error {
|
|||||||
health := "http://" + net.JoinHostPort(
|
health := "http://" + net.JoinHostPort(
|
||||||
"127.0.0.1",
|
"127.0.0.1",
|
||||||
port,
|
port,
|
||||||
) + wafprotocol.HealthPath
|
) + waf.HealthPath
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(
|
req, err := http.NewRequestWithContext(
|
||||||
ctx,
|
ctx,
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
"sneak.berlin/go/smallwebwaf/internal/rproxy"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -23,11 +23,11 @@ import (
|
|||||||
// setting for either.
|
// setting for either.
|
||||||
func RunRProxy(ctx context.Context, params Params) int {
|
func RunRProxy(ctx context.Context, params Params) int {
|
||||||
hostname, _ := os.Hostname() // "" when the host has no name to give
|
hostname, _ := os.Hostname() // "" when the host has no name to give
|
||||||
processLog := requestlog.NewProcessLogger(
|
processLog := processlog.New(processlog.Params{
|
||||||
params.Stdout,
|
Output: params.Stdout,
|
||||||
hostname,
|
InstanceName: hostname,
|
||||||
slog.LevelInfo,
|
Level: slog.LevelInfo,
|
||||||
)
|
})
|
||||||
|
|
||||||
cfg, err := config.RProxyFromEnvironment(params.Viper)
|
cfg, err := config.RProxyFromEnvironment(params.Viper)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
+17
-17
@@ -21,7 +21,7 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/sww"
|
"sneak.berlin/go/smallwebwaf/internal/sww"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -164,7 +164,7 @@ func TestABanThroughOneSiteRefusesTheClientOnTheOther(t *testing.T) {
|
|||||||
|
|
||||||
// One request's three lines carry one id, rproxy's with waf's answer.
|
// One request's three lines carry one id, rproxy's with waf's answer.
|
||||||
line := firstOut.line(t, "path", bannedProbe)
|
line := firstOut.line(t, "path", bannedProbe)
|
||||||
if line["disposition"] != wafprotocol.DispositionDeny ||
|
if line["disposition"] != waf.DispositionDeny ||
|
||||||
line["client_ip"] != placed {
|
line["client_ip"] != placed {
|
||||||
t.Errorf("rproxy's line is %v", line)
|
t.Errorf("rproxy's line is %v", line)
|
||||||
}
|
}
|
||||||
@@ -249,7 +249,7 @@ func TestTheSiteAnswersWhileWAFIsStoppedAndStartedAgain(t *testing.T) {
|
|||||||
wantStatus(t, site+"/after-stop", placed, http.StatusOK)
|
wantStatus(t, site+"/after-stop", placed, http.StatusOK)
|
||||||
|
|
||||||
line := out.line(t, "path", "/after-stop")
|
line := out.line(t, "path", "/after-stop")
|
||||||
if line["disposition"] != wafprotocol.DispositionAllow ||
|
if line["disposition"] != waf.DispositionAllow ||
|
||||||
line["waf_backend_no_answer"] != true {
|
line["waf_backend_no_answer"] != true {
|
||||||
t.Errorf("rproxy's line is %v", line)
|
t.Errorf("rproxy's line is %v", line)
|
||||||
}
|
}
|
||||||
@@ -437,7 +437,7 @@ func TestAChunkedBodyStaysABodyAtWAF(t *testing.T) {
|
|||||||
const smuggledID = "smuggled-request"
|
const smuggledID = "smuggled-request"
|
||||||
|
|
||||||
smuggled := "GET " + bannedProbe + " HTTP/1.1\r\nHost: site.example\r\n" +
|
smuggled := "GET " + bannedProbe + " HTTP/1.1\r\nHost: site.example\r\n" +
|
||||||
wafprotocol.ClientHeader + ": " + otherClient + "\r\n" +
|
waf.ClientHeader + ": " + otherClient + "\r\n" +
|
||||||
"X-Request-ID: " + smuggledID + "\r\n\r\n"
|
"X-Request-ID: " + smuggledID + "\r\n\r\n"
|
||||||
|
|
||||||
status := sendRaw(
|
status := sendRaw(
|
||||||
@@ -605,11 +605,10 @@ func TestRProxyAsksAWAFThatSpeaksHTTP2OverHTTP1(t *testing.T) {
|
|||||||
protocols := make(chan string, 10)
|
protocols := make(chan string, 10)
|
||||||
backend := httptest.NewUnstartedServer(http.HandlerFunc(
|
backend := httptest.NewUnstartedServer(http.HandlerFunc(
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.Header.Get(wafprotocol.ClientHeader) != "" {
|
if r.Header.Get(waf.ClientHeader) != "" {
|
||||||
protocols <- r.Proto
|
protocols <- r.Proto
|
||||||
|
|
||||||
w.Header().
|
w.Header().Set(waf.DispositionHeader, waf.DispositionDeny)
|
||||||
Set(wafprotocol.DispositionHeader, wafprotocol.DispositionDeny)
|
|
||||||
w.WriteHeader(http.StatusForbidden)
|
w.WriteHeader(http.StatusForbidden)
|
||||||
}
|
}
|
||||||
}))
|
}))
|
||||||
@@ -691,9 +690,8 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
|
|||||||
release := make(chan struct{})
|
release := make(chan struct{})
|
||||||
backend := httptest.NewServer(http.HandlerFunc(
|
backend := httptest.NewServer(http.HandlerFunc(
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.Header.Get(wafprotocol.ClientHeader) != "" {
|
if r.Header.Get(waf.ClientHeader) != "" {
|
||||||
w.Header().
|
w.Header().Set(waf.DispositionHeader, waf.DispositionAllow)
|
||||||
Set(wafprotocol.DispositionHeader, wafprotocol.DispositionAllow)
|
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -787,7 +785,10 @@ func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
|
|||||||
func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
|
func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const cutOffAfter = 5 * time.Second
|
const (
|
||||||
|
cutOffAfter = 5 * time.Second
|
||||||
|
upstreamSetting = "SWW_RPROXY_UPSTREAM_URL"
|
||||||
|
)
|
||||||
|
|
||||||
asked := make(chan struct{})
|
asked := make(chan struct{})
|
||||||
release := make(chan struct{})
|
release := make(chan struct{})
|
||||||
@@ -801,8 +802,7 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
|
|||||||
|
|
||||||
backend := httptest.NewServer(http.HandlerFunc(
|
backend := httptest.NewServer(http.HandlerFunc(
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
func(w http.ResponseWriter, _ *http.Request) {
|
||||||
w.Header().
|
w.Header().Set(waf.DispositionHeader, waf.DispositionAllow)
|
||||||
Set(wafprotocol.DispositionHeader, wafprotocol.DispositionAllow)
|
|
||||||
}))
|
}))
|
||||||
t.Cleanup(backend.Close)
|
t.Cleanup(backend.Close)
|
||||||
|
|
||||||
@@ -810,9 +810,9 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
|
|||||||
defer stop()
|
defer stop()
|
||||||
|
|
||||||
env := map[string]string{
|
env := map[string]string{
|
||||||
rproxyListenAddr: localhost + ":0",
|
rproxyListenAddr: localhost + ":0",
|
||||||
rproxyUpstreamURL: app.URL,
|
upstreamSetting: app.URL,
|
||||||
wafBackend: backend.URL,
|
wafBackend: backend.URL,
|
||||||
}
|
}
|
||||||
out := &output{}
|
out := &output{}
|
||||||
exited := make(chan int, 1)
|
exited := make(chan int, 1)
|
||||||
@@ -935,7 +935,7 @@ func TestMainRunsRProxy(t *testing.T) {
|
|||||||
addr, stop := startRProxyProcess(t, wafBackend+"=http://127.0.0.1:9")
|
addr, stop := startRProxyProcess(t, wafBackend+"=http://127.0.0.1:9")
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet,
|
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet,
|
||||||
"http://"+addr+wafprotocol.HealthPath, http.NoBody)
|
"http://"+addr+waf.HealthPath, http.NoBody)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("new request: %v", err)
|
t.Fatalf("new request: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
|
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/sww"
|
"sneak.berlin/go/smallwebwaf/internal/sww"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -1813,7 +1813,7 @@ func statusFrom(t *testing.T, url, from string) int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
req.Header.Set("X-Forwarded-For", from)
|
req.Header.Set("X-Forwarded-For", from)
|
||||||
req.Header.Set(wafprotocol.ClientHeader, from)
|
req.Header.Set(waf.ClientHeader, from)
|
||||||
req.Header.Set("X-Request-ID", rand.Text())
|
req.Header.Set("X-Request-ID", rand.Text())
|
||||||
|
|
||||||
transport := &http.Transport{}
|
transport := &http.Transport{}
|
||||||
|
|||||||
+22
-16
@@ -19,10 +19,10 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/state"
|
"sneak.berlin/go/smallwebwaf/internal/state"
|
||||||
@@ -43,8 +43,11 @@ const remoteLogStopTimeout = 2 * time.Second
|
|||||||
func RunWAF(ctx context.Context, params Params) int {
|
func RunWAF(ctx context.Context, params Params) int {
|
||||||
// Until the settings are read, the one message is an invalid setting's
|
// Until the settings are read, the one message is an invalid setting's
|
||||||
// error, which every SWW_WAF_LOG_LEVEL lets through.
|
// error, which every SWW_WAF_LOG_LEVEL lets through.
|
||||||
processLog := requestlog.NewProcessLogger(params.Stdout,
|
processLog := processlog.New(processlog.Params{
|
||||||
config.InstanceName(params.Viper), slog.LevelError)
|
Output: params.Stdout,
|
||||||
|
InstanceName: config.InstanceName(params.Viper),
|
||||||
|
Level: slog.LevelError,
|
||||||
|
})
|
||||||
|
|
||||||
cfg, err := config.WAFFromEnvironment(params.Viper)
|
cfg, err := config.WAFFromEnvironment(params.Viper)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -63,11 +66,11 @@ func RunWAF(ctx context.Context, params Params) int {
|
|||||||
stdout = io.MultiWriter(params.Stdout, remote)
|
stdout = io.MultiWriter(params.Stdout, remote)
|
||||||
}
|
}
|
||||||
|
|
||||||
processLog = requestlog.NewProcessLogger(
|
processLog = processlog.New(processlog.Params{
|
||||||
stdout,
|
Output: stdout,
|
||||||
cfg.InstanceName,
|
InstanceName: cfg.InstanceName,
|
||||||
cfg.LogLevel,
|
Level: cfg.LogLevel,
|
||||||
)
|
})
|
||||||
|
|
||||||
if remote != nil {
|
if remote != nil {
|
||||||
stopSending := startSending(ctx, remote, processLog)
|
stopSending := startSending(ctx, remote, processLog)
|
||||||
@@ -239,14 +242,17 @@ func newLookupFile(
|
|||||||
|
|
||||||
// newLimiter returns the table of clients, with the rate and byte limits.
|
// newLimiter returns the table of clients, with the rate and byte limits.
|
||||||
func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
|
func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
|
||||||
return ratelimit.New(ratelimit.Limits{
|
return ratelimit.New(ratelimit.Params{
|
||||||
PerMinute: cfg.RateLimitPerMinute,
|
Limits: ratelimit.Limits{
|
||||||
PerHour: cfg.RateLimitPerHour,
|
PerMinute: cfg.RateLimitPerMinute,
|
||||||
PerDay: cfg.RateLimitPerDay,
|
PerHour: cfg.RateLimitPerHour,
|
||||||
BytesPerMinute: cfg.BytesLimitPerMinute,
|
PerDay: cfg.RateLimitPerDay,
|
||||||
BytesPerHour: cfg.BytesLimitPerHour,
|
BytesPerMinute: cfg.BytesLimitPerMinute,
|
||||||
BytesPerDay: cfg.BytesLimitPerDay,
|
BytesPerHour: cfg.BytesLimitPerHour,
|
||||||
}, cfg.MaxTrackedClients)
|
BytesPerDay: cfg.BytesLimitPerDay,
|
||||||
|
},
|
||||||
|
MaxClients: cfg.MaxTrackedClients,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// newLedger returns the ledger of the bans, with the settings for them.
|
// newLedger returns the ledger of the bans, with the settings for them.
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ import (
|
|||||||
_ "time/tzdata"
|
_ "time/tzdata"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/sww"
|
"sneak.berlin/go/smallwebwaf/internal/sww"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
|
|
||||||
// commandForMain, while set in the environment, has this test binary run Main
|
// commandForMain, while set in the environment, has this test binary run Main
|
||||||
@@ -106,7 +106,7 @@ func TestMainRunsWAF(t *testing.T) {
|
|||||||
|
|
||||||
starting := out.lineBeforeExit(t, "starting", exited)
|
starting := out.lineBeforeExit(t, "starting", exited)
|
||||||
addr, _ := starting["address"].(string)
|
addr, _ := starting["address"].(string)
|
||||||
ask(t, addr, "/", http.StatusOK, wafprotocol.DispositionAllow)
|
ask(t, addr, "/", http.StatusOK, waf.DispositionAllow)
|
||||||
|
|
||||||
// The files are written as sww stops.
|
// The files are written as sww stops.
|
||||||
err = cmd.Process.Signal(syscall.SIGTERM)
|
err = cmd.Process.Signal(syscall.SIGTERM)
|
||||||
@@ -217,7 +217,7 @@ func ask(t *testing.T, addr, path string, status int, disposition string) {
|
|||||||
t.Fatalf("new request: %v", err)
|
t.Fatalf("new request: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
req.Header.Set(wafprotocol.ClientHeader, placed)
|
req.Header.Set(waf.ClientHeader, placed)
|
||||||
req.Header.Set("X-Request-ID", rand.Text())
|
req.Header.Set("X-Request-ID", rand.Text())
|
||||||
|
|
||||||
transport := &http.Transport{}
|
transport := &http.Transport{}
|
||||||
@@ -232,11 +232,11 @@ func ask(t *testing.T, addr, path string, status int, disposition string) {
|
|||||||
_ = res.Body.Close()
|
_ = res.Body.Close()
|
||||||
|
|
||||||
if res.StatusCode != status ||
|
if res.StatusCode != status ||
|
||||||
res.Header.Get(wafprotocol.DispositionHeader) != disposition {
|
res.Header.Get(waf.DispositionHeader) != disposition {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"answered %d with disposition %q, want %d with %q",
|
"answered %d with disposition %q, want %d with %q",
|
||||||
res.StatusCode,
|
res.StatusCode,
|
||||||
res.Header.Get(wafprotocol.DispositionHeader),
|
res.Header.Get(waf.DispositionHeader),
|
||||||
status,
|
status,
|
||||||
disposition,
|
disposition,
|
||||||
)
|
)
|
||||||
@@ -287,18 +287,12 @@ func TestWAFKeepsItsBansAcrossARestartAndNeedsNoApp(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
runWAF(t, env, func(addr string) {
|
runWAF(t, env, func(addr string) {
|
||||||
ask(t, addr, "/", http.StatusOK, wafprotocol.DispositionAllow)
|
ask(t, addr, "/", http.StatusOK, waf.DispositionAllow)
|
||||||
// The default rule file bans a probe for this.
|
// The default rule file bans a probe for this.
|
||||||
ask(
|
ask(t, addr, "/.env", http.StatusForbidden, waf.DispositionDeny)
|
||||||
t,
|
|
||||||
addr,
|
|
||||||
"/.env",
|
|
||||||
http.StatusForbidden,
|
|
||||||
wafprotocol.DispositionDeny,
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
runWAF(t, env, func(addr string) {
|
runWAF(t, env, func(addr string) {
|
||||||
ask(t, addr, "/", http.StatusForbidden, wafprotocol.DispositionDeny)
|
ask(t, addr, "/", http.StatusForbidden, waf.DispositionDeny)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -26,7 +26,6 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -85,7 +84,7 @@ func TestSixteenThousandCookiesAreRefusedBeforeTheCoreRuleSetReadsThem(
|
|||||||
|
|
||||||
got := do(t, req)
|
got := do(t, req)
|
||||||
|
|
||||||
wantDisposition(t, got, tooLarge, wafprotocol.DispositionDeny)
|
wantDisposition(t, got, tooLarge, waf.DispositionDeny)
|
||||||
|
|
||||||
line := s.out.requestLine(t)
|
line := s.out.requestLine(t)
|
||||||
wantLine(t, line, tooLarge, requestlog.ActionRequestHeaderTooLarge)
|
wantLine(t, line, tooLarge, requestlog.ActionRequestHeaderTooLarge)
|
||||||
@@ -165,14 +164,9 @@ func TestARequestLineOverTheLimitIsRefusedWith414BeforeAnyOtherCheck(
|
|||||||
line := s.out.requestLine(t)
|
line := s.out.requestLine(t)
|
||||||
|
|
||||||
if tc.status == http.StatusOK {
|
if tc.status == http.StatusOK {
|
||||||
wantDisposition(
|
wantDisposition(t, got, tc.status, waf.DispositionAllow)
|
||||||
t,
|
|
||||||
got,
|
|
||||||
tc.status,
|
|
||||||
wafprotocol.DispositionAllow,
|
|
||||||
)
|
|
||||||
} else {
|
} else {
|
||||||
wantDisposition(t, got, tc.status, wafprotocol.DispositionDeny)
|
wantDisposition(t, got, tc.status, waf.DispositionDeny)
|
||||||
}
|
}
|
||||||
|
|
||||||
wantLine(t, line, tc.status, tc.action)
|
wantLine(t, line, tc.status, tc.action)
|
||||||
@@ -225,14 +219,9 @@ func TestAHeaderLineOverTheLimitIsRefusedWith431(t *testing.T) {
|
|||||||
got := do(t, req)
|
got := do(t, req)
|
||||||
|
|
||||||
if tc.refused {
|
if tc.refused {
|
||||||
wantDisposition(
|
wantDisposition(t, got, tooLarge, waf.DispositionDeny)
|
||||||
t,
|
|
||||||
got,
|
|
||||||
tooLarge,
|
|
||||||
wafprotocol.DispositionDeny,
|
|
||||||
)
|
|
||||||
} else {
|
} else {
|
||||||
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -255,7 +244,10 @@ func TestRequestLineRefusalNamesItsHeuristicLengthAndLimit(t *testing.T) {
|
|||||||
func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) {
|
func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
repeated := slices.Repeat([]string{"1"}, headerFieldLimit)
|
repeated := make([]string, headerFieldLimit)
|
||||||
|
for i := range repeated {
|
||||||
|
repeated[i] = "1"
|
||||||
|
}
|
||||||
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name string
|
name string
|
||||||
@@ -302,12 +294,7 @@ func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) {
|
|||||||
line := s.out.requestLine(t)
|
line := s.out.requestLine(t)
|
||||||
|
|
||||||
if tc.refused {
|
if tc.refused {
|
||||||
wantDisposition(
|
wantDisposition(t, got, tooLarge, waf.DispositionDeny)
|
||||||
t,
|
|
||||||
got,
|
|
||||||
tooLarge,
|
|
||||||
wafprotocol.DispositionDeny,
|
|
||||||
)
|
|
||||||
wantLine(
|
wantLine(
|
||||||
t,
|
t,
|
||||||
line,
|
line,
|
||||||
@@ -318,12 +305,7 @@ func TestEachHeaderLimitRefusesARequestOverIt(t *testing.T) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
wantDisposition(
|
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||||
t,
|
|
||||||
got,
|
|
||||||
http.StatusOK,
|
|
||||||
wafprotocol.DispositionAllow,
|
|
||||||
)
|
|
||||||
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
|
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
|
||||||
|
|
||||||
if line.DurationCoreRuleSetMS == nil {
|
if line.DurationCoreRuleSetMS == nil {
|
||||||
@@ -350,12 +332,7 @@ func TestHeaderLimitsHoldWhateverTheCoreRuleSetSettingsSay(t *testing.T) {
|
|||||||
req := s.question(siteA, client, "/")
|
req := s.question(siteA, client, "/")
|
||||||
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
||||||
|
|
||||||
wantDisposition(
|
wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny)
|
||||||
t,
|
|
||||||
do(t, req),
|
|
||||||
tooLarge,
|
|
||||||
wafprotocol.DispositionDeny,
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -371,12 +348,7 @@ func TestClientInTheAllowNetsSkipsTheHeaderLimits(t *testing.T) {
|
|||||||
req := s.question(siteA, client, "/")
|
req := s.question(siteA, client, "/")
|
||||||
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
||||||
|
|
||||||
wantDisposition(
|
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow)
|
||||||
t,
|
|
||||||
do(t, req),
|
|
||||||
http.StatusOK,
|
|
||||||
wafprotocol.DispositionAllow,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestObserveModeAllowsARequestOverTheHeaderLimitsUnread(t *testing.T) {
|
func TestObserveModeAllowsARequestOverTheHeaderLimitsUnread(t *testing.T) {
|
||||||
@@ -392,7 +364,7 @@ func TestObserveModeAllowsARequestOverTheHeaderLimitsUnread(t *testing.T) {
|
|||||||
|
|
||||||
got := do(t, req)
|
got := do(t, req)
|
||||||
|
|
||||||
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||||
|
|
||||||
line := s.out.requestLine(t)
|
line := s.out.requestLine(t)
|
||||||
if line.WouldAction != requestlog.ActionRequestHeaderTooLarge ||
|
if line.WouldAction != requestlog.ActionRequestHeaderTooLarge ||
|
||||||
@@ -414,14 +386,14 @@ func TestTheRateLimitsDoNotCountRequestsOverTheHeaderLimits(t *testing.T) {
|
|||||||
for range 2 {
|
for range 2 {
|
||||||
req := s.question(siteA, client, "/")
|
req := s.question(siteA, client, "/")
|
||||||
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
||||||
wantDisposition(t, do(t, req), tooLarge, wafprotocol.DispositionDeny)
|
wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny)
|
||||||
}
|
}
|
||||||
|
|
||||||
wantDisposition(
|
wantDisposition(
|
||||||
t,
|
t,
|
||||||
s.ask(client, "/"),
|
s.ask(client, "/"),
|
||||||
http.StatusOK,
|
http.StatusOK,
|
||||||
wafprotocol.DispositionAllow,
|
waf.DispositionAllow,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -494,12 +466,7 @@ func TestTheHeaderLimitsComeBeforeEveryCheckThatRefuses(t *testing.T) {
|
|||||||
req := s.question(siteA, client, tc.path)
|
req := s.question(siteA, client, tc.path)
|
||||||
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
||||||
|
|
||||||
wantDisposition(
|
wantDisposition(t, do(t, req), tooLarge, waf.DispositionDeny)
|
||||||
t,
|
|
||||||
do(t, req),
|
|
||||||
tooLarge,
|
|
||||||
wafprotocol.DispositionDeny,
|
|
||||||
)
|
|
||||||
wantLine(
|
wantLine(
|
||||||
t,
|
t,
|
||||||
s.out.requestLine(t),
|
s.out.requestLine(t),
|
||||||
@@ -536,16 +503,11 @@ func TestARequestOverTheHeaderLimitsRunsNoOtherCheck(t *testing.T) {
|
|||||||
|
|
||||||
over := s.question(siteA, client, "/")
|
over := s.question(siteA, client, "/")
|
||||||
over.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
over.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
||||||
wantDisposition(t, do(t, over), tooLarge, wafprotocol.DispositionDeny)
|
wantDisposition(t, do(t, over), tooLarge, waf.DispositionDeny)
|
||||||
|
|
||||||
within := s.question(siteA, client, "/")
|
within := s.question(siteA, client, "/")
|
||||||
within.Header.Set(cookieHeader, cookies(cookieLimit))
|
within.Header.Set(cookieHeader, cookies(cookieLimit))
|
||||||
wantDisposition(
|
wantDisposition(t, do(t, within), http.StatusOK, waf.DispositionAllow)
|
||||||
t,
|
|
||||||
do(t, within),
|
|
||||||
http.StatusOK,
|
|
||||||
wafprotocol.DispositionAllow,
|
|
||||||
)
|
|
||||||
|
|
||||||
lines := s.out.requestLines(t, 2)
|
lines := s.out.requestLines(t, 2)
|
||||||
|
|
||||||
@@ -602,7 +564,7 @@ func TestNotWaitingForGeoJSNeverLowersAClientsLimit(t *testing.T) {
|
|||||||
t,
|
t,
|
||||||
s.ask(fromDE, "/"),
|
s.ask(fromDE, "/"),
|
||||||
http.StatusForbidden,
|
http.StatusForbidden,
|
||||||
wafprotocol.DispositionDeny,
|
waf.DispositionDeny,
|
||||||
)
|
)
|
||||||
|
|
||||||
line := s.out.requestLine(t)
|
line := s.out.requestLine(t)
|
||||||
@@ -639,7 +601,7 @@ func TestAClientGeoJSPlacedInNoCountryCountsAsUnknownWithoutWaiting(
|
|||||||
t,
|
t,
|
||||||
s.ask(client, "/"),
|
s.ask(client, "/"),
|
||||||
http.StatusForbidden,
|
http.StatusForbidden,
|
||||||
wafprotocol.DispositionDeny,
|
waf.DispositionDeny,
|
||||||
)
|
)
|
||||||
|
|
||||||
line := s.out.requestLine(t)
|
line := s.out.requestLine(t)
|
||||||
@@ -668,7 +630,7 @@ func TestObserveModeAsksGeoJSNothingForARequestOverTheHeaderLimits(
|
|||||||
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
req.Header.Set(cookieHeader, cookies(cookieLimit+1))
|
||||||
|
|
||||||
got := do(t, req)
|
got := do(t, req)
|
||||||
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||||
|
|
||||||
if len(asked()) != 0 || got.header.Get("X-Client-Asn") != "" {
|
if len(asked()) != 0 || got.header.Get("X-Client-Asn") != "" {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
@@ -710,7 +672,7 @@ func askDirectly(
|
|||||||
http.NoBody,
|
http.NoBody,
|
||||||
)
|
)
|
||||||
req.RemoteAddr = net.JoinHostPort(rproxyPeer, "1234")
|
req.RemoteAddr = net.JoinHostPort(rproxyPeer, "1234")
|
||||||
req.Header.Set(wafprotocol.ClientHeader, client)
|
req.Header.Set(waf.ClientHeader, client)
|
||||||
req.Header.Set(requestIDHeader, "the-id")
|
req.Header.Set(requestIDHeader, "the-id")
|
||||||
change(req)
|
change(req)
|
||||||
|
|
||||||
|
|||||||
@@ -53,13 +53,13 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
if country == "" && !rq.skippedLookupWait {
|
if country == "" && !rq.skippedLookupWait {
|
||||||
unknown = percentage{
|
unknown = percentage{
|
||||||
cfg.UnknownCountryLimitPercent,
|
cfg.UnknownCountryLimitPercent,
|
||||||
config.UnknownCountryLimitPercentSetting,
|
"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fetched := percentage{percent: whole}
|
fetched := percentage{percent: whole}
|
||||||
if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed {
|
if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed {
|
||||||
fetched = percentage{percent, config.ASNLimitPercentURLSetting}
|
fetched = percentage{percent, "SWW_WAF_ASN_LIMIT_PERCENT_URL"}
|
||||||
}
|
}
|
||||||
|
|
||||||
blocklisted := percentage{percent: whole}
|
blocklisted := percentage{percent: whole}
|
||||||
@@ -67,7 +67,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
cfg.BlocklistAction == config.ReputationActionLimit {
|
cfg.BlocklistAction == config.ReputationActionLimit {
|
||||||
blocklisted = percentage{
|
blocklisted = percentage{
|
||||||
cfg.BlocklistLimitPercent,
|
cfg.BlocklistLimitPercent,
|
||||||
config.BlocklistActionSetting,
|
"SWW_WAF_BLOCKLIST_ACTION",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -76,11 +76,11 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit {
|
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit {
|
||||||
dnsblAbuseIPDBListed = percentage{
|
dnsblAbuseIPDBListed = percentage{
|
||||||
cfg.DNSBLAbuseIPDBLimitPercent,
|
cfg.DNSBLAbuseIPDBLimitPercent,
|
||||||
config.DNSBLAbuseIPDBActionSetting,
|
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const tagSetting = config.TagLimitPercentSetting
|
const tagSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
|
||||||
|
|
||||||
asnRequests := lowest(
|
asnRequests := lowest(
|
||||||
given(cfg.TagLimitPercent, asnTag(asn), tagSetting),
|
given(cfg.TagLimitPercent, asnTag(asn), tagSetting),
|
||||||
@@ -108,7 +108,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
asnBytes = given(
|
asnBytes = given(
|
||||||
cfg.ASNBytesLimitPercent,
|
cfg.ASNBytesLimitPercent,
|
||||||
asn,
|
asn,
|
||||||
config.ASNBytesLimitPercentSetting,
|
"SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -116,7 +116,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
countryBytes = given(
|
countryBytes = given(
|
||||||
cfg.CountryBytesLimitPercent,
|
cfg.CountryBytesLimitPercent,
|
||||||
country,
|
country,
|
||||||
config.CountryBytesLimitPercentSetting,
|
"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -19,7 +19,6 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
|
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// The biased thresholds.
|
// The biased thresholds.
|
||||||
@@ -590,7 +589,7 @@ func serveFromDE(
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(t.Context(), method, "/", body)
|
req := httptest.NewRequestWithContext(t.Context(), method, "/", body)
|
||||||
req.Header.Set(wafprotocol.ClientHeader, fromDE)
|
req.Header.Set(waf.ClientHeader, fromDE)
|
||||||
req.Header.Set(requestIDHeader, "the-id")
|
req.Header.Set(requestIDHeader, "the-id")
|
||||||
|
|
||||||
server.Handler.ServeHTTP(httptest.NewRecorder(), req)
|
server.Handler.ServeHTTP(httptest.NewRecorder(), req)
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// peerAddress is the address of the request's TCP peer.
|
// peerAddress is the address of the request's TCP peer.
|
||||||
@@ -20,11 +19,14 @@ func peerAddress(r *http.Request) netip.Addr {
|
|||||||
return addrPort.Addr().Unmap()
|
return addrPort.Addr().Unmap()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// requestIDHeader carries the request's id, from rproxy.
|
||||||
|
const requestIDHeader = "X-Request-ID"
|
||||||
|
|
||||||
// requestID is the request's id: the one a question gives, or for any other
|
// requestID is the request's id: the one a question gives, or for any other
|
||||||
// request a new random one.
|
// request a new random one.
|
||||||
func requestID(r *http.Request, isQuestion bool) string {
|
func requestID(r *http.Request, isQuestion bool) string {
|
||||||
if isQuestion {
|
if isQuestion {
|
||||||
return r.Header.Get(wafprotocol.RequestIDHeader)
|
return r.Header.Get(requestIDHeader)
|
||||||
}
|
}
|
||||||
|
|
||||||
return rand.Text()
|
return rand.Text()
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The Core Rule Set's settings the tests set, besides SWW_WAF_CORE_RULE_SET_MODE, and
|
// The Core Rule Set's settings the tests set, besides SWW_WAF_CORE_RULE_SET_MODE, and
|
||||||
@@ -395,9 +395,9 @@ func TestGiteasFixesApplyOnlyToAHostnameThatServesGitea(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
wantDisposition(t, do(t, s.question(siteA, client, fileName)),
|
wantDisposition(t, do(t, s.question(siteA, client, fileName)),
|
||||||
http.StatusOK, wafprotocol.DispositionAllow)
|
http.StatusOK, waf.DispositionAllow)
|
||||||
wantDisposition(t, do(t, s.question(siteB, client, fileName)),
|
wantDisposition(t, do(t, s.question(siteB, client, fileName)),
|
||||||
http.StatusForbidden, wafprotocol.DispositionDeny)
|
http.StatusForbidden, waf.DispositionDeny)
|
||||||
|
|
||||||
lines := s.out.requestLines(t, 2)
|
lines := s.out.requestLines(t, 2)
|
||||||
wantLine(t, lines[0], http.StatusOK, requestlog.ActionForward)
|
wantLine(t, lines[0], http.StatusOK, requestlog.ActionForward)
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -34,8 +33,7 @@ func (countries) check(
|
|||||||
return &verdict{
|
return &verdict{
|
||||||
action: requestlog.ActionCountryDenied,
|
action: requestlog.ActionCountryDenied,
|
||||||
figures: map[string]any{
|
figures: map[string]any{
|
||||||
"country": country,
|
"country": country, "setting": "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
|
||||||
"setting": config.ExclusivelyAllowedCountriesSetting,
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -5,7 +5,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestHealthEndpointCountsTowardNoLimit(t *testing.T) {
|
func TestHealthEndpointCountsTowardNoLimit(t *testing.T) {
|
||||||
@@ -25,13 +25,8 @@ func TestHealthEndpointCountsTowardNoLimit(t *testing.T) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
for range healthChecks {
|
for range healthChecks {
|
||||||
got := s.ownEndpointRequest(
|
got := s.ownEndpointRequest("", http.MethodGet, waf.HealthPath, "",
|
||||||
"",
|
http.StatusOK)
|
||||||
http.MethodGet,
|
|
||||||
wafprotocol.HealthPath,
|
|
||||||
"",
|
|
||||||
http.StatusOK,
|
|
||||||
)
|
|
||||||
|
|
||||||
if string(got.body) != "ok\n" ||
|
if string(got.body) != "ok\n" ||
|
||||||
got.header.Get("Content-Type") != contentType {
|
got.header.Get("Content-Type") != contentType {
|
||||||
|
|||||||
@@ -104,6 +104,12 @@ func onReport() []heuristic {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// runAfterAnswer runs the heuristics of afterAnswer on the request, at the time
|
||||||
|
// it is called.
|
||||||
|
func (rq *request) runAfterAnswer(ctx context.Context) {
|
||||||
|
rq.runBanningHeuristics(ctx, afterAnswer())
|
||||||
|
}
|
||||||
|
|
||||||
// runHeuristics runs list, one of those checkClient runs, on the request at now,
|
// runHeuristics runs list, one of those checkClient runs, on the request at now,
|
||||||
// and returns the first verdict, or nil when none gives one. The log line's
|
// and returns the first verdict, or nil when none gives one. The log line's
|
||||||
// heuristic and heuristic_figures name the heuristic that gave it, and give its
|
// heuristic and heuristic_figures name the heuristic that gave it, and give its
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestHistoryKeepsEachRequestOfTheClient(t *testing.T) {
|
func TestHistoryKeepsEachRequestOfTheClient(t *testing.T) {
|
||||||
@@ -119,7 +118,7 @@ func TestOwnEndpointsAreNotInTheHistory(t *testing.T) {
|
|||||||
s.ownEndpointRequest(
|
s.ownEndpointRequest(
|
||||||
"",
|
"",
|
||||||
http.MethodGet,
|
http.MethodGet,
|
||||||
wafprotocol.HealthPath,
|
waf.HealthPath,
|
||||||
"",
|
"",
|
||||||
http.StatusOK,
|
http.StatusOK,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package waf
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -15,6 +16,15 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// The headers in which an answer of allow gives rproxy the client's AS number
|
||||||
|
// and country, for the app, while SWW_WAF_ADD_LOOKUP_HEADERS is true. Go writes
|
||||||
|
// every header name in this form, as it sends it and as it receives it, so
|
||||||
|
// X-Client-ASN arrives as X-Client-Asn; header names are not case-sensitive.
|
||||||
|
const (
|
||||||
|
asnHeader = "X-Client-Asn"
|
||||||
|
countryHeader = "X-Client-Country"
|
||||||
|
)
|
||||||
|
|
||||||
// lookupResult is what the lookup gave for a request: whether its client was
|
// lookupResult is what the lookup gave for a request: whether its client was
|
||||||
// looked up, the answer, and whether the request skipped the wait for GeoJS.
|
// looked up, the answer, and whether the request skipped the wait for GeoJS.
|
||||||
type lookupResult struct {
|
type lookupResult struct {
|
||||||
@@ -157,6 +167,18 @@ func GeoJSFailed(
|
|||||||
m.GeoJSFailures.Inc()
|
m.GeoJSFailures.Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// setLookupHeaders sets the headers that give the client's AS number and
|
||||||
|
// country, leaving out one that is unknown.
|
||||||
|
func setLookupHeaders(header http.Header, asn, country string) {
|
||||||
|
if asn != "" {
|
||||||
|
header.Set(asnHeader, asn)
|
||||||
|
}
|
||||||
|
|
||||||
|
if country != "" {
|
||||||
|
header.Set(countryHeader, country)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// canBePlaced reports whether a lookup can place addr: private, loopback
|
// canBePlaced reports whether a lookup can place addr: private, loopback
|
||||||
// and link-local addresses have no AS number or country.
|
// and link-local addresses have no AS number or country.
|
||||||
func canBePlaced(addr netip.Addr) bool {
|
func canBePlaced(addr netip.Addr) bool {
|
||||||
|
|||||||
@@ -16,7 +16,6 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -67,7 +66,7 @@ func TestMetricsNeedTheTokenAndOtherPathsAreNotFound(t *testing.T) {
|
|||||||
{http.MethodPost, waf.MetricsPath, bearer, http.StatusNotFound},
|
{http.MethodPost, waf.MetricsPath, bearer, http.StatusNotFound},
|
||||||
{http.MethodGet, waf.MetricsPath + "/", bearer, http.StatusNotFound},
|
{http.MethodGet, waf.MetricsPath + "/", bearer, http.StatusNotFound},
|
||||||
{http.MethodGet, "/_sww/bans", bearer, http.StatusNotFound},
|
{http.MethodGet, "/_sww/bans", bearer, http.StatusNotFound},
|
||||||
{http.MethodPost, wafprotocol.HealthPath, "", http.StatusNotFound},
|
{http.MethodPost, waf.HealthPath, "", http.StatusNotFound},
|
||||||
} {
|
} {
|
||||||
got := s.ownEndpointRequest(
|
got := s.ownEndpointRequest(
|
||||||
tc.authorization,
|
tc.authorization,
|
||||||
|
|||||||
@@ -28,6 +28,14 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/rules"
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ownEndpointPrefix starts the path of each of waf's own endpoints. The app
|
||||||
|
// never sees a path under it.
|
||||||
|
const ownEndpointPrefix = "/_sww/"
|
||||||
|
|
||||||
|
// HealthPath is the health endpoint of waf and of rproxy, which the
|
||||||
|
// container's health check asks.
|
||||||
|
const HealthPath = "/_sww/healthz"
|
||||||
|
|
||||||
// MetricsPath is where the metrics are, for a request that carries
|
// MetricsPath is where the metrics are, for a request that carries
|
||||||
// SWW_WAF_METRICS_TOKEN.
|
// SWW_WAF_METRICS_TOKEN.
|
||||||
const MetricsPath = "/_sww/metrics"
|
const MetricsPath = "/_sww/metrics"
|
||||||
|
|||||||
+22
-18
@@ -24,6 +24,7 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/processlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
@@ -398,11 +399,11 @@ func startRProxy(
|
|||||||
server := rproxy.New(rproxy.Params{
|
server := rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: io.Discard,
|
RequestLog: io.Discard,
|
||||||
ProcessLog: requestlog.NewProcessLogger(
|
ProcessLog: processlog.New(processlog.Params{
|
||||||
io.Discard,
|
Output: io.Discard,
|
||||||
"rproxy",
|
InstanceName: "rproxy",
|
||||||
slog.LevelInfo,
|
Level: slog.LevelInfo,
|
||||||
),
|
}),
|
||||||
})
|
})
|
||||||
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(
|
listener, err := (&net.ListenConfig{}).Listen(
|
||||||
@@ -471,11 +472,11 @@ func newWAFWith(
|
|||||||
}
|
}
|
||||||
|
|
||||||
out := &output{}
|
out := &output{}
|
||||||
processLog := requestlog.NewProcessLogger(
|
processLog := processlog.New(processlog.Params{
|
||||||
out,
|
Output: out,
|
||||||
cfg.InstanceName,
|
InstanceName: cfg.InstanceName,
|
||||||
cfg.LogLevel,
|
Level: cfg.LogLevel,
|
||||||
)
|
})
|
||||||
|
|
||||||
ruleFiles, err := rules.Load(rules.Params{
|
ruleFiles, err := rules.Load(rules.Params{
|
||||||
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
|
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
|
||||||
@@ -561,14 +562,17 @@ func addCounters(params *waf.Params) {
|
|||||||
cfg.MetricsBusiestCountriesAndASNs,
|
cfg.MetricsBusiestCountriesAndASNs,
|
||||||
cfg.InstanceName,
|
cfg.InstanceName,
|
||||||
)
|
)
|
||||||
params.Limiter = ratelimit.New(ratelimit.Limits{
|
params.Limiter = ratelimit.New(ratelimit.Params{
|
||||||
PerMinute: cfg.RateLimitPerMinute,
|
Limits: ratelimit.Limits{
|
||||||
PerHour: cfg.RateLimitPerHour,
|
PerMinute: cfg.RateLimitPerMinute,
|
||||||
PerDay: cfg.RateLimitPerDay,
|
PerHour: cfg.RateLimitPerHour,
|
||||||
BytesPerMinute: cfg.BytesLimitPerMinute,
|
PerDay: cfg.RateLimitPerDay,
|
||||||
BytesPerHour: cfg.BytesLimitPerHour,
|
BytesPerMinute: cfg.BytesLimitPerMinute,
|
||||||
BytesPerDay: cfg.BytesLimitPerDay,
|
BytesPerHour: cfg.BytesLimitPerHour,
|
||||||
}, cfg.MaxTrackedClients)
|
BytesPerDay: cfg.BytesLimitPerDay,
|
||||||
|
},
|
||||||
|
MaxClients: cfg.MaxTrackedClients,
|
||||||
|
})
|
||||||
params.Ledger = bans.New(bans.Params{
|
params.Ledger = bans.New(bans.Params{
|
||||||
LimitBanDuration: cfg.LimitBanDuration,
|
LimitBanDuration: cfg.LimitBanDuration,
|
||||||
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
|
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
|
||||||
|
|||||||
@@ -345,16 +345,6 @@ func timing(start, end time.Time) *float64 {
|
|||||||
return new(requestlog.Milliseconds(end.Sub(start)))
|
return new(requestlog.Milliseconds(end.Sub(start)))
|
||||||
}
|
}
|
||||||
|
|
||||||
// banAndCount runs list, afterAnswer or onReport, whose heuristics only ban.
|
|
||||||
// It then counts the request for the anomaly thresholds and adds it to its
|
|
||||||
// client's history. list runs first, so that a refusal limit it finds broken
|
|
||||||
// is in that history. ctx is the request's own context.
|
|
||||||
func (rq *request) banAndCount(ctx context.Context, list []heuristic) {
|
|
||||||
rq.runBanningHeuristics(ctx, list)
|
|
||||||
rq.countAnomalies()
|
|
||||||
rq.addToHistory()
|
|
||||||
}
|
|
||||||
|
|
||||||
// addToHistory adds the request, which has ended, to its client's
|
// addToHistory adds the request, which has ended, to its client's
|
||||||
// history, and counts its offences in the metrics, and then the lookup's
|
// history, and counts its offences in the metrics, and then the lookup's
|
||||||
// answer about the client, as answerAtTheEnd gives it, to that history and
|
// answer about the client, as answerAtTheEnd gives it, to that history and
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import (
|
|||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -248,7 +248,7 @@ func TestTimingsAreInOrder(t *testing.T) {
|
|||||||
wantStatus(t, do(t, req), http.StatusForbidden)
|
wantStatus(t, do(t, req), http.StatusForbidden)
|
||||||
refused := out.endedLines(t, 2)[1]
|
refused := out.endedLines(t, 2)[1]
|
||||||
|
|
||||||
wantStatus(t, get(t, server.Addr, wafprotocol.HealthPath), http.StatusOK)
|
wantStatus(t, get(t, server.Addr, waf.HealthPath), http.StatusOK)
|
||||||
health := out.endedLines(t, 3)[2]
|
health := out.endedLines(t, 3)[2]
|
||||||
|
|
||||||
// A request passed to the app has every timing; one refused, none of
|
// A request passed to the app has every timing; one refused, none of
|
||||||
|
|||||||
+253
-74
@@ -1,21 +1,75 @@
|
|||||||
package waf
|
package waf
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/netip"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// The headers of rproxy's question and of waf's answer, as
|
||||||
|
// "waf" in SPEC.md gives them.
|
||||||
|
const (
|
||||||
|
// ClientHeader marks a request as rproxy's question, and gives the
|
||||||
|
// visitor's address.
|
||||||
|
ClientHeader = "X-Sww-Client"
|
||||||
|
// PeerHeader gives the address of rproxy's own TCP peer.
|
||||||
|
PeerHeader = "X-Sww-Peer"
|
||||||
|
// ContentLengthHeader gives the body length the visitor announced.
|
||||||
|
ContentLengthHeader = "X-Sww-Content-Length"
|
||||||
|
// DispositionHeader gives waf's answer.
|
||||||
|
DispositionHeader = "X-Sww-Disposition"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The values of DispositionHeader.
|
||||||
|
const (
|
||||||
|
// DispositionAllow tells rproxy to pass the request to the app.
|
||||||
|
DispositionAllow = "allow"
|
||||||
|
// DispositionDeny tells rproxy to give the visitor the answer's status,
|
||||||
|
// headers and body.
|
||||||
|
DispositionDeny = "deny"
|
||||||
|
// DispositionClose tells rproxy to close the visitor's connection
|
||||||
|
// without an answer, which SWW_WAF_BAN_RESPONSE asks for when it is close.
|
||||||
|
// waf closing the question's connection instead would read as
|
||||||
|
// waf being down, and rproxy would then let the request through.
|
||||||
|
DispositionClose = "close"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ReportPath is where rproxy sends the report of each request waf
|
||||||
|
// answered.
|
||||||
|
const ReportPath = "/_sww/report"
|
||||||
|
|
||||||
|
// QuestionBodyMaxBytes is the most of a visitor's body a question carries.
|
||||||
|
const QuestionBodyMaxBytes = 64 << 10
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// maxAwaitingReport is the most answered requests held until their
|
// maxAwaitingReport is the most answered requests held until their
|
||||||
// reports come. Past it, the request answered longest ago is dropped.
|
// reports come. Past it, the request answered longest ago is dropped.
|
||||||
maxAwaitingReport = 10000
|
maxAwaitingReport = 10000
|
||||||
|
// maxRequestIDBytes is the longest id a question may give.
|
||||||
|
maxRequestIDBytes = 128
|
||||||
|
// reportBodyMaxBytes is the most of a report's body that is read. It is
|
||||||
|
// far above any report's.
|
||||||
|
reportBodyMaxBytes = 1 << 20
|
||||||
|
// maxHTTPStatus is the highest status a report may give.
|
||||||
|
maxHTTPStatus = 599
|
||||||
|
// maxReportedBytes is the most bytes a report may give for each way, 1
|
||||||
|
// PiB, so that adding them up cannot overflow.
|
||||||
|
maxReportedBytes = 1 << 50
|
||||||
|
// maxReportedMilliseconds is the longest duration a report may give,
|
||||||
|
// about 31 years, so that it fits a time.Duration.
|
||||||
|
maxReportedMilliseconds = 1e12
|
||||||
// wafTimeout bounds how long a request to waf takes to
|
// wafTimeout bounds how long a request to waf takes to
|
||||||
// arrive, and its answer to leave, and wafIdleTimeout how long a
|
// arrive, and its answer to leave, and wafIdleTimeout how long a
|
||||||
// kept-open connection waits for its next request.
|
// kept-open connection waits for its next request.
|
||||||
@@ -23,6 +77,26 @@ const (
|
|||||||
wafIdleTimeout = 120 * time.Second
|
wafIdleTimeout = 120 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
errReportFieldMissing = errors.New("a field is missing")
|
||||||
|
errReportValue = errors.New("a value is out of range")
|
||||||
|
)
|
||||||
|
|
||||||
|
// Report is the JSON object rproxy posts to ReportPath once the visitor's
|
||||||
|
// answer has ended, its fields named as the request log's of the same
|
||||||
|
// meaning. RequestID is the question's id. Status is what the visitor got, 0
|
||||||
|
// if nothing. The durations are in milliseconds.
|
||||||
|
//
|
||||||
|
//nolint:tagliatelle // docs/request-log.md names the fields in snake_case
|
||||||
|
type Report struct {
|
||||||
|
RequestID string `json:"request_id"`
|
||||||
|
Status int `json:"status"`
|
||||||
|
RequestBytes int64 `json:"request_bytes"`
|
||||||
|
ResponseBytes int64 `json:"response_bytes"`
|
||||||
|
DurationTotalMS float64 `json:"duration_total_ms"`
|
||||||
|
DurationUpstreamTotalMS float64 `json:"duration_upstream_total_ms"`
|
||||||
|
}
|
||||||
|
|
||||||
// New returns the server `sww waf` runs. It answers
|
// New returns the server `sww waf` runs. It answers
|
||||||
// rproxy's questions, takes rproxy's reports and answers its own endpoints. It
|
// rproxy's questions, takes rproxy's reports and answers its own endpoints. It
|
||||||
// passes nothing to an app. rproxy applies the size and time limits to the
|
// passes nothing to an app. rproxy applies the size and time limits to the
|
||||||
@@ -58,10 +132,10 @@ func New(params Params) *Server {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ServeHTTP handles one request to waf: a request carrying
|
// ServeHTTP handles one request to waf: a request carrying
|
||||||
// wafprotocol.ClientHeader is rproxy's question, and any other is for waf
|
// ClientHeader is rproxy's question, and any other is for waf
|
||||||
// itself.
|
// itself.
|
||||||
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
if len(r.Header.Values(wafprotocol.ClientHeader)) > 0 {
|
if len(r.Header.Values(ClientHeader)) > 0 {
|
||||||
h.answerQuestion(w, r)
|
h.answerQuestion(w, r)
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -88,76 +162,104 @@ func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
|
|||||||
// carries, and answers rproxy what to do with it. A denied request's counting
|
// carries, and answers rproxy what to do with it. A denied request's counting
|
||||||
// is finished here, and an allowed one's by its report.
|
// is finished here, and an allowed one's by its report.
|
||||||
func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
|
||||||
question, ok := h.readQuestion(w, r)
|
client, body, ok := h.readQuestion(w, r)
|
||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// One that cannot be read leaves the log line's peer_ip invalid.
|
||||||
|
peer, _ := netip.ParseAddr(r.Header.Get(PeerHeader))
|
||||||
|
|
||||||
rq := h.startRequest(
|
rq := h.startRequest(
|
||||||
w,
|
w,
|
||||||
question.Visitor,
|
asTheVisitorSentIt(r, body),
|
||||||
question.Client,
|
client.Unmap(),
|
||||||
question.Peer,
|
peer.Unmap(),
|
||||||
true,
|
true,
|
||||||
)
|
)
|
||||||
// These run even when a check panics. finish runs first, so that what is
|
// The last deferred runs first, so that what is held has the finished log
|
||||||
// held has the finished log line. Go's server sends the answer only once
|
// line. Go's server sends the answer only once this function has
|
||||||
// this function has returned, so the report cannot come before it is held.
|
// returned, so the report cannot come before it is held.
|
||||||
defer func() {
|
defer rq.holdForReport()
|
||||||
rq.finish()
|
defer rq.finish()
|
||||||
rq.holdForReport()
|
|
||||||
}()
|
|
||||||
|
|
||||||
rq.site = r.Host
|
rq.site = r.Host
|
||||||
rq.line.WAFLine = requestlog.WAFLineDecision
|
rq.line.WAFLine = requestlog.WAFLineDecision
|
||||||
refused := rq.check(r.Context())
|
refused := rq.check(r.Context())
|
||||||
rq.checked = time.Now()
|
rq.checked = time.Now()
|
||||||
underOwnPrefix := strings.HasPrefix(
|
underOwnPrefix := strings.HasPrefix(r.URL.Path, ownEndpointPrefix)
|
||||||
r.URL.Path,
|
|
||||||
wafprotocol.OwnEndpointPrefix,
|
|
||||||
)
|
|
||||||
|
|
||||||
if refused == nil && !underOwnPrefix {
|
if refused == nil && !underOwnPrefix {
|
||||||
rq.allow()
|
|
||||||
// In observe mode, what enforce mode would have refused counts here.
|
// In observe mode, what enforce mode would have refused counts here.
|
||||||
rq.runBanningHeuristics(r.Context(), afterAnswer())
|
defer rq.runAfterAnswer(r.Context())
|
||||||
|
|
||||||
|
rq.allow()
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The last deferred runs first: runAfterAnswer before addToHistory, so that
|
||||||
|
// a broken refusal limit is in the client's history.
|
||||||
|
defer rq.addToHistory()
|
||||||
|
defer rq.countAnomalies()
|
||||||
|
defer rq.runAfterAnswer(r.Context())
|
||||||
|
|
||||||
if refused != nil {
|
if refused != nil {
|
||||||
rq.deny(*refused)
|
rq.deny(*refused)
|
||||||
} else {
|
|
||||||
// The app never sees this prefix. waf's own endpoints are at its
|
return
|
||||||
// own address, so this answers as a path with no endpoint does.
|
|
||||||
rq.line.Action = requestlog.ActionOwnEndpoint
|
|
||||||
wafprotocol.SetAnswerHeaders(
|
|
||||||
rq.out.Header(),
|
|
||||||
wafprotocol.Answer{Disposition: wafprotocol.DispositionDeny},
|
|
||||||
)
|
|
||||||
http.Error(
|
|
||||||
rq.out,
|
|
||||||
http.StatusText(http.StatusNotFound),
|
|
||||||
http.StatusNotFound,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
rq.banAndCount(r.Context(), afterAnswer())
|
// The app never sees this prefix. waf's own endpoints are at its
|
||||||
|
// own address, so this answers as a path with no endpoint does.
|
||||||
|
rq.line.Action = requestlog.ActionOwnEndpoint
|
||||||
|
rq.out.Header().Set(DispositionHeader, DispositionDeny)
|
||||||
|
http.Error(
|
||||||
|
rq.out,
|
||||||
|
http.StatusText(http.StatusNotFound),
|
||||||
|
http.StatusNotFound,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// readQuestion reads a question, and reports whether waf takes it. It answers
|
// readQuestion returns the visitor's address and the body a question carries,
|
||||||
// a question it does not take with 400 and no disposition. rproxy passes that
|
// and whether waf takes the question. It answers a question it does not
|
||||||
// 400 to the visitor, and SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION does
|
// take with 400 and no disposition. rproxy passes that 400 to the visitor,
|
||||||
// not apply. The id of a question waf takes is held from here on, without a
|
// and SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION does not apply. The id of
|
||||||
// decision yet, so that another question with that id is refused while this
|
// a question waf takes is held from here on, without a decision yet, so
|
||||||
// one is being handled.
|
// that another question with that id is refused while this one is being
|
||||||
|
// handled.
|
||||||
//
|
//
|
||||||
// A question whose body stops arriving gets no answer, and is not looked at.
|
// The body is read before any check, since Go's server does not send an answer
|
||||||
|
// until it has read the body. A question whose body stops arriving gets no
|
||||||
|
// answer, and is not looked at.
|
||||||
func (h *handler) readQuestion(
|
func (h *handler) readQuestion(
|
||||||
w http.ResponseWriter, r *http.Request,
|
w http.ResponseWriter, r *http.Request,
|
||||||
) (wafprotocol.Question, bool) {
|
) (netip.Addr, []byte, bool) {
|
||||||
question, err := wafprotocol.ReadQuestion(w, r)
|
client, err := netip.ParseAddr(r.Header.Get(ClientHeader))
|
||||||
if errors.Is(err, wafprotocol.ErrBodyStoppedArriving) {
|
if err != nil || client.Zone() != "" {
|
||||||
|
h.refuseQuestion(w, ClientHeader+" is not an address without a zone")
|
||||||
|
|
||||||
|
return netip.Addr{}, nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
id := r.Header.Get(requestIDHeader)
|
||||||
|
if id == "" || len(id) > maxRequestIDBytes || !utf8.ValidString(id) {
|
||||||
|
h.refuseQuestion(w, requestIDHeader+" is not a valid id")
|
||||||
|
|
||||||
|
return netip.Addr{}, nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := io.ReadAll(
|
||||||
|
http.MaxBytesReader(w, r.Body, QuestionBodyMaxBytes),
|
||||||
|
)
|
||||||
|
|
||||||
|
if _, tooLarge := errors.AsType[*http.MaxBytesError](err); tooLarge {
|
||||||
|
h.refuseQuestion(w, "the body is over 64 KiB")
|
||||||
|
|
||||||
|
return netip.Addr{}, nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
h.processLog.Warn(
|
h.processLog.Warn(
|
||||||
"a question's body stopped arriving",
|
"a question's body stopped arriving",
|
||||||
"error",
|
"error",
|
||||||
@@ -168,23 +270,17 @@ func (h *handler) readQuestion(
|
|||||||
panic(http.ErrAbortHandler)
|
panic(http.ErrAbortHandler)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
held, _ := h.awaitingReport.ContainsOrAdd(id, nil)
|
||||||
h.refuseQuestion(w, err.Error())
|
|
||||||
|
|
||||||
return question, false
|
|
||||||
}
|
|
||||||
|
|
||||||
held, _ := h.awaitingReport.ContainsOrAdd(question.RequestID, nil)
|
|
||||||
if held {
|
if held {
|
||||||
h.refuseQuestion(
|
h.refuseQuestion(
|
||||||
w,
|
w,
|
||||||
wafprotocol.RequestIDHeader+" is the id of a request waf holds",
|
requestIDHeader+" is the id of a request waf holds",
|
||||||
)
|
)
|
||||||
|
|
||||||
return question, false
|
return netip.Addr{}, nil, false
|
||||||
}
|
}
|
||||||
|
|
||||||
return question, true
|
return client, body, true
|
||||||
}
|
}
|
||||||
|
|
||||||
// refuseQuestion answers a question waf does not take, for reason.
|
// refuseQuestion answers a question waf does not take, for reason.
|
||||||
@@ -193,18 +289,48 @@ func (h *handler) refuseQuestion(w http.ResponseWriter, reason string) {
|
|||||||
http.Error(w, reason, http.StatusBadRequest)
|
http.Error(w, reason, http.StatusBadRequest)
|
||||||
}
|
}
|
||||||
|
|
||||||
// allow answers a question with wafprotocol.DispositionAllow and, while
|
// asTheVisitorSentIt returns a copy of r, a question, as the checks and the
|
||||||
|
// log line are to see it: without the headers rproxy adds for waf,
|
||||||
|
// with body, which has been read of r, and with the length the visitor
|
||||||
|
// announced, as -1 when it announced none. The question's own length covers
|
||||||
|
// only the part of the body it carries.
|
||||||
|
func asTheVisitorSentIt(r *http.Request, body []byte) *http.Request {
|
||||||
|
sent := r.Clone(r.Context())
|
||||||
|
announced := sent.Header.Get(ContentLengthHeader)
|
||||||
|
|
||||||
|
sent.Header.Del(ClientHeader)
|
||||||
|
sent.Header.Del(PeerHeader)
|
||||||
|
sent.Header.Del(ContentLengthHeader)
|
||||||
|
sent.Header.Del("Content-Length")
|
||||||
|
|
||||||
|
sent.ContentLength = -1
|
||||||
|
|
||||||
|
length, err := strconv.ParseInt(announced, 10, 64)
|
||||||
|
if err == nil && length >= 0 {
|
||||||
|
sent.ContentLength = length
|
||||||
|
sent.Header.Set("Content-Length", announced)
|
||||||
|
}
|
||||||
|
|
||||||
|
sent.Body = http.NoBody
|
||||||
|
if len(body) > 0 {
|
||||||
|
sent.Body = io.NopCloser(bytes.NewReader(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
return sent
|
||||||
|
}
|
||||||
|
|
||||||
|
// allow answers a question with DispositionAllow and, while
|
||||||
// SWW_WAF_ADD_LOOKUP_HEADERS is true, the client's AS number and country.
|
// SWW_WAF_ADD_LOOKUP_HEADERS is true, the client's AS number and country.
|
||||||
func (rq *request) allow() {
|
func (rq *request) allow() {
|
||||||
rq.allowed = true
|
rq.allowed = true
|
||||||
|
|
||||||
answer := wafprotocol.Answer{Disposition: wafprotocol.DispositionAllow}
|
header := rq.out.Header()
|
||||||
|
header.Set(DispositionHeader, DispositionAllow)
|
||||||
|
|
||||||
if rq.h.config.AddLookupHeaders {
|
if rq.h.config.AddLookupHeaders {
|
||||||
answer.ASN = rq.line.ASN
|
setLookupHeaders(header, rq.line.ASN, rq.line.Country)
|
||||||
answer.Country = rq.line.Country
|
|
||||||
}
|
}
|
||||||
|
|
||||||
wafprotocol.SetAnswerHeaders(rq.out.Header(), answer)
|
|
||||||
rq.out.WriteHeader(http.StatusOK)
|
rq.out.WriteHeader(http.StatusOK)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -222,19 +348,13 @@ func (rq *request) deny(r refusal) {
|
|||||||
if r.status == 0 {
|
if r.status == 0 {
|
||||||
// SWW_WAF_BAN_RESPONSE is close. Written past rq.out, since the log line's
|
// SWW_WAF_BAN_RESPONSE is close. Written past rq.out, since the log line's
|
||||||
// status stays 0: the visitor is sent nothing.
|
// status stays 0: the visitor is sent nothing.
|
||||||
wafprotocol.SetAnswerHeaders(
|
rq.out.Header().Set(DispositionHeader, DispositionClose)
|
||||||
rq.out.Header(),
|
|
||||||
wafprotocol.Answer{Disposition: wafprotocol.DispositionClose},
|
|
||||||
)
|
|
||||||
rq.out.ResponseWriter.WriteHeader(http.StatusOK)
|
rq.out.ResponseWriter.WriteHeader(http.StatusOK)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
wafprotocol.SetAnswerHeaders(
|
rq.out.Header().Set(DispositionHeader, DispositionDeny)
|
||||||
rq.out.Header(),
|
|
||||||
wafprotocol.Answer{Disposition: wafprotocol.DispositionDeny},
|
|
||||||
)
|
|
||||||
http.Error(rq.out, http.StatusText(r.status), r.status)
|
http.Error(rq.out, http.StatusText(r.status), r.status)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -242,7 +362,7 @@ func (rq *request) deny(r refusal) {
|
|||||||
// runs on it: the address is on the private network, where the peers are
|
// runs on it: the address is on the private network, where the peers are
|
||||||
// rproxy and the admin's tools, not visitors.
|
// rproxy and the admin's tools, not visitors.
|
||||||
func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.Method == http.MethodPost && r.URL.Path == wafprotocol.ReportPath {
|
if r.Method == http.MethodPost && r.URL.Path == ReportPath {
|
||||||
h.takeReport(w, r)
|
h.takeReport(w, r)
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -253,7 +373,7 @@ func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) {
|
|||||||
rq := h.startRequest(w, r, peer, peer, false)
|
rq := h.startRequest(w, r, peer, peer, false)
|
||||||
defer rq.finish()
|
defer rq.finish()
|
||||||
|
|
||||||
if r.Method == http.MethodGet && r.URL.Path == wafprotocol.HealthPath {
|
if r.Method == http.MethodGet && r.URL.Path == HealthPath {
|
||||||
rq.answerHealth()
|
rq.answerHealth()
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -262,11 +382,11 @@ func (h *handler) answerOwnEndpoint(w http.ResponseWriter, r *http.Request) {
|
|||||||
rq.answerAdmin()
|
rq.answerAdmin()
|
||||||
}
|
}
|
||||||
|
|
||||||
// takeReport answers POST wafprotocol.ReportPath: it finishes the request the
|
// takeReport answers POST ReportPath: it finishes the request the report
|
||||||
// report names, if it holds that request's decision, and ignores the report
|
// names, if it holds that request's decision, and ignores the report
|
||||||
// otherwise.
|
// otherwise.
|
||||||
func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) {
|
||||||
report, err := wafprotocol.ReadReport(w, r)
|
report, err := readReport(w, r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.processLog.Warn("a report is refused", "error", err.Error())
|
h.processLog.Warn("a report is refused", "error", err.Error())
|
||||||
http.Error(w, "the body is not a report", http.StatusBadRequest)
|
http.Error(w, "the body is not a report", http.StatusBadRequest)
|
||||||
@@ -284,6 +404,63 @@ func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) {
|
|||||||
w.WriteHeader(http.StatusNoContent)
|
w.WriteHeader(http.StatusNoContent)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// readReport reads a report from r's body, which must be one JSON object with
|
||||||
|
// all of Report's fields, in the ranges SPEC.md gives.
|
||||||
|
func readReport(w http.ResponseWriter, r *http.Request) (Report, error) {
|
||||||
|
var report Report
|
||||||
|
|
||||||
|
body, err := io.ReadAll(
|
||||||
|
http.MaxBytesReader(w, r.Body, reportBodyMaxBytes),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return report, fmt.Errorf("read the body: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var fields map[string]json.RawMessage
|
||||||
|
|
||||||
|
err = json.Unmarshal(body, &fields)
|
||||||
|
if err != nil {
|
||||||
|
return report, fmt.Errorf("decode the body: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, name := range []string{
|
||||||
|
"request_id", "status", "request_bytes", "response_bytes",
|
||||||
|
"duration_total_ms", "duration_upstream_total_ms",
|
||||||
|
} {
|
||||||
|
value, there := fields[name]
|
||||||
|
if !there || string(value) == "null" {
|
||||||
|
return report, fmt.Errorf("%w: %s", errReportFieldMissing, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
err = json.Unmarshal(body, &report)
|
||||||
|
if err != nil {
|
||||||
|
return report, fmt.Errorf("decode the body: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return report, report.checkRanges()
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkRanges returns an error for a value of the report that cannot be true.
|
||||||
|
func (report *Report) checkRanges() error {
|
||||||
|
switch {
|
||||||
|
case report.Status != 0 &&
|
||||||
|
(report.Status < http.StatusContinue || report.Status > maxHTTPStatus):
|
||||||
|
return fmt.Errorf("%w: status", errReportValue)
|
||||||
|
case report.RequestBytes < 0 || report.RequestBytes > maxReportedBytes:
|
||||||
|
return fmt.Errorf("%w: request_bytes", errReportValue)
|
||||||
|
case report.ResponseBytes < 0 || report.ResponseBytes > maxReportedBytes:
|
||||||
|
return fmt.Errorf("%w: response_bytes", errReportValue)
|
||||||
|
case report.DurationTotalMS < 0 || report.DurationTotalMS > maxReportedMilliseconds:
|
||||||
|
return fmt.Errorf("%w: duration_total_ms", errReportValue)
|
||||||
|
case report.DurationUpstreamTotalMS < 0 ||
|
||||||
|
report.DurationUpstreamTotalMS > maxReportedMilliseconds:
|
||||||
|
return fmt.Errorf("%w: duration_upstream_total_ms", errReportValue)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// finishWith finishes a request with what its report says, and writes the
|
// finishWith finishes a request with what its report says, and writes the
|
||||||
// request's last log line, which holds its decision's fields and the report's.
|
// request's last log line, which holds its decision's fields and the report's.
|
||||||
// For an allowed request it first runs the heuristics of onReport, and adds the
|
// For an allowed request it first runs the heuristics of onReport, and adds the
|
||||||
@@ -293,7 +470,7 @@ func (h *handler) takeReport(w http.ResponseWriter, r *http.Request) {
|
|||||||
func (h *handler) finishWith(
|
func (h *handler) finishWith(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
held *decision,
|
held *decision,
|
||||||
report wafprotocol.Report,
|
report Report,
|
||||||
) {
|
) {
|
||||||
rq := &request{
|
rq := &request{
|
||||||
decision: *held,
|
decision: *held,
|
||||||
@@ -314,7 +491,9 @@ func (h *handler) finishWith(
|
|||||||
if rq.allowed {
|
if rq.allowed {
|
||||||
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
|
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
|
||||||
|
|
||||||
rq.banAndCount(ctx, onReport())
|
rq.runBanningHeuristics(ctx, onReport())
|
||||||
|
rq.countAnomalies()
|
||||||
|
rq.addToHistory()
|
||||||
|
|
||||||
rq.h.metrics.RequestReported(
|
rq.h.metrics.RequestReported(
|
||||||
&rq.line,
|
&rq.line,
|
||||||
|
|||||||
+86
-195
@@ -22,7 +22,6 @@ import (
|
|||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -107,8 +106,8 @@ func (s *WAF) questionWith(
|
|||||||
|
|
||||||
req := newRequest(s.t, method, s.addr, path, body)
|
req := newRequest(s.t, method, s.addr, path, body)
|
||||||
req.Host = host
|
req.Host = host
|
||||||
req.Header.Set(wafprotocol.ClientHeader, from)
|
req.Header.Set(waf.ClientHeader, from)
|
||||||
req.Header.Set(wafprotocol.PeerHeader, rproxyPeer)
|
req.Header.Set(waf.PeerHeader, rproxyPeer)
|
||||||
req.Header.Set(requestIDHeader, rand.Text())
|
req.Header.Set(requestIDHeader, rand.Text())
|
||||||
|
|
||||||
return req
|
return req
|
||||||
@@ -135,18 +134,13 @@ func (s *WAF) allowed(from string) string {
|
|||||||
s.t.Helper()
|
s.t.Helper()
|
||||||
|
|
||||||
req := s.question(siteA, from, "/")
|
req := s.question(siteA, from, "/")
|
||||||
wantDisposition(
|
wantDisposition(s.t, do(s.t, req), http.StatusOK, waf.DispositionAllow)
|
||||||
s.t,
|
|
||||||
do(s.t, req),
|
|
||||||
http.StatusOK,
|
|
||||||
wafprotocol.DispositionAllow,
|
|
||||||
)
|
|
||||||
|
|
||||||
return req.Header.Get(requestIDHeader)
|
return req.Header.Get(requestIDHeader)
|
||||||
}
|
}
|
||||||
|
|
||||||
// report sends rproxy's report and returns the answer.
|
// report sends rproxy's report and returns the answer.
|
||||||
func (s *WAF) report(report wafprotocol.Report) answer {
|
func (s *WAF) report(report waf.Report) answer {
|
||||||
s.t.Helper()
|
s.t.Helper()
|
||||||
|
|
||||||
body, err := json.Marshal(report)
|
body, err := json.Marshal(report)
|
||||||
@@ -161,11 +155,8 @@ func (s *WAF) report(report wafprotocol.Report) answer {
|
|||||||
func (s *WAF) reportBody(body string) answer {
|
func (s *WAF) reportBody(body string) answer {
|
||||||
s.t.Helper()
|
s.t.Helper()
|
||||||
|
|
||||||
return do(
|
return do(s.t, newRequest(s.t, http.MethodPost, s.addr, waf.ReportPath,
|
||||||
s.t,
|
strings.NewReader(body)))
|
||||||
newRequest(s.t, http.MethodPost, s.addr, wafprotocol.ReportPath,
|
|
||||||
strings.NewReader(body)),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// passedOn asks about a request from the visitor at from on siteA, and reports
|
// passedOn asks about a request from the visitor at from on siteA, and reports
|
||||||
@@ -173,7 +164,7 @@ func (s *WAF) reportBody(body string) answer {
|
|||||||
func (s *WAF) passedOn(from string) {
|
func (s *WAF) passedOn(from string) {
|
||||||
s.t.Helper()
|
s.t.Helper()
|
||||||
|
|
||||||
got := s.report(wafprotocol.Report{
|
got := s.report(waf.Report{
|
||||||
RequestID: s.allowed(from),
|
RequestID: s.allowed(from),
|
||||||
Status: http.StatusOK,
|
Status: http.StatusOK,
|
||||||
RequestBytes: bodyBytes,
|
RequestBytes: bodyBytes,
|
||||||
@@ -219,7 +210,7 @@ func wantDisposition(
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
wantStatus(t, got, status)
|
wantStatus(t, got, status)
|
||||||
|
|
||||||
if answered := got.header.Get(wafprotocol.DispositionHeader); answered !=
|
if answered := got.header.Get(waf.DispositionHeader); answered !=
|
||||||
disposition {
|
disposition {
|
||||||
t.Errorf(
|
t.Errorf(
|
||||||
"answered with disposition %q, want %q",
|
"answered with disposition %q, want %q",
|
||||||
@@ -281,10 +272,10 @@ func TestQuestionAllowedIsAnsweredAndLogged(t *testing.T) {
|
|||||||
req := s.question(siteA, client, "/page?q=1&token=visible")
|
req := s.question(siteA, client, "/page?q=1&token=visible")
|
||||||
req.Header.Set(requestIDHeader, "the-id")
|
req.Header.Set(requestIDHeader, "the-id")
|
||||||
req.Header.Set("X-Forwarded-Proto", "https")
|
req.Header.Set("X-Forwarded-Proto", "https")
|
||||||
req.Header.Set(wafprotocol.ContentLengthHeader, "1234")
|
req.Header.Set(waf.ContentLengthHeader, "1234")
|
||||||
|
|
||||||
got := do(t, req)
|
got := do(t, req)
|
||||||
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||||
wantBody(t, got, "")
|
wantBody(t, got, "")
|
||||||
|
|
||||||
if id := got.header.Get(requestIDHeader); id != "" {
|
if id := got.header.Get(requestIDHeader); id != "" {
|
||||||
@@ -317,12 +308,7 @@ func TestIPv6VisitorIsCountedInItsGroup(t *testing.T) {
|
|||||||
s.allowed("2001:db8::1")
|
s.allowed("2001:db8::1")
|
||||||
|
|
||||||
got := s.ask("2001:db8::2", "/")
|
got := s.ask("2001:db8::2", "/")
|
||||||
wantDisposition(
|
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
||||||
t,
|
|
||||||
got,
|
|
||||||
http.StatusForbidden,
|
|
||||||
wafprotocol.DispositionDeny,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestIPv4VisitorInIPv6FormIsTheIPv4Client(t *testing.T) {
|
func TestIPv4VisitorInIPv6FormIsTheIPv4Client(t *testing.T) {
|
||||||
@@ -331,14 +317,9 @@ func TestIPv4VisitorInIPv6FormIsTheIPv4Client(t *testing.T) {
|
|||||||
s := startWAF(t, "", map[string]string{denyNets: client})
|
s := startWAF(t, "", map[string]string{denyNets: client})
|
||||||
|
|
||||||
req := s.question(siteA, "::ffff:"+client, "/")
|
req := s.question(siteA, "::ffff:"+client, "/")
|
||||||
req.Header.Set(wafprotocol.PeerHeader, "::ffff:"+rproxyPeer)
|
req.Header.Set(waf.PeerHeader, "::ffff:"+rproxyPeer)
|
||||||
|
|
||||||
wantDisposition(
|
wantDisposition(t, do(t, req), http.StatusForbidden, waf.DispositionDeny)
|
||||||
t,
|
|
||||||
do(t, req),
|
|
||||||
http.StatusForbidden,
|
|
||||||
wafprotocol.DispositionDeny,
|
|
||||||
)
|
|
||||||
|
|
||||||
line := s.out.requestLine(t)
|
line := s.out.requestLine(t)
|
||||||
if line.ClientIP != client || line.PeerIP != rproxyPeer {
|
if line.ClientIP != client || line.PeerIP != rproxyPeer {
|
||||||
@@ -358,13 +339,13 @@ func TestMalformedQuestionGets400AndAWarningAndNoLine(t *testing.T) {
|
|||||||
change func(*http.Request)
|
change func(*http.Request)
|
||||||
}{
|
}{
|
||||||
{"client not an address", func(req *http.Request) {
|
{"client not an address", func(req *http.Request) {
|
||||||
req.Header.Set(wafprotocol.ClientHeader, "not an address")
|
req.Header.Set(waf.ClientHeader, "not an address")
|
||||||
}},
|
}},
|
||||||
{"client empty", func(req *http.Request) {
|
{"client empty", func(req *http.Request) {
|
||||||
req.Header.Set(wafprotocol.ClientHeader, "")
|
req.Header.Set(waf.ClientHeader, "")
|
||||||
}},
|
}},
|
||||||
{"client address with a zone", func(req *http.Request) {
|
{"client address with a zone", func(req *http.Request) {
|
||||||
req.Header.Set(wafprotocol.ClientHeader, "2001:db8::1%eth0")
|
req.Header.Set(waf.ClientHeader, "2001:db8::1%eth0")
|
||||||
}},
|
}},
|
||||||
{"id absent", func(req *http.Request) {
|
{"id absent", func(req *http.Request) {
|
||||||
req.Header.Del(requestIDHeader)
|
req.Header.Del(requestIDHeader)
|
||||||
@@ -409,12 +390,7 @@ func TestLongestValidIDIsTaken(t *testing.T) {
|
|||||||
|
|
||||||
req := s.question(siteA, client, "/")
|
req := s.question(siteA, client, "/")
|
||||||
req.Header.Set(requestIDHeader, strings.Repeat("x", 128))
|
req.Header.Set(requestIDHeader, strings.Repeat("x", 128))
|
||||||
wantDisposition(
|
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow)
|
||||||
t,
|
|
||||||
do(t, req),
|
|
||||||
http.StatusOK,
|
|
||||||
wafprotocol.DispositionAllow,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestQuestionWithABodyOver64KiBGets400(t *testing.T) {
|
func TestQuestionWithABodyOver64KiBGets400(t *testing.T) {
|
||||||
@@ -423,18 +399,16 @@ func TestQuestionWithABodyOver64KiBGets400(t *testing.T) {
|
|||||||
s := startWAF(t, "", nil)
|
s := startWAF(t, "", nil)
|
||||||
|
|
||||||
over := strings.NewReader(
|
over := strings.NewReader(
|
||||||
strings.Repeat("x", wafprotocol.QuestionBodyMaxBytes+1),
|
strings.Repeat("x", waf.QuestionBodyMaxBytes+1),
|
||||||
)
|
)
|
||||||
got := do(t, s.questionWith(http.MethodPost, siteA, client, "/", over))
|
got := do(t, s.questionWith(http.MethodPost, siteA, client, "/", over))
|
||||||
wantDisposition(t, got, http.StatusBadRequest, "")
|
wantDisposition(t, got, http.StatusBadRequest, "")
|
||||||
wantProcessMessages(t, s, "a question is refused", 1)
|
wantProcessMessages(t, s, "a question is refused", 1)
|
||||||
wantNoRequestLines(t, s)
|
wantNoRequestLines(t, s)
|
||||||
|
|
||||||
whole := strings.NewReader(
|
whole := strings.NewReader(strings.Repeat("x", waf.QuestionBodyMaxBytes))
|
||||||
strings.Repeat("x", wafprotocol.QuestionBodyMaxBytes),
|
|
||||||
)
|
|
||||||
got = do(t, s.questionWith(http.MethodPost, siteA, client, "/", whole))
|
got = do(t, s.questionWith(http.MethodPost, siteA, client, "/", whole))
|
||||||
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestQuestionWithAHeldIDIsRefused(t *testing.T) {
|
func TestQuestionWithAHeldIDIsRefused(t *testing.T) {
|
||||||
@@ -450,12 +424,12 @@ func TestQuestionWithAHeldIDIsRefused(t *testing.T) {
|
|||||||
wantProcessMessages(t, s, "a question is refused", 1)
|
wantProcessMessages(t, s, "a question is refused", 1)
|
||||||
|
|
||||||
// A denied request's id is held as well.
|
// A denied request's id is held as well.
|
||||||
denied := s.question(siteA, otherClient, wafprotocol.HealthPath)
|
denied := s.question(siteA, otherClient, waf.HealthPath)
|
||||||
wantDisposition(
|
wantDisposition(
|
||||||
t,
|
t,
|
||||||
do(t, denied),
|
do(t, denied),
|
||||||
http.StatusNotFound,
|
http.StatusNotFound,
|
||||||
wafprotocol.DispositionDeny,
|
waf.DispositionDeny,
|
||||||
)
|
)
|
||||||
|
|
||||||
again = s.question(siteA, client, "/")
|
again = s.question(siteA, client, "/")
|
||||||
@@ -463,9 +437,7 @@ func TestQuestionWithAHeldIDIsRefused(t *testing.T) {
|
|||||||
wantDisposition(t, do(t, again), http.StatusBadRequest, "")
|
wantDisposition(t, do(t, again), http.StatusBadRequest, "")
|
||||||
|
|
||||||
// The report ends the holding, and the refused questions changed nothing.
|
// The report ends the holding, and the refused questions changed nothing.
|
||||||
got := s.report(
|
got := s.report(waf.Report{RequestID: id, ResponseBytes: answerBytes})
|
||||||
wafprotocol.Report{RequestID: id, ResponseBytes: answerBytes},
|
|
||||||
)
|
|
||||||
wantStatus(t, got, http.StatusNoContent)
|
wantStatus(t, got, http.StatusNoContent)
|
||||||
|
|
||||||
if history := s.history(); history.ResponseBytes != answerBytes {
|
if history := s.history(); history.ResponseBytes != answerBytes {
|
||||||
@@ -474,12 +446,7 @@ func TestQuestionWithAHeldIDIsRefused(t *testing.T) {
|
|||||||
|
|
||||||
again = s.question(siteA, client, "/")
|
again = s.question(siteA, client, "/")
|
||||||
again.Header.Set(requestIDHeader, id)
|
again.Header.Set(requestIDHeader, id)
|
||||||
wantDisposition(
|
wantDisposition(t, do(t, again), http.StatusOK, waf.DispositionAllow)
|
||||||
t,
|
|
||||||
do(t, again),
|
|
||||||
http.StatusOK,
|
|
||||||
wafprotocol.DispositionAllow,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
|
func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
|
||||||
@@ -514,7 +481,7 @@ func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
|
|||||||
|
|
||||||
// A report for a question not yet answered is ignored, and that question's
|
// A report for a question not yet answered is ignored, and that question's
|
||||||
// id is taken.
|
// id is taken.
|
||||||
got := s.report(wafprotocol.Report{RequestID: id, ResponseBytes: 1000})
|
got := s.report(waf.Report{RequestID: id, ResponseBytes: 1000})
|
||||||
wantStatus(t, got, http.StatusNoContent)
|
wantStatus(t, got, http.StatusNoContent)
|
||||||
|
|
||||||
// This would wait on the same lookup were the id not taken.
|
// This would wait on the same lookup were the id not taken.
|
||||||
@@ -529,7 +496,7 @@ func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
|
|||||||
|
|
||||||
select {
|
select {
|
||||||
case got := <-answered:
|
case got := <-answered:
|
||||||
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||||
case <-time.After(waitLimit):
|
case <-time.After(waitLimit):
|
||||||
t.Fatal("the question got no answer")
|
t.Fatal("the question got no answer")
|
||||||
}
|
}
|
||||||
@@ -548,9 +515,7 @@ func TestReportNeverReachesAQuestionStillBeingHandled(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The true report comes after, and is the one that counts.
|
// The true report comes after, and is the one that counts.
|
||||||
got = s.report(
|
got = s.report(waf.Report{RequestID: id, ResponseBytes: answerBytes})
|
||||||
wafprotocol.Report{RequestID: id, ResponseBytes: answerBytes},
|
|
||||||
)
|
|
||||||
wantStatus(t, got, http.StatusNoContent)
|
wantStatus(t, got, http.StatusNoContent)
|
||||||
|
|
||||||
if history := s.history(); history.ResponseBytes != answerBytes {
|
if history := s.history(); history.ResponseBytes != answerBytes {
|
||||||
@@ -568,17 +533,17 @@ func TestPeerAbsentOrUnreadableIsAnsweredAndLoggedInvalid(t *testing.T) {
|
|||||||
s := startWAF(t, "", nil)
|
s := startWAF(t, "", nil)
|
||||||
|
|
||||||
req := s.question(siteA, client, "/")
|
req := s.question(siteA, client, "/")
|
||||||
req.Header.Del(wafprotocol.PeerHeader)
|
req.Header.Del(waf.PeerHeader)
|
||||||
|
|
||||||
if peer != "" {
|
if peer != "" {
|
||||||
req.Header.Set(wafprotocol.PeerHeader, peer)
|
req.Header.Set(waf.PeerHeader, peer)
|
||||||
}
|
}
|
||||||
|
|
||||||
wantDisposition(
|
wantDisposition(
|
||||||
t,
|
t,
|
||||||
do(t, req),
|
do(t, req),
|
||||||
http.StatusOK,
|
http.StatusOK,
|
||||||
wafprotocol.DispositionAllow,
|
waf.DispositionAllow,
|
||||||
)
|
)
|
||||||
|
|
||||||
if line := s.out.requestLine(t); line.PeerIP != "invalid IP" {
|
if line := s.out.requestLine(t); line.PeerIP != "invalid IP" {
|
||||||
@@ -606,12 +571,12 @@ func TestContentLengthUnreadableOrNegativeCountsAsAbsent(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
req := s.question(siteA, client, "/")
|
req := s.question(siteA, client, "/")
|
||||||
req.Header.Set(wafprotocol.ContentLengthHeader, announced)
|
req.Header.Set(waf.ContentLengthHeader, announced)
|
||||||
wantDisposition(
|
wantDisposition(
|
||||||
t,
|
t,
|
||||||
do(t, req),
|
do(t, req),
|
||||||
http.StatusOK,
|
http.StatusOK,
|
||||||
wafprotocol.DispositionAllow,
|
waf.DispositionAllow,
|
||||||
)
|
)
|
||||||
|
|
||||||
if line := s.out.requestLine(t); line.ContentLength != 0 {
|
if line := s.out.requestLine(t); line.ContentLength != 0 {
|
||||||
@@ -630,18 +595,13 @@ func TestHeaderSentTwiceUsesItsFirstValue(t *testing.T) {
|
|||||||
s := startWAF(t, "", nil)
|
s := startWAF(t, "", nil)
|
||||||
|
|
||||||
req := s.question(siteA, client, "/")
|
req := s.question(siteA, client, "/")
|
||||||
req.Header.Add(wafprotocol.ClientHeader, otherClient)
|
req.Header.Add(waf.ClientHeader, otherClient)
|
||||||
req.Header.Add(wafprotocol.PeerHeader, "172.18.0.3")
|
req.Header.Add(waf.PeerHeader, "172.18.0.3")
|
||||||
req.Header.Set(requestIDHeader, "first")
|
req.Header.Set(requestIDHeader, "first")
|
||||||
req.Header.Add(requestIDHeader, "second")
|
req.Header.Add(requestIDHeader, "second")
|
||||||
req.Header.Set(wafprotocol.ContentLengthHeader, "1234")
|
req.Header.Set(waf.ContentLengthHeader, "1234")
|
||||||
req.Header.Add(wafprotocol.ContentLengthHeader, "5678")
|
req.Header.Add(waf.ContentLengthHeader, "5678")
|
||||||
wantDisposition(
|
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow)
|
||||||
t,
|
|
||||||
do(t, req),
|
|
||||||
http.StatusOK,
|
|
||||||
wafprotocol.DispositionAllow,
|
|
||||||
)
|
|
||||||
|
|
||||||
line := s.out.requestLine(t)
|
line := s.out.requestLine(t)
|
||||||
if line.ClientIP != client || line.PeerIP != rproxyPeer ||
|
if line.ClientIP != client || line.PeerIP != rproxyPeer ||
|
||||||
@@ -670,7 +630,7 @@ func sendQuestionWithPartOfItsBody(t *testing.T, conn net.Conn) {
|
|||||||
_, err := fmt.Fprintf(conn,
|
_, err := fmt.Fprintf(conn,
|
||||||
"POST / HTTP/1.1\r\nHost: %s\r\n%s: %s\r\n%s: an-id\r\n"+
|
"POST / HTTP/1.1\r\nHost: %s\r\n%s: %s\r\n%s: an-id\r\n"+
|
||||||
"Content-Length: 100\r\n\r\npart",
|
"Content-Length: 100\r\n\r\npart",
|
||||||
siteA, wafprotocol.ClientHeader, client, requestIDHeader)
|
siteA, waf.ClientHeader, client, requestIDHeader)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("send the question: %v", err)
|
t.Fatalf("send the question: %v", err)
|
||||||
}
|
}
|
||||||
@@ -739,7 +699,7 @@ func TestQuestionWhoseBodyStopsArrivingGetsNoAnswer(t *testing.T) {
|
|||||||
t,
|
t,
|
||||||
do(t, again),
|
do(t, again),
|
||||||
http.StatusOK,
|
http.StatusOK,
|
||||||
wafprotocol.DispositionAllow,
|
waf.DispositionAllow,
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -757,7 +717,7 @@ func TestQuestionDeniedGetsTheBanResponse(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
got := s.ask(client, "/")
|
got := s.ask(client, "/")
|
||||||
wantDisposition(t, got, status, wafprotocol.DispositionDeny)
|
wantDisposition(t, got, status, waf.DispositionDeny)
|
||||||
wantBody(t, got, http.StatusText(status)+"\n")
|
wantBody(t, got, http.StatusText(status)+"\n")
|
||||||
wantLine(
|
wantLine(
|
||||||
t,
|
t,
|
||||||
@@ -777,7 +737,7 @@ func TestBanResponseCloseIsAnAnswerOfItsOwn(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
got := s.ask(client, "/")
|
got := s.ask(client, "/")
|
||||||
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionClose)
|
wantDisposition(t, got, http.StatusOK, waf.DispositionClose)
|
||||||
wantBody(t, got, "")
|
wantBody(t, got, "")
|
||||||
// The visitor is sent nothing.
|
// The visitor is sent nothing.
|
||||||
wantLine(t, s.out.requestLine(t), 0, requestlog.ActionClientDenied)
|
wantLine(t, s.out.requestLine(t), 0, requestlog.ActionClientDenied)
|
||||||
@@ -801,22 +761,12 @@ func TestBlockRuleIsDeniedWith403AndABanRuleWithTheBanResponse(
|
|||||||
})
|
})
|
||||||
|
|
||||||
got := s.ask(client, "/blocked")
|
got := s.ask(client, "/blocked")
|
||||||
wantDisposition(
|
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
||||||
t,
|
|
||||||
got,
|
|
||||||
http.StatusForbidden,
|
|
||||||
wafprotocol.DispositionDeny,
|
|
||||||
)
|
|
||||||
wantLine(t, s.out.requestLine(t), http.StatusForbidden,
|
wantLine(t, s.out.requestLine(t), http.StatusForbidden,
|
||||||
requestlog.ActionBlockedByBlockRule)
|
requestlog.ActionBlockedByBlockRule)
|
||||||
|
|
||||||
got = s.ask(client, "/.env")
|
got = s.ask(client, "/.env")
|
||||||
wantDisposition(
|
wantDisposition(t, got, http.StatusTooManyRequests, waf.DispositionDeny)
|
||||||
t,
|
|
||||||
got,
|
|
||||||
http.StatusTooManyRequests,
|
|
||||||
wafprotocol.DispositionDeny,
|
|
||||||
)
|
|
||||||
wantLine(t, s.out.requestLines(t, 2)[1], http.StatusTooManyRequests,
|
wantLine(t, s.out.requestLines(t, 2)[1], http.StatusTooManyRequests,
|
||||||
requestlog.ActionTemporarilyBanned)
|
requestlog.ActionTemporarilyBanned)
|
||||||
}
|
}
|
||||||
@@ -831,15 +781,10 @@ func TestQuestionUnderTheOwnPrefixIsDeniedWith404AfterTheChecks(
|
|||||||
})
|
})
|
||||||
|
|
||||||
for _, path := range []string{
|
for _, path := range []string{
|
||||||
wafprotocol.HealthPath, waf.MetricsPath, wafprotocol.ReportPath,
|
waf.HealthPath, waf.MetricsPath, waf.ReportPath,
|
||||||
} {
|
} {
|
||||||
got := s.ask(client, path)
|
got := s.ask(client, path)
|
||||||
wantDisposition(
|
wantDisposition(t, got, http.StatusNotFound, waf.DispositionDeny)
|
||||||
t,
|
|
||||||
got,
|
|
||||||
http.StatusNotFound,
|
|
||||||
wafprotocol.DispositionDeny,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
wantLine(
|
wantLine(
|
||||||
@@ -850,12 +795,7 @@ func TestQuestionUnderTheOwnPrefixIsDeniedWith404AfterTheChecks(
|
|||||||
)
|
)
|
||||||
|
|
||||||
got := s.ask(otherClient, waf.MetricsPath)
|
got := s.ask(otherClient, waf.MetricsPath)
|
||||||
wantDisposition(
|
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
||||||
t,
|
|
||||||
got,
|
|
||||||
http.StatusForbidden,
|
|
||||||
wafprotocol.DispositionDeny,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestObserveModeAllowsAndLogsWhatEnforceModeWouldHaveDone(t *testing.T) {
|
func TestObserveModeAllowsAndLogsWhatEnforceModeWouldHaveDone(t *testing.T) {
|
||||||
@@ -864,7 +804,7 @@ func TestObserveModeAllowsAndLogsWhatEnforceModeWouldHaveDone(t *testing.T) {
|
|||||||
s := startWAF(t, "", map[string]string{mode: observe, denyNets: client})
|
s := startWAF(t, "", map[string]string{mode: observe, denyNets: client})
|
||||||
|
|
||||||
got := s.ask(client, "/")
|
got := s.ask(client, "/")
|
||||||
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||||
|
|
||||||
line := s.out.requestLine(t)
|
line := s.out.requestLine(t)
|
||||||
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
|
wantLine(t, line, http.StatusOK, requestlog.ActionForward)
|
||||||
@@ -882,7 +822,7 @@ func TestObserveModeRefusalAnsweredAllowCountsForTheRefusalLimit(
|
|||||||
|
|
||||||
for range 2 {
|
for range 2 {
|
||||||
got := s.ask(client, "/blocked")
|
got := s.ask(client, "/blocked")
|
||||||
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||||
}
|
}
|
||||||
|
|
||||||
lines := s.out.requestLines(t, 2)
|
lines := s.out.requestLines(t, 2)
|
||||||
@@ -906,12 +846,7 @@ func TestDeniedQuestionCountsForTheRefusalLimit(t *testing.T) {
|
|||||||
|
|
||||||
for range 2 {
|
for range 2 {
|
||||||
got := s.ask(client, "/blocked")
|
got := s.ask(client, "/blocked")
|
||||||
wantDisposition(
|
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
||||||
t,
|
|
||||||
got,
|
|
||||||
http.StatusForbidden,
|
|
||||||
wafprotocol.DispositionDeny,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
lines := s.out.requestLines(t, 2)
|
lines := s.out.requestLines(t, 2)
|
||||||
@@ -947,12 +882,7 @@ func TestAllowCarriesTheLookupHeadersWhileTheSettingIsSet(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
got := s.ask(fromDE, "/")
|
got := s.ask(fromDE, "/")
|
||||||
wantDisposition(
|
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||||
t,
|
|
||||||
got,
|
|
||||||
http.StatusOK,
|
|
||||||
wafprotocol.DispositionAllow,
|
|
||||||
)
|
|
||||||
|
|
||||||
asn, country := got.header.Get(
|
asn, country := got.header.Get(
|
||||||
"X-Client-Asn",
|
"X-Client-Asn",
|
||||||
@@ -986,7 +916,7 @@ func TestReportCountsTowardAByteLimitWhichThenBans(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 100 bytes are over the limit of 99.
|
// 100 bytes are over the limit of 99.
|
||||||
got := s.report(wafprotocol.Report{
|
got := s.report(waf.Report{
|
||||||
RequestID: id, Status: http.StatusBadGateway,
|
RequestID: id, Status: http.StatusBadGateway,
|
||||||
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
|
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
|
||||||
})
|
})
|
||||||
@@ -1010,12 +940,7 @@ func TestReportCountsTowardAByteLimitWhichThenBans(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
got = s.ask(client, "/")
|
got = s.ask(client, "/")
|
||||||
wantDisposition(
|
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
||||||
t,
|
|
||||||
got,
|
|
||||||
http.StatusForbidden,
|
|
||||||
wafprotocol.DispositionDeny,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestReportCountsAtThePercentageItsQuestionWasGiven(t *testing.T) {
|
func TestReportCountsAtThePercentageItsQuestionWasGiven(t *testing.T) {
|
||||||
@@ -1033,12 +958,12 @@ func TestReportCountsAtThePercentageItsQuestionWasGiven(t *testing.T) {
|
|||||||
t,
|
t,
|
||||||
s.ask(fromKP, "/"),
|
s.ask(fromKP, "/"),
|
||||||
http.StatusOK,
|
http.StatusOK,
|
||||||
wafprotocol.DispositionAllow,
|
waf.DispositionAllow,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.passedOn(fromDE)
|
s.passedOn(fromDE)
|
||||||
wantDisposition(t, s.ask(fromDE, "/"), http.StatusForbidden,
|
wantDisposition(t, s.ask(fromDE, "/"), http.StatusForbidden,
|
||||||
wafprotocol.DispositionDeny)
|
waf.DispositionDeny)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestReportAddsTheRequestToItsClientsHistoryAndTheAnomalyCounters(
|
func TestReportAddsTheRequestToItsClientsHistoryAndTheAnomalyCounters(
|
||||||
@@ -1064,7 +989,7 @@ func TestReportAddsTheRequestToItsClientsHistoryAndTheAnomalyCounters(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
got := s.report(wafprotocol.Report{
|
got := s.report(waf.Report{
|
||||||
RequestID: id, Status: http.StatusBadGateway,
|
RequestID: id, Status: http.StatusBadGateway,
|
||||||
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
|
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
|
||||||
})
|
})
|
||||||
@@ -1096,7 +1021,7 @@ func TestDeniedRequestIsFinishedAtItsDecision(t *testing.T) {
|
|||||||
t,
|
t,
|
||||||
do(t, denied),
|
do(t, denied),
|
||||||
http.StatusForbidden,
|
http.StatusForbidden,
|
||||||
wafprotocol.DispositionDeny,
|
waf.DispositionDeny,
|
||||||
)
|
)
|
||||||
|
|
||||||
counters := s.server.Anomalies.Snapshot()
|
counters := s.server.Anomalies.Snapshot()
|
||||||
@@ -1114,7 +1039,7 @@ func TestDeniedRequestIsFinishedAtItsDecision(t *testing.T) {
|
|||||||
|
|
||||||
wantMetric(t, s.metrics(), series, 1)
|
wantMetric(t, s.metrics(), series, 1)
|
||||||
|
|
||||||
got := s.report(wafprotocol.Report{
|
got := s.report(waf.Report{
|
||||||
RequestID: denied.Header.Get(
|
RequestID: denied.Header.Get(
|
||||||
requestIDHeader,
|
requestIDHeader,
|
||||||
), Status: http.StatusForbidden,
|
), Status: http.StatusForbidden,
|
||||||
@@ -1157,7 +1082,7 @@ func TestMetricsCountAnAllowedRequestFromItsReport(t *testing.T) {
|
|||||||
1,
|
1,
|
||||||
) // the scrape itself
|
) // the scrape itself
|
||||||
|
|
||||||
got := s.report(wafprotocol.Report{
|
got := s.report(waf.Report{
|
||||||
RequestID: id, Status: http.StatusBadGateway,
|
RequestID: id, Status: http.StatusBadGateway,
|
||||||
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
|
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
|
||||||
DurationTotalMS: 5, DurationUpstreamTotalMS: 4,
|
DurationTotalMS: 5, DurationUpstreamTotalMS: 4,
|
||||||
@@ -1175,7 +1100,7 @@ func TestMetricsCountAnAllowedRequestFromItsReport(t *testing.T) {
|
|||||||
// check needs a server that has not been scraped yet.
|
// check needs a server that has not been scraped yet.
|
||||||
unscraped := startWAF(t, "", map[string]string{metricsToken: token})
|
unscraped := startWAF(t, "", map[string]string{metricsToken: token})
|
||||||
|
|
||||||
got = unscraped.report(wafprotocol.Report{
|
got = unscraped.report(waf.Report{
|
||||||
RequestID: unscraped.allowed(client), Status: http.StatusBadGateway,
|
RequestID: unscraped.allowed(client), Status: http.StatusBadGateway,
|
||||||
DurationTotalMS: 5,
|
DurationTotalMS: 5,
|
||||||
})
|
})
|
||||||
@@ -1193,7 +1118,7 @@ func TestReportForARequestNotHeldIsIgnored(t *testing.T) {
|
|||||||
|
|
||||||
s := startWAF(t, "", map[string]string{bytesLimitPerMinute: byteLimit})
|
s := startWAF(t, "", map[string]string{bytesLimitPerMinute: byteLimit})
|
||||||
|
|
||||||
got := s.report(wafprotocol.Report{
|
got := s.report(waf.Report{
|
||||||
RequestID: "no-such-request", ResponseBytes: 1000,
|
RequestID: "no-such-request", ResponseBytes: 1000,
|
||||||
})
|
})
|
||||||
wantStatus(t, got, http.StatusNoContent)
|
wantStatus(t, got, http.StatusNoContent)
|
||||||
@@ -1212,9 +1137,7 @@ func TestReportForARequestNotHeldIsIgnored(t *testing.T) {
|
|||||||
// A second report of the same request does not count it twice.
|
// A second report of the same request does not count it twice.
|
||||||
id := s.allowed(client)
|
id := s.allowed(client)
|
||||||
for range 2 {
|
for range 2 {
|
||||||
got = s.report(
|
got = s.report(waf.Report{RequestID: id, ResponseBytes: answerBytes})
|
||||||
wafprotocol.Report{RequestID: id, ResponseBytes: answerBytes},
|
|
||||||
)
|
|
||||||
wantStatus(t, got, http.StatusNoContent)
|
wantStatus(t, got, http.StatusNoContent)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1424,7 +1347,7 @@ func TestBytesAtTheirMostEachWayBanRatherThanOverflow(t *testing.T) {
|
|||||||
|
|
||||||
s := startWAF(t, "", map[string]string{bytesLimitPerMinute: byteLimit})
|
s := startWAF(t, "", map[string]string{bytesLimitPerMinute: byteLimit})
|
||||||
|
|
||||||
got := s.report(wafprotocol.Report{
|
got := s.report(waf.Report{
|
||||||
RequestID: s.allowed(client), Status: http.StatusOK,
|
RequestID: s.allowed(client), Status: http.StatusOK,
|
||||||
RequestBytes: reportedBytesLimit, ResponseBytes: reportedBytesLimit,
|
RequestBytes: reportedBytesLimit, ResponseBytes: reportedBytesLimit,
|
||||||
})
|
})
|
||||||
@@ -1518,14 +1441,9 @@ func TestReportWritesTheLastLineWithTheDecisionsAndTheReportsFields(
|
|||||||
req.Header.Set("X-Forwarded-Proto", "https")
|
req.Header.Set("X-Forwarded-Proto", "https")
|
||||||
id := req.Header.Get(requestIDHeader)
|
id := req.Header.Get(requestIDHeader)
|
||||||
|
|
||||||
wantDisposition(
|
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow)
|
||||||
t,
|
|
||||||
do(t, req),
|
|
||||||
http.StatusOK,
|
|
||||||
wafprotocol.DispositionAllow,
|
|
||||||
)
|
|
||||||
|
|
||||||
got := s.report(wafprotocol.Report{
|
got := s.report(waf.Report{
|
||||||
RequestID: id, Status: http.StatusCreated,
|
RequestID: id, Status: http.StatusCreated,
|
||||||
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
|
RequestBytes: bodyBytes, ResponseBytes: answerBytes,
|
||||||
DurationTotalMS: 12.5, DurationUpstreamTotalMS: 10.25,
|
DurationTotalMS: 12.5, DurationUpstreamTotalMS: 10.25,
|
||||||
@@ -1572,7 +1490,7 @@ func TestEveryDeniedRequestsReportGetsItsLastLine(t *testing.T) {
|
|||||||
"close", map[string]string{denyNets: client, banResponse: banResponseClose}, "/",
|
"close", map[string]string{denyNets: client, banResponse: banResponseClose}, "/",
|
||||||
0, requestlog.ActionClientDenied,
|
0, requestlog.ActionClientDenied,
|
||||||
},
|
},
|
||||||
{"own prefix", nil, wafprotocol.HealthPath, http.StatusNotFound,
|
{"own prefix", nil, waf.HealthPath, http.StatusNotFound,
|
||||||
requestlog.ActionOwnEndpoint},
|
requestlog.ActionOwnEndpoint},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
@@ -1584,7 +1502,7 @@ func TestEveryDeniedRequestsReportGetsItsLastLine(t *testing.T) {
|
|||||||
id := req.Header.Get(requestIDHeader)
|
id := req.Header.Get(requestIDHeader)
|
||||||
do(t, req)
|
do(t, req)
|
||||||
|
|
||||||
got := s.report(wafprotocol.Report{
|
got := s.report(waf.Report{
|
||||||
RequestID: id, Status: tc.status, ResponseBytes: 8, DurationTotalMS: 3,
|
RequestID: id, Status: tc.status, ResponseBytes: 8, DurationTotalMS: 3,
|
||||||
})
|
})
|
||||||
wantStatus(t, got, http.StatusNoContent)
|
wantStatus(t, got, http.StatusNoContent)
|
||||||
@@ -1624,13 +1542,13 @@ func TestOldestAnsweredRequestIsDroppedPastTheBound(t *testing.T) {
|
|||||||
t,
|
t,
|
||||||
readAnswer(res),
|
readAnswer(res),
|
||||||
http.StatusOK,
|
http.StatusOK,
|
||||||
wafprotocol.DispositionAllow,
|
waf.DispositionAllow,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// The table is full again with next and those, without oldest.
|
// The table is full again with next and those, without oldest.
|
||||||
got := s.report(
|
got := s.report(
|
||||||
wafprotocol.Report{RequestID: oldest, ResponseBytes: answerBytes},
|
waf.Report{RequestID: oldest, ResponseBytes: answerBytes},
|
||||||
)
|
)
|
||||||
wantStatus(t, got, http.StatusNoContent)
|
wantStatus(t, got, http.StatusNoContent)
|
||||||
|
|
||||||
@@ -1641,9 +1559,7 @@ func TestOldestAnsweredRequestIsDroppedPastTheBound(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
got = s.report(
|
got = s.report(waf.Report{RequestID: next, ResponseBytes: answerBytes})
|
||||||
wafprotocol.Report{RequestID: next, ResponseBytes: answerBytes},
|
|
||||||
)
|
|
||||||
wantStatus(t, got, http.StatusNoContent)
|
wantStatus(t, got, http.StatusNoContent)
|
||||||
|
|
||||||
if counted := s.history().Requests; counted != 1 {
|
if counted := s.history().Requests; counted != 1 {
|
||||||
@@ -1664,20 +1580,10 @@ func TestBanMadeThroughOneHostRefusesTheClientOnAnother(t *testing.T) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
got := do(t, s.question(siteA, client, "/.env"))
|
got := do(t, s.question(siteA, client, "/.env"))
|
||||||
wantDisposition(
|
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
||||||
t,
|
|
||||||
got,
|
|
||||||
http.StatusForbidden,
|
|
||||||
wafprotocol.DispositionDeny,
|
|
||||||
)
|
|
||||||
|
|
||||||
got = do(t, s.question(siteB, client, "/"))
|
got = do(t, s.question(siteB, client, "/"))
|
||||||
wantDisposition(
|
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
||||||
t,
|
|
||||||
got,
|
|
||||||
http.StatusForbidden,
|
|
||||||
wafprotocol.DispositionDeny,
|
|
||||||
)
|
|
||||||
|
|
||||||
lines := s.out.requestLines(t, 2)
|
lines := s.out.requestLines(t, 2)
|
||||||
if lines[0].Host != siteA || lines[1].Host != siteB {
|
if lines[0].Host != siteA || lines[1].Host != siteB {
|
||||||
@@ -1715,12 +1621,7 @@ func TestRequestsOnEverySiteCountTowardOneSetOfLimits(t *testing.T) {
|
|||||||
s.allowed(client)
|
s.allowed(client)
|
||||||
|
|
||||||
got := do(t, s.question(siteB, client, "/"))
|
got := do(t, s.question(siteB, client, "/"))
|
||||||
wantDisposition(
|
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
||||||
t,
|
|
||||||
got,
|
|
||||||
http.StatusForbidden,
|
|
||||||
wafprotocol.DispositionDeny,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOwnEndpointsAnswerOnlyWithoutTheClientHeader(t *testing.T) {
|
func TestOwnEndpointsAnswerOnlyWithoutTheClientHeader(t *testing.T) {
|
||||||
@@ -1735,13 +1636,13 @@ func TestOwnEndpointsAnswerOnlyWithoutTheClientHeader(t *testing.T) {
|
|||||||
path, authorization string
|
path, authorization string
|
||||||
want int
|
want int
|
||||||
}{
|
}{
|
||||||
{wafprotocol.HealthPath, "", http.StatusOK},
|
{waf.HealthPath, "", http.StatusOK},
|
||||||
{waf.MetricsPath, bearer, http.StatusOK},
|
{waf.MetricsPath, bearer, http.StatusOK},
|
||||||
{waf.MetricsPath, "", http.StatusUnauthorized},
|
{waf.MetricsPath, "", http.StatusUnauthorized},
|
||||||
{waf.BansPath, adminBearer, http.StatusOK},
|
{waf.BansPath, adminBearer, http.StatusOK},
|
||||||
{waf.BansPath, "", http.StatusUnauthorized},
|
{waf.BansPath, "", http.StatusUnauthorized},
|
||||||
{waf.ClientsPath + client, adminBearer, http.StatusOK},
|
{waf.ClientsPath + client, adminBearer, http.StatusOK},
|
||||||
{wafprotocol.ReportPath, "", http.StatusNotFound},
|
{waf.ReportPath, "", http.StatusNotFound},
|
||||||
{"/_sww/other", "", http.StatusNotFound},
|
{"/_sww/other", "", http.StatusNotFound},
|
||||||
{"/", "", http.StatusNotFound},
|
{"/", "", http.StatusNotFound},
|
||||||
} {
|
} {
|
||||||
@@ -1758,10 +1659,10 @@ func TestOwnEndpointsAnswerOnlyWithoutTheClientHeader(t *testing.T) {
|
|||||||
t,
|
t,
|
||||||
do(t, req),
|
do(t, req),
|
||||||
http.StatusNotFound,
|
http.StatusNotFound,
|
||||||
wafprotocol.DispositionDeny,
|
waf.DispositionDeny,
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
wantDisposition(t, do(t, req), http.StatusOK, wafprotocol.DispositionAllow)
|
wantDisposition(t, do(t, req), http.StatusOK, waf.DispositionAllow)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1771,11 +1672,11 @@ func TestOwnEndpointsLinesDoNotSayWhichOfTheTwoTheyAre(t *testing.T) {
|
|||||||
|
|
||||||
s := startWAF(t, "", nil)
|
s := startWAF(t, "", nil)
|
||||||
|
|
||||||
wantStatus(t, get(t, s.addr, wafprotocol.HealthPath), http.StatusOK)
|
wantStatus(t, get(t, s.addr, waf.HealthPath), http.StatusOK)
|
||||||
|
|
||||||
line := s.out.requestLine(t)
|
line := s.out.requestLine(t)
|
||||||
if _, there := line.fields["waf_line"]; there ||
|
if _, there := line.fields["waf_line"]; there ||
|
||||||
line.Path != wafprotocol.HealthPath {
|
line.Path != waf.HealthPath {
|
||||||
t.Errorf("the health check's log line is %+v", line.fields)
|
t.Errorf("the health check's log line is %+v", line.fields)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1787,7 +1688,7 @@ func TestALargeBodyAnnouncedAndLongHeadersAreNoReasonToRefuse(t *testing.T) {
|
|||||||
|
|
||||||
req := s.questionWith(http.MethodPost, siteA, client, "/",
|
req := s.questionWith(http.MethodPost, siteA, client, "/",
|
||||||
strings.NewReader("a body"))
|
strings.NewReader("a body"))
|
||||||
req.Header.Set(wafprotocol.ContentLengthHeader, "1073741824")
|
req.Header.Set(waf.ContentLengthHeader, "1073741824")
|
||||||
req.Header.Set(forwardedFor, otherClient)
|
req.Header.Set(forwardedFor, otherClient)
|
||||||
|
|
||||||
// 40,000 bytes of headers, over rproxy's 32 KiB, in lines within the
|
// 40,000 bytes of headers, over rproxy's 32 KiB, in lines within the
|
||||||
@@ -1797,7 +1698,7 @@ func TestALargeBodyAnnouncedAndLongHeadersAreNoReasonToRefuse(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
got := do(t, req)
|
got := do(t, req)
|
||||||
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||||
|
|
||||||
line := s.out.requestLine(t)
|
line := s.out.requestLine(t)
|
||||||
if line.ClientIP != client || line.ContentLength != 1<<30 {
|
if line.ClientIP != client || line.ContentLength != 1<<30 {
|
||||||
@@ -1875,7 +1776,7 @@ carried header:Content-Length block ^6$
|
|||||||
strings.NewReader("a body"))
|
strings.NewReader("a body"))
|
||||||
|
|
||||||
if tc.announced != "" {
|
if tc.announced != "" {
|
||||||
req.Header.Set(wafprotocol.ContentLengthHeader, tc.announced)
|
req.Header.Set(waf.ContentLengthHeader, tc.announced)
|
||||||
}
|
}
|
||||||
|
|
||||||
wantStatus(t, do(t, req), tc.want)
|
wantStatus(t, do(t, req), tc.want)
|
||||||
@@ -1893,24 +1794,19 @@ func TestCoreRuleSetReadsTheBodyAQuestionCarries(t *testing.T) {
|
|||||||
req := s.questionWith(http.MethodPost, siteA, client, "/",
|
req := s.questionWith(http.MethodPost, siteA, client, "/",
|
||||||
strings.NewReader(body))
|
strings.NewReader(body))
|
||||||
req.Header.Set("Content-Type", contentType)
|
req.Header.Set("Content-Type", contentType)
|
||||||
req.Header.Set(wafprotocol.ContentLengthHeader, "100000")
|
req.Header.Set(waf.ContentLengthHeader, "100000")
|
||||||
|
|
||||||
return do(t, req)
|
return do(t, req)
|
||||||
}
|
}
|
||||||
|
|
||||||
got := ask(formData, "id=1'%20OR%20'1'='1")
|
got := ask(formData, "id=1'%20OR%20'1'='1")
|
||||||
wantDisposition(
|
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
||||||
t,
|
|
||||||
got,
|
|
||||||
http.StatusForbidden,
|
|
||||||
wafprotocol.DispositionDeny,
|
|
||||||
)
|
|
||||||
|
|
||||||
// The first 8 KiB of a longer JSON body cannot be parsed, which is no
|
// The first 8 KiB of a longer JSON body cannot be parsed, which is no
|
||||||
// attack, so the Core Rule Set does not read it.
|
// attack, so the Core Rule Set does not read it.
|
||||||
cut := `{"text":"` + strings.Repeat("x", 8<<10)
|
cut := `{"text":"` + strings.Repeat("x", 8<<10)
|
||||||
got = ask("application/json", cut)
|
got = ask("application/json", cut)
|
||||||
wantDisposition(t, got, http.StatusOK, wafprotocol.DispositionAllow)
|
wantDisposition(t, got, http.StatusOK, waf.DispositionAllow)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCoreRuleSetAlertNamesTheSite(t *testing.T) {
|
func TestCoreRuleSetAlertNamesTheSite(t *testing.T) {
|
||||||
@@ -1919,12 +1815,7 @@ func TestCoreRuleSetAlertNamesTheSite(t *testing.T) {
|
|||||||
s := startWAF(t, "", map[string]string{coreRuleSetMode: block})
|
s := startWAF(t, "", map[string]string{coreRuleSetMode: block})
|
||||||
|
|
||||||
got := s.ask(client, sqlInjection)
|
got := s.ask(client, sqlInjection)
|
||||||
wantDisposition(
|
wantDisposition(t, got, http.StatusForbidden, waf.DispositionDeny)
|
||||||
t,
|
|
||||||
got,
|
|
||||||
http.StatusForbidden,
|
|
||||||
wafprotocol.DispositionDeny,
|
|
||||||
)
|
|
||||||
|
|
||||||
waiting := s.queue.Snapshot().Waiting[alerts.DestinationWebhook]
|
waiting := s.queue.Snapshot().Waiting[alerts.DestinationWebhook]
|
||||||
if len(waiting) != 1 ||
|
if len(waiting) != 1 ||
|
||||||
@@ -1945,7 +1836,7 @@ func TestReputationHitAlertNamesTheSite(t *testing.T) {
|
|||||||
loadLists(t, s.server, map[string][]string{dropURL: {client}})
|
loadLists(t, s.server, map[string][]string{dropURL: {client}})
|
||||||
|
|
||||||
wantDisposition(t, s.ask(client, "/"), http.StatusForbidden,
|
wantDisposition(t, s.ask(client, "/"), http.StatusForbidden,
|
||||||
wafprotocol.DispositionDeny)
|
waf.DispositionDeny)
|
||||||
|
|
||||||
waiting := s.queue.Snapshot().Waiting[alerts.DestinationWebhook]
|
waiting := s.queue.Snapshot().Waiting[alerts.DestinationWebhook]
|
||||||
if len(waiting) != 1 || waiting[0].Event != alerts.EventReputationHit ||
|
if len(waiting) != 1 || waiting[0].Event != alerts.EventReputationHit ||
|
||||||
|
|||||||
@@ -1,46 +0,0 @@
|
|||||||
package wafprotocol
|
|
||||||
|
|
||||||
import "net/http"
|
|
||||||
|
|
||||||
// Answer is what waf's answer to a question tells rproxy in its headers.
|
|
||||||
// The status, the other headers and the body of an answer of deny are what
|
|
||||||
// the visitor gets.
|
|
||||||
type Answer struct {
|
|
||||||
// Disposition is what rproxy does with the request: DispositionAllow,
|
|
||||||
// DispositionDeny or DispositionClose.
|
|
||||||
Disposition string
|
|
||||||
// ASN and Country are the client's AS number and country, each empty
|
|
||||||
// when the answer does not give it.
|
|
||||||
ASN, Country string
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetAnswerHeaders sets the headers of waf's answer that tell rproxy answer,
|
|
||||||
// leaving out an ASN or Country that is empty.
|
|
||||||
func SetAnswerHeaders(header http.Header, answer Answer) {
|
|
||||||
header.Set(DispositionHeader, answer.Disposition)
|
|
||||||
|
|
||||||
if answer.ASN != "" {
|
|
||||||
header.Set(ASNHeader, answer.ASN)
|
|
||||||
}
|
|
||||||
|
|
||||||
if answer.Country != "" {
|
|
||||||
header.Set(CountryHeader, answer.Country)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ReadAnswer returns what the headers of waf's answer tell rproxy, and
|
|
||||||
// whether they give a valid disposition.
|
|
||||||
func ReadAnswer(header http.Header) (Answer, bool) {
|
|
||||||
disposition := header.Get(DispositionHeader)
|
|
||||||
|
|
||||||
switch disposition {
|
|
||||||
case DispositionAllow, DispositionDeny, DispositionClose:
|
|
||||||
return Answer{
|
|
||||||
Disposition: disposition,
|
|
||||||
ASN: header.Get(ASNHeader),
|
|
||||||
Country: header.Get(CountryHeader),
|
|
||||||
}, true
|
|
||||||
}
|
|
||||||
|
|
||||||
return Answer{}, false
|
|
||||||
}
|
|
||||||
@@ -1,189 +0,0 @@
|
|||||||
package wafprotocol
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/netip"
|
|
||||||
"net/url"
|
|
||||||
"strconv"
|
|
||||||
"unicode/utf8"
|
|
||||||
)
|
|
||||||
|
|
||||||
// maxRequestIDBytes is the longest id a question may give.
|
|
||||||
const maxRequestIDBytes = 128
|
|
||||||
|
|
||||||
// ErrBodyStoppedArriving is the error ReadQuestion wraps when the question's
|
|
||||||
// body stops arriving. Every other error it returns is why waf refuses the
|
|
||||||
// question.
|
|
||||||
var ErrBodyStoppedArriving = errors.New("the body stopped arriving")
|
|
||||||
|
|
||||||
// Why waf refuses a question. Each is also the body of the refusal.
|
|
||||||
var (
|
|
||||||
errClientNotAnAddress = errors.New(
|
|
||||||
ClientHeader + " is not an address without a zone",
|
|
||||||
)
|
|
||||||
errRequestIDNotValid = errors.New(RequestIDHeader + " is not a valid id")
|
|
||||||
errBodyTooLarge = errors.New("the body is over 64 KiB")
|
|
||||||
)
|
|
||||||
|
|
||||||
// QuestionParams are what NewQuestion needs.
|
|
||||||
type QuestionParams struct {
|
|
||||||
// Visitor is the visitor's request.
|
|
||||||
Visitor *http.Request
|
|
||||||
// PathAsSent is Visitor's path as the visitor sent it, which the question
|
|
||||||
// sends unchanged.
|
|
||||||
PathAsSent string
|
|
||||||
// Body is the part of Visitor's body the question carries, at most
|
|
||||||
// QuestionBodyMaxBytes.
|
|
||||||
Body []byte
|
|
||||||
// WAFBackend is waf's address. The question takes its scheme and host.
|
|
||||||
WAFBackend *url.URL
|
|
||||||
// Client is the visitor's address, and Peer that of rproxy's own TCP
|
|
||||||
// peer.
|
|
||||||
Client, Peer netip.Addr
|
|
||||||
// RequestID is the request's id.
|
|
||||||
RequestID string
|
|
||||||
// Scheme is how the visitor reached rproxy, or the trusted proxy in front
|
|
||||||
// of it.
|
|
||||||
Scheme string
|
|
||||||
}
|
|
||||||
|
|
||||||
// Question is a question as waf reads it.
|
|
||||||
type Question struct {
|
|
||||||
// Client is the visitor's address.
|
|
||||||
Client netip.Addr
|
|
||||||
// Peer is the address of rproxy's own TCP peer. It is not valid when the
|
|
||||||
// question gives none that can be read.
|
|
||||||
Peer netip.Addr
|
|
||||||
// RequestID is the request's id.
|
|
||||||
RequestID string
|
|
||||||
// Visitor is the visitor's request, as asTheVisitorSentIt gives it.
|
|
||||||
Visitor *http.Request
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewQuestion returns the question about a visitor's request, as "The
|
|
||||||
// question" in SPEC.md gives it: the request as the visitor sent it, with the
|
|
||||||
// part of the body it carries and rproxy's headers.
|
|
||||||
func NewQuestion(
|
|
||||||
ctx context.Context,
|
|
||||||
params QuestionParams,
|
|
||||||
) (*http.Request, error) {
|
|
||||||
in := params.Visitor
|
|
||||||
|
|
||||||
question, err := http.NewRequestWithContext(
|
|
||||||
ctx,
|
|
||||||
in.Method,
|
|
||||||
params.WAFBackend.String(),
|
|
||||||
bytes.NewReader(params.Body),
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("make the question: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
backend := question.URL
|
|
||||||
target := *in.URL
|
|
||||||
target.Scheme = backend.Scheme
|
|
||||||
target.Host = backend.Host
|
|
||||||
target.Opaque = params.PathAsSent
|
|
||||||
question.URL = &target
|
|
||||||
question.Host = in.Host
|
|
||||||
|
|
||||||
// The length NewRequestWithContext set is that of the body carried. This
|
|
||||||
// line makes a body whose length the visitor did not announce go chunked,
|
|
||||||
// as "The question" in SPEC.md requires.
|
|
||||||
question.TransferEncoding = in.TransferEncoding
|
|
||||||
|
|
||||||
question.Header = in.Header.Clone()
|
|
||||||
question.Header.Set(ClientHeader, params.Client.String())
|
|
||||||
question.Header.Set(PeerHeader, params.Peer.String())
|
|
||||||
question.Header.Del(ContentLengthHeader)
|
|
||||||
|
|
||||||
announced := in.Header.Get("Content-Length")
|
|
||||||
if announced != "" {
|
|
||||||
question.Header.Set(ContentLengthHeader, announced)
|
|
||||||
}
|
|
||||||
|
|
||||||
question.Header.Set(RequestIDHeader, params.RequestID)
|
|
||||||
question.Header.Set("X-Forwarded-Proto", params.Scheme)
|
|
||||||
|
|
||||||
// Go's client adds a User-Agent of its own to a request without the key.
|
|
||||||
_, sent := question.Header["User-Agent"]
|
|
||||||
if !sent {
|
|
||||||
question.Header["User-Agent"] = nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return question, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ReadQuestion reads the question r, with the body it carries. An error
|
|
||||||
// wrapping ErrBodyStoppedArriving means the body stopped arriving. Any other
|
|
||||||
// error is why waf refuses the question, and the question's body is then not
|
|
||||||
// read. The body is read whole, since Go's server does not send an answer
|
|
||||||
// until it has read the body.
|
|
||||||
func ReadQuestion(w http.ResponseWriter, r *http.Request) (Question, error) {
|
|
||||||
client, err := netip.ParseAddr(r.Header.Get(ClientHeader))
|
|
||||||
if err != nil || client.Zone() != "" {
|
|
||||||
return Question{}, errClientNotAnAddress
|
|
||||||
}
|
|
||||||
|
|
||||||
id := r.Header.Get(RequestIDHeader)
|
|
||||||
if id == "" || len(id) > maxRequestIDBytes || !utf8.ValidString(id) {
|
|
||||||
return Question{}, errRequestIDNotValid
|
|
||||||
}
|
|
||||||
|
|
||||||
body, err := io.ReadAll(
|
|
||||||
http.MaxBytesReader(w, r.Body, QuestionBodyMaxBytes),
|
|
||||||
)
|
|
||||||
|
|
||||||
if _, tooLarge := errors.AsType[*http.MaxBytesError](err); tooLarge {
|
|
||||||
return Question{}, errBodyTooLarge
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return Question{}, fmt.Errorf("%w: %w", ErrBodyStoppedArriving, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// One that cannot be read leaves the log line's peer_ip invalid.
|
|
||||||
peer, _ := netip.ParseAddr(r.Header.Get(PeerHeader))
|
|
||||||
|
|
||||||
return Question{
|
|
||||||
Client: client.Unmap(),
|
|
||||||
Peer: peer.Unmap(),
|
|
||||||
RequestID: id,
|
|
||||||
Visitor: asTheVisitorSentIt(r, body),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// asTheVisitorSentIt returns a copy of r, a question, as the checks and the
|
|
||||||
// log line are to see it: without the headers rproxy adds for waf,
|
|
||||||
// with body, which has been read of r, and with the length the visitor
|
|
||||||
// announced, as -1 when it announced none. The question's own length covers
|
|
||||||
// only the part of the body it carries.
|
|
||||||
func asTheVisitorSentIt(r *http.Request, body []byte) *http.Request {
|
|
||||||
sent := r.Clone(r.Context())
|
|
||||||
announced := sent.Header.Get(ContentLengthHeader)
|
|
||||||
|
|
||||||
sent.Header.Del(ClientHeader)
|
|
||||||
sent.Header.Del(PeerHeader)
|
|
||||||
sent.Header.Del(ContentLengthHeader)
|
|
||||||
sent.Header.Del("Content-Length")
|
|
||||||
|
|
||||||
sent.ContentLength = -1
|
|
||||||
|
|
||||||
length, err := strconv.ParseInt(announced, 10, 64)
|
|
||||||
if err == nil && length >= 0 {
|
|
||||||
sent.ContentLength = length
|
|
||||||
sent.Header.Set("Content-Length", announced)
|
|
||||||
}
|
|
||||||
|
|
||||||
sent.Body = http.NoBody
|
|
||||||
if len(body) > 0 {
|
|
||||||
sent.Body = io.NopCloser(bytes.NewReader(body))
|
|
||||||
}
|
|
||||||
|
|
||||||
return sent
|
|
||||||
}
|
|
||||||
@@ -1,123 +0,0 @@
|
|||||||
package wafprotocol
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// reportBodyMaxBytes is the most of a report's body that is read. It is
|
|
||||||
// far above any report's.
|
|
||||||
reportBodyMaxBytes = 1 << 20
|
|
||||||
// maxReportedBytes is the most bytes a report may give for each way, 1
|
|
||||||
// PiB, so that adding them up cannot overflow.
|
|
||||||
maxReportedBytes = 1 << 50
|
|
||||||
// maxReportedMilliseconds is the longest duration a report may give,
|
|
||||||
// about 31 years, so that it fits a time.Duration.
|
|
||||||
maxReportedMilliseconds = 1e12
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
errReportFieldMissing = errors.New("a field is missing")
|
|
||||||
errReportValue = errors.New("a value is out of range")
|
|
||||||
)
|
|
||||||
|
|
||||||
// Report is the JSON object rproxy posts to ReportPath once the visitor's
|
|
||||||
// answer has ended, its fields named as the request log's of the same
|
|
||||||
// meaning. RequestID is the question's id. Status is what the visitor got, 0
|
|
||||||
// if nothing. The durations are in milliseconds.
|
|
||||||
//
|
|
||||||
//nolint:tagliatelle // docs/request-log.md names the fields in snake_case
|
|
||||||
type Report struct {
|
|
||||||
RequestID string `json:"request_id"`
|
|
||||||
Status int `json:"status"`
|
|
||||||
RequestBytes int64 `json:"request_bytes"`
|
|
||||||
ResponseBytes int64 `json:"response_bytes"`
|
|
||||||
DurationTotalMS float64 `json:"duration_total_ms"`
|
|
||||||
DurationUpstreamTotalMS float64 `json:"duration_upstream_total_ms"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewReportRequest returns the request that posts report to reportURL, which
|
|
||||||
// is waf's ReportPath.
|
|
||||||
func NewReportRequest(
|
|
||||||
ctx context.Context,
|
|
||||||
reportURL string,
|
|
||||||
report *Report,
|
|
||||||
) (*http.Request, error) {
|
|
||||||
body, err := json.Marshal(report)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("encode the report: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(
|
|
||||||
ctx, http.MethodPost, reportURL, bytes.NewReader(body))
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("make the request: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
|
|
||||||
return req, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ReadReport reads a report from r's body, which must be one JSON object with
|
|
||||||
// all of Report's fields, in the ranges SPEC.md gives.
|
|
||||||
func ReadReport(w http.ResponseWriter, r *http.Request) (Report, error) {
|
|
||||||
var report Report
|
|
||||||
|
|
||||||
body, err := io.ReadAll(
|
|
||||||
http.MaxBytesReader(w, r.Body, reportBodyMaxBytes),
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return report, fmt.Errorf("read the body: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var fields map[string]json.RawMessage
|
|
||||||
|
|
||||||
err = json.Unmarshal(body, &fields)
|
|
||||||
if err != nil {
|
|
||||||
return report, fmt.Errorf("decode the body: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, name := range []string{
|
|
||||||
"request_id", "status", "request_bytes", "response_bytes",
|
|
||||||
"duration_total_ms", "duration_upstream_total_ms",
|
|
||||||
} {
|
|
||||||
value, there := fields[name]
|
|
||||||
if !there || string(value) == "null" {
|
|
||||||
return report, fmt.Errorf("%w: %s", errReportFieldMissing, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
err = json.Unmarshal(body, &report)
|
|
||||||
if err != nil {
|
|
||||||
return report, fmt.Errorf("decode the body: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return report, report.checkRanges()
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkRanges returns an error for a value of the report that cannot be true.
|
|
||||||
func (report *Report) checkRanges() error {
|
|
||||||
switch {
|
|
||||||
case report.Status != 0 &&
|
|
||||||
(report.Status < http.StatusContinue || report.Status > ReportStatusMax):
|
|
||||||
return fmt.Errorf("%w: status", errReportValue)
|
|
||||||
case report.RequestBytes < 0 || report.RequestBytes > maxReportedBytes:
|
|
||||||
return fmt.Errorf("%w: request_bytes", errReportValue)
|
|
||||||
case report.ResponseBytes < 0 || report.ResponseBytes > maxReportedBytes:
|
|
||||||
return fmt.Errorf("%w: response_bytes", errReportValue)
|
|
||||||
case report.DurationTotalMS < 0 || report.DurationTotalMS > maxReportedMilliseconds:
|
|
||||||
return fmt.Errorf("%w: duration_total_ms", errReportValue)
|
|
||||||
case report.DurationUpstreamTotalMS < 0 ||
|
|
||||||
report.DurationUpstreamTotalMS > maxReportedMilliseconds:
|
|
||||||
return fmt.Errorf("%w: duration_upstream_total_ms", errReportValue)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,61 +0,0 @@
|
|||||||
// Package wafprotocol holds the three messages rproxy and waf send each
|
|
||||||
// other, as "`waf`" in SPEC.md gives them. rproxy asks waf a question about
|
|
||||||
// each visitor's request, and waf answers it. Once the visitor's answer has
|
|
||||||
// ended, rproxy sends waf a report. This package writes and reads each
|
|
||||||
// message, and declares every name and limit both sides must agree on.
|
|
||||||
package wafprotocol
|
|
||||||
|
|
||||||
// The headers of the question and of the answer. Go writes every header name
|
|
||||||
// in this form, as it sends it and as it receives it, so X-Client-ASN arrives
|
|
||||||
// as X-Client-Asn; header names are not case-sensitive.
|
|
||||||
const (
|
|
||||||
// ClientHeader marks a request as rproxy's question, and gives the
|
|
||||||
// visitor's address.
|
|
||||||
ClientHeader = "X-Sww-Client"
|
|
||||||
// PeerHeader gives the address of rproxy's own TCP peer.
|
|
||||||
PeerHeader = "X-Sww-Peer"
|
|
||||||
// ContentLengthHeader gives the body length the visitor announced.
|
|
||||||
ContentLengthHeader = "X-Sww-Content-Length"
|
|
||||||
// RequestIDHeader gives the request's id, which rproxy makes. rproxy
|
|
||||||
// also sets it on the request it passes to the app.
|
|
||||||
RequestIDHeader = "X-Request-ID"
|
|
||||||
// DispositionHeader gives waf's answer.
|
|
||||||
DispositionHeader = "X-Sww-Disposition"
|
|
||||||
// ASNHeader and CountryHeader give the client's AS number and country in
|
|
||||||
// an answer of allow. rproxy passes them to the app.
|
|
||||||
ASNHeader = "X-Client-Asn"
|
|
||||||
CountryHeader = "X-Client-Country"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The values of DispositionHeader.
|
|
||||||
const (
|
|
||||||
// DispositionAllow tells rproxy to pass the request to the app.
|
|
||||||
DispositionAllow = "allow"
|
|
||||||
// DispositionDeny tells rproxy to give the visitor the answer's status,
|
|
||||||
// headers and body.
|
|
||||||
DispositionDeny = "deny"
|
|
||||||
// DispositionClose tells rproxy to close the visitor's connection
|
|
||||||
// without an answer, which SWW_WAF_BAN_RESPONSE asks for when it is close.
|
|
||||||
// waf closing the question's connection instead would read as
|
|
||||||
// waf being down, and rproxy would then let the request through.
|
|
||||||
DispositionClose = "close"
|
|
||||||
)
|
|
||||||
|
|
||||||
// OwnEndpointPrefix starts the path of every request for sww itself: waf's
|
|
||||||
// own endpoints and rproxy's health endpoint. The app never sees a path under
|
|
||||||
// it.
|
|
||||||
const OwnEndpointPrefix = "/_sww/"
|
|
||||||
|
|
||||||
// HealthPath is the health endpoint of waf and of rproxy, which the
|
|
||||||
// container's health check asks.
|
|
||||||
const HealthPath = "/_sww/healthz"
|
|
||||||
|
|
||||||
// ReportPath is where rproxy sends the report of each request waf
|
|
||||||
// answered.
|
|
||||||
const ReportPath = "/_sww/report"
|
|
||||||
|
|
||||||
// QuestionBodyMaxBytes is the most of a visitor's body a question carries.
|
|
||||||
const QuestionBodyMaxBytes = 64 << 10
|
|
||||||
|
|
||||||
// ReportStatusMax is the highest status a report may give.
|
|
||||||
const ReportStatusMax = 599
|
|
||||||
@@ -1,204 +0,0 @@
|
|||||||
package wafprotocol_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/netip"
|
|
||||||
"net/url"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
|
|
||||||
)
|
|
||||||
|
|
||||||
// requestID is the id the tests' questions and reports give.
|
|
||||||
const requestID = "an-id"
|
|
||||||
|
|
||||||
// questionRead is what the server startQuestionReader starts read of a
|
|
||||||
// question: the question, and the body it carries.
|
|
||||||
type questionRead struct {
|
|
||||||
question wafprotocol.Question
|
|
||||||
body string
|
|
||||||
}
|
|
||||||
|
|
||||||
// startQuestionReader starts a server that reads each question it gets with
|
|
||||||
// ReadQuestion. It returns the server's URL and the channel what it reads goes
|
|
||||||
// to.
|
|
||||||
func startQuestionReader(t *testing.T) (*url.URL, <-chan questionRead) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
read := make(chan questionRead, 1)
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
question, err := wafprotocol.ReadQuestion(w, r)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("read the question: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var body []byte
|
|
||||||
if question.Visitor != nil {
|
|
||||||
body, _ = io.ReadAll(question.Visitor.Body)
|
|
||||||
}
|
|
||||||
|
|
||||||
read <- questionRead{question, string(body)}
|
|
||||||
},
|
|
||||||
))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
serverURL, err := url.Parse(server.URL)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("parse the server's URL: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return serverURL, read
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestQuestionReadsAsWritten sends a question NewQuestion writes over HTTP,
|
|
||||||
// and checks that ReadQuestion gives back the visitor's request and what
|
|
||||||
// rproxy added.
|
|
||||||
func TestQuestionReadsAsWritten(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
target = "/a%2Fb?q=1"
|
|
||||||
body = "name=value"
|
|
||||||
held = "name"
|
|
||||||
)
|
|
||||||
|
|
||||||
client := netip.MustParseAddr("2001:db8::1")
|
|
||||||
peer := netip.MustParseAddr("192.0.2.1")
|
|
||||||
backend, read := startQuestionReader(t)
|
|
||||||
|
|
||||||
visitor := httptest.NewRequestWithContext(
|
|
||||||
t.Context(),
|
|
||||||
http.MethodPost,
|
|
||||||
"http://site.example.com"+target,
|
|
||||||
strings.NewReader(body),
|
|
||||||
)
|
|
||||||
visitor.Header.Set("Content-Length", "10")
|
|
||||||
visitor.Header.Set("User-Agent", "a visitor")
|
|
||||||
visitor.Header.Set(wafprotocol.PeerHeader, "the visitor's own")
|
|
||||||
|
|
||||||
question, err := wafprotocol.NewQuestion(
|
|
||||||
t.Context(),
|
|
||||||
wafprotocol.QuestionParams{
|
|
||||||
Visitor: visitor,
|
|
||||||
PathAsSent: "/a%2Fb",
|
|
||||||
Body: []byte(held),
|
|
||||||
WAFBackend: backend,
|
|
||||||
Client: client,
|
|
||||||
Peer: peer,
|
|
||||||
RequestID: requestID,
|
|
||||||
Scheme: "https",
|
|
||||||
},
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("write the question: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
res, err := http.DefaultTransport.RoundTrip(question)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("send the question: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_ = res.Body.Close()
|
|
||||||
|
|
||||||
got := <-read
|
|
||||||
if got.question.Visitor == nil {
|
|
||||||
t.FailNow()
|
|
||||||
}
|
|
||||||
|
|
||||||
if got.question.Client != client || got.question.Peer != peer ||
|
|
||||||
got.question.RequestID != requestID {
|
|
||||||
t.Errorf("read %+v", got.question)
|
|
||||||
}
|
|
||||||
|
|
||||||
sent := got.question.Visitor
|
|
||||||
if sent.Host != "site.example.com" || sent.RequestURI != target ||
|
|
||||||
sent.ContentLength != int64(len(body)) || got.body != held {
|
|
||||||
t.Errorf(
|
|
||||||
"read %s at %s, of length %d, carrying %q",
|
|
||||||
sent.RequestURI, sent.Host, sent.ContentLength, got.body,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, want := range map[string]string{
|
|
||||||
"Content-Length": "10",
|
|
||||||
"User-Agent": "a visitor",
|
|
||||||
"X-Forwarded-Proto": "https",
|
|
||||||
wafprotocol.RequestIDHeader: requestID,
|
|
||||||
wafprotocol.ClientHeader: "",
|
|
||||||
wafprotocol.PeerHeader: "",
|
|
||||||
wafprotocol.ContentLengthHeader: "",
|
|
||||||
} {
|
|
||||||
if sent.Header.Get(name) != want {
|
|
||||||
t.Errorf("%s is %q, want %q", name, sent.Header.Get(name), want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestAnswerReadsAsWritten checks that ReadAnswer gives back what
|
|
||||||
// SetAnswerHeaders sets, and that headers without a valid disposition give
|
|
||||||
// nothing.
|
|
||||||
func TestAnswerReadsAsWritten(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, want := range []wafprotocol.Answer{
|
|
||||||
{Disposition: wafprotocol.DispositionAllow, ASN: "64496", Country: "DE"},
|
|
||||||
{Disposition: wafprotocol.DispositionAllow, Country: "DE"},
|
|
||||||
{Disposition: wafprotocol.DispositionDeny},
|
|
||||||
{Disposition: wafprotocol.DispositionClose},
|
|
||||||
} {
|
|
||||||
header := http.Header{}
|
|
||||||
wafprotocol.SetAnswerHeaders(header, want)
|
|
||||||
|
|
||||||
got, valid := wafprotocol.ReadAnswer(header)
|
|
||||||
if !valid || got != want {
|
|
||||||
t.Errorf("read %+v (valid: %t), want %+v", got, valid, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
header := http.Header{}
|
|
||||||
header.Set(wafprotocol.DispositionHeader, "maybe")
|
|
||||||
header.Set(wafprotocol.ASNHeader, "64496")
|
|
||||||
|
|
||||||
got, valid := wafprotocol.ReadAnswer(header)
|
|
||||||
if valid || got != (wafprotocol.Answer{}) {
|
|
||||||
t.Errorf("read %+v (valid: %t), want nothing", got, valid)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestReportReadsAsWritten checks that ReadReport gives back the report
|
|
||||||
// NewReportRequest posts.
|
|
||||||
func TestReportReadsAsWritten(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
want := wafprotocol.Report{
|
|
||||||
RequestID: requestID,
|
|
||||||
Status: http.StatusOK,
|
|
||||||
RequestBytes: 10,
|
|
||||||
ResponseBytes: 20,
|
|
||||||
DurationTotalMS: 1.5,
|
|
||||||
DurationUpstreamTotalMS: 0.5,
|
|
||||||
}
|
|
||||||
|
|
||||||
req, err := wafprotocol.NewReportRequest(
|
|
||||||
t.Context(),
|
|
||||||
"http://waf.example.com"+wafprotocol.ReportPath,
|
|
||||||
&want,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("write the report: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if req.Method != http.MethodPost ||
|
|
||||||
req.URL.Path != wafprotocol.ReportPath {
|
|
||||||
t.Errorf("the report is %s %s", req.Method, req.URL.Path)
|
|
||||||
}
|
|
||||||
|
|
||||||
got, err := wafprotocol.ReadReport(httptest.NewRecorder(), req)
|
|
||||||
if err != nil || got != want {
|
|
||||||
t.Errorf("read %+v (%v), want %+v", got, err, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Reference in new issue
Block a user