Compare commits

..
Author SHA1 Message Date
sneak e52edb9f62 requestlog: the package doc no longer says one line per request
check / check (push) Waiting to run
It said the request log is one JSON object per request, then listed
three for a request: two from waf and one from rproxy.

Model: opus-5-5
2026-11-19 18:43:32 +00:00
sneak 5667661dd8 SPEC: the request log is not one line per request
check / check (push) Waiting to run
The package list said requestlog writes one JSON line per request. SPEC's
"The log" section gives a request waf answers three request log lines.

Model: opus-5-5
2026-11-19 18:23:52 +00:00
sneak 5ce35505bc Constructors take Params: ratelimit.New and processlog.New (closes #450)
ratelimit.New takes ratelimit.Params (Limits, MaxClients) in place of
two arguments by position. The process logger moves out of requestlog
into internal/processlog, whose New takes Params (Output, InstanceName,
Level); its two tests move with it unchanged. geojs.GeoJS keeps its
params whole, as DNSBL, AbuseIPDB and routewatch.Client do. SPEC and
README list the new package.

Model: opus-5-5
2026-11-19 18:23:52 +00:00
27 changed files with 434 additions and 408 deletions

No files matched your search

+5 -3
View File
@@ -906,9 +906,11 @@ stops the running process (see [`docs/state-files.md`](docs/state-files.md)).
`internal/coreruleset`, `internal/geojs`, `internal/lookup`, `internal/coreruleset`, `internal/geojs`, `internal/lookup`,
`internal/reputation`, `internal/routewatch`, `internal/anomaly` and `internal/reputation`, `internal/routewatch`, `internal/anomaly` and
`internal/alerts`: one check or source each. `internal/alerts`: one check or source each.
- `internal/state`, `internal/requestlog`, `internal/remotelog` and - `internal/state`: the state and cache files.
`internal/metrics`: the state and cache files, the log lines, the syslog - `internal/requestlog`: the request log.
sender and the metrics. - `internal/processlog`: the process's own messages.
- `internal/remotelog`: the syslog sender.
- `internal/metrics`: the metrics.
## Rationale ## Rationale
+4 -2
View File
@@ -104,8 +104,10 @@ never as a time.
its own. its own.
- `internal/state`: reads the state and cache files at start, takes in edits - `internal/state`: reads the state and cache files at start, takes in edits
while running, and writes the files when due and at the stop. while running, and writes the files when due and at the stop.
- `internal/requestlog` and `internal/remotelog`: the lines on stdout, and - `internal/requestlog`: the request log, as JSON lines on stdout.
sending them to the syslog server. - `internal/processlog`: the process lines on stdout, which hold the
process's own messages.
- `internal/remotelog`: sends the lines on stdout to the syslog server.
- `internal/metrics`: the metrics, in the Prometheus text format. - `internal/metrics`: the metrics, in the Prometheus text format.
- `Dockerfile`: the lint and test phases, then the image. - `Dockerfile`: the lint and test phases, then the image.
`deploy/example-app` is an app built on the image, with `rproxy` in its `deploy/example-app` is an app built on the image, with `rproxy` in its
+17 -30
View File
@@ -27,19 +27,6 @@ import (
"sneak.berlin/go/smallwebwaf/internal/remotelog" "sneak.berlin/go/smallwebwaf/internal/remotelog"
) )
// The names of the settings that waf writes into log lines and ban notes,
// as the setting that refused a request or lowered a client's limits.
const (
ExclusivelyAllowedCountriesSetting = "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES"
TagLimitPercentSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
ASNBytesLimitPercentSetting = "SWW_WAF_ASN_BYTES_LIMIT_PERCENT"
CountryBytesLimitPercentSetting = "SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT"
UnknownCountryLimitPercentSetting = "SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT"
ASNLimitPercentURLSetting = "SWW_WAF_ASN_LIMIT_PERCENT_URL"
BlocklistActionSetting = "SWW_WAF_BLOCKLIST_ACTION"
DNSBLAbuseIPDBActionSetting = "SWW_WAF_DNSBL_ABUSEIPDB_ACTION"
)
// WAF is the settings of `sww waf`. A time or size limit of zero is off. // WAF is the settings of `sww waf`. A time or size limit of zero is off.
type WAF struct { type WAF struct {
// ListenAddr is where waf listens (SWW_WAF_LISTEN_ADDR). // ListenAddr is where waf listens (SWW_WAF_LISTEN_ADDR).
@@ -631,24 +618,24 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
"800", "800",
), ),
ExclusivelyAllowedCountries: env.countries( ExclusivelyAllowedCountries: env.countries(
ExclusivelyAllowedCountriesSetting, ""), "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""),
TagLists: env.tagLists("SWW_WAF_TAG_LISTS"), TagLists: env.tagLists("SWW_WAF_TAG_LISTS"),
TagDeny: env.tags("SWW_WAF_TAG_DENY"), TagDeny: env.tags("SWW_WAF_TAG_DENY"),
TagLimitPercent: env.tagPercents(TagLimitPercentSetting), TagLimitPercent: env.tagPercents("SWW_WAF_TAG_LIMIT_PERCENT"),
RouteWatchURL: env.routeWatchURL("SWW_WAF_ROUTEWATCH_URL"), RouteWatchURL: env.routeWatchURL("SWW_WAF_ROUTEWATCH_URL"),
ASNBytesLimitPercent: env.percents( ASNBytesLimitPercent: env.percents(
ASNBytesLimitPercentSetting, "SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
ParseASN, ParseASN,
), ),
CountryBytesLimitPercent: env.percents( CountryBytesLimitPercent: env.percents(
CountryBytesLimitPercentSetting, "SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
parseCountry, parseCountry,
), ),
UnknownCountryLimitPercent: env.percent( UnknownCountryLimitPercent: env.percent(
UnknownCountryLimitPercentSetting, "SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
"100", "100",
), ),
ASNLimitPercentURL: env.listURL(ASNLimitPercentURLSetting), ASNLimitPercentURL: env.listURL("SWW_WAF_ASN_LIMIT_PERCENT_URL"),
BlocklistURLs: env.listURLs("SWW_WAF_BLOCKLIST_URLS"), BlocklistURLs: env.listURLs("SWW_WAF_BLOCKLIST_URLS"),
BlocklistRefresh: env.refresh("SWW_WAF_BLOCKLIST_REFRESH", "24h"), BlocklistRefresh: env.refresh("SWW_WAF_BLOCKLIST_REFRESH", "24h"),
DNSBLZones: env.zones("SWW_WAF_DNSBL_ZONES"), DNSBLZones: env.zones("SWW_WAF_DNSBL_ZONES"),
@@ -822,9 +809,9 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
strconv.FormatBool(cfg.LookupSource != LookupSourceOff)) strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action( cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
BlocklistActionSetting, string(ReputationActionDeny)) "SWW_WAF_BLOCKLIST_ACTION", string(ReputationActionDeny))
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action( cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
DNSBLAbuseIPDBActionSetting, "limit:25") "SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "limit:25")
env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil) env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
env.checkLookupDBPath(cfg) env.checkLookupDBPath(cfg)
@@ -1233,7 +1220,7 @@ func (e *environment) checkLookupDBPath(cfg *WAF) {
func (e *environment) checkCountriesAndLookups(cfg *WAF) { func (e *environment) checkCountriesAndLookups(cfg *WAF) {
for _, country := range cfg.ExclusivelyAllowedCountries { for _, country := range cfg.ExclusivelyAllowedCountries {
if slices.Contains(cfg.TagDeny, CountryTagPrefix+country) { if slices.Contains(cfg.TagDeny, CountryTagPrefix+country) {
e.check(ExclusivelyAllowedCountriesSetting, e.check("SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
fmt.Errorf("%q %w", country, errOnBothLists)) fmt.Errorf("%q %w", country, errOnBothLists))
} }
} }
@@ -1252,15 +1239,15 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
name string name string
set bool set bool
}{ }{
{ExclusivelyAllowedCountriesSetting, len(cfg.ExclusivelyAllowedCountries) > 0}, {"SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", len(cfg.ExclusivelyAllowedCountries) > 0},
{"SWW_WAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders}, {"SWW_WAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders},
{"SWW_WAF_TAG_DENY", slices.ContainsFunc(cfg.TagDeny, onlyLookupGives)}, {"SWW_WAF_TAG_DENY", slices.ContainsFunc(cfg.TagDeny, onlyLookupGives)},
{TagLimitPercentSetting, slices.ContainsFunc( {"SWW_WAF_TAG_LIMIT_PERCENT", slices.ContainsFunc(
slices.Collect(maps.Keys(cfg.TagLimitPercent)), onlyLookupGives)}, slices.Collect(maps.Keys(cfg.TagLimitPercent)), onlyLookupGives)},
{ASNBytesLimitPercentSetting, len(cfg.ASNBytesLimitPercent) > 0}, {"SWW_WAF_ASN_BYTES_LIMIT_PERCENT", len(cfg.ASNBytesLimitPercent) > 0},
{CountryBytesLimitPercentSetting, len(cfg.CountryBytesLimitPercent) > 0}, {"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT", len(cfg.CountryBytesLimitPercent) > 0},
{UnknownCountryLimitPercentSetting, cfg.UnknownCountryLimitPercent < 100}, {"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT", cfg.UnknownCountryLimitPercent < 100},
{ASNLimitPercentURLSetting, cfg.ASNLimitPercentURL != ""}, {"SWW_WAF_ASN_LIMIT_PERCENT_URL", cfg.ASNLimitPercentURL != ""},
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0}, {"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0},
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0}, {"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0},
{"SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0}, {"SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0},
@@ -1280,7 +1267,7 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
// blocklist too: the file at a URL is fetched as one list or the other. // blocklist too: the file at a URL is fetched as one list or the other.
func (e *environment) checkASNLimitPercentURL(cfg *WAF) { func (e *environment) checkASNLimitPercentURL(cfg *WAF) {
if slices.Contains(cfg.BlocklistURLs, cfg.ASNLimitPercentURL) { if slices.Contains(cfg.BlocklistURLs, cfg.ASNLimitPercentURL) {
e.check(ASNLimitPercentURLSetting, e.check("SWW_WAF_ASN_LIMIT_PERCENT_URL",
fmt.Errorf("%q %w", cfg.ASNLimitPercentURL, errInBlocklistURLs)) fmt.Errorf("%q %w", cfg.ASNLimitPercentURL, errInBlocklistURLs))
} }
} }
@@ -1307,7 +1294,7 @@ func (e *environment) checkTags(cfg *WAF) {
} }
for tag := range cfg.TagLimitPercent { for tag := range cfg.TagLimitPercent {
e.check(TagLimitPercentSetting, checkTag(cfg, tag)) e.check("SWW_WAF_TAG_LIMIT_PERCENT", checkTag(cfg, tag))
} }
} }
+14 -24
View File
@@ -84,12 +84,7 @@ type Params struct {
// one request to GeoJS is under way at a time, and it asks about every // one request to GeoJS is under way at a time, and it asks about every
// client waiting, up to maxPerRequest. It is safe for concurrent use. // client waiting, up to maxPerRequest. It is safe for concurrent use.
type GeoJS struct { type GeoJS struct {
url string params Params
answered func(Answer)
now func() time.Time
asked func()
failed func(err error, askingAgainIn time.Duration)
wentWithoutAnswer func()
// httpClient follows no redirect, so that visitors' addresses go to // httpClient follows no redirect, so that visitors' addresses go to
// GeoJS alone: a redirect is a failure. // GeoJS alone: a redirect is a failure.
httpClient *http.Client httpClient *http.Client
@@ -145,12 +140,7 @@ func New(params Params) *GeoJS {
} }
return &GeoJS{ return &GeoJS{
url: params.URL, params: params,
answered: params.Answered,
now: params.Now,
asked: params.Asked,
failed: params.Failed,
wentWithoutAnswer: params.WentWithoutAnswer,
httpClient: &http.Client{ httpClient: &http.Client{
CheckRedirect: func(*http.Request, []*http.Request) error { CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse return http.ErrUseLastResponse
@@ -193,7 +183,7 @@ func (g *GeoJS) LookUp(
answer, found := g.kept(client) answer, found := g.kept(client)
if !found { if !found {
g.wentWithoutAnswer() g.params.WentWithoutAnswer()
} }
w, waiting := g.waiting[client] w, waiting := g.waiting[client]
@@ -273,7 +263,7 @@ func (g *GeoJS) Load(answers []Answer) {
g.answers.Purge() g.answers.Purge()
now := g.now() now := g.params.Now()
for _, answer := range answers { for _, answer := range answers {
if now.Sub(answer.Answered) < keepFor { if now.Sub(answer.Answered) < keepFor {
@@ -310,7 +300,7 @@ func (g *GeoJS) answerOrWait(
} }
if w == nil { if w == nil {
g.wentWithoutAnswer() g.params.WentWithoutAnswer()
return Answer{}, nil // too many clients wait already return Answer{}, nil // too many clients wait already
} }
@@ -322,7 +312,7 @@ func (g *GeoJS) answerOrWait(
} }
if w.late { if w.late {
g.wentWithoutAnswer() g.params.WentWithoutAnswer()
return Answer{}, nil return Answer{}, nil
} }
@@ -333,7 +323,7 @@ func (g *GeoJS) answerOrWait(
// kept returns client's answer, if GeoJS gave it less than keepFor ago, // kept returns client's answer, if GeoJS gave it less than keepFor ago,
// and notes that it was used. // and notes that it was used.
func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) { func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
now := g.now() now := g.params.Now()
kept, found := g.answers.Get(client) kept, found := g.answers.Get(client)
if !found || now.Sub(kept.Answered) >= keepFor { if !found || now.Sub(kept.Answered) >= keepFor {
@@ -350,7 +340,7 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
// been called. The requests to GeoJS are for every client waiting, so they // been called. The requests to GeoJS are for every client waiting, so they
// go on when the caller whose ctx is given stops waiting, until StopAsking. // go on when the caller whose ctx is given stops waiting, until StopAsking.
func (g *GeoJS) ask(ctx context.Context) { func (g *GeoJS) ask(ctx context.Context) {
if g.asking || g.stopped || g.now().Before(g.retryAt) { if g.asking || g.stopped || g.params.Now().Before(g.retryAt) {
return return
} }
@@ -382,9 +372,9 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
kept, answered := g.keep(clients, given, err) kept, answered := g.keep(clients, given, err)
if g.answered != nil { if g.params.Answered != nil {
for _, answer := range kept { for _, answer := range kept {
g.answered(answer) g.params.Answered(answer)
} }
} }
@@ -433,7 +423,7 @@ func (g *GeoJS) keep(
g.mu.Lock() g.mu.Lock()
defer g.mu.Unlock() defer g.mu.Unlock()
now := g.now() now := g.params.Now()
kept := make([]Answer, 0, len(clients)) kept := make([]Answer, 0, len(clients))
leftOut := 0 leftOut := 0
@@ -477,7 +467,7 @@ func (g *GeoJS) keep(
w.late = true w.late = true
} }
g.failed(err, g.retryDelay) g.params.Failed(err, g.retryDelay)
return kept, false return kept, false
} }
@@ -507,7 +497,7 @@ func (g *GeoJS) request(
req, err := http.NewRequestWithContext( req, err := http.NewRequestWithContext(
ctx, ctx,
http.MethodGet, http.MethodGet,
g.url, g.params.URL,
http.NoBody, http.NoBody,
) )
if err != nil { if err != nil {
@@ -516,7 +506,7 @@ func (g *GeoJS) request(
req.URL.RawQuery = "ip=" + strings.Join(addrs, ",") req.URL.RawQuery = "ip=" + strings.Join(addrs, ",")
g.asked() g.params.Asked()
res, err := g.httpClient.Do(req) res, err := g.httpClient.Do(req)
if err != nil { if err != nil {
+44
View File
@@ -0,0 +1,44 @@
// Package processlog builds the logger for the process's own messages,
// which it writes as JSON lines marked "type":"process". The request log
// is package requestlog's.
package processlog
import (
"io"
"log/slog"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
// Params are what New needs.
type Params struct {
// Output is where the lines are written.
Output io.Writer
// InstanceName, SWW_WAF_INSTANCE_NAME, is each line's instance.
InstanceName string
// Level, SWW_WAF_LOG_LEVEL, is the least severe level written.
Level slog.Level
}
// New returns the logger for the process's own messages: JSON lines on
// params.Output, marked "type":"process", with the time in the same form
// as a request line's. It writes only the messages at params.Level or more
// severe; the request lines requestlog.Write writes are never held back.
func New(params Params) *slog.Logger {
handler := slog.NewJSONHandler(params.Output, &slog.HandlerOptions{
Level: params.Level,
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
if attr.Key == slog.TimeKey && len(groups) == 0 {
return slog.String(
slog.TimeKey,
requestlog.FormatTime(attr.Value.Time()),
)
}
return attr
},
})
return slog.New(handler).
With("type", "process", "instance", params.InstanceName)
}
+105
View File
@@ -0,0 +1,105 @@
package processlog_test
import (
"bytes"
"encoding/json"
"log/slog"
"slices"
"strings"
"testing"
"time"
"sneak.berlin/go/smallwebwaf/internal/processlog"
)
// instanceName is the instance the tests' loggers are built with.
const instanceName = "host1/blog"
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processlog.New(processlog.Params{
Output: &out,
InstanceName: instanceName,
Level: slog.LevelInfo,
}).
Info("starting",
"version", "v1")
var fields map[string]any
err := json.Unmarshal(out.Bytes(), &fields)
if err != nil {
t.Fatalf("decode %q: %v", out.String(), err)
}
if fields["type"] != "process" ||
fields["instance"] != "host1/blog" ||
fields["msg"] != "starting" ||
fields["level"] != "INFO" ||
fields["version"] != "v1" {
t.Errorf("process line %v", fields)
}
timeText, _ := fields["time"].(string)
logged, err := time.Parse(time.RFC3339, timeText)
if err != nil || !strings.HasSuffix(timeText, "Z") ||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
time.Since(logged) > time.Minute {
t.Errorf(
"process line time %q, want now in UTC with milliseconds",
timeText,
)
}
}
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
t.Parallel()
levels := []slog.Level{
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
}
for i, level := range levels {
t.Run(level.String(), func(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processLog := processlog.New(processlog.Params{
Output: &out,
InstanceName: instanceName,
Level: level,
})
for _, at := range levels {
processLog.Log(t.Context(), at, "message")
}
var got, want []string
for line := range strings.Lines(out.String()) {
var fields struct {
Level string `json:"level"`
}
err := json.Unmarshal([]byte(line), &fields)
if err != nil {
t.Fatalf("decode %q: %v", line, err)
}
got = append(got, fields.Level)
}
for _, written := range levels[i:] {
want = append(want, written.String())
}
if !slices.Equal(got, want) {
t.Errorf("lines at %v, want %v", got, want)
}
})
}
}
+7 -4
View File
@@ -11,7 +11,7 @@ import (
func TestHistoryKeepsEveryRequest(t *testing.T) { func TestHistoryKeepsEveryRequest(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -57,7 +57,7 @@ func TestHistoryKeepsEveryRequest(t *testing.T) {
func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) { func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
other := netip.MustParsePrefix("198.51.100.7/32") other := netip.MustParsePrefix("198.51.100.7/32")
start := midnight() start := midnight()
@@ -100,7 +100,10 @@ func TestLookupReachesTheHistoryOfAClientInTheTable(t *testing.T) {
func TestResetKeepsTheHistory(t *testing.T) { func TestResetKeepsTheHistory(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize) limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -123,7 +126,7 @@ func TestResetKeepsTheHistory(t *testing.T) {
func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) { func TestRequestsAddsUpTheClientsInsideTheNetblock(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
for client, requests := range map[string]int{ for client, requests := range map[string]int{
"198.51.100.9/32": 2, "198.51.100.9/32": 2,
+18 -6
View File
@@ -156,16 +156,28 @@ type Request struct {
BlockedByCoreRuleSet bool BlockedByCoreRuleSet bool
} }
// New returns a Limiter for limits, with no client counted yet, whose // Params are what New needs.
// table holds at most maxClients clients (SWW_WAF_MAX_TRACKED_CLIENTS). Past type Params struct {
// it, the least recently seen client is dropped, with its history, and // Limits are what each client's requests and bytes are counted against.
// starts afresh if it comes back. Limits Limits
func New(limits Limits, maxClients int) *Limiter { // MaxClients is the most clients the table holds,
clients, err := simplelru.NewLRU[netip.Prefix, *Client](maxClients, nil) // SWW_WAF_MAX_TRACKED_CLIENTS. Past it, the least recently seen client
// is dropped, with its history, and starts afresh if it comes back.
MaxClients int
}
// New returns a Limiter with no client counted yet.
func New(params Params) *Limiter {
clients, err := simplelru.NewLRU[netip.Prefix, *Client](
params.MaxClients,
nil,
)
if err != nil { if err != nil {
panic(err) // NewLRU fails only for a size below one panic(err) // NewLRU fails only for a size below one
} }
limits := params.Limits
return &Limiter{ return &Limiter{
windows: [3]window{ windows: [3]window{
{ {
+63 -39
View File
@@ -33,10 +33,10 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
now := midnight() now := midnight()
limiter := ratelimit.New( limiter := ratelimit.New(ratelimit.Params{
ratelimit.Limits{PerMinute: limit, PerDay: limit + 1}, Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit + 1},
tableSize, MaxClients: tableSize,
) })
if limiter.OverARateLimit(client, now) { if limiter.OverARateLimit(client, now) {
t.Error("a client never seen is over a rate limit") t.Error("a client never seen is over a rate limit")
@@ -72,7 +72,7 @@ func TestOverARateLimitSaysWhatTheNextRequestWouldDoAndCountsNothing(
wantCount(t, limiter, client, now, minute) wantCount(t, limiter, client, now, minute)
// A limit that is off is never gone over. // A limit that is off is never gone over.
off := ratelimit.New(ratelimit.Limits{}, tableSize) off := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
off.Count(client, now, whole) off.Count(client, now, whole)
if off.OverARateLimit(client, now) { if off.OverARateLimit(client, now) {
@@ -95,7 +95,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
t.Run(tc.window, func(t *testing.T) { t.Run(tc.window, func(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(tc.limits, tableSize) limiter := ratelimit.New(ratelimit.Params{
Limits: tc.limits,
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
quarter := tc.length / 4 quarter := tc.length / 4
@@ -126,10 +129,10 @@ func TestEachWindowRefusesAtItsLimitAndLetsTheClientBack(t *testing.T) {
func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) { func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New( limiter := ratelimit.New(ratelimit.Params{
ratelimit.Limits{PerMinute: limit, PerHour: limit}, Limits: ratelimit.Limits{PerMinute: limit, PerHour: limit},
tableSize, MaxClients: tableSize,
) })
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -159,10 +162,13 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) { func TestClientGetsItsPercentageOfEachLimitRoundedDown(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New( limiter := ratelimit.New(ratelimit.Params{
ratelimit.Limits{PerMinute: 5, BytesPerDay: math.MaxInt64}, Limits: ratelimit.Limits{
tableSize, PerMinute: 5,
) BytesPerDay: math.MaxInt64,
},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -203,10 +209,10 @@ func TestZeroPercentIsAZeroAllowanceAndALimitOffStaysOff(t *testing.T) {
t.Parallel() t.Parallel()
// Only the hour has limits: the minute's and the day's are off. // Only the hour has limits: the minute's and the day's are off.
limiter := ratelimit.New( limiter := ratelimit.New(ratelimit.Params{
ratelimit.Limits{PerHour: limit, BytesPerHour: 1000}, Limits: ratelimit.Limits{PerHour: limit, BytesPerHour: 1000},
tableSize, MaxClients: tableSize,
) })
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -253,7 +259,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
t.Run(tc.window, func(t *testing.T) { t.Run(tc.window, func(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(tc.limits, tableSize) limiter := ratelimit.New(ratelimit.Params{
Limits: tc.limits,
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
// 600 bytes are within the limit, 600 more over it. // 600 bytes are within the limit, 600 more over it.
@@ -287,10 +296,10 @@ func TestEachByteLimitIsBrokenByTheBytesCounted(t *testing.T) {
func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) { func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New( limiter := ratelimit.New(ratelimit.Params{
ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000}, Limits: ratelimit.Limits{PerMinute: 2, BytesPerMinute: 1000},
tableSize, MaxClients: tableSize,
) })
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
other := netip.MustParsePrefix("203.0.113.10/32") other := netip.MustParsePrefix("203.0.113.10/32")
start := midnight() start := midnight()
@@ -314,7 +323,7 @@ func TestALimitIsBrokenOnlyByWhatIsAddedToIt(t *testing.T) {
func TestCountGivesTheBytesInEachWindow(t *testing.T) { func TestCountGivesTheBytesInEachWindow(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -340,7 +349,10 @@ func TestCountGivesTheBytesInEachWindow(t *testing.T) {
func TestResetSetsTheBytesBackToZero(t *testing.T) { func TestResetSetsTheBytesBackToZero(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{BytesPerDay: 1000}, tableSize) limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{BytesPerDay: 1000},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -356,7 +368,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
) { ) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -411,7 +423,7 @@ func TestRefusalsOverTheLimitInAMinuteBreakTheRefusalLimit(
func TestCountGivesTheRequestsInEachWindow(t *testing.T) { func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -438,10 +450,10 @@ func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
func TestResetSetsTheCountsBackToZero(t *testing.T) { func TestResetSetsTheCountsBackToZero(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New( limiter := ratelimit.New(ratelimit.Params{
ratelimit.Limits{PerMinute: limit, PerDay: limit}, Limits: ratelimit.Limits{PerMinute: limit, PerDay: limit},
tableSize, MaxClients: tableSize,
) })
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -463,7 +475,10 @@ func TestResetSetsTheCountsBackToZero(t *testing.T) {
func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) { func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize) limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -482,10 +497,10 @@ func TestClientBackAfterAWholeBucketIsWithinTheLimitAtOnce(t *testing.T) {
func TestRefusedRequestsCount(t *testing.T) { func TestRefusedRequestsCount(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New( limiter := ratelimit.New(ratelimit.Params{
ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit}, Limits: ratelimit.Limits{PerMinute: limit, PerHour: 2 * limit},
tableSize, MaxClients: tableSize,
) })
refused := netip.MustParsePrefix("203.0.113.9/32") refused := netip.MustParsePrefix("203.0.113.9/32")
within := netip.MustParsePrefix("203.0.113.10/32") within := netip.MustParsePrefix("203.0.113.10/32")
start := midnight() start := midnight()
@@ -518,7 +533,10 @@ func TestRefusedRequestsCount(t *testing.T) {
func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) { func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{PerMinute: limit}, tableSize) limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: limit},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -534,7 +552,10 @@ func TestRequestCountedLateGoesInTheBucketUnderWay(t *testing.T) {
func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) { func TestClockSetBackStartsTheBucketsAfresh(t *testing.T) {
t.Parallel() t.Parallel()
limiter := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize) limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
@@ -562,7 +583,10 @@ func TestKeepsAtMostMaxClientsDroppingTheLeastRecentlySeen(t *testing.T) {
const maxClients = 3 const maxClients = 3
limiter := ratelimit.New(ratelimit.Limits{PerMinute: 1}, maxClients) limiter := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerMinute: 1},
MaxClients: maxClients,
})
now := midnight() now := midnight()
clients := make([]netip.Prefix, maxClients+1) clients := make([]netip.Prefix, maxClients+1)
+12 -6
View File
@@ -19,7 +19,7 @@ func TestSnapshotListsTheClientsByAddress(t *testing.T) {
"2001:db8::/64", "2001:db8::/64",
} }
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
for _, i := range []int{2, 3, 0, 1} { for _, i := range []int{2, 3, 0, 1} {
limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole) limiter.Count(netip.MustParsePrefix(want[i]), midnight(), whole)
} }
@@ -52,7 +52,10 @@ func TestLoadedCountsCarryOn(t *testing.T) {
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
before := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize) before := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
for range limit { for range limit {
wantCount(t, before, client, start, "") wantCount(t, before, client, start, "")
} }
@@ -60,7 +63,10 @@ func TestLoadedCountsCarryOn(t *testing.T) {
// Loaded into a new limiter, as across a restart, the client has no // Loaded into a new limiter, as across a restart, the client has no
// fresh allowance. // fresh allowance.
later := start.Add(time.Minute) later := start.Add(time.Minute)
after := ratelimit.New(ratelimit.Limits{PerHour: limit}, tableSize) after := ratelimit.New(ratelimit.Params{
Limits: ratelimit.Limits{PerHour: limit},
MaxClients: tableSize,
})
after.Load(before.Snapshot(), later) after.Load(before.Snapshot(), later)
wantCount(t, after, client, later, hour) wantCount(t, after, client, later, hour)
} }
@@ -71,7 +77,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
client := netip.MustParsePrefix("203.0.113.9/32") client := netip.MustParsePrefix("203.0.113.9/32")
start := midnight() start := midnight()
limiter := ratelimit.New(ratelimit.Limits{}, tableSize) limiter := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
limiter.Count(client, start, whole) limiter.Count(client, start, whole)
limiter.CountBytes(client, start, 5, whole) limiter.CountBytes(client, start, 5, whole)
limiter.CountRefusal(client, start, limit) limiter.CountRefusal(client, start, limit)
@@ -80,7 +86,7 @@ func TestLoadEmptiesBucketsWhoseTimeHasPassed(t *testing.T) {
loaded := func(now time.Time) ratelimit.Client { loaded := func(now time.Time) ratelimit.Client {
t.Helper() t.Helper()
after := ratelimit.New(ratelimit.Limits{}, tableSize) after := ratelimit.New(ratelimit.Params{MaxClients: tableSize})
after.Load(limiter.Snapshot(), now) after.Load(limiter.Snapshot(), now)
return after.Snapshot()[0] return after.Snapshot()[0]
@@ -142,7 +148,7 @@ func TestLoadDropsTheLeastRecentlySeenFirst(t *testing.T) {
addr = addr.Next() addr = addr.Next()
} }
limiter := ratelimit.New(ratelimit.Limits{}, maxClients) limiter := ratelimit.New(ratelimit.Params{MaxClients: maxClients})
limiter.Load(clients, midnight()) limiter.Load(clients, midnight())
got := limiter.Snapshot() got := limiter.Snapshot()
+4 -33
View File
@@ -1,7 +1,7 @@
// Package requestlog writes the lines sww prints on stdout: one // Package requestlog writes the request log sww prints on stdout: JSON
// JSON object per request, two from waf and a short one from rproxy, // lines marked "type":"request", at most two from waf and a short one
// marked "type":"request", and the process's own messages as JSON lines marked // from rproxy for each request. Package processlog writes the process's
// "type":"process". // own messages.
package requestlog package requestlog
import ( import (
@@ -9,7 +9,6 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"log/slog"
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/ratelimit"
@@ -300,31 +299,3 @@ func Milliseconds(d time.Duration) float64 {
time.Millisecond/time.Microsecond, time.Millisecond/time.Microsecond,
) )
} }
// NewProcessLogger returns the logger for the process's own messages:
// JSON lines on w, marked "type":"process", with the time in the same form
// as a request line's, and instanceName, SWW_WAF_INSTANCE_NAME, as instance.
// It writes only the messages at level, SWW_WAF_LOG_LEVEL, or more severe;
// the request lines Write writes are never held back.
func NewProcessLogger(
w io.Writer,
instanceName string,
level slog.Level,
) *slog.Logger {
handler := slog.NewJSONHandler(w, &slog.HandlerOptions{
Level: level,
ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
if attr.Key == slog.TimeKey && len(groups) == 0 {
return slog.String(
slog.TimeKey,
FormatTime(attr.Value.Time()),
)
}
return attr
},
})
return slog.New(handler).
With("type", "process", "instance", instanceName)
}
-87
View File
@@ -3,8 +3,6 @@ package requestlog_test
import ( import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"log/slog"
"slices"
"strings" "strings"
"testing" "testing"
"time" "time"
@@ -134,88 +132,3 @@ func TestWriteRProxyLineWritesOneJSONLineWithOnlyItsFields(t *testing.T) {
t.Errorf("wrote %s, want %s", out.String(), want) t.Errorf("wrote %s, want %s", out.String(), want)
} }
} }
func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
t.Parallel()
var out bytes.Buffer
requestlog.NewProcessLogger(&out, "host1/blog", slog.LevelInfo).
Info("starting",
"version", "v1")
var fields map[string]any
err := json.Unmarshal(out.Bytes(), &fields)
if err != nil {
t.Fatalf("decode %q: %v", out.String(), err)
}
if fields["type"] != "process" ||
fields["instance"] != "host1/blog" ||
fields["msg"] != "starting" ||
fields["level"] != "INFO" ||
fields["version"] != "v1" {
t.Errorf("process line %v", fields)
}
timeText, _ := fields["time"].(string)
logged, err := time.Parse(time.RFC3339, timeText)
if err != nil || !strings.HasSuffix(timeText, "Z") ||
len(timeText) != len("2006-01-02T15:04:05.000Z") ||
time.Since(logged) > time.Minute {
t.Errorf(
"process line time %q, want now in UTC with milliseconds",
timeText,
)
}
}
func TestProcessLoggerWritesTheMessagesAtItsLevelOrMoreSevere(t *testing.T) {
t.Parallel()
levels := []slog.Level{
slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError,
}
for i, level := range levels {
t.Run(level.String(), func(t *testing.T) {
t.Parallel()
var out bytes.Buffer
processLog := requestlog.NewProcessLogger(
&out,
"host1/blog",
level,
)
for _, at := range levels {
processLog.Log(t.Context(), at, "message")
}
var got, want []string
for line := range strings.Lines(out.String()) {
var fields struct {
Level string `json:"level"`
}
err := json.Unmarshal([]byte(line), &fields)
if err != nil {
t.Fatalf("decode %q: %v", line, err)
}
got = append(got, fields.Level)
}
for _, written := range levels[i:] {
want = append(want, written.String())
}
if !slices.Equal(got, want) {
t.Errorf("lines at %v, want %v", got, want)
}
})
}
}
-6
View File
@@ -2,12 +2,6 @@ package rproxy
import "time" import "time"
// StopBreakerClock has the circuit breaker of s read at as the time, so that how
// long requests take cannot matter to it. It is called before s serves.
func (s *Server) StopBreakerClock(at time.Time) {
s.h.breakerClock = func() time.Time { return at }
}
// Breaker is the circuit breaker, for tests that give it the time. // Breaker is the circuit breaker, for tests that give it the time.
type Breaker struct { type Breaker struct {
b breaker b breaker
+6 -6
View File
@@ -18,7 +18,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest" "sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy" "sneak.berlin/go/smallwebwaf/internal/rproxy"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/waf"
) )
@@ -51,11 +51,11 @@ func TestServerHasTheFixedLimits(t *testing.T) {
server := rproxy.New(rproxy.Params{ server := rproxy.New(rproxy.Params{
Config: cfg, Config: cfg,
RequestLog: io.Discard, RequestLog: io.Discard,
ProcessLog: requestlog.NewProcessLogger( ProcessLog: processlog.New(processlog.Params{
io.Discard, Output: io.Discard,
"test", InstanceName: "test",
slog.LevelInfo, Level: slog.LevelInfo,
), }),
}) })
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 || if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
+13 -8
View File
@@ -20,6 +20,7 @@ import (
"net/url" "net/url"
"slices" "slices"
"strings" "strings"
"sync"
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
@@ -93,7 +94,6 @@ func New(params Params) *Server {
reportURL: (&url.URL{ reportURL: (&url.URL{
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath, Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath,
}).String(), }).String(),
breakerClock: time.Now,
} }
return &Server{ return &Server{
@@ -113,7 +113,11 @@ func New(params Params) *Server {
// which SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS bounds. Any report after this call // which SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS bounds. Any report after this call
// is dropped: that of a WebSocket that closes while rproxy stops, for one. // is dropped: that of a WebSocket that closes while rproxy stops, for one.
func (s *Server) WaitForReports() { func (s *Server) WaitForReports() {
s.h.reports.stopAndWait() s.h.reportsMu.Lock()
s.h.reportsStopped = true
s.h.reportsMu.Unlock()
s.h.reports.Wait()
} }
// handler is the proxy. It holds what every request shares; what belongs to // handler is the proxy. It holds what every request shares; what belongs to
@@ -129,12 +133,13 @@ type handler struct {
wafTransport http.RoundTripper wafTransport http.RoundTripper
// reportURL is where the reports go. // reportURL is where the reports go.
reportURL string reportURL string
// breaker is the circuit breaker for asks to waf, and breakerClock gives it // breaker is the circuit breaker for asks to waf.
// the time. breaker breaker
breaker breaker // reports waits for the reports being sent. reportsMu guards starting one
breakerClock func() time.Time // against reportsStopped, which is set once nothing is to wait for another.
// reports are the goroutines sending reports to waf. reports sync.WaitGroup
reports reportGoroutines reportsMu sync.Mutex
reportsStopped bool
} }
// newTransport returns what carries requests to the app. It never goes through // newTransport returns what carries requests to the app. It never goes through
+11 -33
View File
@@ -21,7 +21,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/config/configtest" "sneak.berlin/go/smallwebwaf/internal/config/configtest"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy" "sneak.berlin/go/smallwebwaf/internal/rproxy"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/waf"
) )
@@ -356,19 +356,6 @@ func startServerLogging(
) (string, *rproxy.Server) { ) (string, *rproxy.Server) {
t.Helper() t.Helper()
server := newServer(t, backend, appURL, env, requestLog, processLog)
return serve(t, server), server
}
// newServer returns rproxy, not yet serving, with its request lines going to
// requestLog and its process lines to processLog.
func newServer(
t *testing.T, backend, appURL string, env map[string]string,
requestLog, processLog io.Writer,
) *rproxy.Server {
t.Helper()
settings := map[string]string{ settings := map[string]string{
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL, "SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
wafBackendTimeout: "60000", wafBackendTimeout: "60000",
@@ -380,20 +367,15 @@ func newServer(
t.Fatalf("settings %v: %v", settings, err) t.Fatalf("settings %v: %v", settings, err)
} }
return rproxy.New(rproxy.Params{ server := rproxy.New(rproxy.Params{
Config: cfg, Config: cfg,
RequestLog: requestLog, RequestLog: requestLog,
ProcessLog: requestlog.NewProcessLogger( ProcessLog: processlog.New(processlog.Params{
processLog, Output: processLog,
"test", InstanceName: "test",
slog.LevelInfo, Level: slog.LevelInfo,
), }),
}) })
}
// serve has server serve until the test ends, and returns its address.
func serve(t *testing.T, server *rproxy.Server) string {
t.Helper()
listener, err := (&net.ListenConfig{}).Listen( listener, err := (&net.ListenConfig{}).Listen(
t.Context(), t.Context(),
@@ -414,7 +396,7 @@ func serve(t *testing.T, server *rproxy.Server) string {
server.WaitForReports() server.WaitForReports()
}) })
return listener.Addr().String() return listener.Addr().String(), server
} }
// answered is the visitor's answer. // answered is the visitor's answer.
@@ -1550,13 +1532,9 @@ func TestAnswersThatCannotBePassedOnOpenTheBreaker(t *testing.T) {
tooLarge(w, r) tooLarge(w, r)
}) })
app := startApp(t, greet) app := startApp(t, greet)
out := &output{} addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
server := newServer(t, standIn.URL, app.URL, nil, out, out) wafBackendTimeout: "60000",
// The breaker's clock stands still. On a slow host, the five failures could })
// otherwise take more than the five seconds that open the breaker. The next
// request could come more than a second later, when it may ask waf again.
server.StopBreakerClock(time.Now())
addr := serve(t, server)
for range 5 { for range 5 {
if got := get(t, addr, "/"); got.status != http.StatusBadGateway { if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
+11 -35
View File
@@ -10,7 +10,7 @@ import (
"maps" "maps"
"net/http" "net/http"
"strings" "strings"
"sync" "time"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/waf"
) )
@@ -97,7 +97,7 @@ func (rq *request) askUnlessBreakerOpen(
ctx context.Context, ctx context.Context,
) (*wafAnswer, error) { ) (*wafAnswer, error) {
b := &rq.h.breaker b := &rq.h.breaker
if !b.mayAsk(rq.h.breakerClock()) { if !b.mayAsk(time.Now()) {
return nil, errBreakerOpen return nil, errBreakerOpen
} }
@@ -107,7 +107,7 @@ func (rq *request) askUnlessBreakerOpen(
case err == nil: case err == nil:
b.noteAnswer() b.noteAnswer()
case ctx.Err() == nil: case ctx.Err() == nil:
b.noteFailure(rq.h.breakerClock()) b.noteFailure(time.Now())
} }
return answer, err return answer, err
@@ -254,7 +254,14 @@ func (rq *request) answerWith(answer *wafAnswer) {
// visitor does not wait for it. The report goes on after ctx, the request's own // visitor does not wait for it. The report goes on after ctx, the request's own
// context, has ended. // context, has ended.
func (h *handler) sendReport(ctx context.Context, report *waf.Report) { func (h *handler) sendReport(ctx context.Context, report *waf.Report) {
h.reports.startUnlessStopped(func() { h.reportsMu.Lock()
defer h.reportsMu.Unlock()
if h.reportsStopped {
return
}
h.reports.Go(func() {
err := h.postReport(context.WithoutCancel(ctx), report) err := h.postReport(context.WithoutCancel(ctx), report)
if err != nil { if err != nil {
h.processLog.Warn("a report is dropped", "error", err.Error()) h.processLog.Warn("a report is dropped", "error", err.Error())
@@ -262,37 +269,6 @@ func (h *handler) sendReport(ctx context.Context, report *waf.Report) {
}) })
} }
// reportGoroutines are the goroutines sending reports to waf. None starts once
// stopAndWait has been called.
type reportGoroutines struct {
mu sync.Mutex
stopped bool
running sync.WaitGroup
}
// startUnlessStopped runs send on a goroutine of its own, or drops it once
// stopAndWait has been called.
func (g *reportGoroutines) startUnlessStopped(send func()) {
g.mu.Lock()
defer g.mu.Unlock()
if g.stopped {
return
}
g.running.Go(send)
}
// stopAndWait stops any more goroutines from starting, and returns once those
// running have ended.
func (g *reportGoroutines) stopAndWait() {
g.mu.Lock()
g.stopped = true
g.mu.Unlock()
g.running.Wait()
}
// postReport posts report to waf, and waits for its answer for up to // postReport posts report to waf, and waits for its answer for up to
// SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS. // SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS.
func (h *handler) postReport(ctx context.Context, report *waf.Report) error { func (h *handler) postReport(ctx context.Context, report *waf.Report) error {
+1 -1
View File
@@ -2448,7 +2448,7 @@ func newParams(dir string) state.Params {
AttackBanDuration: 7 * 24 * time.Hour, AttackBanDuration: 7 * 24 * time.Hour,
MaxBans: 5000, MaxBans: 5000,
}), }),
Limiter: ratelimit.New(ratelimit.Limits{}, 20000), Limiter: ratelimit.New(ratelimit.Params{MaxClients: 20000}),
GeoJS: geojs.New(geojs.Params{Now: midnight}), GeoJS: geojs.New(geojs.Params{Now: midnight}),
Lists: reputation.New(reputation.Params{ Lists: reputation.New(reputation.Params{
BlocklistURLs: []string{ BlocklistURLs: []string{
+6 -6
View File
@@ -10,7 +10,7 @@ import (
"go.uber.org/fx" "go.uber.org/fx"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/rproxy" "sneak.berlin/go/smallwebwaf/internal/rproxy"
) )
@@ -23,11 +23,11 @@ import (
// setting for either. // setting for either.
func RunRProxy(ctx context.Context, params Params) int { func RunRProxy(ctx context.Context, params Params) int {
hostname, _ := os.Hostname() // "" when the host has no name to give hostname, _ := os.Hostname() // "" when the host has no name to give
processLog := requestlog.NewProcessLogger( processLog := processlog.New(processlog.Params{
params.Stdout, Output: params.Stdout,
hostname, InstanceName: hostname,
slog.LevelInfo, Level: slog.LevelInfo,
) })
cfg, err := config.RProxyFromEnvironment(params.Viper) cfg, err := config.RProxyFromEnvironment(params.Viper)
if err != nil { if err != nil {
+7 -4
View File
@@ -755,7 +755,8 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
} }
} }
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection. // sendAndKeepOpen sends request to rproxy at addr, and returns the connection. A
// read from it ends with an error once waitLimit is up.
func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn { func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
t.Helper() t.Helper()
@@ -768,6 +769,11 @@ func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
_ = conn.Close() _ = conn.Close()
}) })
err = conn.SetDeadline(time.Now().Add(waitLimit))
if err != nil {
t.Fatalf("set the deadline: %v", err)
}
_, err = io.WriteString(conn, request) _, err = io.WriteString(conn, request)
if err != nil { if err != nil {
t.Fatalf("send: %v", err) t.Fatalf("send: %v", err)
@@ -832,9 +838,6 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
stop() stop()
// Counted from the stop, so a slow wait for the app cannot shorten it.
_ = conn.SetReadDeadline(stopped.Add(cutOffAfter + waitLimit))
text, err := io.ReadAll(conn) text, err := io.ReadAll(conn)
if err != nil || len(text) != 0 { if err != nil || len(text) != 0 {
t.Errorf("read %q (%v), want the connection cut off", text, err) t.Errorf("read %q (%v), want the connection cut off", text, err)
+22 -16
View File
@@ -19,10 +19,10 @@ import (
"sneak.berlin/go/smallwebwaf/internal/geojs" "sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics" "sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/remotelog" "sneak.berlin/go/smallwebwaf/internal/remotelog"
"sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/routewatch" "sneak.berlin/go/smallwebwaf/internal/routewatch"
"sneak.berlin/go/smallwebwaf/internal/rules" "sneak.berlin/go/smallwebwaf/internal/rules"
"sneak.berlin/go/smallwebwaf/internal/state" "sneak.berlin/go/smallwebwaf/internal/state"
@@ -43,8 +43,11 @@ const remoteLogStopTimeout = 2 * time.Second
func RunWAF(ctx context.Context, params Params) int { func RunWAF(ctx context.Context, params Params) int {
// Until the settings are read, the one message is an invalid setting's // Until the settings are read, the one message is an invalid setting's
// error, which every SWW_WAF_LOG_LEVEL lets through. // error, which every SWW_WAF_LOG_LEVEL lets through.
processLog := requestlog.NewProcessLogger(params.Stdout, processLog := processlog.New(processlog.Params{
config.InstanceName(params.Viper), slog.LevelError) Output: params.Stdout,
InstanceName: config.InstanceName(params.Viper),
Level: slog.LevelError,
})
cfg, err := config.WAFFromEnvironment(params.Viper) cfg, err := config.WAFFromEnvironment(params.Viper)
if err != nil { if err != nil {
@@ -63,11 +66,11 @@ func RunWAF(ctx context.Context, params Params) int {
stdout = io.MultiWriter(params.Stdout, remote) stdout = io.MultiWriter(params.Stdout, remote)
} }
processLog = requestlog.NewProcessLogger( processLog = processlog.New(processlog.Params{
stdout, Output: stdout,
cfg.InstanceName, InstanceName: cfg.InstanceName,
cfg.LogLevel, Level: cfg.LogLevel,
) })
if remote != nil { if remote != nil {
stopSending := startSending(ctx, remote, processLog) stopSending := startSending(ctx, remote, processLog)
@@ -239,14 +242,17 @@ func newLookupFile(
// newLimiter returns the table of clients, with the rate and byte limits. // newLimiter returns the table of clients, with the rate and byte limits.
func newLimiter(cfg *config.WAF) *ratelimit.Limiter { func newLimiter(cfg *config.WAF) *ratelimit.Limiter {
return ratelimit.New(ratelimit.Limits{ return ratelimit.New(ratelimit.Params{
PerMinute: cfg.RateLimitPerMinute, Limits: ratelimit.Limits{
PerHour: cfg.RateLimitPerHour, PerMinute: cfg.RateLimitPerMinute,
PerDay: cfg.RateLimitPerDay, PerHour: cfg.RateLimitPerHour,
BytesPerMinute: cfg.BytesLimitPerMinute, PerDay: cfg.RateLimitPerDay,
BytesPerHour: cfg.BytesLimitPerHour, BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerDay: cfg.BytesLimitPerDay, BytesPerHour: cfg.BytesLimitPerHour,
}, cfg.MaxTrackedClients) BytesPerDay: cfg.BytesLimitPerDay,
},
MaxClients: cfg.MaxTrackedClients,
})
} }
// newLedger returns the ledger of the bans, with the settings for them. // newLedger returns the ledger of the bans, with the settings for them.
+7 -7
View File
@@ -53,13 +53,13 @@ func (rq *request) limitPercentages() (percentage, percentage) {
if country == "" && !rq.skippedLookupWait { if country == "" && !rq.skippedLookupWait {
unknown = percentage{ unknown = percentage{
cfg.UnknownCountryLimitPercent, cfg.UnknownCountryLimitPercent,
config.UnknownCountryLimitPercentSetting, "SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
} }
} }
fetched := percentage{percent: whole} fetched := percentage{percent: whole}
if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed { if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed {
fetched = percentage{percent, config.ASNLimitPercentURLSetting} fetched = percentage{percent, "SWW_WAF_ASN_LIMIT_PERCENT_URL"}
} }
blocklisted := percentage{percent: whole} blocklisted := percentage{percent: whole}
@@ -67,7 +67,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
cfg.BlocklistAction == config.ReputationActionLimit { cfg.BlocklistAction == config.ReputationActionLimit {
blocklisted = percentage{ blocklisted = percentage{
cfg.BlocklistLimitPercent, cfg.BlocklistLimitPercent,
config.BlocklistActionSetting, "SWW_WAF_BLOCKLIST_ACTION",
} }
} }
@@ -76,11 +76,11 @@ func (rq *request) limitPercentages() (percentage, percentage) {
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit { cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit {
dnsblAbuseIPDBListed = percentage{ dnsblAbuseIPDBListed = percentage{
cfg.DNSBLAbuseIPDBLimitPercent, cfg.DNSBLAbuseIPDBLimitPercent,
config.DNSBLAbuseIPDBActionSetting, "SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
} }
} }
const tagSetting = config.TagLimitPercentSetting const tagSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
asnRequests := lowest( asnRequests := lowest(
given(cfg.TagLimitPercent, asnTag(asn), tagSetting), given(cfg.TagLimitPercent, asnTag(asn), tagSetting),
@@ -108,7 +108,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
asnBytes = given( asnBytes = given(
cfg.ASNBytesLimitPercent, cfg.ASNBytesLimitPercent,
asn, asn,
config.ASNBytesLimitPercentSetting, "SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
) )
} }
@@ -116,7 +116,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
countryBytes = given( countryBytes = given(
cfg.CountryBytesLimitPercent, cfg.CountryBytesLimitPercent,
country, country,
config.CountryBytesLimitPercentSetting, "SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
) )
} }
+1 -3
View File
@@ -7,7 +7,6 @@ import (
"slices" "slices"
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
) )
@@ -34,8 +33,7 @@ func (countries) check(
return &verdict{ return &verdict{
action: requestlog.ActionCountryDenied, action: requestlog.ActionCountryDenied,
figures: map[string]any{ figures: map[string]any{
"country": country, "country": country, "setting": "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
"setting": config.ExclusivelyAllowedCountriesSetting,
}, },
} }
} }
+6
View File
@@ -104,6 +104,12 @@ func onReport() []heuristic {
} }
} }
// runAfterAnswer runs the heuristics of afterAnswer on the request, at the time
// it is called.
func (rq *request) runAfterAnswer(ctx context.Context) {
rq.runBanningHeuristics(ctx, afterAnswer())
}
// runHeuristics runs list, one of those checkClient runs, on the request at now, // runHeuristics runs list, one of those checkClient runs, on the request at now,
// and returns the first verdict, or nil when none gives one. The log line's // and returns the first verdict, or nil when none gives one. The log line's
// heuristic and heuristic_figures name the heuristic that gave it, and give its // heuristic and heuristic_figures name the heuristic that gave it, and give its
+22 -18
View File
@@ -24,6 +24,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/geojs" "sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics" "sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/processlog"
"sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/requestlog"
@@ -398,11 +399,11 @@ func startRProxy(
server := rproxy.New(rproxy.Params{ server := rproxy.New(rproxy.Params{
Config: cfg, Config: cfg,
RequestLog: io.Discard, RequestLog: io.Discard,
ProcessLog: requestlog.NewProcessLogger( ProcessLog: processlog.New(processlog.Params{
io.Discard, Output: io.Discard,
"rproxy", InstanceName: "rproxy",
slog.LevelInfo, Level: slog.LevelInfo,
), }),
}) })
listener, err := (&net.ListenConfig{}).Listen( listener, err := (&net.ListenConfig{}).Listen(
@@ -471,11 +472,11 @@ func newWAFWith(
} }
out := &output{} out := &output{}
processLog := requestlog.NewProcessLogger( processLog := processlog.New(processlog.Params{
out, Output: out,
cfg.InstanceName, InstanceName: cfg.InstanceName,
cfg.LogLevel, Level: cfg.LogLevel,
) })
ruleFiles, err := rules.Load(rules.Params{ ruleFiles, err := rules.Load(rules.Params{
Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog, Dir: cfg.RuleFilesDir, Enabled: cfg.RuleFilesEnabled, ProcessLog: processLog,
@@ -561,14 +562,17 @@ func addCounters(params *waf.Params) {
cfg.MetricsBusiestCountriesAndASNs, cfg.MetricsBusiestCountriesAndASNs,
cfg.InstanceName, cfg.InstanceName,
) )
params.Limiter = ratelimit.New(ratelimit.Limits{ params.Limiter = ratelimit.New(ratelimit.Params{
PerMinute: cfg.RateLimitPerMinute, Limits: ratelimit.Limits{
PerHour: cfg.RateLimitPerHour, PerMinute: cfg.RateLimitPerMinute,
PerDay: cfg.RateLimitPerDay, PerHour: cfg.RateLimitPerHour,
BytesPerMinute: cfg.BytesLimitPerMinute, PerDay: cfg.RateLimitPerDay,
BytesPerHour: cfg.BytesLimitPerHour, BytesPerMinute: cfg.BytesLimitPerMinute,
BytesPerDay: cfg.BytesLimitPerDay, BytesPerHour: cfg.BytesLimitPerHour,
}, cfg.MaxTrackedClients) BytesPerDay: cfg.BytesLimitPerDay,
},
MaxClients: cfg.MaxTrackedClients,
})
params.Ledger = bans.New(bans.Params{ params.Ledger = bans.New(bans.Params{
LimitBanDuration: cfg.LimitBanDuration, LimitBanDuration: cfg.LimitBanDuration,
LimitBanRepeatWindow: cfg.LimitBanRepeatWindow, LimitBanRepeatWindow: cfg.LimitBanRepeatWindow,
-10
View File
@@ -345,16 +345,6 @@ func timing(start, end time.Time) *float64 {
return new(requestlog.Milliseconds(end.Sub(start))) return new(requestlog.Milliseconds(end.Sub(start)))
} }
// banAndCount runs list, afterAnswer or onReport, whose heuristics only ban.
// It then counts the request for the anomaly thresholds and adds it to its
// client's history. list runs first, so that a refusal limit it finds broken
// is in that history. ctx is the request's own context.
func (rq *request) banAndCount(ctx context.Context, list []heuristic) {
rq.runBanningHeuristics(ctx, list)
rq.countAnomalies()
rq.addToHistory()
}
// addToHistory adds the request, which has ended, to its client's // addToHistory adds the request, which has ended, to its client's
// history, and counts its offences in the metrics, and then the lookup's // history, and counts its offences in the metrics, and then the lookup's
// answer about the client, as answerAtTheEnd gives it, to that history and // answer about the client, as answerAtTheEnd gives it, to that history and
+28 -21
View File
@@ -177,13 +177,11 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
peer.Unmap(), peer.Unmap(),
true, true,
) )
// These run even when a check panics. finish runs first, so that what is // The last deferred runs first, so that what is held has the finished log
// held has the finished log line. Go's server sends the answer only once // line. Go's server sends the answer only once this function has
// this function has returned, so the report cannot come before it is held. // returned, so the report cannot come before it is held.
defer func() { defer rq.holdForReport()
rq.finish() defer rq.finish()
rq.holdForReport()
}()
rq.site = r.Host rq.site = r.Host
rq.line.WAFLine = requestlog.WAFLineDecision rq.line.WAFLine = requestlog.WAFLineDecision
@@ -192,28 +190,35 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
underOwnPrefix := strings.HasPrefix(r.URL.Path, ownEndpointPrefix) underOwnPrefix := strings.HasPrefix(r.URL.Path, ownEndpointPrefix)
if refused == nil && !underOwnPrefix { if refused == nil && !underOwnPrefix {
rq.allow()
// In observe mode, what enforce mode would have refused counts here. // In observe mode, what enforce mode would have refused counts here.
rq.runBanningHeuristics(r.Context(), afterAnswer()) defer rq.runAfterAnswer(r.Context())
rq.allow()
return return
} }
// The last deferred runs first: runAfterAnswer before addToHistory, so that
// a broken refusal limit is in the client's history.
defer rq.addToHistory()
defer rq.countAnomalies()
defer rq.runAfterAnswer(r.Context())
if refused != nil { if refused != nil {
rq.deny(*refused) rq.deny(*refused)
} else {
// The app never sees this prefix. waf's own endpoints are at its return
// own address, so this answers as a path with no endpoint does.
rq.line.Action = requestlog.ActionOwnEndpoint
rq.out.Header().Set(DispositionHeader, DispositionDeny)
http.Error(
rq.out,
http.StatusText(http.StatusNotFound),
http.StatusNotFound,
)
} }
rq.banAndCount(r.Context(), afterAnswer()) // The app never sees this prefix. waf's own endpoints are at its
// own address, so this answers as a path with no endpoint does.
rq.line.Action = requestlog.ActionOwnEndpoint
rq.out.Header().Set(DispositionHeader, DispositionDeny)
http.Error(
rq.out,
http.StatusText(http.StatusNotFound),
http.StatusNotFound,
)
} }
// readQuestion returns the visitor's address and the body a question carries, // readQuestion returns the visitor's address and the body a question carries,
@@ -486,7 +491,9 @@ func (h *handler) finishWith(
if rq.allowed { if rq.allowed {
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
rq.banAndCount(ctx, onReport()) rq.runBanningHeuristics(ctx, onReport())
rq.countAnomalies()
rq.addToHistory()
rq.h.metrics.RequestReported( rq.h.metrics.RequestReported(
&rq.line, &rq.line,