Compare commits

...
Author SHA1 Message Date
clawbot 123f34679f rproxy, sww: keep the breaker and stop tests from failing under load (closes #474)
check / check (push) Waiting to run
The circuit breaker reads the time through a clock the handler holds,
which is time.Now outside tests. The test that answers waf's
questions with answers rproxy cannot pass on stops that clock, so how
long its requests take on a slow host no longer decides whether the
breaker opens or turns the next request away.

The test that stops rproxy mid-request sets its read deadline from the
stop, not from the send, so a slow wait for the app no longer shortens
it.

Model: opus-5-5
2026-11-19 18:30:44 +00:00
clawbot 11ac8aee50 config: export the setting names waf logs (closes #448)
check / check (push) Waiting to run
waf wrote eight setting names as string literals into log lines and
ban notes, and config wrote each again where it reads the setting. A
rename that missed one would log a setting that does not exist. config
now exports a constant for each name, used by its reader and by waf.

Model: opus-5-5
2026-11-19 19:29:49 +01:00
8 changed files with 84 additions and 42 deletions

No files matched your search

+30 -17
View File
@@ -27,6 +27,19 @@ import (
"sneak.berlin/go/smallwebwaf/internal/remotelog"
)
// The names of the settings that waf writes into log lines and ban notes,
// as the setting that refused a request or lowered a client's limits.
const (
ExclusivelyAllowedCountriesSetting = "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES"
TagLimitPercentSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
ASNBytesLimitPercentSetting = "SWW_WAF_ASN_BYTES_LIMIT_PERCENT"
CountryBytesLimitPercentSetting = "SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT"
UnknownCountryLimitPercentSetting = "SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT"
ASNLimitPercentURLSetting = "SWW_WAF_ASN_LIMIT_PERCENT_URL"
BlocklistActionSetting = "SWW_WAF_BLOCKLIST_ACTION"
DNSBLAbuseIPDBActionSetting = "SWW_WAF_DNSBL_ABUSEIPDB_ACTION"
)
// WAF is the settings of `sww waf`. A time or size limit of zero is off.
type WAF struct {
// ListenAddr is where waf listens (SWW_WAF_LISTEN_ADDR).
@@ -618,24 +631,24 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
"800",
),
ExclusivelyAllowedCountries: env.countries(
"SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""),
ExclusivelyAllowedCountriesSetting, ""),
TagLists: env.tagLists("SWW_WAF_TAG_LISTS"),
TagDeny: env.tags("SWW_WAF_TAG_DENY"),
TagLimitPercent: env.tagPercents("SWW_WAF_TAG_LIMIT_PERCENT"),
TagLimitPercent: env.tagPercents(TagLimitPercentSetting),
RouteWatchURL: env.routeWatchURL("SWW_WAF_ROUTEWATCH_URL"),
ASNBytesLimitPercent: env.percents(
"SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
ASNBytesLimitPercentSetting,
ParseASN,
),
CountryBytesLimitPercent: env.percents(
"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
CountryBytesLimitPercentSetting,
parseCountry,
),
UnknownCountryLimitPercent: env.percent(
"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
UnknownCountryLimitPercentSetting,
"100",
),
ASNLimitPercentURL: env.listURL("SWW_WAF_ASN_LIMIT_PERCENT_URL"),
ASNLimitPercentURL: env.listURL(ASNLimitPercentURLSetting),
BlocklistURLs: env.listURLs("SWW_WAF_BLOCKLIST_URLS"),
BlocklistRefresh: env.refresh("SWW_WAF_BLOCKLIST_REFRESH", "24h"),
DNSBLZones: env.zones("SWW_WAF_DNSBL_ZONES"),
@@ -809,9 +822,9 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
"SWW_WAF_BLOCKLIST_ACTION", string(ReputationActionDeny))
BlocklistActionSetting, string(ReputationActionDeny))
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "limit:25")
DNSBLAbuseIPDBActionSetting, "limit:25")
env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
env.checkLookupDBPath(cfg)
@@ -1220,7 +1233,7 @@ func (e *environment) checkLookupDBPath(cfg *WAF) {
func (e *environment) checkCountriesAndLookups(cfg *WAF) {
for _, country := range cfg.ExclusivelyAllowedCountries {
if slices.Contains(cfg.TagDeny, CountryTagPrefix+country) {
e.check("SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
e.check(ExclusivelyAllowedCountriesSetting,
fmt.Errorf("%q %w", country, errOnBothLists))
}
}
@@ -1239,15 +1252,15 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
name string
set bool
}{
{"SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", len(cfg.ExclusivelyAllowedCountries) > 0},
{ExclusivelyAllowedCountriesSetting, len(cfg.ExclusivelyAllowedCountries) > 0},
{"SWW_WAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders},
{"SWW_WAF_TAG_DENY", slices.ContainsFunc(cfg.TagDeny, onlyLookupGives)},
{"SWW_WAF_TAG_LIMIT_PERCENT", slices.ContainsFunc(
{TagLimitPercentSetting, slices.ContainsFunc(
slices.Collect(maps.Keys(cfg.TagLimitPercent)), onlyLookupGives)},
{"SWW_WAF_ASN_BYTES_LIMIT_PERCENT", len(cfg.ASNBytesLimitPercent) > 0},
{"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT", len(cfg.CountryBytesLimitPercent) > 0},
{"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT", cfg.UnknownCountryLimitPercent < 100},
{"SWW_WAF_ASN_LIMIT_PERCENT_URL", cfg.ASNLimitPercentURL != ""},
{ASNBytesLimitPercentSetting, len(cfg.ASNBytesLimitPercent) > 0},
{CountryBytesLimitPercentSetting, len(cfg.CountryBytesLimitPercent) > 0},
{UnknownCountryLimitPercentSetting, cfg.UnknownCountryLimitPercent < 100},
{ASNLimitPercentURLSetting, cfg.ASNLimitPercentURL != ""},
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0},
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0},
{"SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0},
@@ -1267,7 +1280,7 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
// blocklist too: the file at a URL is fetched as one list or the other.
func (e *environment) checkASNLimitPercentURL(cfg *WAF) {
if slices.Contains(cfg.BlocklistURLs, cfg.ASNLimitPercentURL) {
e.check("SWW_WAF_ASN_LIMIT_PERCENT_URL",
e.check(ASNLimitPercentURLSetting,
fmt.Errorf("%q %w", cfg.ASNLimitPercentURL, errInBlocklistURLs))
}
}
@@ -1294,7 +1307,7 @@ func (e *environment) checkTags(cfg *WAF) {
}
for tag := range cfg.TagLimitPercent {
e.check("SWW_WAF_TAG_LIMIT_PERCENT", checkTag(cfg, tag))
e.check(TagLimitPercentSetting, checkTag(cfg, tag))
}
}
+6
View File
@@ -2,6 +2,12 @@ package rproxy
import "time"
// StopBreakerClock has the circuit breaker of s read at as the time, so that how
// long requests take cannot matter to it. It is called before s serves.
func (s *Server) StopBreakerClock(at time.Time) {
s.h.breakerClock = func() time.Time { return at }
}
// Breaker is the circuit breaker, for tests that give it the time.
type Breaker struct {
b breaker
+5 -2
View File
@@ -94,6 +94,7 @@ func New(params Params) *Server {
reportURL: (&url.URL{
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath,
}).String(),
breakerClock: time.Now,
}
return &Server{
@@ -133,8 +134,10 @@ type handler struct {
wafTransport http.RoundTripper
// reportURL is where the reports go.
reportURL string
// breaker is the circuit breaker for asks to waf.
breaker breaker
// breaker is the circuit breaker for asks to waf, and breakerClock gives it
// the time.
breaker breaker
breakerClock func() time.Time
// reports waits for the reports being sent. reportsMu guards starting one
// against reportsStopped, which is set once nothing is to wait for another.
reports sync.WaitGroup
+27 -5
View File
@@ -356,6 +356,19 @@ func startServerLogging(
) (string, *rproxy.Server) {
t.Helper()
server := newServer(t, backend, appURL, env, requestLog, processLog)
return serve(t, server), server
}
// newServer returns rproxy, not yet serving, with its request lines going to
// requestLog and its process lines to processLog.
func newServer(
t *testing.T, backend, appURL string, env map[string]string,
requestLog, processLog io.Writer,
) *rproxy.Server {
t.Helper()
settings := map[string]string{
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
wafBackendTimeout: "60000",
@@ -367,7 +380,7 @@ func startServerLogging(
t.Fatalf("settings %v: %v", settings, err)
}
server := rproxy.New(rproxy.Params{
return rproxy.New(rproxy.Params{
Config: cfg,
RequestLog: requestLog,
ProcessLog: requestlog.NewProcessLogger(
@@ -376,6 +389,11 @@ func startServerLogging(
slog.LevelInfo,
),
})
}
// serve has server serve until the test ends, and returns its address.
func serve(t *testing.T, server *rproxy.Server) string {
t.Helper()
listener, err := (&net.ListenConfig{}).Listen(
t.Context(),
@@ -396,7 +414,7 @@ func startServerLogging(
server.WaitForReports()
})
return listener.Addr().String(), server
return listener.Addr().String()
}
// answered is the visitor's answer.
@@ -1532,9 +1550,13 @@ func TestAnswersThatCannotBePassedOnOpenTheBreaker(t *testing.T) {
tooLarge(w, r)
})
app := startApp(t, greet)
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
wafBackendTimeout: "60000",
})
out := &output{}
server := newServer(t, standIn.URL, app.URL, nil, out, out)
// The breaker's clock stands still. On a slow host, the five failures could
// otherwise take more than the five seconds that open the breaker. The next
// request could come more than a second later, when it may ask waf again.
server.StopBreakerClock(time.Now())
addr := serve(t, server)
for range 5 {
if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
+2 -3
View File
@@ -10,7 +10,6 @@ import (
"maps"
"net/http"
"strings"
"time"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
@@ -97,7 +96,7 @@ func (rq *request) askUnlessBreakerOpen(
ctx context.Context,
) (*wafAnswer, error) {
b := &rq.h.breaker
if !b.mayAsk(time.Now()) {
if !b.mayAsk(rq.h.breakerClock()) {
return nil, errBreakerOpen
}
@@ -107,7 +106,7 @@ func (rq *request) askUnlessBreakerOpen(
case err == nil:
b.noteAnswer()
case ctx.Err() == nil:
b.noteFailure(time.Now())
b.noteFailure(rq.h.breakerClock())
}
return answer, err
+4 -7
View File
@@ -755,8 +755,7 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
}
}
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection. A
// read from it ends with an error once waitLimit is up.
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection.
func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
t.Helper()
@@ -769,11 +768,6 @@ func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
_ = conn.Close()
})
err = conn.SetDeadline(time.Now().Add(waitLimit))
if err != nil {
t.Fatalf("set the deadline: %v", err)
}
_, err = io.WriteString(conn, request)
if err != nil {
t.Fatalf("send: %v", err)
@@ -838,6 +832,9 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
stop()
// Counted from the stop, so a slow wait for the app cannot shorten it.
_ = conn.SetReadDeadline(stopped.Add(cutOffAfter + waitLimit))
text, err := io.ReadAll(conn)
if err != nil || len(text) != 0 {
t.Errorf("read %q (%v), want the connection cut off", text, err)
+7 -7
View File
@@ -53,13 +53,13 @@ func (rq *request) limitPercentages() (percentage, percentage) {
if country == "" && !rq.skippedLookupWait {
unknown = percentage{
cfg.UnknownCountryLimitPercent,
"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
config.UnknownCountryLimitPercentSetting,
}
}
fetched := percentage{percent: whole}
if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed {
fetched = percentage{percent, "SWW_WAF_ASN_LIMIT_PERCENT_URL"}
fetched = percentage{percent, config.ASNLimitPercentURLSetting}
}
blocklisted := percentage{percent: whole}
@@ -67,7 +67,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
cfg.BlocklistAction == config.ReputationActionLimit {
blocklisted = percentage{
cfg.BlocklistLimitPercent,
"SWW_WAF_BLOCKLIST_ACTION",
config.BlocklistActionSetting,
}
}
@@ -76,11 +76,11 @@ func (rq *request) limitPercentages() (percentage, percentage) {
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit {
dnsblAbuseIPDBListed = percentage{
cfg.DNSBLAbuseIPDBLimitPercent,
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
config.DNSBLAbuseIPDBActionSetting,
}
}
const tagSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
const tagSetting = config.TagLimitPercentSetting
asnRequests := lowest(
given(cfg.TagLimitPercent, asnTag(asn), tagSetting),
@@ -108,7 +108,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
asnBytes = given(
cfg.ASNBytesLimitPercent,
asn,
"SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
config.ASNBytesLimitPercentSetting,
)
}
@@ -116,7 +116,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
countryBytes = given(
cfg.CountryBytesLimitPercent,
country,
"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
config.CountryBytesLimitPercentSetting,
)
}
+3 -1
View File
@@ -7,6 +7,7 @@ import (
"slices"
"time"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
@@ -33,7 +34,8 @@ func (countries) check(
return &verdict{
action: requestlog.ActionCountryDenied,
figures: map[string]any{
"country": country, "setting": "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
"country": country,
"setting": config.ExclusivelyAllowedCountriesSetting,
},
}
}