Compare commits

..
Author SHA1 Message Date
clawbot 123f34679f rproxy, sww: keep the breaker and stop tests from failing under load (closes #474)
check / check (push) Waiting to run
The circuit breaker reads the time through a clock the handler holds,
which is time.Now outside tests. The test that answers waf's
questions with answers rproxy cannot pass on stops that clock, so how
long its requests take on a slow host no longer decides whether the
breaker opens or turns the next request away.

The test that stops rproxy mid-request sets its read deadline from the
stop, not from the send, so a slow wait for the app no longer shortens
it.

Model: opus-5-5
2026-11-19 18:30:44 +00:00
clawbot 11ac8aee50 config: export the setting names waf logs (closes #448)
check / check (push) Waiting to run
waf wrote eight setting names as string literals into log lines and
ban notes, and config wrote each again where it reads the setting. A
rename that missed one would log a setting that does not exist. config
now exports a constant for each name, used by its reader and by waf.

Model: opus-5-5
2026-11-19 19:29:49 +01:00
clawbot e7cc0adbdb waf: end the asks to GeoJS, the DNSBL zones, AbuseIPDB and routewatch at the stop (closes #439)
check / check (push) Waiting to run
Each of the four keeps the cancel function of every ask it started, and a
wait group of their goroutines. Its new method (GeoJS.StopAsking,
DNSBL.StopAsking, AbuseIPDB.StopChecking, routewatch Client.StopAsking)
refuses further asks, ends those under way, and returns once their
goroutines have. An ask ended that way is no failure: no alert, no failure
count, no wait before the next. Each provider in internal/sww adds the
method as a stop hook, so the last write of the state files follows it.

Model: opus-5-5
2026-11-19 19:02:12 +01:00
19 changed files with 475 additions and 115 deletions

No files matched your search

+30 -17
View File
@@ -27,6 +27,19 @@ import (
"sneak.berlin/go/smallwebwaf/internal/remotelog"
)
// The names of the settings that waf writes into log lines and ban notes,
// as the setting that refused a request or lowered a client's limits.
const (
ExclusivelyAllowedCountriesSetting = "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES"
TagLimitPercentSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
ASNBytesLimitPercentSetting = "SWW_WAF_ASN_BYTES_LIMIT_PERCENT"
CountryBytesLimitPercentSetting = "SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT"
UnknownCountryLimitPercentSetting = "SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT"
ASNLimitPercentURLSetting = "SWW_WAF_ASN_LIMIT_PERCENT_URL"
BlocklistActionSetting = "SWW_WAF_BLOCKLIST_ACTION"
DNSBLAbuseIPDBActionSetting = "SWW_WAF_DNSBL_ABUSEIPDB_ACTION"
)
// WAF is the settings of `sww waf`. A time or size limit of zero is off.
type WAF struct {
// ListenAddr is where waf listens (SWW_WAF_LISTEN_ADDR).
@@ -618,24 +631,24 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
"800",
),
ExclusivelyAllowedCountries: env.countries(
"SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""),
ExclusivelyAllowedCountriesSetting, ""),
TagLists: env.tagLists("SWW_WAF_TAG_LISTS"),
TagDeny: env.tags("SWW_WAF_TAG_DENY"),
TagLimitPercent: env.tagPercents("SWW_WAF_TAG_LIMIT_PERCENT"),
TagLimitPercent: env.tagPercents(TagLimitPercentSetting),
RouteWatchURL: env.routeWatchURL("SWW_WAF_ROUTEWATCH_URL"),
ASNBytesLimitPercent: env.percents(
"SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
ASNBytesLimitPercentSetting,
ParseASN,
),
CountryBytesLimitPercent: env.percents(
"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
CountryBytesLimitPercentSetting,
parseCountry,
),
UnknownCountryLimitPercent: env.percent(
"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
UnknownCountryLimitPercentSetting,
"100",
),
ASNLimitPercentURL: env.listURL("SWW_WAF_ASN_LIMIT_PERCENT_URL"),
ASNLimitPercentURL: env.listURL(ASNLimitPercentURLSetting),
BlocklistURLs: env.listURLs("SWW_WAF_BLOCKLIST_URLS"),
BlocklistRefresh: env.refresh("SWW_WAF_BLOCKLIST_REFRESH", "24h"),
DNSBLZones: env.zones("SWW_WAF_DNSBL_ZONES"),
@@ -809,9 +822,9 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
"SWW_WAF_BLOCKLIST_ACTION", string(ReputationActionDeny))
BlocklistActionSetting, string(ReputationActionDeny))
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "limit:25")
DNSBLAbuseIPDBActionSetting, "limit:25")
env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
env.checkLookupDBPath(cfg)
@@ -1220,7 +1233,7 @@ func (e *environment) checkLookupDBPath(cfg *WAF) {
func (e *environment) checkCountriesAndLookups(cfg *WAF) {
for _, country := range cfg.ExclusivelyAllowedCountries {
if slices.Contains(cfg.TagDeny, CountryTagPrefix+country) {
e.check("SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
e.check(ExclusivelyAllowedCountriesSetting,
fmt.Errorf("%q %w", country, errOnBothLists))
}
}
@@ -1239,15 +1252,15 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
name string
set bool
}{
{"SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", len(cfg.ExclusivelyAllowedCountries) > 0},
{ExclusivelyAllowedCountriesSetting, len(cfg.ExclusivelyAllowedCountries) > 0},
{"SWW_WAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders},
{"SWW_WAF_TAG_DENY", slices.ContainsFunc(cfg.TagDeny, onlyLookupGives)},
{"SWW_WAF_TAG_LIMIT_PERCENT", slices.ContainsFunc(
{TagLimitPercentSetting, slices.ContainsFunc(
slices.Collect(maps.Keys(cfg.TagLimitPercent)), onlyLookupGives)},
{"SWW_WAF_ASN_BYTES_LIMIT_PERCENT", len(cfg.ASNBytesLimitPercent) > 0},
{"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT", len(cfg.CountryBytesLimitPercent) > 0},
{"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT", cfg.UnknownCountryLimitPercent < 100},
{"SWW_WAF_ASN_LIMIT_PERCENT_URL", cfg.ASNLimitPercentURL != ""},
{ASNBytesLimitPercentSetting, len(cfg.ASNBytesLimitPercent) > 0},
{CountryBytesLimitPercentSetting, len(cfg.CountryBytesLimitPercent) > 0},
{UnknownCountryLimitPercentSetting, cfg.UnknownCountryLimitPercent < 100},
{ASNLimitPercentURLSetting, cfg.ASNLimitPercentURL != ""},
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0},
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0},
{"SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0},
@@ -1267,7 +1280,7 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
// blocklist too: the file at a URL is fetched as one list or the other.
func (e *environment) checkASNLimitPercentURL(cfg *WAF) {
if slices.Contains(cfg.BlocklistURLs, cfg.ASNLimitPercentURL) {
e.check("SWW_WAF_ASN_LIMIT_PERCENT_URL",
e.check(ASNLimitPercentURLSetting,
fmt.Errorf("%q %w", cfg.ASNLimitPercentURL, errInBlocklistURLs))
}
}
@@ -1294,7 +1307,7 @@ func (e *environment) checkTags(cfg *WAF) {
}
for tag := range cfg.TagLimitPercent {
e.check("SWW_WAF_TAG_LIMIT_PERCENT", checkTag(cfg, tag))
e.check(TagLimitPercentSetting, checkTag(cfg, tag))
}
}
+47 -14
View File
@@ -99,8 +99,13 @@ type GeoJS struct {
// waiting are the clients without an answer: those to ask GeoJS about,
// and those it is being asked about.
waiting map[netip.Prefix]*wait
// asking is true while a request to GeoJS is under way.
asking bool
// asking is true while a request to GeoJS is under way, and endAsking
// ends it. stopped is true once StopAsking has been called, and
// askGoroutines waits for the goroutine that makes the requests.
asking bool
endAsking context.CancelFunc
stopped bool
askGoroutines sync.WaitGroup
// retryDelay is how long GeoJS is left alone after its last failure,
// zero after an answer; retryAt is when it may be asked again.
retryDelay time.Duration
@@ -210,6 +215,23 @@ func (g *GeoJS) LookUpWithoutWaiting(
return answer
}
// StopAsking ends the request to GeoJS under way, and returns once the
// goroutine that made it has. A request it ends is no failure: Failed is not
// called, and GeoJS is not left alone after it. After StopAsking, GeoJS is
// asked nothing.
func (g *GeoJS) StopAsking() {
g.mu.Lock()
g.stopped = true
if g.asking {
g.endAsking()
}
g.mu.Unlock()
g.askGoroutines.Wait()
}
// Kept returns client's answer, if one is kept, without asking GeoJS.
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
g.mu.Lock()
@@ -294,7 +316,8 @@ func (g *GeoJS) answerOrWait(
}
if !g.asking {
// GeoJS is left alone after a failure, so no answer can come.
// GeoJS is left alone after a failure, or asked nothing after
// StopAsking, so no answer can come.
w.late = true
}
@@ -323,22 +346,28 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
}
// ask starts asking GeoJS about the waiting clients, unless a request to
// it is under way or it is left alone after a failure. The requests to
// GeoJS are for every client waiting, so they go on when the caller whose
// ctx is given stops waiting.
// it is under way, it is left alone after a failure, or StopAsking has
// been called. The requests to GeoJS are for every client waiting, so they
// go on when the caller whose ctx is given stops waiting, until StopAsking.
func (g *GeoJS) ask(ctx context.Context) {
if g.asking || g.now().Before(g.retryAt) {
if g.asking || g.stopped || g.now().Before(g.retryAt) {
return
}
g.asking = true
askCtx, end := context.WithCancel(context.WithoutCancel(ctx))
g.asking, g.endAsking = true, end
go g.askAboutWaiting(context.WithoutCancel(ctx))
g.askGoroutines.Go(func() {
defer end()
g.askAboutWaiting(askCtx)
})
}
// askAboutWaiting asks GeoJS about the waiting clients, one request at a
// time, until none is left or GeoJS fails. Each answer kept is given to
// Answered, outside the lock, since Answered takes locks of its own.
// time, until none is left, GeoJS fails, or StopAsking ends ctx. Each
// answer kept is given to Answered, outside the lock, since Answered takes
// locks of its own.
func (g *GeoJS) askAboutWaiting(ctx context.Context) {
for {
clients := g.nextClients()
@@ -347,6 +376,10 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
}
given, err := g.request(ctx, clients)
if err != nil && ctx.Err() != nil {
continue // ended by StopAsking, which is no failure
}
kept, answered := g.keep(clients, given, err)
if g.answered != nil {
@@ -362,13 +395,13 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
}
// nextClients returns up to maxPerRequest of the waiting clients. When
// none is waiting, it returns none and notes that no request to GeoJS is
// under way.
// none is waiting, or StopAsking has been called, it returns none and
// notes that no request to GeoJS is under way.
func (g *GeoJS) nextClients() []netip.Prefix {
g.mu.Lock()
defer g.mu.Unlock()
if len(g.waiting) == 0 {
if len(g.waiting) == 0 || g.stopped {
g.asking = false
return nil
+38
View File
@@ -318,6 +318,44 @@ func TestCallerWhoseContextEndsEarlyDoesNotStopTheRequest(t *testing.T) {
})
}
func TestStopAskingEndsTheRequestUnderWayAsNoFailureAndAsksNoMore(
t *testing.T,
) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
service, _, g, reports := startWithReports()
clients := newClients()
service.set(hanging)
g.LookUpWithoutWaiting(t.Context(), clients())
waitForRequests(t, service, 1)
// The request is ended, not waited out, and is no failure.
began := time.Now()
g.StopAsking()
if waited := time.Since(began); waited != 0 {
t.Errorf("StopAsking returned after %s, want at once", waited)
}
reports.wantFailures(t)
// A new client is then not asked about, and its lookup does not
// wait.
service.set(answering)
wantCountry(t, g, clients(), "")
if waited := time.Since(began); waited != 0 {
t.Errorf("the lookup returned after %s, want at once", waited)
}
waitForRequests(t, service, 1)
reports.wantFailures(t)
})
}
func TestRequestOver10SecondsIsAFailure(t *testing.T) {
t.Parallel()
+48 -16
View File
@@ -110,8 +110,12 @@ type AbuseIPDB struct {
// scores are by client. Each is added as it is fetched and never moved
// up, so that the one fetched longest ago is the first dropped.
scores *simplelru.LRU[netip.Prefix, Score]
// checking are the clients whose check is under way.
checking map[netip.Prefix]bool
// checking are the clients whose check is under way, each with the
// function that ends it. stopped is true once StopChecking has been
// called, and checkGoroutines waits for the goroutine of each check.
checking map[netip.Prefix]context.CancelFunc
stopped bool
checkGoroutines sync.WaitGroup
// day is the day, in UTC, of the checks spent counts.
day civil.Date
spent int
@@ -134,7 +138,7 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
params: params,
httpClient: &http.Client{},
scores: scores,
checking: map[netip.Prefix]bool{},
checking: map[netip.Prefix]context.CancelFunc{},
}
}
@@ -144,11 +148,11 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
// addresses its request comes from. A client without one is checked in
// the background, by addr, the address its request came from, if
// offender, if it has committed an offence, unless its check is under
// way, a check failed less than failureDelay ago, or the day's checks
// have used up DailyBudget; Hit never waits for a check. The check that
// uses the budget up is logged and raised as a source_failure alert. ctx
// is the context of the client's request, and a check goes on after the
// request ends.
// way, a check failed less than failureDelay ago, the day's checks have
// used up DailyBudget, or StopChecking has been called; Hit never waits
// for a check. The check that uses the budget up is logged and raised as a
// source_failure alert. ctx is the context of the client's request, and a
// check goes on after the request ends, until StopChecking.
func (a *AbuseIPDB) Hit(
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
) (int64, bool) {
@@ -167,14 +171,21 @@ func (a *AbuseIPDB) Hit(
a.day, a.spent = today, 0
}
check := offender && !a.checking[client] && !now.Before(a.retryAt) &&
a.spent < a.params.DailyBudget
_, checking := a.checking[client]
check := offender && !checking && !a.stopped &&
!now.Before(a.retryAt) && a.spent < a.params.DailyBudget
if check {
a.checking[client] = true
checkCtx, end := context.WithCancel(context.WithoutCancel(ctx))
a.checking[client] = end
a.checks++
a.spent++
go a.check(context.WithoutCancel(ctx), client, addr)
a.checkGoroutines.Go(func() {
defer end()
a.check(checkCtx, client, addr)
})
}
usedUp := check && a.spent == a.params.DailyBudget
@@ -189,6 +200,24 @@ func (a *AbuseIPDB) Hit(
return 0, false
}
// StopChecking ends the checks under way, and returns once their goroutines
// have. A check it ends gives no score and is no failure: it is neither
// counted, logged nor raised as an alert, and checks are not left alone
// after it. The ended check stays spent from the day's budget. After
// StopChecking, Hit checks no client.
func (a *AbuseIPDB) StopChecking() {
a.mu.Lock()
a.stopped = true
for _, end := range a.checking {
end()
}
a.mu.Unlock()
a.checkGoroutines.Wait()
}
// Checked returns how many checks were made.
func (a *AbuseIPDB) Checked() int {
a.mu.Lock()
@@ -315,7 +344,8 @@ func (a *AbuseIPDB) RunBlacklist(ctx context.Context) {
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
// the score as client's, and notes the check as no longer under way. A
// check that fails gives no score: it is counted, logged and raised as a
// source_failure alert, and no client is checked for failureDelay.
// source_failure alert, and no client is checked for failureDelay. A check
// ended as ctx ends, by StopChecking, is no failure.
func (a *AbuseIPDB) check(
ctx context.Context,
client netip.Prefix,
@@ -323,24 +353,26 @@ func (a *AbuseIPDB) check(
) {
score, err := a.ask(ctx, addr)
now := a.params.Now()
isFailure := err != nil && ctx.Err() == nil
a.mu.Lock()
delete(a.checking, client)
if err == nil {
switch {
case err == nil:
a.scores.Add(
client,
Score{Client: client, Score: score, Fetched: now},
)
} else {
case isFailure:
a.failures++
a.retryAt = now.Add(failureDelay)
}
a.mu.Unlock()
if err != nil {
if isFailure {
a.alert("checking a client with AbuseIPDB failed", err)
}
}
+37
View File
@@ -394,6 +394,43 @@ func TestCheckNotAnsweredWithinTheTimeoutFailsAndHitNeverWaits(
})
}
func TestStopCheckingEndsTheCheckUnderWayAsNoFailureAndChecksNoMore(
t *testing.T,
) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
queue := newQueue()
p := abuseIPDBParams()
p.Alerts = queue
abuseIPDB := &abuseIPDBStandIn{hanging: true}
checker := newAbuseIPDB(abuseIPDB, p)
wantScore(t, checker, suspect, true, 0, false)
synctest.Wait()
// The check is ended, not waited out, and is no failure.
began := time.Now()
checker.StopChecking()
if waited := time.Since(began); waited != 0 {
t.Errorf("StopChecking returned after %s, want at once", waited)
}
wantFailures(t, checker, 0)
if got := waiting(queue); len(got) != 0 {
t.Errorf("alerts waiting %+v, want none", got)
}
// An offender without a score is then not checked.
wantScore(t, checker, other, true, 0, false)
synctest.Wait()
wantChecked(t, abuseIPDB, suspect)
})
}
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
t.Parallel()
+44 -13
View File
@@ -82,8 +82,12 @@ type DNSBL struct {
// verdicts are by query. Each is added as it is fetched and never moved
// up, so that the one fetched longest ago is the first dropped.
verdicts *simplelru.LRU[query, Verdict]
// asking are the queries under way.
asking map[query]bool
// asking are the queries under way, each with the function that ends
// it. stopped is true once StopAsking has been called, and
// queryGoroutines waits for the goroutine of each query.
asking map[query]context.CancelFunc
stopped bool
queryGoroutines sync.WaitGroup
// queries and failures count, by zone, the queries made and those that
// failed, and retryAt is when a zone whose last query failed may be
// asked again.
@@ -120,7 +124,7 @@ func NewDNSBL(params DNSBLParams) *DNSBL {
params: params,
resolver: resolver,
verdicts: verdicts,
asking: map[query]bool{},
asking: map[query]context.CancelFunc{},
queries: map[string]int{},
failures: map[string]int{},
retryAt: map[string]time.Time{},
@@ -138,9 +142,10 @@ func (d *DNSBL) Zones() []string {
// log, the alerts and the metrics. A verdict is used until CacheTTL has
// passed since it was fetched. Each zone without one is asked about addr
// in the background, unless a query about addr to it is under way, the
// zone is left alone after a failure, or maxQueries are under way;
// ListedBy never waits for a query. ctx is the context of the client's
// request, and a query goes on after the request ends.
// zone is left alone after a failure, maxQueries are under way, or
// StopAsking has been called; ListedBy never waits for a query. ctx is the
// context of the client's request, and a query goes on after the request
// ends, until StopAsking.
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
d.mu.Lock()
defer d.mu.Unlock()
@@ -153,23 +158,47 @@ func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
q := query{zone: zone, client: addr}
kept, found := d.verdicts.Peek(q)
_, asking := d.asking[q]
switch {
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
if kept.Listed {
listedBy = append(listedBy, config.MaskZoneKey(zone))
}
case !d.asking[q] && !now.Before(d.retryAt[zone]) && len(d.asking) < maxQueries:
d.asking[q] = true
case !asking && !d.stopped && !now.Before(d.retryAt[zone]) &&
len(d.asking) < maxQueries:
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
d.asking[q] = end
d.queries[zone]++
go d.ask(context.WithoutCancel(ctx), q)
d.queryGoroutines.Go(func() {
defer end()
d.ask(queryCtx, q)
})
}
}
return listedBy
}
// StopAsking ends the queries under way, and returns once their
// goroutines have. A query it ends gives no verdict and is no failure: it
// is neither counted, logged nor raised as an alert, and the zone is not
// left alone after it. After StopAsking, ListedBy asks no zone.
func (d *DNSBL) StopAsking() {
d.mu.Lock()
d.stopped = true
for _, end := range d.asking {
end()
}
d.mu.Unlock()
d.queryGoroutines.Wait()
}
// Queries returns how many queries were made to zone.
func (d *DNSBL) Queries(zone string) int {
d.mu.Lock()
@@ -238,27 +267,29 @@ func (d *DNSBL) Load(verdicts []Verdict) {
// query as no longer under way. A query that fails gives no verdict: it
// is counted, logged and raised as a source_failure alert, which show the
// zone with its key masked, and the zone is not asked again for
// failureDelay.
// failureDelay. A query ended as ctx ends, by StopAsking, is no failure.
func (d *DNSBL) ask(ctx context.Context, q query) {
listed, err := d.lookUp(ctx, q)
now := d.params.Now()
isFailure := err != nil && ctx.Err() == nil
d.mu.Lock()
delete(d.asking, q)
if err == nil {
switch {
case err == nil:
d.verdicts.Add(q, Verdict{
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
})
} else {
case isFailure:
d.failures[q.zone]++
d.retryAt[q.zone] = now.Add(failureDelay)
}
d.mu.Unlock()
if err != nil {
if isFailure {
const failed = "asking a DNSBL zone failed"
shown := config.MaskZoneKey(q.zone)
+39
View File
@@ -210,6 +210,45 @@ func TestQueryNotAnsweredWithinTheTimeoutFails(t *testing.T) {
})
}
//
//nolint:paralleltest // one at a time, as the comment at the top of this file says
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
t *testing.T,
) {
synctest.Test(t, func(t *testing.T) {
queue := newQueue()
p := dnsblParams(zone)
p.Alerts = queue
resolver := &resolverStandIn{hanging: true}
dnsbl := newDNSBL(resolver, p)
wantZones(t, dnsbl, listed)
synctest.Wait()
// The query is ended, not waited out, and is no failure.
began := time.Now()
dnsbl.StopAsking()
if waited := time.Since(began); waited != 0 {
t.Errorf("StopAsking returned after %s, want at once", waited)
}
wantQueries(t, dnsbl, 1, 0)
if got := waiting(queue); len(got) != 0 {
t.Errorf("alerts waiting %+v, want none", got)
}
// A client without a verdict is then asked about in no zone.
wantZones(t, dnsbl, unlisted)
synctest.Wait()
wantQueries(t, dnsbl, 1, 0)
wantAsked(t, resolver, listedName)
waitForTheResolver()
})
}
//
//nolint:paralleltest // one at a time, as the comment at the top of this file says
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(
+43 -12
View File
@@ -99,8 +99,12 @@ type Client struct {
// answers are by client. Each is added as it comes and never moved up,
// so that the one that came longest ago is the first dropped.
answers *simplelru.LRU[netip.Prefix, Answer]
// asking are the clients whose query is under way.
asking map[netip.Prefix]bool
// asking are the clients whose query is under way, each with the
// function that ends it. stopped is true once StopAsking has been
// called, and queryGoroutines waits for the goroutine of each query.
asking map[netip.Prefix]context.CancelFunc
stopped bool
queryGoroutines sync.WaitGroup
// retryAt is when routewatch may be asked again after a query failed.
retryAt time.Time
}
@@ -116,7 +120,7 @@ func New(params Params) *Client {
params: params,
httpClient: &http.Client{},
answers: answers,
asking: map[netip.Prefix]bool{},
asking: map[netip.Prefix]context.CancelFunc{},
}
}
@@ -124,8 +128,9 @@ func New(params Params) *Client {
// IPv6 group, or why it has no route. It never waits for a query. A client
// without an answer is asked about in the background, by addr, the address its
// request came from, unless its query is under way, a query failed less than
// failureKept ago, or maxQueries are under way. ctx is the context of the
// client's request, and a query goes on after the request ends.
// failureKept ago, maxQueries are under way, or StopAsking has been called. ctx
// is the context of the client's request, and a query goes on after the request
// ends, until StopAsking.
func (c *Client) Route(
ctx context.Context, client netip.Prefix, addr netip.Addr,
) (Route, Missing) {
@@ -147,15 +152,38 @@ func (c *Client) Route(
return Route{}, RequestFailed
}
if !c.asking[client] && len(c.asking) < maxQueries {
c.asking[client] = true
_, asking := c.asking[client]
if !asking && !c.stopped && len(c.asking) < maxQueries {
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
c.asking[client] = end
go c.ask(context.WithoutCancel(ctx), client, addr)
c.queryGoroutines.Go(func() {
defer end()
c.ask(queryCtx, client, addr)
})
}
return Route{}, NoAnswerYet
}
// StopAsking ends the queries under way, and returns once their goroutines
// have. A query it ends gives no answer and is no failure: it is not logged,
// and routewatch is not left alone after it. After StopAsking, Route asks
// routewatch nothing.
func (c *Client) StopAsking() {
c.mu.Lock()
c.stopped = true
for _, end := range c.asking {
end()
}
c.mu.Unlock()
c.queryGoroutines.Wait()
}
// Snapshot returns every answer still in use, sorted by client, as
// routewatch.json lists them.
func (c *Client) Snapshot() []Answer {
@@ -205,7 +233,8 @@ func (c *Client) Load(answers []Answer) {
// ask asks routewatch about addr, keeps the answer as client's, and notes the
// query as no longer under way. A query that fails gives no answer, and
// routewatch is not asked again for failureKept. Only the first failure of
// those under way is logged.
// those under way is logged. A query ended as ctx ends, by StopAsking, is no
// failure.
func (c *Client) ask(
ctx context.Context,
client netip.Prefix,
@@ -213,18 +242,20 @@ func (c *Client) ask(
) {
route, err := c.query(ctx, addr)
now := c.params.Now()
isFailure := err != nil && ctx.Err() == nil
c.mu.Lock()
delete(c.asking, client)
firstFailure := err != nil && !now.Before(c.retryAt)
firstFailure := isFailure && !now.Before(c.retryAt)
if err == nil {
switch {
case err == nil:
c.answers.Add(client, Answer{
Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
})
} else {
case isFailure:
c.retryAt = now.Add(failureKept)
}
+48
View File
@@ -361,6 +361,54 @@ func TestHangingRoutewatchNeverDelaysAndFailsTheQuery(t *testing.T) {
})
}
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
t *testing.T,
) {
t.Parallel()
// In a synctest bubble, as TestHangingRoutewatchNeverDelaysAndFailsTheQuery
// says.
synctest.Test(t, func(t *testing.T) {
f := &fixture{t: t, log: &syncBuffer{}}
f.client = routewatch.New(routewatch.Params{
URL: silentURL, Now: time.Now,
ProcessLog: slog.New(slog.NewTextHandler(f.log, nil)),
})
f.route()
synctest.Wait()
// The query is ended, not waited out.
began := time.Now()
f.client.StopAsking()
if waited := time.Since(began); waited != 0 {
t.Errorf("StopAsking returned after %s, want at once", waited)
}
// A query started now would fail within waitLimit. None is, and the
// one ended is no failure: nothing is logged, and routewatch is not
// left alone.
for range 2 {
if _, missing := f.route(); missing != routewatch.NoAnswerYet {
t.Errorf(
"Route gave %q, want %q",
missing,
routewatch.NoAnswerYet,
)
}
time.Sleep(waitLimit)
synctest.Wait()
}
if f.log.String() != "" {
t.Errorf("process log %q, want nothing", f.log.String())
}
})
}
func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
t.Parallel()
+6
View File
@@ -2,6 +2,12 @@ package rproxy
import "time"
// StopBreakerClock has the circuit breaker of s read at as the time, so that how
// long requests take cannot matter to it. It is called before s serves.
func (s *Server) StopBreakerClock(at time.Time) {
s.h.breakerClock = func() time.Time { return at }
}
// Breaker is the circuit breaker, for tests that give it the time.
type Breaker struct {
b breaker
+5 -2
View File
@@ -94,6 +94,7 @@ func New(params Params) *Server {
reportURL: (&url.URL{
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath,
}).String(),
breakerClock: time.Now,
}
return &Server{
@@ -133,8 +134,10 @@ type handler struct {
wafTransport http.RoundTripper
// reportURL is where the reports go.
reportURL string
// breaker is the circuit breaker for asks to waf.
breaker breaker
// breaker is the circuit breaker for asks to waf, and breakerClock gives it
// the time.
breaker breaker
breakerClock func() time.Time
// reports waits for the reports being sent. reportsMu guards starting one
// against reportsStopped, which is set once nothing is to wait for another.
reports sync.WaitGroup
+27 -5
View File
@@ -356,6 +356,19 @@ func startServerLogging(
) (string, *rproxy.Server) {
t.Helper()
server := newServer(t, backend, appURL, env, requestLog, processLog)
return serve(t, server), server
}
// newServer returns rproxy, not yet serving, with its request lines going to
// requestLog and its process lines to processLog.
func newServer(
t *testing.T, backend, appURL string, env map[string]string,
requestLog, processLog io.Writer,
) *rproxy.Server {
t.Helper()
settings := map[string]string{
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
wafBackendTimeout: "60000",
@@ -367,7 +380,7 @@ func startServerLogging(
t.Fatalf("settings %v: %v", settings, err)
}
server := rproxy.New(rproxy.Params{
return rproxy.New(rproxy.Params{
Config: cfg,
RequestLog: requestLog,
ProcessLog: requestlog.NewProcessLogger(
@@ -376,6 +389,11 @@ func startServerLogging(
slog.LevelInfo,
),
})
}
// serve has server serve until the test ends, and returns its address.
func serve(t *testing.T, server *rproxy.Server) string {
t.Helper()
listener, err := (&net.ListenConfig{}).Listen(
t.Context(),
@@ -396,7 +414,7 @@ func startServerLogging(
server.WaitForReports()
})
return listener.Addr().String(), server
return listener.Addr().String()
}
// answered is the visitor's answer.
@@ -1532,9 +1550,13 @@ func TestAnswersThatCannotBePassedOnOpenTheBreaker(t *testing.T) {
tooLarge(w, r)
})
app := startApp(t, greet)
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
wafBackendTimeout: "60000",
})
out := &output{}
server := newServer(t, standIn.URL, app.URL, nil, out, out)
// The breaker's clock stands still. On a slow host, the five failures could
// otherwise take more than the five seconds that open the breaker. The next
// request could come more than a second later, when it may ask waf again.
server.StopBreakerClock(time.Now())
addr := serve(t, server)
for range 5 {
if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
+2 -3
View File
@@ -10,7 +10,6 @@ import (
"maps"
"net/http"
"strings"
"time"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
@@ -97,7 +96,7 @@ func (rq *request) askUnlessBreakerOpen(
ctx context.Context,
) (*wafAnswer, error) {
b := &rq.h.breaker
if !b.mayAsk(time.Now()) {
if !b.mayAsk(rq.h.breakerClock()) {
return nil, errBreakerOpen
}
@@ -107,7 +106,7 @@ func (rq *request) askUnlessBreakerOpen(
case err == nil:
b.noteAnswer()
case ctx.Err() == nil:
b.noteFailure(time.Now())
b.noteFailure(rq.h.breakerClock())
}
return answer, err
+4 -7
View File
@@ -755,8 +755,7 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
}
}
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection. A
// read from it ends with an error once waitLimit is up.
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection.
func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
t.Helper()
@@ -769,11 +768,6 @@ func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
_ = conn.Close()
})
err = conn.SetDeadline(time.Now().Add(waitLimit))
if err != nil {
t.Fatalf("set the deadline: %v", err)
}
_, err = io.WriteString(conn, request)
if err != nil {
t.Fatalf("send: %v", err)
@@ -838,6 +832,9 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
stop()
// Counted from the stop, so a slow wait for the app cannot shorten it.
_ = conn.SetReadDeadline(stopped.Add(cutOffAfter + waitLimit))
text, err := io.ReadAll(conn)
if err != nil || len(text) != 0 {
t.Errorf("read %q (%v), want the connection cut off", text, err)
+36 -12
View File
@@ -118,7 +118,8 @@ func RunWAF(ctx context.Context, params Params) int {
"settings", cfg)
// The last write follows the stop of every hook, and so the end of every
// loop that changes the state or raises an alert. A hook of the state
// loop that changes the state or raises an alert, and of every ask of
// GeoJS, the DNSBL zones, AbuseIPDB and routewatch. A hook of the state
// files would stop before the loops of the parts built ahead of them.
return run(ctx, app, failed, processLog, func() error {
err := files.WriteAll()
@@ -297,11 +298,13 @@ func newLists(
return lists
}
// newDNSBL returns what asks the DNSBL zones, none asked yet.
// newDNSBL returns what asks the DNSBL zones, none asked yet. At the stop it
// ends the queries under way, as reputation.DNSBL.StopAsking says.
func newDNSBL(
cfg *config.WAF, processLog *slog.Logger, alertQueue *alerts.Queue,
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
alertQueue *alerts.Queue,
) *reputation.DNSBL {
return reputation.NewDNSBL(reputation.DNSBLParams{
dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{
Zones: cfg.DNSBLZones,
Resolver: cfg.DNSBLResolver,
CacheTTL: cfg.IPReputationCacheTTL,
@@ -310,12 +313,17 @@ func newDNSBL(
ProcessLog: processLog,
Alerts: alertQueue,
})
lc.Append(fx.StopHook(dnsbl.StopAsking))
return dnsbl
}
// newAbuseIPDB returns what checks clients with AbuseIPDB while
// SWW_WAF_ABUSEIPDB_KEY is set, none checked yet. Until waf stops, it
// fetches AbuseIPDB's blacklist as it is due, and bans the clients on it in
// ledger, as waf.BanAbuseIPDBBlacklist says.
// ledger, as waf.BanAbuseIPDBBlacklist says. At the stop it ends the checks
// under way, as reputation.AbuseIPDB.StopChecking says.
func newAbuseIPDB(
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
alertQueue *alerts.Queue, ledger *bans.Ledger,
@@ -341,24 +349,30 @@ func newAbuseIPDB(
})
lc.Append(background(abuseIPDB.RunBlacklist))
lc.Append(fx.StopHook(abuseIPDB.StopChecking))
return abuseIPDB
}
// newRouteWatch returns what asks routewatch about clients, or nil while
// SWW_WAF_ROUTEWATCH_URL is unset.
// SWW_WAF_ROUTEWATCH_URL is unset. At the stop it ends the queries under
// way, as routewatch.Client.StopAsking says.
func newRouteWatch(
cfg *config.WAF, processLog *slog.Logger,
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
) *routewatch.Client {
if cfg.RouteWatchURL == "" {
return nil
}
return routewatch.New(routewatch.Params{
routeWatch := routewatch.New(routewatch.Params{
URL: cfg.RouteWatchURL,
Now: utcNow,
ProcessLog: processLog,
})
lc.Append(fx.StopHook(routeWatch.StopAsking))
return routeWatch
}
// newMetrics returns the metrics, with those read from the other parts as
@@ -396,12 +410,18 @@ func newMetrics(
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
// answer goes to the limiter and the ledger, as waf.AddLookup says, and each
// failure is handled as waf.GeoJSFailed says.
// failure is handled as waf.GeoJSFailed says. At the stop it ends the request
// under way, as geojs.GeoJS.StopAsking says.
func newGeoJS(
cfg *config.WAF, m *metrics.Metrics, limiter *ratelimit.Limiter,
ledger *bans.Ledger, alertQueue *alerts.Queue, processLog *slog.Logger,
lc fx.Lifecycle,
cfg *config.WAF,
m *metrics.Metrics,
limiter *ratelimit.Limiter,
ledger *bans.Ledger,
alertQueue *alerts.Queue,
processLog *slog.Logger,
) *geojs.GeoJS {
return geojs.New(geojs.Params{
geoJS := geojs.New(geojs.Params{
URL: geojs.URL,
Answered: func(answer geojs.Answer) {
waf.AddLookup(cfg, limiter, ledger, answer)
@@ -413,6 +433,10 @@ func newGeoJS(
},
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
})
lc.Append(fx.StopHook(geoJS.StopAsking))
return geoJS
}
// newWAFServer returns the server waf runs, with its parts and the time
+5 -4
View File
@@ -548,10 +548,6 @@ func (s *sender) requestWithHeader(
// body is sent as it is after the headers, header holding its
// Content-Length or Transfer-Encoding. header may hold several lines,
// separated by "\r\n". It returns the whole answer.
//
// It waits for the answer as long as it takes, since a busy host can slow
// the Core Rule Set past any shorter wait. go test's -timeout ends a test
// whose answer never comes.
func (s *sender) requestWithBody(
method, from, path, header, body string, status int, action string,
) (logLine, answer) {
@@ -570,6 +566,11 @@ func (s *sender) requestWithBody(
send(s.t, conn, method+" "+path+" HTTP/1.1\r\nHost: "+appHost+"\r\n"+
forwardedFor+": "+from+"\r\n"+header+"\r\n"+body)
err := conn.SetReadDeadline(time.Now().Add(waitLimit))
if err != nil {
s.t.Fatalf("set read deadline: %v", err)
}
var got answer
res, err := http.ReadResponse(bufio.NewReader(conn), nil)
+7 -7
View File
@@ -53,13 +53,13 @@ func (rq *request) limitPercentages() (percentage, percentage) {
if country == "" && !rq.skippedLookupWait {
unknown = percentage{
cfg.UnknownCountryLimitPercent,
"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
config.UnknownCountryLimitPercentSetting,
}
}
fetched := percentage{percent: whole}
if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed {
fetched = percentage{percent, "SWW_WAF_ASN_LIMIT_PERCENT_URL"}
fetched = percentage{percent, config.ASNLimitPercentURLSetting}
}
blocklisted := percentage{percent: whole}
@@ -67,7 +67,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
cfg.BlocklistAction == config.ReputationActionLimit {
blocklisted = percentage{
cfg.BlocklistLimitPercent,
"SWW_WAF_BLOCKLIST_ACTION",
config.BlocklistActionSetting,
}
}
@@ -76,11 +76,11 @@ func (rq *request) limitPercentages() (percentage, percentage) {
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit {
dnsblAbuseIPDBListed = percentage{
cfg.DNSBLAbuseIPDBLimitPercent,
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
config.DNSBLAbuseIPDBActionSetting,
}
}
const tagSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
const tagSetting = config.TagLimitPercentSetting
asnRequests := lowest(
given(cfg.TagLimitPercent, asnTag(asn), tagSetting),
@@ -108,7 +108,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
asnBytes = given(
cfg.ASNBytesLimitPercent,
asn,
"SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
config.ASNBytesLimitPercentSetting,
)
}
@@ -116,7 +116,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
countryBytes = given(
cfg.CountryBytesLimitPercent,
country,
"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
config.CountryBytesLimitPercentSetting,
)
}
+3 -1
View File
@@ -7,6 +7,7 @@ import (
"slices"
"time"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
@@ -33,7 +34,8 @@ func (countries) check(
return &verdict{
action: requestlog.ActionCountryDenied,
figures: map[string]any{
"country": country, "setting": "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
"country": country,
"setting": config.ExclusivelyAllowedCountriesSetting,
},
}
}
+6 -2
View File
@@ -193,11 +193,11 @@ func (o *output) requestLine(t *testing.T) logLine {
// endedLines waits for count requests that went through rproxy, or to waf's
// own endpoints, to have ended, and returns the last log line of each, in the
// order waf took them. A question's last line is the one its report writes.
// It waits as long as it takes, as requestWithBody does.
func (o *output) endedLines(t *testing.T, count int) []logLine {
t.Helper()
for {
deadline := time.Now().Add(waitLimit)
for time.Now().Before(deadline) {
found := o.lastLines(t)
ended := len(found) >= count
@@ -213,6 +213,10 @@ func (o *output) endedLines(t *testing.T, count int) []logLine {
time.Sleep(pollInterval)
}
t.Fatalf("fewer than %d requests have ended after %s", count, waitLimit)
return nil
}
// lastLines returns the last request log line written so far of each request,