Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
69e62c3e37 | ||
|
|
aaa28f541b | ||
|
|
98d7432077 | ||
|
|
11ac8aee50 | ||
|
|
e7cc0adbdb |
No files matched your search
+30
-17
@@ -27,6 +27,19 @@ import (
|
||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||
)
|
||||
|
||||
// The names of the settings that waf writes into log lines and ban notes,
|
||||
// as the setting that refused a request or lowered a client's limits.
|
||||
const (
|
||||
ExclusivelyAllowedCountriesSetting = "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES"
|
||||
TagLimitPercentSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
|
||||
ASNBytesLimitPercentSetting = "SWW_WAF_ASN_BYTES_LIMIT_PERCENT"
|
||||
CountryBytesLimitPercentSetting = "SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT"
|
||||
UnknownCountryLimitPercentSetting = "SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT"
|
||||
ASNLimitPercentURLSetting = "SWW_WAF_ASN_LIMIT_PERCENT_URL"
|
||||
BlocklistActionSetting = "SWW_WAF_BLOCKLIST_ACTION"
|
||||
DNSBLAbuseIPDBActionSetting = "SWW_WAF_DNSBL_ABUSEIPDB_ACTION"
|
||||
)
|
||||
|
||||
// WAF is the settings of `sww waf`. A time or size limit of zero is off.
|
||||
type WAF struct {
|
||||
// ListenAddr is where waf listens (SWW_WAF_LISTEN_ADDR).
|
||||
@@ -618,24 +631,24 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
||||
"800",
|
||||
),
|
||||
ExclusivelyAllowedCountries: env.countries(
|
||||
"SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""),
|
||||
ExclusivelyAllowedCountriesSetting, ""),
|
||||
TagLists: env.tagLists("SWW_WAF_TAG_LISTS"),
|
||||
TagDeny: env.tags("SWW_WAF_TAG_DENY"),
|
||||
TagLimitPercent: env.tagPercents("SWW_WAF_TAG_LIMIT_PERCENT"),
|
||||
TagLimitPercent: env.tagPercents(TagLimitPercentSetting),
|
||||
RouteWatchURL: env.routeWatchURL("SWW_WAF_ROUTEWATCH_URL"),
|
||||
ASNBytesLimitPercent: env.percents(
|
||||
"SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
|
||||
ASNBytesLimitPercentSetting,
|
||||
ParseASN,
|
||||
),
|
||||
CountryBytesLimitPercent: env.percents(
|
||||
"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
|
||||
CountryBytesLimitPercentSetting,
|
||||
parseCountry,
|
||||
),
|
||||
UnknownCountryLimitPercent: env.percent(
|
||||
"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
|
||||
UnknownCountryLimitPercentSetting,
|
||||
"100",
|
||||
),
|
||||
ASNLimitPercentURL: env.listURL("SWW_WAF_ASN_LIMIT_PERCENT_URL"),
|
||||
ASNLimitPercentURL: env.listURL(ASNLimitPercentURLSetting),
|
||||
BlocklistURLs: env.listURLs("SWW_WAF_BLOCKLIST_URLS"),
|
||||
BlocklistRefresh: env.refresh("SWW_WAF_BLOCKLIST_REFRESH", "24h"),
|
||||
DNSBLZones: env.zones("SWW_WAF_DNSBL_ZONES"),
|
||||
@@ -809,9 +822,9 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
||||
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
|
||||
strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
|
||||
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
|
||||
"SWW_WAF_BLOCKLIST_ACTION", string(ReputationActionDeny))
|
||||
BlocklistActionSetting, string(ReputationActionDeny))
|
||||
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
|
||||
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "limit:25")
|
||||
DNSBLAbuseIPDBActionSetting, "limit:25")
|
||||
|
||||
env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
|
||||
env.checkLookupDBPath(cfg)
|
||||
@@ -1220,7 +1233,7 @@ func (e *environment) checkLookupDBPath(cfg *WAF) {
|
||||
func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
||||
for _, country := range cfg.ExclusivelyAllowedCountries {
|
||||
if slices.Contains(cfg.TagDeny, CountryTagPrefix+country) {
|
||||
e.check("SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
|
||||
e.check(ExclusivelyAllowedCountriesSetting,
|
||||
fmt.Errorf("%q %w", country, errOnBothLists))
|
||||
}
|
||||
}
|
||||
@@ -1239,15 +1252,15 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
||||
name string
|
||||
set bool
|
||||
}{
|
||||
{"SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", len(cfg.ExclusivelyAllowedCountries) > 0},
|
||||
{ExclusivelyAllowedCountriesSetting, len(cfg.ExclusivelyAllowedCountries) > 0},
|
||||
{"SWW_WAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders},
|
||||
{"SWW_WAF_TAG_DENY", slices.ContainsFunc(cfg.TagDeny, onlyLookupGives)},
|
||||
{"SWW_WAF_TAG_LIMIT_PERCENT", slices.ContainsFunc(
|
||||
{TagLimitPercentSetting, slices.ContainsFunc(
|
||||
slices.Collect(maps.Keys(cfg.TagLimitPercent)), onlyLookupGives)},
|
||||
{"SWW_WAF_ASN_BYTES_LIMIT_PERCENT", len(cfg.ASNBytesLimitPercent) > 0},
|
||||
{"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT", len(cfg.CountryBytesLimitPercent) > 0},
|
||||
{"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT", cfg.UnknownCountryLimitPercent < 100},
|
||||
{"SWW_WAF_ASN_LIMIT_PERCENT_URL", cfg.ASNLimitPercentURL != ""},
|
||||
{ASNBytesLimitPercentSetting, len(cfg.ASNBytesLimitPercent) > 0},
|
||||
{CountryBytesLimitPercentSetting, len(cfg.CountryBytesLimitPercent) > 0},
|
||||
{UnknownCountryLimitPercentSetting, cfg.UnknownCountryLimitPercent < 100},
|
||||
{ASNLimitPercentURLSetting, cfg.ASNLimitPercentURL != ""},
|
||||
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0},
|
||||
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0},
|
||||
{"SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0},
|
||||
@@ -1267,7 +1280,7 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
||||
// blocklist too: the file at a URL is fetched as one list or the other.
|
||||
func (e *environment) checkASNLimitPercentURL(cfg *WAF) {
|
||||
if slices.Contains(cfg.BlocklistURLs, cfg.ASNLimitPercentURL) {
|
||||
e.check("SWW_WAF_ASN_LIMIT_PERCENT_URL",
|
||||
e.check(ASNLimitPercentURLSetting,
|
||||
fmt.Errorf("%q %w", cfg.ASNLimitPercentURL, errInBlocklistURLs))
|
||||
}
|
||||
}
|
||||
@@ -1294,7 +1307,7 @@ func (e *environment) checkTags(cfg *WAF) {
|
||||
}
|
||||
|
||||
for tag := range cfg.TagLimitPercent {
|
||||
e.check("SWW_WAF_TAG_LIMIT_PERCENT", checkTag(cfg, tag))
|
||||
e.check(TagLimitPercentSetting, checkTag(cfg, tag))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+47
-14
@@ -99,8 +99,13 @@ type GeoJS struct {
|
||||
// waiting are the clients without an answer: those to ask GeoJS about,
|
||||
// and those it is being asked about.
|
||||
waiting map[netip.Prefix]*wait
|
||||
// asking is true while a request to GeoJS is under way.
|
||||
asking bool
|
||||
// asking is true while a request to GeoJS is under way, and endAsking
|
||||
// ends it. stopped is true once StopAsking has been called, and
|
||||
// askGoroutines waits for the goroutine that makes the requests.
|
||||
asking bool
|
||||
endAsking context.CancelFunc
|
||||
stopped bool
|
||||
askGoroutines sync.WaitGroup
|
||||
// retryDelay is how long GeoJS is left alone after its last failure,
|
||||
// zero after an answer; retryAt is when it may be asked again.
|
||||
retryDelay time.Duration
|
||||
@@ -210,6 +215,23 @@ func (g *GeoJS) LookUpWithoutWaiting(
|
||||
return answer
|
||||
}
|
||||
|
||||
// StopAsking ends the request to GeoJS under way, and returns once the
|
||||
// goroutine that made it has. A request it ends is no failure: Failed is not
|
||||
// called, and GeoJS is not left alone after it. After StopAsking, GeoJS is
|
||||
// asked nothing.
|
||||
func (g *GeoJS) StopAsking() {
|
||||
g.mu.Lock()
|
||||
|
||||
g.stopped = true
|
||||
if g.asking {
|
||||
g.endAsking()
|
||||
}
|
||||
|
||||
g.mu.Unlock()
|
||||
|
||||
g.askGoroutines.Wait()
|
||||
}
|
||||
|
||||
// Kept returns client's answer, if one is kept, without asking GeoJS.
|
||||
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
|
||||
g.mu.Lock()
|
||||
@@ -294,7 +316,8 @@ func (g *GeoJS) answerOrWait(
|
||||
}
|
||||
|
||||
if !g.asking {
|
||||
// GeoJS is left alone after a failure, so no answer can come.
|
||||
// GeoJS is left alone after a failure, or asked nothing after
|
||||
// StopAsking, so no answer can come.
|
||||
w.late = true
|
||||
}
|
||||
|
||||
@@ -323,22 +346,28 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
||||
}
|
||||
|
||||
// ask starts asking GeoJS about the waiting clients, unless a request to
|
||||
// it is under way or it is left alone after a failure. The requests to
|
||||
// GeoJS are for every client waiting, so they go on when the caller whose
|
||||
// ctx is given stops waiting.
|
||||
// it is under way, it is left alone after a failure, or StopAsking has
|
||||
// been called. The requests to GeoJS are for every client waiting, so they
|
||||
// go on when the caller whose ctx is given stops waiting, until StopAsking.
|
||||
func (g *GeoJS) ask(ctx context.Context) {
|
||||
if g.asking || g.now().Before(g.retryAt) {
|
||||
if g.asking || g.stopped || g.now().Before(g.retryAt) {
|
||||
return
|
||||
}
|
||||
|
||||
g.asking = true
|
||||
askCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||
g.asking, g.endAsking = true, end
|
||||
|
||||
go g.askAboutWaiting(context.WithoutCancel(ctx))
|
||||
g.askGoroutines.Go(func() {
|
||||
defer end()
|
||||
|
||||
g.askAboutWaiting(askCtx)
|
||||
})
|
||||
}
|
||||
|
||||
// askAboutWaiting asks GeoJS about the waiting clients, one request at a
|
||||
// time, until none is left or GeoJS fails. Each answer kept is given to
|
||||
// Answered, outside the lock, since Answered takes locks of its own.
|
||||
// time, until none is left, GeoJS fails, or StopAsking ends ctx. Each
|
||||
// answer kept is given to Answered, outside the lock, since Answered takes
|
||||
// locks of its own.
|
||||
func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
||||
for {
|
||||
clients := g.nextClients()
|
||||
@@ -347,6 +376,10 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
||||
}
|
||||
|
||||
given, err := g.request(ctx, clients)
|
||||
if err != nil && ctx.Err() != nil {
|
||||
continue // ended by StopAsking, which is no failure
|
||||
}
|
||||
|
||||
kept, answered := g.keep(clients, given, err)
|
||||
|
||||
if g.answered != nil {
|
||||
@@ -362,13 +395,13 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
||||
}
|
||||
|
||||
// nextClients returns up to maxPerRequest of the waiting clients. When
|
||||
// none is waiting, it returns none and notes that no request to GeoJS is
|
||||
// under way.
|
||||
// none is waiting, or StopAsking has been called, it returns none and
|
||||
// notes that no request to GeoJS is under way.
|
||||
func (g *GeoJS) nextClients() []netip.Prefix {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
|
||||
if len(g.waiting) == 0 {
|
||||
if len(g.waiting) == 0 || g.stopped {
|
||||
g.asking = false
|
||||
|
||||
return nil
|
||||
|
||||
@@ -318,6 +318,44 @@ func TestCallerWhoseContextEndsEarlyDoesNotStopTheRequest(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestStopAskingEndsTheRequestUnderWayAsNoFailureAndAsksNoMore(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
service, _, g, reports := startWithReports()
|
||||
clients := newClients()
|
||||
|
||||
service.set(hanging)
|
||||
g.LookUpWithoutWaiting(t.Context(), clients())
|
||||
waitForRequests(t, service, 1)
|
||||
|
||||
// The request is ended, not waited out, and is no failure.
|
||||
began := time.Now()
|
||||
|
||||
g.StopAsking()
|
||||
|
||||
if waited := time.Since(began); waited != 0 {
|
||||
t.Errorf("StopAsking returned after %s, want at once", waited)
|
||||
}
|
||||
|
||||
reports.wantFailures(t)
|
||||
|
||||
// A new client is then not asked about, and its lookup does not
|
||||
// wait.
|
||||
service.set(answering)
|
||||
wantCountry(t, g, clients(), "")
|
||||
|
||||
if waited := time.Since(began); waited != 0 {
|
||||
t.Errorf("the lookup returned after %s, want at once", waited)
|
||||
}
|
||||
|
||||
waitForRequests(t, service, 1)
|
||||
reports.wantFailures(t)
|
||||
})
|
||||
}
|
||||
|
||||
func TestRequestOver10SecondsIsAFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -110,8 +110,12 @@ type AbuseIPDB struct {
|
||||
// scores are by client. Each is added as it is fetched and never moved
|
||||
// up, so that the one fetched longest ago is the first dropped.
|
||||
scores *simplelru.LRU[netip.Prefix, Score]
|
||||
// checking are the clients whose check is under way.
|
||||
checking map[netip.Prefix]bool
|
||||
// checking are the clients whose check is under way, each with the
|
||||
// function that ends it. stopped is true once StopChecking has been
|
||||
// called, and checkGoroutines waits for the goroutine of each check.
|
||||
checking map[netip.Prefix]context.CancelFunc
|
||||
stopped bool
|
||||
checkGoroutines sync.WaitGroup
|
||||
// day is the day, in UTC, of the checks spent counts.
|
||||
day civil.Date
|
||||
spent int
|
||||
@@ -134,7 +138,7 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
|
||||
params: params,
|
||||
httpClient: &http.Client{},
|
||||
scores: scores,
|
||||
checking: map[netip.Prefix]bool{},
|
||||
checking: map[netip.Prefix]context.CancelFunc{},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -144,11 +148,11 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
|
||||
// addresses its request comes from. A client without one is checked in
|
||||
// the background, by addr, the address its request came from, if
|
||||
// offender, if it has committed an offence, unless its check is under
|
||||
// way, a check failed less than failureDelay ago, or the day's checks
|
||||
// have used up DailyBudget; Hit never waits for a check. The check that
|
||||
// uses the budget up is logged and raised as a source_failure alert. ctx
|
||||
// is the context of the client's request, and a check goes on after the
|
||||
// request ends.
|
||||
// way, a check failed less than failureDelay ago, the day's checks have
|
||||
// used up DailyBudget, or StopChecking has been called; Hit never waits
|
||||
// for a check. The check that uses the budget up is logged and raised as a
|
||||
// source_failure alert. ctx is the context of the client's request, and a
|
||||
// check goes on after the request ends, until StopChecking.
|
||||
func (a *AbuseIPDB) Hit(
|
||||
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
|
||||
) (int64, bool) {
|
||||
@@ -167,14 +171,21 @@ func (a *AbuseIPDB) Hit(
|
||||
a.day, a.spent = today, 0
|
||||
}
|
||||
|
||||
check := offender && !a.checking[client] && !now.Before(a.retryAt) &&
|
||||
a.spent < a.params.DailyBudget
|
||||
_, checking := a.checking[client]
|
||||
|
||||
check := offender && !checking && !a.stopped &&
|
||||
!now.Before(a.retryAt) && a.spent < a.params.DailyBudget
|
||||
if check {
|
||||
a.checking[client] = true
|
||||
checkCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||
a.checking[client] = end
|
||||
a.checks++
|
||||
a.spent++
|
||||
|
||||
go a.check(context.WithoutCancel(ctx), client, addr)
|
||||
a.checkGoroutines.Go(func() {
|
||||
defer end()
|
||||
|
||||
a.check(checkCtx, client, addr)
|
||||
})
|
||||
}
|
||||
|
||||
usedUp := check && a.spent == a.params.DailyBudget
|
||||
@@ -189,6 +200,24 @@ func (a *AbuseIPDB) Hit(
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// StopChecking ends the checks under way, and returns once their goroutines
|
||||
// have. A check it ends gives no score and is no failure: it is neither
|
||||
// counted, logged nor raised as an alert, and checks are not left alone
|
||||
// after it. The ended check stays spent from the day's budget. After
|
||||
// StopChecking, Hit checks no client.
|
||||
func (a *AbuseIPDB) StopChecking() {
|
||||
a.mu.Lock()
|
||||
|
||||
a.stopped = true
|
||||
for _, end := range a.checking {
|
||||
end()
|
||||
}
|
||||
|
||||
a.mu.Unlock()
|
||||
|
||||
a.checkGoroutines.Wait()
|
||||
}
|
||||
|
||||
// Checked returns how many checks were made.
|
||||
func (a *AbuseIPDB) Checked() int {
|
||||
a.mu.Lock()
|
||||
@@ -315,7 +344,8 @@ func (a *AbuseIPDB) RunBlacklist(ctx context.Context) {
|
||||
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
|
||||
// the score as client's, and notes the check as no longer under way. A
|
||||
// check that fails gives no score: it is counted, logged and raised as a
|
||||
// source_failure alert, and no client is checked for failureDelay.
|
||||
// source_failure alert, and no client is checked for failureDelay. A check
|
||||
// ended as ctx ends, by StopChecking, is no failure.
|
||||
func (a *AbuseIPDB) check(
|
||||
ctx context.Context,
|
||||
client netip.Prefix,
|
||||
@@ -323,24 +353,26 @@ func (a *AbuseIPDB) check(
|
||||
) {
|
||||
score, err := a.ask(ctx, addr)
|
||||
now := a.params.Now()
|
||||
isFailure := err != nil && ctx.Err() == nil
|
||||
|
||||
a.mu.Lock()
|
||||
|
||||
delete(a.checking, client)
|
||||
|
||||
if err == nil {
|
||||
switch {
|
||||
case err == nil:
|
||||
a.scores.Add(
|
||||
client,
|
||||
Score{Client: client, Score: score, Fetched: now},
|
||||
)
|
||||
} else {
|
||||
case isFailure:
|
||||
a.failures++
|
||||
a.retryAt = now.Add(failureDelay)
|
||||
}
|
||||
|
||||
a.mu.Unlock()
|
||||
|
||||
if err != nil {
|
||||
if isFailure {
|
||||
a.alert("checking a client with AbuseIPDB failed", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -394,6 +394,43 @@ func TestCheckNotAnsweredWithinTheTimeoutFailsAndHitNeverWaits(
|
||||
})
|
||||
}
|
||||
|
||||
func TestStopCheckingEndsTheCheckUnderWayAsNoFailureAndChecksNoMore(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
queue := newQueue()
|
||||
p := abuseIPDBParams()
|
||||
p.Alerts = queue
|
||||
abuseIPDB := &abuseIPDBStandIn{hanging: true}
|
||||
checker := newAbuseIPDB(abuseIPDB, p)
|
||||
|
||||
wantScore(t, checker, suspect, true, 0, false)
|
||||
synctest.Wait()
|
||||
|
||||
// The check is ended, not waited out, and is no failure.
|
||||
began := time.Now()
|
||||
|
||||
checker.StopChecking()
|
||||
|
||||
if waited := time.Since(began); waited != 0 {
|
||||
t.Errorf("StopChecking returned after %s, want at once", waited)
|
||||
}
|
||||
|
||||
wantFailures(t, checker, 0)
|
||||
|
||||
if got := waiting(queue); len(got) != 0 {
|
||||
t.Errorf("alerts waiting %+v, want none", got)
|
||||
}
|
||||
|
||||
// An offender without a score is then not checked.
|
||||
wantScore(t, checker, other, true, 0, false)
|
||||
synctest.Wait()
|
||||
wantChecked(t, abuseIPDB, suspect)
|
||||
})
|
||||
}
|
||||
|
||||
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -82,8 +82,12 @@ type DNSBL struct {
|
||||
// verdicts are by query. Each is added as it is fetched and never moved
|
||||
// up, so that the one fetched longest ago is the first dropped.
|
||||
verdicts *simplelru.LRU[query, Verdict]
|
||||
// asking are the queries under way.
|
||||
asking map[query]bool
|
||||
// asking are the queries under way, each with the function that ends
|
||||
// it. stopped is true once StopAsking has been called, and
|
||||
// queryGoroutines waits for the goroutine of each query.
|
||||
asking map[query]context.CancelFunc
|
||||
stopped bool
|
||||
queryGoroutines sync.WaitGroup
|
||||
// queries and failures count, by zone, the queries made and those that
|
||||
// failed, and retryAt is when a zone whose last query failed may be
|
||||
// asked again.
|
||||
@@ -120,7 +124,7 @@ func NewDNSBL(params DNSBLParams) *DNSBL {
|
||||
params: params,
|
||||
resolver: resolver,
|
||||
verdicts: verdicts,
|
||||
asking: map[query]bool{},
|
||||
asking: map[query]context.CancelFunc{},
|
||||
queries: map[string]int{},
|
||||
failures: map[string]int{},
|
||||
retryAt: map[string]time.Time{},
|
||||
@@ -138,9 +142,10 @@ func (d *DNSBL) Zones() []string {
|
||||
// log, the alerts and the metrics. A verdict is used until CacheTTL has
|
||||
// passed since it was fetched. Each zone without one is asked about addr
|
||||
// in the background, unless a query about addr to it is under way, the
|
||||
// zone is left alone after a failure, or maxQueries are under way;
|
||||
// ListedBy never waits for a query. ctx is the context of the client's
|
||||
// request, and a query goes on after the request ends.
|
||||
// zone is left alone after a failure, maxQueries are under way, or
|
||||
// StopAsking has been called; ListedBy never waits for a query. ctx is the
|
||||
// context of the client's request, and a query goes on after the request
|
||||
// ends, until StopAsking.
|
||||
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
@@ -153,23 +158,47 @@ func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
||||
q := query{zone: zone, client: addr}
|
||||
|
||||
kept, found := d.verdicts.Peek(q)
|
||||
_, asking := d.asking[q]
|
||||
|
||||
switch {
|
||||
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
|
||||
if kept.Listed {
|
||||
listedBy = append(listedBy, config.MaskZoneKey(zone))
|
||||
}
|
||||
case !d.asking[q] && !now.Before(d.retryAt[zone]) && len(d.asking) < maxQueries:
|
||||
d.asking[q] = true
|
||||
case !asking && !d.stopped && !now.Before(d.retryAt[zone]) &&
|
||||
len(d.asking) < maxQueries:
|
||||
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||
d.asking[q] = end
|
||||
d.queries[zone]++
|
||||
|
||||
go d.ask(context.WithoutCancel(ctx), q)
|
||||
d.queryGoroutines.Go(func() {
|
||||
defer end()
|
||||
|
||||
d.ask(queryCtx, q)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return listedBy
|
||||
}
|
||||
|
||||
// StopAsking ends the queries under way, and returns once their
|
||||
// goroutines have. A query it ends gives no verdict and is no failure: it
|
||||
// is neither counted, logged nor raised as an alert, and the zone is not
|
||||
// left alone after it. After StopAsking, ListedBy asks no zone.
|
||||
func (d *DNSBL) StopAsking() {
|
||||
d.mu.Lock()
|
||||
|
||||
d.stopped = true
|
||||
for _, end := range d.asking {
|
||||
end()
|
||||
}
|
||||
|
||||
d.mu.Unlock()
|
||||
|
||||
d.queryGoroutines.Wait()
|
||||
}
|
||||
|
||||
// Queries returns how many queries were made to zone.
|
||||
func (d *DNSBL) Queries(zone string) int {
|
||||
d.mu.Lock()
|
||||
@@ -238,27 +267,29 @@ func (d *DNSBL) Load(verdicts []Verdict) {
|
||||
// query as no longer under way. A query that fails gives no verdict: it
|
||||
// is counted, logged and raised as a source_failure alert, which show the
|
||||
// zone with its key masked, and the zone is not asked again for
|
||||
// failureDelay.
|
||||
// failureDelay. A query ended as ctx ends, by StopAsking, is no failure.
|
||||
func (d *DNSBL) ask(ctx context.Context, q query) {
|
||||
listed, err := d.lookUp(ctx, q)
|
||||
now := d.params.Now()
|
||||
isFailure := err != nil && ctx.Err() == nil
|
||||
|
||||
d.mu.Lock()
|
||||
|
||||
delete(d.asking, q)
|
||||
|
||||
if err == nil {
|
||||
switch {
|
||||
case err == nil:
|
||||
d.verdicts.Add(q, Verdict{
|
||||
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
|
||||
})
|
||||
} else {
|
||||
case isFailure:
|
||||
d.failures[q.zone]++
|
||||
d.retryAt[q.zone] = now.Add(failureDelay)
|
||||
}
|
||||
|
||||
d.mu.Unlock()
|
||||
|
||||
if err != nil {
|
||||
if isFailure {
|
||||
const failed = "asking a DNSBL zone failed"
|
||||
|
||||
shown := config.MaskZoneKey(q.zone)
|
||||
|
||||
@@ -210,6 +210,45 @@ func TestQueryNotAnsweredWithinTheTimeoutFails(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
//
|
||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
|
||||
t *testing.T,
|
||||
) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
queue := newQueue()
|
||||
p := dnsblParams(zone)
|
||||
p.Alerts = queue
|
||||
resolver := &resolverStandIn{hanging: true}
|
||||
dnsbl := newDNSBL(resolver, p)
|
||||
|
||||
wantZones(t, dnsbl, listed)
|
||||
synctest.Wait()
|
||||
|
||||
// The query is ended, not waited out, and is no failure.
|
||||
began := time.Now()
|
||||
|
||||
dnsbl.StopAsking()
|
||||
|
||||
if waited := time.Since(began); waited != 0 {
|
||||
t.Errorf("StopAsking returned after %s, want at once", waited)
|
||||
}
|
||||
|
||||
wantQueries(t, dnsbl, 1, 0)
|
||||
|
||||
if got := waiting(queue); len(got) != 0 {
|
||||
t.Errorf("alerts waiting %+v, want none", got)
|
||||
}
|
||||
|
||||
// A client without a verdict is then asked about in no zone.
|
||||
wantZones(t, dnsbl, unlisted)
|
||||
synctest.Wait()
|
||||
wantQueries(t, dnsbl, 1, 0)
|
||||
wantAsked(t, resolver, listedName)
|
||||
waitForTheResolver()
|
||||
})
|
||||
}
|
||||
|
||||
//
|
||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(
|
||||
|
||||
@@ -99,8 +99,12 @@ type Client struct {
|
||||
// answers are by client. Each is added as it comes and never moved up,
|
||||
// so that the one that came longest ago is the first dropped.
|
||||
answers *simplelru.LRU[netip.Prefix, Answer]
|
||||
// asking are the clients whose query is under way.
|
||||
asking map[netip.Prefix]bool
|
||||
// asking are the clients whose query is under way, each with the
|
||||
// function that ends it. stopped is true once StopAsking has been
|
||||
// called, and queryGoroutines waits for the goroutine of each query.
|
||||
asking map[netip.Prefix]context.CancelFunc
|
||||
stopped bool
|
||||
queryGoroutines sync.WaitGroup
|
||||
// retryAt is when routewatch may be asked again after a query failed.
|
||||
retryAt time.Time
|
||||
}
|
||||
@@ -116,7 +120,7 @@ func New(params Params) *Client {
|
||||
params: params,
|
||||
httpClient: &http.Client{},
|
||||
answers: answers,
|
||||
asking: map[netip.Prefix]bool{},
|
||||
asking: map[netip.Prefix]context.CancelFunc{},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -124,8 +128,9 @@ func New(params Params) *Client {
|
||||
// IPv6 group, or why it has no route. It never waits for a query. A client
|
||||
// without an answer is asked about in the background, by addr, the address its
|
||||
// request came from, unless its query is under way, a query failed less than
|
||||
// failureKept ago, or maxQueries are under way. ctx is the context of the
|
||||
// client's request, and a query goes on after the request ends.
|
||||
// failureKept ago, maxQueries are under way, or StopAsking has been called. ctx
|
||||
// is the context of the client's request, and a query goes on after the request
|
||||
// ends, until StopAsking.
|
||||
func (c *Client) Route(
|
||||
ctx context.Context, client netip.Prefix, addr netip.Addr,
|
||||
) (Route, Missing) {
|
||||
@@ -147,15 +152,38 @@ func (c *Client) Route(
|
||||
return Route{}, RequestFailed
|
||||
}
|
||||
|
||||
if !c.asking[client] && len(c.asking) < maxQueries {
|
||||
c.asking[client] = true
|
||||
_, asking := c.asking[client]
|
||||
if !asking && !c.stopped && len(c.asking) < maxQueries {
|
||||
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||
c.asking[client] = end
|
||||
|
||||
go c.ask(context.WithoutCancel(ctx), client, addr)
|
||||
c.queryGoroutines.Go(func() {
|
||||
defer end()
|
||||
|
||||
c.ask(queryCtx, client, addr)
|
||||
})
|
||||
}
|
||||
|
||||
return Route{}, NoAnswerYet
|
||||
}
|
||||
|
||||
// StopAsking ends the queries under way, and returns once their goroutines
|
||||
// have. A query it ends gives no answer and is no failure: it is not logged,
|
||||
// and routewatch is not left alone after it. After StopAsking, Route asks
|
||||
// routewatch nothing.
|
||||
func (c *Client) StopAsking() {
|
||||
c.mu.Lock()
|
||||
|
||||
c.stopped = true
|
||||
for _, end := range c.asking {
|
||||
end()
|
||||
}
|
||||
|
||||
c.mu.Unlock()
|
||||
|
||||
c.queryGoroutines.Wait()
|
||||
}
|
||||
|
||||
// Snapshot returns every answer still in use, sorted by client, as
|
||||
// routewatch.json lists them.
|
||||
func (c *Client) Snapshot() []Answer {
|
||||
@@ -205,7 +233,8 @@ func (c *Client) Load(answers []Answer) {
|
||||
// ask asks routewatch about addr, keeps the answer as client's, and notes the
|
||||
// query as no longer under way. A query that fails gives no answer, and
|
||||
// routewatch is not asked again for failureKept. Only the first failure of
|
||||
// those under way is logged.
|
||||
// those under way is logged. A query ended as ctx ends, by StopAsking, is no
|
||||
// failure.
|
||||
func (c *Client) ask(
|
||||
ctx context.Context,
|
||||
client netip.Prefix,
|
||||
@@ -213,18 +242,20 @@ func (c *Client) ask(
|
||||
) {
|
||||
route, err := c.query(ctx, addr)
|
||||
now := c.params.Now()
|
||||
isFailure := err != nil && ctx.Err() == nil
|
||||
|
||||
c.mu.Lock()
|
||||
|
||||
delete(c.asking, client)
|
||||
|
||||
firstFailure := err != nil && !now.Before(c.retryAt)
|
||||
firstFailure := isFailure && !now.Before(c.retryAt)
|
||||
|
||||
if err == nil {
|
||||
switch {
|
||||
case err == nil:
|
||||
c.answers.Add(client, Answer{
|
||||
Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
|
||||
})
|
||||
} else {
|
||||
case isFailure:
|
||||
c.retryAt = now.Add(failureKept)
|
||||
}
|
||||
|
||||
|
||||
@@ -361,6 +361,54 @@ func TestHangingRoutewatchNeverDelaysAndFailsTheQuery(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
// In a synctest bubble, as TestHangingRoutewatchNeverDelaysAndFailsTheQuery
|
||||
// says.
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
f := &fixture{t: t, log: &syncBuffer{}}
|
||||
f.client = routewatch.New(routewatch.Params{
|
||||
URL: silentURL, Now: time.Now,
|
||||
ProcessLog: slog.New(slog.NewTextHandler(f.log, nil)),
|
||||
})
|
||||
|
||||
f.route()
|
||||
synctest.Wait()
|
||||
|
||||
// The query is ended, not waited out.
|
||||
began := time.Now()
|
||||
|
||||
f.client.StopAsking()
|
||||
|
||||
if waited := time.Since(began); waited != 0 {
|
||||
t.Errorf("StopAsking returned after %s, want at once", waited)
|
||||
}
|
||||
|
||||
// A query started now would fail within waitLimit. None is, and the
|
||||
// one ended is no failure: nothing is logged, and routewatch is not
|
||||
// left alone.
|
||||
for range 2 {
|
||||
if _, missing := f.route(); missing != routewatch.NoAnswerYet {
|
||||
t.Errorf(
|
||||
"Route gave %q, want %q",
|
||||
missing,
|
||||
routewatch.NoAnswerYet,
|
||||
)
|
||||
}
|
||||
|
||||
time.Sleep(waitLimit)
|
||||
synctest.Wait()
|
||||
}
|
||||
|
||||
if f.log.String() != "" {
|
||||
t.Errorf("process log %q, want nothing", f.log.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -2,6 +2,12 @@ package rproxy
|
||||
|
||||
import "time"
|
||||
|
||||
// StopBreakerClock has the circuit breaker of s read at as the time, so that how
|
||||
// long requests take cannot matter to it. It is called before s serves.
|
||||
func (s *Server) StopBreakerClock(at time.Time) {
|
||||
s.h.breakerClock = func() time.Time { return at }
|
||||
}
|
||||
|
||||
// Breaker is the circuit breaker, for tests that give it the time.
|
||||
type Breaker struct {
|
||||
b breaker
|
||||
|
||||
@@ -20,7 +20,6 @@ import (
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
@@ -94,6 +93,7 @@ func New(params Params) *Server {
|
||||
reportURL: (&url.URL{
|
||||
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath,
|
||||
}).String(),
|
||||
breakerClock: time.Now,
|
||||
}
|
||||
|
||||
return &Server{
|
||||
@@ -113,11 +113,7 @@ func New(params Params) *Server {
|
||||
// which SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS bounds. Any report after this call
|
||||
// is dropped: that of a WebSocket that closes while rproxy stops, for one.
|
||||
func (s *Server) WaitForReports() {
|
||||
s.h.reportsMu.Lock()
|
||||
s.h.reportsStopped = true
|
||||
s.h.reportsMu.Unlock()
|
||||
|
||||
s.h.reports.Wait()
|
||||
s.h.reports.stopAndWait()
|
||||
}
|
||||
|
||||
// handler is the proxy. It holds what every request shares; what belongs to
|
||||
@@ -133,13 +129,12 @@ type handler struct {
|
||||
wafTransport http.RoundTripper
|
||||
// reportURL is where the reports go.
|
||||
reportURL string
|
||||
// breaker is the circuit breaker for asks to waf.
|
||||
breaker breaker
|
||||
// reports waits for the reports being sent. reportsMu guards starting one
|
||||
// against reportsStopped, which is set once nothing is to wait for another.
|
||||
reports sync.WaitGroup
|
||||
reportsMu sync.Mutex
|
||||
reportsStopped bool
|
||||
// breaker is the circuit breaker for asks to waf, and breakerClock gives it
|
||||
// the time.
|
||||
breaker breaker
|
||||
breakerClock func() time.Time
|
||||
// reports are the goroutines sending reports to waf.
|
||||
reports reportGoroutines
|
||||
}
|
||||
|
||||
// newTransport returns what carries requests to the app. It never goes through
|
||||
|
||||
@@ -356,6 +356,19 @@ func startServerLogging(
|
||||
) (string, *rproxy.Server) {
|
||||
t.Helper()
|
||||
|
||||
server := newServer(t, backend, appURL, env, requestLog, processLog)
|
||||
|
||||
return serve(t, server), server
|
||||
}
|
||||
|
||||
// newServer returns rproxy, not yet serving, with its request lines going to
|
||||
// requestLog and its process lines to processLog.
|
||||
func newServer(
|
||||
t *testing.T, backend, appURL string, env map[string]string,
|
||||
requestLog, processLog io.Writer,
|
||||
) *rproxy.Server {
|
||||
t.Helper()
|
||||
|
||||
settings := map[string]string{
|
||||
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
|
||||
wafBackendTimeout: "60000",
|
||||
@@ -367,7 +380,7 @@ func startServerLogging(
|
||||
t.Fatalf("settings %v: %v", settings, err)
|
||||
}
|
||||
|
||||
server := rproxy.New(rproxy.Params{
|
||||
return rproxy.New(rproxy.Params{
|
||||
Config: cfg,
|
||||
RequestLog: requestLog,
|
||||
ProcessLog: requestlog.NewProcessLogger(
|
||||
@@ -376,6 +389,11 @@ func startServerLogging(
|
||||
slog.LevelInfo,
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
// serve has server serve until the test ends, and returns its address.
|
||||
func serve(t *testing.T, server *rproxy.Server) string {
|
||||
t.Helper()
|
||||
|
||||
listener, err := (&net.ListenConfig{}).Listen(
|
||||
t.Context(),
|
||||
@@ -396,7 +414,7 @@ func startServerLogging(
|
||||
server.WaitForReports()
|
||||
})
|
||||
|
||||
return listener.Addr().String(), server
|
||||
return listener.Addr().String()
|
||||
}
|
||||
|
||||
// answered is the visitor's answer.
|
||||
@@ -1532,9 +1550,13 @@ func TestAnswersThatCannotBePassedOnOpenTheBreaker(t *testing.T) {
|
||||
tooLarge(w, r)
|
||||
})
|
||||
app := startApp(t, greet)
|
||||
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
||||
wafBackendTimeout: "60000",
|
||||
})
|
||||
out := &output{}
|
||||
server := newServer(t, standIn.URL, app.URL, nil, out, out)
|
||||
// The breaker's clock stands still. On a slow host, the five failures could
|
||||
// otherwise take more than the five seconds that open the breaker. The next
|
||||
// request could come more than a second later, when it may ask waf again.
|
||||
server.StopBreakerClock(time.Now())
|
||||
addr := serve(t, server)
|
||||
|
||||
for range 5 {
|
||||
if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
|
||||
|
||||
+35
-11
@@ -10,7 +10,7 @@ import (
|
||||
"maps"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
"sync"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
)
|
||||
@@ -97,7 +97,7 @@ func (rq *request) askUnlessBreakerOpen(
|
||||
ctx context.Context,
|
||||
) (*wafAnswer, error) {
|
||||
b := &rq.h.breaker
|
||||
if !b.mayAsk(time.Now()) {
|
||||
if !b.mayAsk(rq.h.breakerClock()) {
|
||||
return nil, errBreakerOpen
|
||||
}
|
||||
|
||||
@@ -107,7 +107,7 @@ func (rq *request) askUnlessBreakerOpen(
|
||||
case err == nil:
|
||||
b.noteAnswer()
|
||||
case ctx.Err() == nil:
|
||||
b.noteFailure(time.Now())
|
||||
b.noteFailure(rq.h.breakerClock())
|
||||
}
|
||||
|
||||
return answer, err
|
||||
@@ -254,14 +254,7 @@ func (rq *request) answerWith(answer *wafAnswer) {
|
||||
// visitor does not wait for it. The report goes on after ctx, the request's own
|
||||
// context, has ended.
|
||||
func (h *handler) sendReport(ctx context.Context, report *waf.Report) {
|
||||
h.reportsMu.Lock()
|
||||
defer h.reportsMu.Unlock()
|
||||
|
||||
if h.reportsStopped {
|
||||
return
|
||||
}
|
||||
|
||||
h.reports.Go(func() {
|
||||
h.reports.startUnlessStopped(func() {
|
||||
err := h.postReport(context.WithoutCancel(ctx), report)
|
||||
if err != nil {
|
||||
h.processLog.Warn("a report is dropped", "error", err.Error())
|
||||
@@ -269,6 +262,37 @@ func (h *handler) sendReport(ctx context.Context, report *waf.Report) {
|
||||
})
|
||||
}
|
||||
|
||||
// reportGoroutines are the goroutines sending reports to waf. None starts once
|
||||
// stopAndWait has been called.
|
||||
type reportGoroutines struct {
|
||||
mu sync.Mutex
|
||||
stopped bool
|
||||
running sync.WaitGroup
|
||||
}
|
||||
|
||||
// startUnlessStopped runs send on a goroutine of its own, or drops it once
|
||||
// stopAndWait has been called.
|
||||
func (g *reportGoroutines) startUnlessStopped(send func()) {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
|
||||
if g.stopped {
|
||||
return
|
||||
}
|
||||
|
||||
g.running.Go(send)
|
||||
}
|
||||
|
||||
// stopAndWait stops any more goroutines from starting, and returns once those
|
||||
// running have ended.
|
||||
func (g *reportGoroutines) stopAndWait() {
|
||||
g.mu.Lock()
|
||||
g.stopped = true
|
||||
g.mu.Unlock()
|
||||
|
||||
g.running.Wait()
|
||||
}
|
||||
|
||||
// postReport posts report to waf, and waits for its answer for up to
|
||||
// SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS.
|
||||
func (h *handler) postReport(ctx context.Context, report *waf.Report) error {
|
||||
|
||||
@@ -755,8 +755,7 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection. A
|
||||
// read from it ends with an error once waitLimit is up.
|
||||
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection.
|
||||
func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
|
||||
t.Helper()
|
||||
|
||||
@@ -769,11 +768,6 @@ func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
|
||||
_ = conn.Close()
|
||||
})
|
||||
|
||||
err = conn.SetDeadline(time.Now().Add(waitLimit))
|
||||
if err != nil {
|
||||
t.Fatalf("set the deadline: %v", err)
|
||||
}
|
||||
|
||||
_, err = io.WriteString(conn, request)
|
||||
if err != nil {
|
||||
t.Fatalf("send: %v", err)
|
||||
@@ -838,6 +832,9 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
|
||||
|
||||
stop()
|
||||
|
||||
// Counted from the stop, so a slow wait for the app cannot shorten it.
|
||||
_ = conn.SetReadDeadline(stopped.Add(cutOffAfter + waitLimit))
|
||||
|
||||
text, err := io.ReadAll(conn)
|
||||
if err != nil || len(text) != 0 {
|
||||
t.Errorf("read %q (%v), want the connection cut off", text, err)
|
||||
|
||||
+36
-12
@@ -118,7 +118,8 @@ func RunWAF(ctx context.Context, params Params) int {
|
||||
"settings", cfg)
|
||||
|
||||
// The last write follows the stop of every hook, and so the end of every
|
||||
// loop that changes the state or raises an alert. A hook of the state
|
||||
// loop that changes the state or raises an alert, and of every ask of
|
||||
// GeoJS, the DNSBL zones, AbuseIPDB and routewatch. A hook of the state
|
||||
// files would stop before the loops of the parts built ahead of them.
|
||||
return run(ctx, app, failed, processLog, func() error {
|
||||
err := files.WriteAll()
|
||||
@@ -297,11 +298,13 @@ func newLists(
|
||||
return lists
|
||||
}
|
||||
|
||||
// newDNSBL returns what asks the DNSBL zones, none asked yet.
|
||||
// newDNSBL returns what asks the DNSBL zones, none asked yet. At the stop it
|
||||
// ends the queries under way, as reputation.DNSBL.StopAsking says.
|
||||
func newDNSBL(
|
||||
cfg *config.WAF, processLog *slog.Logger, alertQueue *alerts.Queue,
|
||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||
alertQueue *alerts.Queue,
|
||||
) *reputation.DNSBL {
|
||||
return reputation.NewDNSBL(reputation.DNSBLParams{
|
||||
dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{
|
||||
Zones: cfg.DNSBLZones,
|
||||
Resolver: cfg.DNSBLResolver,
|
||||
CacheTTL: cfg.IPReputationCacheTTL,
|
||||
@@ -310,12 +313,17 @@ func newDNSBL(
|
||||
ProcessLog: processLog,
|
||||
Alerts: alertQueue,
|
||||
})
|
||||
|
||||
lc.Append(fx.StopHook(dnsbl.StopAsking))
|
||||
|
||||
return dnsbl
|
||||
}
|
||||
|
||||
// newAbuseIPDB returns what checks clients with AbuseIPDB while
|
||||
// SWW_WAF_ABUSEIPDB_KEY is set, none checked yet. Until waf stops, it
|
||||
// fetches AbuseIPDB's blacklist as it is due, and bans the clients on it in
|
||||
// ledger, as waf.BanAbuseIPDBBlacklist says.
|
||||
// ledger, as waf.BanAbuseIPDBBlacklist says. At the stop it ends the checks
|
||||
// under way, as reputation.AbuseIPDB.StopChecking says.
|
||||
func newAbuseIPDB(
|
||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||
alertQueue *alerts.Queue, ledger *bans.Ledger,
|
||||
@@ -341,24 +349,30 @@ func newAbuseIPDB(
|
||||
})
|
||||
|
||||
lc.Append(background(abuseIPDB.RunBlacklist))
|
||||
lc.Append(fx.StopHook(abuseIPDB.StopChecking))
|
||||
|
||||
return abuseIPDB
|
||||
}
|
||||
|
||||
// newRouteWatch returns what asks routewatch about clients, or nil while
|
||||
// SWW_WAF_ROUTEWATCH_URL is unset.
|
||||
// SWW_WAF_ROUTEWATCH_URL is unset. At the stop it ends the queries under
|
||||
// way, as routewatch.Client.StopAsking says.
|
||||
func newRouteWatch(
|
||||
cfg *config.WAF, processLog *slog.Logger,
|
||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||
) *routewatch.Client {
|
||||
if cfg.RouteWatchURL == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
return routewatch.New(routewatch.Params{
|
||||
routeWatch := routewatch.New(routewatch.Params{
|
||||
URL: cfg.RouteWatchURL,
|
||||
Now: utcNow,
|
||||
ProcessLog: processLog,
|
||||
})
|
||||
|
||||
lc.Append(fx.StopHook(routeWatch.StopAsking))
|
||||
|
||||
return routeWatch
|
||||
}
|
||||
|
||||
// newMetrics returns the metrics, with those read from the other parts as
|
||||
@@ -396,12 +410,18 @@ func newMetrics(
|
||||
|
||||
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
|
||||
// answer goes to the limiter and the ledger, as waf.AddLookup says, and each
|
||||
// failure is handled as waf.GeoJSFailed says.
|
||||
// failure is handled as waf.GeoJSFailed says. At the stop it ends the request
|
||||
// under way, as geojs.GeoJS.StopAsking says.
|
||||
func newGeoJS(
|
||||
cfg *config.WAF, m *metrics.Metrics, limiter *ratelimit.Limiter,
|
||||
ledger *bans.Ledger, alertQueue *alerts.Queue, processLog *slog.Logger,
|
||||
lc fx.Lifecycle,
|
||||
cfg *config.WAF,
|
||||
m *metrics.Metrics,
|
||||
limiter *ratelimit.Limiter,
|
||||
ledger *bans.Ledger,
|
||||
alertQueue *alerts.Queue,
|
||||
processLog *slog.Logger,
|
||||
) *geojs.GeoJS {
|
||||
return geojs.New(geojs.Params{
|
||||
geoJS := geojs.New(geojs.Params{
|
||||
URL: geojs.URL,
|
||||
Answered: func(answer geojs.Answer) {
|
||||
waf.AddLookup(cfg, limiter, ledger, answer)
|
||||
@@ -413,6 +433,10 @@ func newGeoJS(
|
||||
},
|
||||
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
||||
})
|
||||
|
||||
lc.Append(fx.StopHook(geoJS.StopAsking))
|
||||
|
||||
return geoJS
|
||||
}
|
||||
|
||||
// newWAFServer returns the server waf runs, with its parts and the time
|
||||
|
||||
@@ -53,13 +53,13 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
||||
if country == "" && !rq.skippedLookupWait {
|
||||
unknown = percentage{
|
||||
cfg.UnknownCountryLimitPercent,
|
||||
"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
|
||||
config.UnknownCountryLimitPercentSetting,
|
||||
}
|
||||
}
|
||||
|
||||
fetched := percentage{percent: whole}
|
||||
if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed {
|
||||
fetched = percentage{percent, "SWW_WAF_ASN_LIMIT_PERCENT_URL"}
|
||||
fetched = percentage{percent, config.ASNLimitPercentURLSetting}
|
||||
}
|
||||
|
||||
blocklisted := percentage{percent: whole}
|
||||
@@ -67,7 +67,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
||||
cfg.BlocklistAction == config.ReputationActionLimit {
|
||||
blocklisted = percentage{
|
||||
cfg.BlocklistLimitPercent,
|
||||
"SWW_WAF_BLOCKLIST_ACTION",
|
||||
config.BlocklistActionSetting,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -76,11 +76,11 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
||||
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit {
|
||||
dnsblAbuseIPDBListed = percentage{
|
||||
cfg.DNSBLAbuseIPDBLimitPercent,
|
||||
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
|
||||
config.DNSBLAbuseIPDBActionSetting,
|
||||
}
|
||||
}
|
||||
|
||||
const tagSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
|
||||
const tagSetting = config.TagLimitPercentSetting
|
||||
|
||||
asnRequests := lowest(
|
||||
given(cfg.TagLimitPercent, asnTag(asn), tagSetting),
|
||||
@@ -108,7 +108,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
||||
asnBytes = given(
|
||||
cfg.ASNBytesLimitPercent,
|
||||
asn,
|
||||
"SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
|
||||
config.ASNBytesLimitPercentSetting,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -116,7 +116,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
||||
countryBytes = given(
|
||||
cfg.CountryBytesLimitPercent,
|
||||
country,
|
||||
"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
|
||||
config.CountryBytesLimitPercentSetting,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||
)
|
||||
|
||||
@@ -33,7 +34,8 @@ func (countries) check(
|
||||
return &verdict{
|
||||
action: requestlog.ActionCountryDenied,
|
||||
figures: map[string]any{
|
||||
"country": country, "setting": "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
|
||||
"country": country,
|
||||
"setting": config.ExclusivelyAllowedCountriesSetting,
|
||||
},
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user