geojs: a visitor's wait no longer cuts off the request to GeoJS (closes #404)
check / check (push) Failing after 3m22s

The GeoJS lookup and its cache move from internal/lookup to internal/geojs,
which imports no other package of this repository. waf hands it functions
to count requests, report failures and count lookups that went without an
answer; waf keeps the log line, the source_failure alert and the metrics.

The caller now passes its wait to each lookup. waf passes
SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS. A request to GeoJS has its own
fixed limit of 10 seconds, and only passing it or an error from GeoJS is a
failure. An answer that comes after the caller stopped waiting is kept.

internal/lookup keeps the lookup database and returns geojs.Answer.

Model: opus-5-5
This commit is contained in:
sneak committed 2026-11-16 23:18:27 +00:00
1 parent a2fcc00ded
commit 56f93d4f74
21 files changed
+645 -484

No files matched your search

+9 -4
View File
@@ -20,7 +20,7 @@ import (
"github.com/prometheus/client_golang/prometheus/testutil"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/reputation"
@@ -591,7 +591,7 @@ func TestAClientGeoJSPlacedInNoCountryCountsAsUnknownWithoutWaiting(
s := startWAF(t, geojsURL, map[string]string{
unknownCountryLimitPercent: "0", rateLimitPerMinute: "1",
})
s.server.GeoJS.Load([]lookup.Answer{{
s.server.GeoJS.Load([]geojs.Answer{{
Client: netip.MustParsePrefix(client + "/32"), ASN: "AS64512",
Answered: s.clock.Now(), Used: s.clock.Now(),
}})
@@ -770,6 +770,8 @@ func TestARefusedRequestDoesNotWaitForASlowGeoJS(t *testing.T) {
// TestRequestWaitsAsLongAsTheASNAndCountryLookupTimeoutSays says,
// so that a request that does not wait takes no time at all.
synctest.Test(t, func(t *testing.T) {
defer waitForGeoJSRequestsToBeAbandoned()
server, out := newWAFInBubble(t, tc.env)
tc.prepare(t, server, time.Now())
@@ -838,6 +840,8 @@ func TestAQuestionWaitsForASlowGeoJSOnlyAsLongAsTheDefaultSays(
t.Parallel()
synctest.Test(t, func(t *testing.T) {
defer waitForGeoJSRequestsToBeAbandoned()
server, out := newWAFInBubble(t, tc.env)
tc.prepare(t, server, time.Now())
@@ -950,10 +954,11 @@ func TestWAFAnswersWithinTheTimeLimitWhileEveryOutsideSourceHangs(
t.Errorf("%d checks with AbuseIPDB, want 1", checked)
}
// The query is answered, and the check with AbuseIPDB abandoned,
// before the bubble ends.
// The query is answered, and the check with AbuseIPDB and the request
// to GeoJS abandoned, before the bubble ends.
releaseResolver()
time.Sleep(defaultDNSBLAbuseIPDBWait)
waitForGeoJSRequestsToBeAbandoned()
})
}
+4
View File
@@ -339,6 +339,8 @@ func TestClientWithoutAnAnswerInTimeHasTheUnknownCountryLimitPercent(
// TestRequestWaitsAsLongAsTheASNAndCountryLookupTimeoutSays says, with
// a GeoJS that never answers.
synctest.Test(t, func(t *testing.T) {
defer waitForGeoJSRequestsToBeAbandoned()
server, out, _ := newWAF(t, silentGeoJSURL, time.Now,
map[string]string{unknownCountryLimitPercent: "0"})
@@ -390,6 +392,8 @@ func TestRequestWaitsForItsLookupWhileABiasedThresholdIsSet(t *testing.T) {
// with a GeoJS that never answers. The headers, which need the
// answer too, are off.
synctest.Test(t, func(t *testing.T) {
defer waitForGeoJSRequestsToBeAbandoned()
server, out, _ := newWAF(t, silentGeoJSURL, time.Now,
map[string]string{
asnAndCountryLookupTimeout: strconv.FormatInt(
+26 -6
View File
@@ -7,8 +7,9 @@ import (
"strings"
"time"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/geojs"
)
// The headers in which an answer of allow gives rproxy the client's AS number
@@ -26,9 +27,9 @@ const (
// SWW_WAF_LOOKUP_SOURCE is off or the client is on a private, loopback or
// link-local address, which no lookup can place. The lookup database
// answers at once. With GeoJS, while a setting needs the answer, such as a
// country list or a biased threshold, a new client's request waits for it,
// unless refusedWhateverTheAnswer reports true. ctx is the request's own
// context.
// country list or a biased threshold, a new client's request waits for it up
// to SWW_WAF_ASN_AND_COUNTRY_LOOKUP_TIMEOUT_MS, unless refusedWhateverTheAnswer
// reports true. ctx is the request's own context.
func (rq *request) lookUp(ctx context.Context, now time.Time) {
if rq.h.config.LookupSource == "off" || !canBePlaced(rq.client) {
return
@@ -42,8 +43,11 @@ func (rq *request) lookUp(ctx context.Context, now time.Time) {
case rq.refusedWhateverTheAnswer(now):
rq.lookupAnswer = rq.h.geojs.LookUpWithoutWaiting(ctx, client)
rq.skippedLookupWait = rq.lookupAnswer.Answered.IsZero()
case !rq.h.waitForGeoJS:
rq.lookupAnswer = rq.h.geojs.LookUpWithoutWaiting(ctx, client)
default:
rq.lookupAnswer = rq.h.geojs.LookUp(ctx, client)
rq.lookupAnswer = rq.h.geojs.LookUp(ctx, client,
rq.h.config.ASNAndCountryLookupTimeout)
}
rq.lookedUp = true
@@ -83,13 +87,29 @@ func countryTag(country string) string {
// addLookup adds answer, an answer about a client from the lookup
// database or GeoJS, to the client's history, and to the notes of the bans
// on its netblock that have no AS number, AS name or country yet.
func (h *handler) addLookup(answer lookup.Answer) {
func (h *handler) addLookup(answer geojs.Answer) {
h.limiter.AddLookup(answer.Client, answer.Answered,
answer.ASN, answer.ASName, answer.Country)
h.ledger.AddLookup(h.netblock(answer.Client.Addr()),
answer.ASN, answer.ASName, answer.Country)
}
// geojsFailed logs a failure of GeoJS, raises a source_failure alert for it
// and counts it. askingAgainIn is how long GeoJS is left alone after it.
func (h *handler) geojsFailed(err error, askingAgainIn time.Duration) {
h.processLog.Warn("asking GeoJS failed",
"error", err.Error(), "asking_again_in", askingAgainIn.String())
h.alerts.Raise(alerts.Alert{
Event: alerts.EventSourceFailure,
Reason: "asking GeoJS failed",
Detail: map[string]any{
"source": "geojs", "error": err.Error(),
"asking_again_in": askingAgainIn.String(),
},
})
h.metrics.GeoJSFailures.Inc()
}
// setLookupHeaders sets the headers that give the client's AS number and
// country, leaving out one that is unknown.
func setLookupHeaders(header http.Header, asn, country string) {
+17 -4
View File
@@ -12,7 +12,7 @@ import (
"time"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
@@ -138,7 +138,7 @@ func TestRequestThatWaitedForItsLookupIsCountedWhenItGoesOn(t *testing.T) {
// of lookups.json would keep it, while the first request still waits
// for GeoJS. The second request goes on at once, and is counted.
clk.advance(2500 * time.Millisecond)
server.GeoJS.Load([]lookup.Answer{{
server.GeoJS.Load([]geojs.Answer{{
Client: netip.MustParsePrefix(
fromDE + "/32",
), ASN: asnDE, ASName: asNameDE,
@@ -199,7 +199,7 @@ func TestASNumberAndNameInTheLogLineTheHistoryTheBanNotesAndTheAlert(
// The answer is kept before the requests, so GeoJS is not asked, and
// gives no answer of its own.
netblock := netip.MustParsePrefix(fromDE + "/32")
server.GeoJS.Load([]lookup.Answer{{
server.GeoJS.Load([]geojs.Answer{{
Client: netblock, ASN: asnDE, ASName: asNameDE, Country: "DE",
Answered: clk.Now(), Used: clk.Now(),
}})
@@ -249,7 +249,7 @@ func TestLookupSourceOffLooksNoClientUp(t *testing.T) {
)
// Even an answer kept from before is not used.
server.GeoJS.Load([]lookup.Answer{{
server.GeoJS.Load([]geojs.Answer{{
Client: netip.MustParsePrefix(
fromDE + "/32",
), ASN: asnDE, ASName: asNameDE,
@@ -450,6 +450,8 @@ func TestRequestWaitsAsLongAsTheASNAndCountryLookupTimeoutSays(
// network, which would keep that clock from moving on: the request is
// handed to waf's handler, and GeoJS is one that never answers.
synctest.Test(t, func(t *testing.T) {
defer waitForGeoJSRequestsToBeAbandoned()
// Not the default. The exclusive list needs the answer, and the app
// is never reached.
const timeout = 3 * time.Second
@@ -495,6 +497,17 @@ func TestMain(m *testing.M) {
m.Run()
}
// geojsRequestTimeout is how long a request to GeoJS may take before it is
// abandoned.
const geojsRequestTimeout = 10 * time.Second
// waitForGeoJSRequestsToBeAbandoned sleeps until every request to the GeoJS at
// silentGeoJSURL made so far has been abandoned. A synctest bubble that asks
// it defers this: the bubble cannot end while a request waits on it.
func waitForGeoJSRequestsToBeAbandoned() {
time.Sleep(geojsRequestTimeout)
}
// silentGeoJS is the GeoJS at silentGeoJSURL.
type silentGeoJS struct{}
+50 -14
View File
@@ -1,6 +1,7 @@
package waf_test
import (
"fmt"
"io"
"net/http"
"net/http/httptest"
@@ -10,8 +11,9 @@ import (
"testing"
"time"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
@@ -248,7 +250,7 @@ func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
// The answers are kept before the requests, so that none waits for
// GeoJS.
server.GeoJS.Load([]lookup.Answer{
server.GeoJS.Load([]geojs.Answer{
keptAnswer(fromKP, "KP"), keptAnswer(fromDE, "DE"),
keptAnswer(fromFR, "FR"),
})
@@ -329,13 +331,13 @@ func TestMetricsByASNumberKeepTheBusiestAndCountTheRestAsOther(
// The answers are kept before the requests, so that GeoJS gives none
// of its own. Each client is in an AS of its own.
answer := func(addr, asn string) lookup.Answer {
return lookup.Answer{
answer := func(addr, asn string) geojs.Answer {
return geojs.Answer{
Client: netip.MustParsePrefix(addr + "/32"), ASN: asn,
Answered: clk.Now(), Used: clk.Now(),
}
}
server.GeoJS.Load([]lookup.Answer{
server.GeoJS.Load([]geojs.Answer{
answer(fromDE, "AS64501"), answer(fromKP, "AS64502"),
})
@@ -357,21 +359,23 @@ func TestMetricsByASNumberKeepTheBusiestAndCountTheRestAsOther(
`sww_asn_requests_total{asn="AS64502",instance="app"}`)
}
func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) {
func TestGeoJSFailureIsCountedLoggedAndAlerted(t *testing.T) {
t.Parallel()
geojs := httptest.NewServer(http.HandlerFunc(
failingGeoJS := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusServiceUnavailable)
}))
t.Cleanup(geojs.Close)
t.Cleanup(failingGeoJS.Close)
app := startApp(t, func(http.ResponseWriter, *http.Request) {})
addr, _, server := startProxyWithClock(t, app.URL, geojs.URL, time.Now,
addr, out, server, queue := startProxyWithAlerts(t, app.URL,
failingGeoJS.URL, time.Now,
map[string]string{
trustedProxies: trustLocalhost,
metricsToken: token,
tagDeny: denyKP,
trustedProxies: trustLocalhost,
metricsToken: token,
tagDeny: denyKP,
alertWebhookURL: "https://alerts.example/waf",
})
// GeoJS fails, so the client has no country= tag, and SWW_WAF_TAG_DENY does
@@ -401,14 +405,46 @@ func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) {
`sww_requests_without_geojs_answer_total{instance="app"}`,
1,
)
// The process log's line names the failure but not the address asked
// about.
logged := ""
for line := range strings.Lines(out.text()) {
if strings.Contains(line, "asking GeoJS failed") {
logged = line
}
}
if logged == "" || strings.Contains(logged, fromDE) {
t.Errorf(
"logged %q, want the failure without the address asked about",
logged,
)
}
// fmt prints a map sorted by its keys.
const want = "map[asking_again_in:1s " +
"error:GeoJS answered 503 Service Unavailable source:geojs]"
waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
if len(waiting) != 1 || waiting[0].Event != alerts.EventSourceFailure ||
waiting[0].Reason != "asking GeoJS failed" ||
fmt.Sprint(waiting[0].Detail) != want {
t.Errorf(
"alerts waiting %+v, want one source_failure alert with %v",
waiting,
want,
)
}
}
// keptAnswer returns GeoJS's answer that the client at addr is in
// country, given now.
func keptAnswer(addr, country string) lookup.Answer {
func keptAnswer(addr, country string) geojs.Answer {
now := time.Now()
return lookup.Answer{
return geojs.Answer{
Client: netip.MustParsePrefix(addr + "/32"), Country: country,
Answered: now, Used: now,
}
+20 -17
View File
@@ -17,6 +17,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
@@ -55,7 +56,7 @@ type Params struct {
// ProcessLog receives the process's own messages.
ProcessLog *slog.Logger
// GeoJSURL is where clients' AS numbers and countries are looked up
// while SWW_WAF_LOOKUP_SOURCE is geojs, normally lookup.URL.
// while SWW_WAF_LOOKUP_SOURCE is geojs, normally geojs.URL.
GeoJSURL string
// AbuseIPDBURL is where clients are checked with AbuseIPDB while
// SWW_WAF_ABUSEIPDB_KEY is set, normally reputation.AbuseIPDBURL.
@@ -93,7 +94,7 @@ type Server struct {
Ledger *bans.Ledger
Limiter *ratelimit.Limiter
GeoJS *lookup.GeoJS
GeoJS *geojs.GeoJS
Anomalies *anomaly.Counters
LookupFile *lookup.File
Lists *reputation.Lists
@@ -164,20 +165,19 @@ func newHandler(params Params) *handler {
})
}
h.geojs = lookup.New(lookup.Params{
URL: params.GeoJSURL,
Timeout: params.Config.ASNAndCountryLookupTimeout,
// The country list, the headers, the tag settings that name an asn=
// or country= tag and the biased thresholds act on the answer before
// the request goes on.
Wait: len(params.Config.ExclusivelyAllowedCountries) > 0 ||
params.Config.AddLookupHeaders || params.Config.TagsNeedLookup() ||
biasedThresholdsSet(params.Config),
Answered: h.addLookup,
Now: params.Now,
ProcessLog: params.ProcessLog,
Metrics: m,
Alerts: params.Alerts,
// The country list, the headers, the tag settings that name an asn= or
// country= tag and the biased thresholds act on the answer before the
// request goes on.
h.waitForGeoJS = len(params.Config.ExclusivelyAllowedCountries) > 0 ||
params.Config.AddLookupHeaders || params.Config.TagsNeedLookup() ||
biasedThresholdsSet(params.Config)
h.geojs = geojs.New(geojs.Params{
URL: params.GeoJSURL,
Answered: h.addLookup,
Now: params.Now,
Asked: m.GeoJSRequests.Inc,
Failed: h.geojsFailed,
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
})
m.AddBansAndClients(h.ledger, h.limiter, params.Now)
m.AddRules(params.Rules)
@@ -268,7 +268,7 @@ type handler struct {
metrics *metrics.Metrics
limiter *ratelimit.Limiter
ledger *bans.Ledger
geojs *lookup.GeoJS
geojs *geojs.GeoJS
anomalies *anomaly.Counters
lookupFile *lookup.File
lists *reputation.Lists
@@ -278,6 +278,9 @@ type handler struct {
rules *rules.Files
coreRuleSet *coreruleset.CoreRuleSet
alerts *alerts.Queue
// waitForGeoJS is true when a setting needs each request's answer from
// GeoJS before the request goes on.
waitForGeoJS bool
// awaitingReport holds the decision on each answered question by its id
// until its report comes, and nil for a question still being handled.
awaitingReport *lru.Cache[string, *decision]
+2 -2
View File
@@ -5,7 +5,7 @@ import (
"sync/atomic"
"testing"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
)
@@ -162,7 +162,7 @@ func TestRateLimitExemptPathsAreNeitherCountedNorRefused(t *testing.T) {
// The answers are kept before the requests, so that none waits for
// GeoJS.
server.GeoJS.Load([]lookup.Answer{
server.GeoJS.Load([]geojs.Answer{
keptAnswer(client, "DE"), keptAnswer(fromKP, "KP"),
})
+4 -4
View File
@@ -11,7 +11,7 @@ import (
"sneak.berlin/go/smallwebwaf/internal/anomaly"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/rules"
@@ -42,7 +42,7 @@ type decision struct {
// looked up, whether or not an answer was there, and lookupAnswer is
// what the lookup gave then, the zero Answer while GeoJS had given none.
lookedUp bool
lookupAnswer lookup.Answer
lookupAnswer geojs.Answer
// skippedLookupWait is true for a request that did not wait for GeoJS and
// found no answer kept. SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT does not apply to it,
// so that not waiting never lowers its client's limits.
@@ -382,9 +382,9 @@ func anomalyThresholdsSet(cfg *config.WAF) bool {
// answer about it as the request ends, and whether there is one: the
// lookup database's, which was there at once, or the one GeoJS has given
// by then, which a request does not wait for unless a setting needs it.
func (rq *request) answerAtTheEnd() (lookup.Answer, bool) {
func (rq *request) answerAtTheEnd() (geojs.Answer, bool) {
if !rq.lookedUp {
return lookup.Answer{}, false
return geojs.Answer{}, false
}
if rq.h.config.LookupSource == "file" {