sneak/smallwebwaf
1
0
Fork 0
Code Issues 32 Pull Requests 3 Actions 24 Packages Projects Releases Wiki Activity

Milestones 1 and 2: proxy with timeouts and size limits, rate limits, country lists and the image #40

Open
clawbot wants to merge 25 commits from next into main
pull from: next
merge into: :main
Conversation 0 Commits 25 Files Changed 83
+283 -242
3 changed files with 283 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 68f687cb0c - Show all commits
internal/lookup/export_test.go
+9
View File
@@ -0,0 +1,9 @@
package lookup
import "net/http"
// SetTransport has g's requests to GeoJS go through transport instead of
// the network.
func (g *GeoJS) SetTransport(transport http.RoundTripper) {
g.httpClient.Transport = transport
}
internal/lookup/lookup_test.go
+254 -225
View File
@@ -10,6 +10,7 @@ import (
"strings"
"sync"
"testing"
"testing/synctest"
"time"
"sneak.berlin/go/smallwebwaf/internal/lookup"
@@ -26,80 +27,87 @@ const (
leftOut = "203.0.113.7"
// timeout is how long a new client waits for its answer.
timeout = time.Second
// waitLimit bounds how long a test waits for what should happen.
waitLimit = 10 * time.Second
// pollInterval is how often a test looks again.
pollInterval = 10 * time.Millisecond
// week is how long an answer is kept.
week = 7 * 24 * time.Hour
)
// The tests that have GeoJS asked run in a synctest bubble, where the time
// package runs on a clock of the test's own: a wait lasts exactly as long
// as it should, however slowly the test process runs, and synctest.Wait
// returns once g has done all it can before time passes. The stand-in for
// GeoJS answers without the network, since a request waiting on the
// network would keep that clock from moving on.
func TestKeptAnswerIsUsedFor7DaysThenAskedAgain(t *testing.T) {
t.Parallel()
geojs, clock, g := start(t)
placed := netip.MustParsePrefix("203.0.113.9/32")
notPlaced := netip.MustParsePrefix(unplaced + "/32")
synctest.Test(t, func(t *testing.T) {
geojs, clock, g := start()
placed := netip.MustParsePrefix("203.0.113.9/32")
notPlaced := netip.MustParsePrefix(unplaced + "/32")
wantCountry(t, g, placed, germany)
wantCountry(t, g, notPlaced, "")
wantRequests(t, geojs, 2)
wantCountry(t, g, placed, germany)
wantCountry(t, g, notPlaced, "")
wantRequests(t, geojs, 2)
// An answer without a country is kept too.
clock.advance(week - time.Second)
wantCountry(t, g, placed, germany)
wantCountry(t, g, notPlaced, "")
wantRequests(t, geojs, 2)
// An answer without a country is kept too.
clock.advance(week - time.Second)
wantCountry(t, g, placed, germany)
wantCountry(t, g, notPlaced, "")
wantRequests(t, geojs, 2)
clock.advance(time.Second)
wantCountry(t, g, placed, germany)
wantRequests(t, geojs, 3)
wantAsked(t, geojs, 2, "203.0.113.9")
clock.advance(time.Second)
wantCountry(t, g, placed, germany)
wantRequests(t, geojs, 3)
wantAsked(t, geojs, 2, "203.0.113.9")
})
}
func TestNewClientWaitsAtMostOneSecondThenCountsAsNotFound(t *testing.T) {
t.Parallel()
geojs, clock, g := start(t)
client := netip.MustParsePrefix("203.0.113.9/32")
synctest.Test(t, func(t *testing.T) {
geojs, clock, g := start()
client := netip.MustParsePrefix("203.0.113.9/32")
// The client comes while GeoJS is asked about an earlier client, which
// it answers most of a second later. It is then asked about the client
// and does not answer: that request is abandoned a second after it
// began, well after the client's wait is over.
geojs.set(answeringSlowly)
// The client comes while GeoJS is asked about an earlier client, which
// it answers most of a second later. It is then asked about the client
// and does not answer: that request is abandoned a second after it
// began, well after the client's wait is over.
geojs.set(answeringSlowly)
var earlier sync.WaitGroup
var earlier sync.WaitGroup
earlier.Go(func() { g.Country(t.Context(), netip.MustParsePrefix("203.0.113.1/32")) })
defer earlier.Wait()
earlier.Go(func() { g.Country(t.Context(), netip.MustParsePrefix("203.0.113.1/32")) })
defer earlier.Wait()
waitForRequests(t, geojs, 1)
geojs.set(hanging)
waitForRequests(t, geojs, 1)
geojs.set(hanging)
began := time.Now()
began := time.Now()
wantCountry(t, g, client, "")
wantCountry(t, g, client, "")
took := time.Since(began)
if took < timeout || took > timeout+timeout/2 {
t.Errorf("waited %s for the answer, want %s", took, timeout)
}
took := time.Since(began)
if took != timeout {
t.Errorf("waited %s for the answer, want %s", took, timeout)
}
// Its next request does not wait.
began = time.Now()
// Its next request does not wait.
began = time.Now()
wantCountry(t, g, client, "")
wantCountry(t, g, client, "")
took = time.Since(began)
if took > timeout/2 {
t.Errorf("waited %s again, want no wait", took)
}
took = time.Since(began)
if took != 0 {
t.Errorf("waited %s again, want no wait", took)
}
// Once GeoJS answers, the client is asked about again in the
// background, and has its country.
geojs.set(answering)
waitForCountry(t, g, clock, client, germany)
// Once GeoJS answers, the client is asked about again in the
// background, and has its country.
geojs.set(answering)
waitForCountry(t, g, clock, client, germany)
})
}
func TestAddressLeftOutOfAnAnswerIsAskedAboutAgain(t *testing.T) {
@@ -118,33 +126,35 @@ func TestAddressLeftOutOfAnAnswerIsAskedAboutAgain(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
geojs, clock, g := start(t)
other := netip.MustParsePrefix("203.0.113.1/32")
client := netip.MustParsePrefix(leftOut + "/32")
synctest.Test(t, func(t *testing.T) {
geojs, clock, g := start()
other := netip.MustParsePrefix("203.0.113.1/32")
client := netip.MustParsePrefix(leftOut + "/32")
// GeoJS fails, and is left alone for a second while the client
// comes too, so that the next request asks about both.
geojs.set(failing)
wantCountry(t, g, other, "")
wantCountry(t, g, client, "")
// GeoJS fails, and is left alone for a second while the client
// comes too, so that the next request asks about both.
geojs.set(failing)
wantCountry(t, g, other, "")
wantCountry(t, g, client, "")
geojs.set(tc.answers)
clock.advance(time.Second)
wantCountry(t, g, other, "")
waitForRequests(t, geojs, 2)
geojs.set(tc.answers)
clock.advance(time.Second)
wantCountry(t, g, other, "")
waitForRequests(t, geojs, 2)
// The answer counts as a failure, and the client is asked about
// again, with the other client only if the answer left it out too.
geojs.set(answering)
waitForCountry(t, g, clock, client, germany)
wantCountry(t, g, other, germany)
wantRequests(t, geojs, 3)
// The answer counts as a failure, and the client is asked about
// again, with the other client only if the answer left it out too.
geojs.set(answering)
waitForCountry(t, g, clock, client, germany)
wantCountry(t, g, other, germany)
wantRequests(t, geojs, 3)
if tc.named {
wantAsked(t, geojs, 2, leftOut)
} else {
wantAsked(t, geojs, 2, leftOut, "203.0.113.1")
}
if tc.named {
wantAsked(t, geojs, 2, leftOut)
} else {
wantAsked(t, geojs, 2, leftOut, "203.0.113.1")
}
})
})
}
}
@@ -152,173 +162,190 @@ func TestAddressLeftOutOfAnAnswerIsAskedAboutAgain(t *testing.T) {
func TestRedirectCountsAsFailure(t *testing.T) {
t.Parallel()
geojs, _, g := start(t)
geojs.set(redirecting)
synctest.Test(t, func(t *testing.T) {
geojs, _, g := start()
geojs.set(redirecting)
wantCountry(t, g, netip.MustParsePrefix("203.0.113.9/32"), "")
wantRequests(t, geojs, 1)
wantCountry(t, g, netip.MustParsePrefix("203.0.113.9/32"), "")
wantRequests(t, geojs, 1)
})
}
func TestCountryIsKeptInCapitals(t *testing.T) {
t.Parallel()
geojs, _, g := start(t)
geojs.set(answeringInLowerCase)
synctest.Test(t, func(t *testing.T) {
geojs, _, g := start()
geojs.set(answeringInLowerCase)
wantCountry(t, g, netip.MustParsePrefix("203.0.113.9/32"), germany)
wantCountry(t, g, netip.MustParsePrefix("203.0.113.9/32"), germany)
})
}
func TestFailureIsLoggedWithoutTheAddressesAskedAbout(t *testing.T) {
t.Parallel()
var log strings.Builder
synctest.Test(t, func(t *testing.T) {
var log strings.Builder
// Nothing listens on port 1, so asking GeoJS fails.
g := lookup.New(lookup.Params{
URL: "http://127.0.0.1:1",
Now: time.Now,
ProcessLog: slog.New(slog.NewTextHandler(&log, nil)),
// GeoJS does not answer, so the request to it is abandoned, and fails.
geojs := &standIn{answers: hanging}
g := lookup.New(lookup.Params{
URL: lookup.URL,
Now: time.Now,
ProcessLog: slog.New(slog.NewTextHandler(&log, nil)),
})
g.SetTransport(geojs)
wantCountry(t, g, netip.MustParsePrefix("203.0.113.9/32"), "")
synctest.Wait()
logged := log.String()
if !strings.Contains(logged, "asking GeoJS failed") ||
strings.Contains(logged, "203.0.113.9") {
t.Errorf("logged %q, want the failure without the address asked about", logged)
}
})
wantCountry(t, g, netip.MustParsePrefix("203.0.113.9/32"), "")
logged := log.String()
if !strings.Contains(logged, "asking GeoJS failed") ||
strings.Contains(logged, "203.0.113.9") {
t.Errorf("logged %q, want the failure without the address asked about", logged)
}
}
func TestWaitingClientsAreAskedAboutInOneRequest(t *testing.T) {
t.Parallel()
geojs, clock, g := start(t)
synctest.Test(t, func(t *testing.T) {
geojs, clock, g := start()
// GeoJS fails, and is then left alone for a second, while three more
// clients come. An IPv6 client is a /64, and GeoJS is asked about its
// first address.
geojs.set(failing)
wantCountry(t, g, netip.MustParsePrefix("203.0.113.1/32"), "")
wantCountry(t, g, netip.MustParsePrefix("203.0.113.2/32"), "")
wantCountry(t, g, netip.MustParsePrefix("2001:db8:1:2::/64"), "")
wantRequests(t, geojs, 1)
// GeoJS fails, and is then left alone for a second, while three more
// clients come. An IPv6 client is a /64, and GeoJS is asked about its
// first address.
geojs.set(failing)
wantCountry(t, g, netip.MustParsePrefix("203.0.113.1/32"), "")
wantCountry(t, g, netip.MustParsePrefix("203.0.113.2/32"), "")
wantCountry(t, g, netip.MustParsePrefix("2001:db8:1:2::/64"), "")
wantRequests(t, geojs, 1)
geojs.set(answering)
clock.advance(time.Second)
wantCountry(t, g, netip.MustParsePrefix("203.0.113.3/32"), germany)
wantRequests(t, geojs, 2)
wantAsked(t, geojs, 1, "203.0.113.1", "203.0.113.2", "2001:db8:1:2::", "203.0.113.3")
geojs.set(answering)
clock.advance(time.Second)
wantCountry(t, g, netip.MustParsePrefix("203.0.113.3/32"), germany)
wantRequests(t, geojs, 2)
wantAsked(t, geojs, 1, "203.0.113.1", "203.0.113.2", "2001:db8:1:2::", "203.0.113.3")
})
}
func TestKeptAnswersUnaffectedWhileGeoJSFailsAndAskedAgainWithBackoff(t *testing.T) {
t.Parallel()
geojs, clock, g := start(t)
clients := newClients()
kept := clients()
synctest.Test(t, func(t *testing.T) {
geojs, clock, g := start()
clients := newClients()
kept := clients()
wantCountry(t, g, kept, germany)
geojs.set(failing)
wantCountry(t, g, kept, germany)
wantRequests(t, geojs, 1)
// Each failure leaves GeoJS alone twice as long as the one before, up
// to five minutes. New clients meanwhile count as not found, and the
// client with a kept answer still gets its country, without GeoJS being
// asked.
requests := 1
for _, delay := range []time.Duration{
time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second,
16 * time.Second, 32 * time.Second, 64 * time.Second, 128 * time.Second,
256 * time.Second, 5 * time.Minute, 5 * time.Minute,
} {
wantCountry(t, g, clients(), "")
requests++
wantRequests(t, geojs, requests)
clock.advance(delay - time.Millisecond)
wantCountry(t, g, clients(), "")
wantCountry(t, g, kept, germany)
wantRequests(t, geojs, requests)
clock.advance(time.Millisecond)
}
geojs.set(failing)
wantCountry(t, g, kept, germany)
wantRequests(t, geojs, 1)
// Once GeoJS answers again, it is asked about every client waiting.
geojs.set(answering)
wantCountry(t, g, clients(), germany)
wantRequests(t, geojs, requests+1)
// Each failure leaves GeoJS alone twice as long as the one before, up
// to five minutes. New clients meanwhile count as not found, and the
// client with a kept answer still gets its country, without GeoJS being
// asked.
requests := 1
asked := waitForRequests(t, geojs, requests+1)
if len(asked[requests]) != 23 {
t.Errorf("GeoJS was asked about %d clients, want 23", len(asked[requests]))
}
for _, delay := range []time.Duration{
time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second,
16 * time.Second, 32 * time.Second, 64 * time.Second, 128 * time.Second,
256 * time.Second, 5 * time.Minute, 5 * time.Minute,
} {
wantCountry(t, g, clients(), "")
requests++
wantRequests(t, geojs, requests)
clock.advance(delay - time.Millisecond)
wantCountry(t, g, clients(), "")
wantCountry(t, g, kept, germany)
wantRequests(t, geojs, requests)
clock.advance(time.Millisecond)
}
// Once GeoJS answers again, it is asked about every client waiting.
geojs.set(answering)
wantCountry(t, g, clients(), germany)
wantRequests(t, geojs, requests+1)
asked := waitForRequests(t, geojs, requests+1)
if len(asked[requests]) != 23 {
t.Errorf("GeoJS was asked about %d clients, want 23", len(asked[requests]))
}
})
}
func TestAtMost200AddressesInOneRequest(t *testing.T) {
t.Parallel()
geojs, clock, g := start(t)
clients := newClients()
first := clients()
synctest.Test(t, func(t *testing.T) {
geojs, clock, g := start()
clients := newClients()
first := clients()
// 201 clients wait while GeoJS is left alone after a failure.
geojs.set(failing)
wantCountry(t, g, first, "")
// 201 clients wait while GeoJS is left alone after a failure.
geojs.set(failing)
wantCountry(t, g, first, "")
for range 200 {
wantCountry(t, g, clients(), "")
}
for range 200 {
wantCountry(t, g, clients(), "")
}
// The first one's next request has GeoJS asked again.
geojs.set(answering)
clock.advance(time.Second)
wantCountry(t, g, first, "")
// The first one's next request has GeoJS asked again.
geojs.set(answering)
clock.advance(time.Second)
wantCountry(t, g, first, "")
asked := waitForRequests(t, geojs, 3)
if len(asked[1]) != 200 || len(asked[2]) != 1 {
t.Errorf("GeoJS was asked about %d and then %d clients, want 200 and 1",
len(asked[1]), len(asked[2]))
}
asked := waitForRequests(t, geojs, 3)
if len(asked[1]) != 200 || len(asked[2]) != 1 {
t.Errorf("GeoJS was asked about %d and then %d clients, want 200 and 1",
len(asked[1]), len(asked[2]))
}
})
}
func TestAtMost10000ClientsWait(t *testing.T) {
t.Parallel()
geojs, clock, g := start(t)
clients := newClients()
first := clients()
synctest.Test(t, func(t *testing.T) {
geojs, clock, g := start()
clients := newClients()
first := clients()
// 10,000 clients wait while GeoJS is left alone after a failure, and
// one more cannot join them.
geojs.set(failing)
wantCountry(t, g, first, "")
// 10,000 clients wait while GeoJS is left alone after a failure, and
// one more cannot join them.
geojs.set(failing)
wantCountry(t, g, first, "")
for range 9999 {
wantCountry(t, g, clients(), "")
}
extra := clients()
wantCountry(t, g, extra, "")
// The first one's next request has GeoJS asked about the 10,000, 200
// at a time, and not about the one more.
geojs.set(answering)
clock.advance(time.Second)
wantCountry(t, g, first, "")
asked := waitForRequests(t, geojs, 51)
for i, request := range asked {
if slices.Contains(request, extra.Addr().String()) {
t.Errorf("request %d asked about %s", i, extra.Addr())
for range 9999 {
wantCountry(t, g, clients(), "")
}
}
// With room among those waiting, it is asked about.
wantCountry(t, g, extra, germany)
extra := clients()
wantCountry(t, g, extra, "")
// The first one's next request has GeoJS asked about the 10,000, 200
// at a time, and not about the one more.
geojs.set(answering)
clock.advance(time.Second)
wantCountry(t, g, first, "")
asked := waitForRequests(t, geojs, 51)
for i, request := range asked {
if slices.Contains(request, extra.Addr().String()) {
t.Errorf("request %d asked about %s", i, extra.Addr())
}
}
// With room among those waiting, it is asked about.
wantCountry(t, g, extra, germany)
})
}
// How the stand-in for GeoJS answers.
@@ -337,13 +364,25 @@ const (
// standIn is a stand-in for GeoJS. It notes the addresses each request
// asks about.
type standIn struct {
server *httptest.Server
mu sync.Mutex
answers int
requests [][]string
}
// RoundTrip has the stand-in answer req, in place of the network. A request
// abandoned before the stand-in answers fails, as over the network.
func (s *standIn) RoundTrip(req *http.Request) (*http.Response, error) {
answer := httptest.NewRecorder()
s.ServeHTTP(answer, req)
err := req.Context().Err()
if err != nil {
return nil, err
}
return answer.Result(), nil
}
// ServeHTTP answers a request about the addresses in its ip parameter.
func (s *standIn) ServeHTTP(w http.ResponseWriter, r *http.Request) {
addrs := strings.Split(r.URL.Query().Get("ip"), ",")
@@ -422,7 +461,8 @@ func (s *standIn) asked() [][]string {
return slices.Clone(s.requests)
}
// testClock is a clock the test sets.
// testClock is a clock the test sets. GeoJS tells the time by it, while
// waits run on the bubble's clock.
type testClock struct {
mu sync.Mutex
now time.Time
@@ -444,21 +484,17 @@ func (c *testClock) advance(d time.Duration) {
c.now = c.now.Add(d)
}
// start starts a stand-in for GeoJS that answers, and returns it, a
// clock, and a GeoJS asking it by that clock.
func start(t *testing.T) (*standIn, *testClock, *lookup.GeoJS) {
t.Helper()
// start returns a stand-in for GeoJS that answers, a clock, and a GeoJS
// asking the stand-in by that clock.
func start() (*standIn, *testClock, *lookup.GeoJS) {
geojs := &standIn{}
geojs.server = httptest.NewServer(geojs)
t.Cleanup(geojs.server.Close)
clock := &testClock{now: time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC)}
g := lookup.New(lookup.Params{
URL: geojs.server.URL,
URL: lookup.URL,
Now: clock.Now,
ProcessLog: slog.New(slog.DiscardHandler),
})
g.SetTransport(geojs)
return geojs, clock, g
}
@@ -513,41 +549,34 @@ func wantAsked(t *testing.T, geojs *standIn, i int, want ...string) {
}
}
// waitForRequests waits for GeoJS to have had count requests, and returns
// the addresses each asked about.
// waitForRequests waits until g has done all it can before time passes,
// checks that GeoJS has had count requests, and returns the addresses each
// asked about.
func waitForRequests(t *testing.T, geojs *standIn, count int) [][]string {
t.Helper()
deadline := time.Now().Add(waitLimit)
for time.Now().Before(deadline) {
asked := geojs.asked()
if len(asked) >= count {
return asked
}
synctest.Wait()
time.Sleep(pollInterval)
asked := geojs.asked()
if len(asked) != count {
t.Fatalf("GeoJS had %d requests, want %d", len(asked), count)
}
t.Fatalf("fewer than %d requests to GeoJS after %s", count, waitLimit)
return nil
return asked
}
// waitForCountry waits for g to give client the country want, moving the
// clock on a minute at a time, so that GeoJS is asked again after a
// failure.
// waitForCountry lets a request to GeoJS under way be abandoned, and moves
// the clock on a minute, so that GeoJS may be asked again after a failure.
// It then checks that client's next request does not wait but has it asked
// about again in the background, after which g gives it the country want.
func waitForCountry(
t *testing.T, g *lookup.GeoJS, clock *testClock, client netip.Prefix, want string,
) {
t.Helper()
deadline := time.Now().Add(waitLimit)
for g.Country(t.Context(), client) != want {
if time.Now().After(deadline) {
t.Fatalf("%s is not in %q after %s", client, want, waitLimit)
}
clock.advance(time.Minute)
time.Sleep(pollInterval)
}
time.Sleep(timeout)
clock.advance(time.Minute)
wantCountry(t, g, client, "")
synctest.Wait()
wantCountry(t, g, client, want)
}
internal/lookup/snapshot_test.go
+20 -17
View File
@@ -4,6 +4,7 @@ import (
"net/netip"
"slices"
"testing"
"testing/synctest"
"time"
"sneak.berlin/go/smallwebwaf/internal/lookup"
@@ -12,30 +13,32 @@ import (
func TestSnapshotHoldsEachAnswerAndWhenItWasLastUsed(t *testing.T) {
t.Parallel()
_, clock, g := start(t)
placed := netip.MustParsePrefix("203.0.113.9/32")
notPlaced := netip.MustParsePrefix(unplaced + "/32")
asked := clock.Now()
synctest.Test(t, func(t *testing.T) {
_, clock, g := start()
placed := netip.MustParsePrefix("203.0.113.9/32")
notPlaced := netip.MustParsePrefix(unplaced + "/32")
asked := clock.Now()
wantCountry(t, g, placed, germany)
wantCountry(t, g, notPlaced, "")
wantCountry(t, g, placed, germany)
wantCountry(t, g, notPlaced, "")
clock.advance(time.Hour)
wantCountry(t, g, placed, germany)
clock.advance(time.Hour)
wantCountry(t, g, placed, germany)
want := []lookup.Answer{
{Client: notPlaced, Country: "", Answered: asked, Used: asked},
{Client: placed, Country: germany, Answered: asked, Used: asked.Add(time.Hour)},
}
if got := g.Snapshot(); !slices.Equal(got, want) {
t.Errorf("snapshot\n%+v\nwant\n%+v", got, want)
}
want := []lookup.Answer{
{Client: notPlaced, Country: "", Answered: asked, Used: asked},
{Client: placed, Country: germany, Answered: asked, Used: asked.Add(time.Hour)},
}
if got := g.Snapshot(); !slices.Equal(got, want) {
t.Errorf("snapshot\n%+v\nwant\n%+v", got, want)
}
})
}
func TestLoadedAnswersAreKeptFor7DaysFromWhenGeoJSGaveThem(t *testing.T) {
t.Parallel()
geojs, clock, g := start(t)
geojs, clock, g := start()
now := clock.Now()
kept := lookup.Answer{
Client: netip.MustParsePrefix("203.0.113.9/32"),
@@ -65,7 +68,7 @@ func TestLoadDropsTheAnswerUsedLongestAgoFirst(t *testing.T) {
const maxAnswers = 100000
_, clock, g := start(t)
_, clock, g := start()
now := clock.Now()
// lookups.json lists the answers by client. Here each was last used a