Keep the bans, the clients and GeoJS's answers in state files (closes #17)

smallwebwaf now copies its state to bans.json, clients.json and lookups.json in SWWAF_STATE_DIR, as "Persistent state" in SPEC.md describes, and reads them back at start, so a restart lifts no ban and gives no client a fresh allowance. Each client gains a history, and a ban's notes count the netblock's requests. bans.json is written SWWAF_STATE_WRITE_DELAY after a ban, and every file every SWWAF_STATE_COUNTER_INTERVAL and at the stop. A ban read back is masked to its netblock and refuses every client in it. A file that does not parse, an unknown version, an entry without a field it needs, or an unwritable directory stops the start. Deviation: no AS number or name, and no ban cause, reason or lifting yet. Model: opus-5-5
2026-10-06 08:31:52 +02:00
parent 73ca94f850
commit df2c5042d2
27 changed files with 2859 additions and 272 deletions
@@ -1,6 +1,7 @@
 // Package lookup looks up each client's country through the GeoJS web
 // service, and keeps the answers in memory, for at most 100,000 clients
-// and for 7 days each.
+// and for 7 days each. The answers are written to lookups.json and read
+// from it by the state package.
 package lookup

 import (
@@ -12,6 +13,7 @@ import (
 	"log/slog"
 	"net/http"
 	"net/netip"
+	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -76,7 +78,7 @@ type GeoJS struct {
 	httpClient *http.Client

 	mu      sync.Mutex
-	answers *simplelru.LRU[netip.Prefix, answer]
+	answers *simplelru.LRU[netip.Prefix, *Answer]
 	// waiting are the clients without an answer: those to ask GeoJS about,
 	// and those it is being asked about.
 	waiting map[netip.Prefix]*wait
@@ -88,11 +90,14 @@ type GeoJS struct {
 	retryAt    time.Time
 }

-// answer is what GeoJS said about a client: its country, "" when GeoJS
-// cannot place it, and when GeoJS said so.
-type answer struct {
-	country  string
-	received time.Time
+// Answer is what GeoJS said about a client, as lookups.json holds it: its
+// country, "" when GeoJS cannot place it, when GeoJS said so, and when
+// the answer was last used.
+type Answer struct {
+	Client   netip.Prefix `json:"client"`
+	Country  string       `json:"country"`
+	Answered time.Time    `json:"answered"`
+	Used     time.Time    `json:"used"`
 }

 // wait is a client waiting for its answer.
@@ -107,7 +112,7 @@ type wait struct {

 // New returns a GeoJS with no answer kept yet.
 func New(params Params) *GeoJS {
-	answers, err := simplelru.NewLRU[netip.Prefix, answer](maxAnswers, nil)
+	answers, err := simplelru.NewLRU[netip.Prefix, *Answer](maxAnswers, nil)
 	if err != nil {
 		panic(err) // NewLRU fails only for a size below one
 	}
@@ -164,6 +169,47 @@ func (g *GeoJS) Country(ctx context.Context, client netip.Prefix) string {
 	return country
 }

+// Snapshot returns every answer kept, sorted by client, as lookups.json
+// lists them.
+func (g *GeoJS) Snapshot() []Answer {
+	g.mu.Lock()
+
+	answers := make([]Answer, 0, g.answers.Len())
+	for _, kept := range g.answers.Values() {
+		answers = append(answers, *kept)
+	}
+
+	g.mu.Unlock()
+
+	slices.SortFunc(answers, func(a, b Answer) int {
+		return a.Client.Compare(b.Client)
+	})
+
+	return answers
+}
+
+// Load keeps answers read from lookups.json, in a GeoJS that keeps none
+// yet, in the order they were last used, so that the one used longest
+// ago is dropped first. Answers GeoJS gave keepFor ago or more are
+// dropped.
+func (g *GeoJS) Load(answers []Answer) {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+
+	answers = slices.Clone(answers)
+	slices.SortStableFunc(answers, func(a, b Answer) int {
+		return a.Used.Compare(b.Used)
+	})
+
+	now := g.now()
+
+	for _, answer := range answers {
+		if now.Sub(answer.Answered) < keepFor {
+			g.answers.Add(answer.Client, &answer)
+		}
+	}
+}
+
 // answerOrWait returns client's kept answer if it has one. Otherwise it
 // puts the client among those waiting if there is room, has GeoJS asked
 // about them if it can be, and returns what to wait on for the answer, or
@@ -203,15 +249,19 @@ func (g *GeoJS) answerOrWait(
 	return "", w.asked
 }

-// kept returns client's answer, if one was received less than keepFor
-// ago.
+// kept returns client's answer, if GeoJS gave it less than keepFor ago,
+// and notes that it was used.
 func (g *GeoJS) kept(client netip.Prefix) (string, bool) {
+	now := g.now()
+
 	kept, found := g.answers.Get(client)
-	if !found || g.now().Sub(kept.received) >= keepFor {
+	if !found || now.Sub(kept.Answered) >= keepFor {
 		return "", false
 	}

-	return kept.country, true
+	kept.Used = now
+
+	return kept.Country, true
 }

 // ask starts asking GeoJS about the waiting clients, unless a request to
@@ -293,7 +343,9 @@ func (g *GeoJS) keep(
 			continue
 		}

-		g.answers.Add(client, answer{country: country, received: now})
+		g.answers.Add(client, &Answer{
+			Client: client, Country: country, Answered: now, Used: now,
+		})
 		close(g.waiting[client].asked)
 		delete(g.waiting, client)
 	}
@@ -0,0 +1,96 @@
+package lookup_test
+
+import (
+	"net/netip"
+	"slices"
+	"testing"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/lookup"
+)
+
+func TestSnapshotHoldsEachAnswerAndWhenItWasLastUsed(t *testing.T) {
+	t.Parallel()
+
+	_, clock, g := start(t)
+	placed := netip.MustParsePrefix("203.0.113.9/32")
+	notPlaced := netip.MustParsePrefix(unplaced + "/32")
+	asked := clock.Now()
+
+	wantCountry(t, g, placed, germany)
+	wantCountry(t, g, notPlaced, "")
+
+	clock.advance(time.Hour)
+	wantCountry(t, g, placed, germany)
+
+	want := []lookup.Answer{
+		{Client: notPlaced, Country: "", Answered: asked, Used: asked},
+		{Client: placed, Country: germany, Answered: asked, Used: asked.Add(time.Hour)},
+	}
+	if got := g.Snapshot(); !slices.Equal(got, want) {
+		t.Errorf("snapshot\n%+v\nwant\n%+v", got, want)
+	}
+}
+
+func TestLoadedAnswersAreKeptFor7DaysFromWhenGeoJSGaveThem(t *testing.T) {
+	t.Parallel()
+
+	geojs, clock, g := start(t)
+	now := clock.Now()
+	kept := lookup.Answer{
+		Client:   netip.MustParsePrefix("203.0.113.9/32"),
+		Country:  "FR",
+		Answered: now.Add(-week + time.Second),
+		Used:     now.Add(-time.Hour),
+	}
+	stale := lookup.Answer{
+		Client:   netip.MustParsePrefix("203.0.113.10/32"),
+		Country:  "FR",
+		Answered: now.Add(-week),
+		Used:     now.Add(-time.Hour),
+	}
+
+	g.Load([]lookup.Answer{kept, stale})
+
+	if got := g.Snapshot(); !slices.Equal(got, []lookup.Answer{kept}) {
+		t.Errorf("kept %+v, want only the answer GeoJS gave less than 7 days ago", got)
+	}
+
+	wantCountry(t, g, kept.Client, "FR")
+	wantRequests(t, geojs, 0)
+}
+
+func TestLoadDropsTheAnswerUsedLongestAgoFirst(t *testing.T) {
+	t.Parallel()
+
+	const maxAnswers = 100000
+
+	_, clock, g := start(t)
+	now := clock.Now()
+
+	// lookups.json lists the answers by client. Here each was last used a
+	// second before the one listed before it, so the last listed is the
+	// one used longest ago, and the one dropped.
+	answers := make([]lookup.Answer, maxAnswers+1)
+	addr := netip.MustParseAddr("10.0.0.0")
+
+	for i := range answers {
+		answers[i] = lookup.Answer{
+			Client:   netip.PrefixFrom(addr, addr.BitLen()),
+			Country:  germany,
+			Answered: now,
+			Used:     now.Add(-time.Duration(i) * time.Second),
+		}
+		addr = addr.Next()
+	}
+
+	g.Load(answers)
+
+	got := g.Snapshot()
+	if len(got) != maxAnswers || got[0] != answers[0] ||
+		got[maxAnswers-1] != answers[maxAnswers-1] {
+		t.Errorf("%d answers kept, from %s to %s; want %d, from %s to %s",
+			len(got), got[0].Client, got[len(got)-1].Client, maxAnswers,
+			answers[0].Client, answers[maxAnswers-1].Client)
+	}
+}