Caches of outside APIs live in /var/cache/sww (closes #314)

The answers of GeoJS, the lists fetched by URL, the DNS blocklists and
AbuseIPDB each go in a cache file in SWW_CACHE_DIR (default
/var/cache/sww): lookups.json, lists.json, dnsbl.json and
abuseipdb-scores.json. reputation.json is gone. The AbuseIPDB checks
spent today are a count, not a cache, and stay in SWW_STATE_DIR as
abuseipdb-checks.json.

A cache file never stops the start: one that does not parse is set
aside as an edit is, one that cannot be read is written over, and its
cache starts empty. An unwritable SWW_CACHE_DIR stops the start, as an
unwritable SWW_STATE_DIR does. The run script makes the cache directory
and gives it to sww.

Model: opus-5-5

internal/reputation/abuseipdb.go

@@ -36,22 +36,21 @@ var (
 		"checks spent; none is made until the day ends at 00:00 UTC")
 )
 
-// Score is what AbuseIPDB said about a client, as reputation.json holds
-// it: the client, an IPv4 address or an IPv6 group, its abuse confidence
-// score, from 0 to 100, and when AbuseIPDB answered.
+// Score is what AbuseIPDB said about a client, as abuseipdb-scores.json
+// holds it: the client, an IPv4 address or an IPv6 group, its abuse
+// confidence score, from 0 to 100, and when AbuseIPDB answered.
 type Score struct {
 	Client  netip.Prefix `json:"client"`
 	Score   int64        `json:"score"`
 	Fetched time.Time    `json:"fetched"`
 }
 
-// Checks are what reputation.json keeps of the checks of clients with
-// AbuseIPDB: the day, in UTC, of the checks Spent counts, zero before the
-// first, and the scores still in use.
-type Checks struct {
-	Day    civil.Date `json:"day,omitzero"`
-	Spent  int        `json:"spent"`
-	Scores []Score    `json:"scores"`
+// ChecksSpent is what abuseipdb-checks.json keeps of the checks of
+// clients with AbuseIPDB: the day, in UTC, of the checks Spent counts,
+// zero before the first.
+type ChecksSpent struct {
+	Day   civil.Date
+	Spent int
 }
 
 // AbuseIPDBParams are what NewAbuseIPDB needs.
@@ -196,35 +195,35 @@ func (a *AbuseIPDB) BudgetLeft() int {
 	return max(a.params.DailyBudget-a.spent, 0)
 }
 
-// Snapshot returns the checks spent and every score still in use, sorted
-// by client, as reputation.json keeps them.
-func (a *AbuseIPDB) Snapshot() Checks {
+// Snapshot returns every score still in use, sorted by client, as
+// abuseipdb-scores.json keeps them.
+func (a *AbuseIPDB) Snapshot() []Score {
 	a.mu.Lock()
 
 	now := a.params.Now()
-	checks := Checks{Day: a.day, Spent: a.spent, Scores: make([]Score, 0, a.scores.Len())}
+	scores := make([]Score, 0, a.scores.Len())
 
 	for _, kept := range a.scores.Values() {
 		if now.Sub(kept.Fetched) < a.params.CacheTTL {
-			checks.Scores = append(checks.Scores, kept)
+			scores = append(scores, kept)
 		}
 	}
 
 	a.mu.Unlock()
 
-	slices.SortFunc(checks.Scores, func(x, y Score) int {
+	slices.SortFunc(scores, func(x, y Score) int {
 		return x.Client.Compare(y.Client)
 	})
 
-	return checks
+	return scores
 }
 
-// Load keeps checks, read from reputation.json, in place of those it
-// keeps, but for the scores past maxVerdicts, those fetched longest ago.
-// One fetched CacheTTL ago or more is neither used nor written, as for any
-// score.
-func (a *AbuseIPDB) Load(checks Checks) {
-	scores := slices.Clone(checks.Scores)
+// Load keeps scores, read from abuseipdb-scores.json, in place of those
+// it keeps, but for the scores past maxVerdicts, those fetched longest
+// ago. One fetched CacheTTL ago or more is neither used nor written, as
+// for any score.
+func (a *AbuseIPDB) Load(scores []Score) {
+	scores = slices.Clone(scores)
 	slices.SortStableFunc(scores, func(x, y Score) int {
 		return x.Fetched.Compare(y.Fetched)
 	})
@@ -232,7 +231,6 @@ func (a *AbuseIPDB) Load(checks Checks) {
 	a.mu.Lock()
 	defer a.mu.Unlock()
 
-	a.day, a.spent = checks.Day, checks.Spent
 	a.scores.Purge()
 
 	for _, kept := range scores {
@@ -240,6 +238,24 @@ func (a *AbuseIPDB) Load(checks Checks) {
 	}
 }
 
+// ChecksSpent returns the checks spent, as abuseipdb-checks.json keeps
+// them.
+func (a *AbuseIPDB) ChecksSpent() ChecksSpent {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	return ChecksSpent{Day: a.day, Spent: a.spent}
+}
+
+// LoadChecksSpent keeps checks, read from abuseipdb-checks.json, in place
+// of the checks spent.
+func (a *AbuseIPDB) LoadChecksSpent(checks ChecksSpent) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	a.day, a.spent = checks.Day, checks.Spent
+}
+
 // check checks client with AbuseIPDB by addr, one of its addresses, keeps
 // the score as client's, and notes the check as no longer under way. A
 // check that fails gives no score: it is counted, logged and raised as a

internal/reputation/abuseipdb_test.go

@@ -17,6 +17,7 @@ import (
 	"testing/synctest"
 	"time"
 
+	"cloud.google.com/go/civil"
 	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/metrics"
 	"sneak.berlin/go/smallwebwaf/internal/reputation"
@@ -179,11 +180,13 @@ func TestDailyBudgetKeptAcrossARestartAndWholeAgainAsTheDayEnds(t *testing.T) {
 			t.Errorf("logged\n%s\nwant the budget used up", log.String())
 		}
 
-		// Restarted with what reputation.json keeps, it uses the scores, and
-		// checks no client until the day ends.
+		// Restarted with what abuseipdb-scores.json and
+		// abuseipdb-checks.json keep, it uses the scores, and checks no
+		// client until the day ends.
 		restarted := &abuseIPDBStandIn{}
 		again := newAbuseIPDB(restarted, p)
 		again.Load(checker.Snapshot())
+		again.LoadChecksSpent(checker.ChecksSpent())
 
 		wantScore(t, again, suspect, true, 100, true)
 		wantBudgetLeft(t, again, 0)
@@ -224,13 +227,9 @@ func TestChecksAreCountedInUTCDaysWhateverTheClocksZone(t *testing.T) {
 		time.Sleep(2 * time.Hour)
 		wantBudgetLeft(t, checker, 0)
 
-		encoded, err := json.Marshal(checker.Snapshot())
-		if err != nil {
-			t.Fatalf("encode: %v", err)
-		}
-
-		if !strings.Contains(string(encoded), `"day":"2000-01-01"`) {
-			t.Errorf("reputation.json would hold %s, want the day 2000-01-01", encoded)
+		want := civil.Date{Year: 2000, Month: 1, Day: 1}
+		if day := checker.ChecksSpent().Day; day != want {
+			t.Errorf("checks spent on %s, want on %s", day, want)
 		}
 
 		// At 00:00 UTC the budget is whole again.
@@ -302,7 +301,7 @@ func TestFailedCheckGivesNoScoreAndNoClientIsCheckedForAMinute(t *testing.T) {
 				wantChecked(t, abuseIPDB, suspect, other)
 				wantFailures(t, checker, 2)
 
-				if scores := checker.Snapshot().Scores; len(scores) != 0 {
+				if scores := checker.Snapshot(); len(scores) != 0 {
 					t.Errorf("scores %+v, want none", scores)
 				}
 
@@ -399,7 +398,7 @@ func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
 
 		for name, shown := range map[string]string{
 			"the log": log.String(), "the alerts": string(alerted),
-			"the metrics": scrapeMetrics(t, m), "reputation.json": string(kept),
+			"the metrics": scrapeMetrics(t, m), "abuseipdb-scores.json": string(kept),
 		} {
 			if strings.Contains(shown, key) {
 				t.Errorf("%s shows the key:\n%s", name, shown)
@@ -462,12 +461,12 @@ func TestScoreFetchedATTLAgoIsNeitherUsedNorKept(t *testing.T) {
 		Fetched: now.Add(-cacheTTL),
 	}
 
-	checker.Load(reputation.Checks{Scores: []reputation.Score{stale, inUse}})
+	checker.Load([]reputation.Score{stale, inUse})
 
 	wantScore(t, checker, suspect, false, 100, true)
 	wantScore(t, checker, other, false, 0, false)
 
-	got := checker.Snapshot().Scores
+	got := checker.Snapshot()
 	if !reflect.DeepEqual(got, []reputation.Score{inUse}) {
 		t.Errorf("scores %+v, want only %+v", got, inUse)
 	}
@@ -481,7 +480,7 @@ func TestAtMost100000ScoresKeptTheOneFetchedLongestAgoDroppedFirst(t *testing.T)
 	p.Now = func() time.Time { return now }
 	checker := reputation.NewAbuseIPDB(p)
 
-	// 100,001 scores, listed by client, as reputation.json lists them, each
+	// 100,001 scores, listed by client, as abuseipdb-scores.json lists them, each
 	// fetched a millisecond before the one before it: the last is one too
 	// many.
 	const count = 100001
@@ -497,9 +496,9 @@ func TestAtMost100000ScoresKeptTheOneFetchedLongestAgoDroppedFirst(t *testing.T)
 		addr = addr.Next()
 	}
 
-	checker.Load(reputation.Checks{Scores: scores})
+	checker.Load(scores)
 
-	got := checker.Snapshot().Scores
+	got := checker.Snapshot()
 	if len(got) != count-1 || !slices.Contains(got, scores[0]) ||
 		slices.Contains(got, scores[count-1]) {
 		t.Errorf("%d scores kept, want all but the one fetched longest ago", len(got))

internal/reputation/crowdsec_test.go

@@ -291,7 +291,7 @@ func crowdSecFailures() []crowdSecFailure {
 // other than its decision list, such as the one a redirect names, and that
 // the key is in none of what the fetches leave behind: log, the process
 // log, the alerts waiting in queue, and the copies of lists, which
-// reputation.json keeps.
+// lists.json keeps.
 func wantKeyNotShown(
 	t *testing.T, e *engine, log string, lists *reputation.Lists, queue *alerts.Queue,
 ) {
@@ -326,7 +326,7 @@ func TestCrowdSecDecisionListKeptAcrossARestartEndsWhenItsDecisionsDo(t *testing
 		lists := start(t, e, crowdSecParams())
 		kept := lists.Snapshot()
 
-		// Restarted half an hour later with what reputation.json keeps, and
+		// Restarted half an hour later with what lists.json keeps, and
 		// the engine down, the decision still bans, until the end it had at
 		// the fetch, half an hour on.
 		time.Sleep(30 * time.Minute)

internal/reputation/dnsbl.go

@@ -40,7 +40,7 @@ var (
 	errNotListing = errors.New("the answer is outside 127.0.0.0/8")
 )
 
-// Verdict is what a zone said about a client, as reputation.json holds
+// Verdict is what a zone said about a client, as dnsbl.json holds
 // it: the zone, the client's address, whether the zone lists it, and when
 // the zone answered.
 type Verdict struct {
@@ -187,7 +187,7 @@ func (d *DNSBL) Failures(zone string) int {
 }
 
 // Snapshot returns every verdict still in use, sorted by client, then by
-// zone, as reputation.json lists them.
+// zone, as dnsbl.json lists them.
 func (d *DNSBL) Snapshot() []Verdict {
 	d.mu.Lock()
 
@@ -209,7 +209,7 @@ func (d *DNSBL) Snapshot() []Verdict {
 	return verdicts
 }
 
-// Load keeps verdicts, read from reputation.json, in place of those it
+// Load keeps verdicts, read from dnsbl.json, in place of those it
 // keeps, but for those of a zone SWW_DNSBL_ZONES does not name, and,
 // past maxVerdicts, those fetched longest ago. One fetched CacheTTL ago or
 // more is neither used nor written, as for any verdict.

internal/reputation/dnsbl_test.go

@@ -389,7 +389,7 @@ func TestVerdictsKeptAcrossARestart(t *testing.T) {
 			t.Errorf("verdicts %+v, want %+v", kept, want)
 		}
 
-		// Restarted an hour later with what reputation.json keeps, it uses
+		// Restarted an hour later with what dnsbl.json keeps, it uses
 		// the verdicts, and asks the zone nothing, until the TTL has passed
 		// since they were fetched.
 		time.Sleep(time.Hour)
@@ -446,7 +446,7 @@ func TestAtMost100000VerdictsKeptTheOneFetchedLongestAgoDroppedFirst(t *testing.
 	p.Now = func() time.Time { return now }
 	dnsbl := reputation.NewDNSBL(p)
 
-	// 100,001 verdicts, listed by client, as reputation.json lists them,
+	// 100,001 verdicts, listed by client, as dnsbl.json lists them,
 	// each fetched a millisecond before the one before it: the last is one
 	// too many.
 	const count = 100001

internal/reputation/reputation.go

@@ -7,8 +7,8 @@
 // and when each was last tried. It also asks the DNSBL zones of
 // SWW_DNSBL_ZONES about clients, and keeps their verdicts, and checks
 // clients with AbuseIPDB, and keeps their scores and the checks spent
-// today. The state package writes all of these to reputation.json and
-// reads them from it, so that a restart keeps them too.
+// today. The state package writes all of these to files and reads them
+// from there, so that a restart keeps them too.
 package reputation
 
 import (
@@ -54,7 +54,7 @@ var (
 		"does not give an address or a netblock and a duration, such as 4h0m0s")
 )
 
-// List is a list as reputation.json holds it: the URL it is fetched from,
+// List is a list as lists.json holds it: the URL it is fetched from,
 // when it was last tried, the fetch failed or not, and its last good copy:
 // when that was fetched, and its lines, as fetched, comment lines
 // included, both left out while no fetch of it has succeeded.
@@ -106,7 +106,7 @@ type Lists struct {
 	lists map[string]*list
 }
 
-// list is one list: what reputation.json keeps of it, its last try, zero
+// list is one list: what lists.json keeps of it, its last try, zero
 // before the first, and its last good copy, what that copy says, and how
 // many fetches of it failed.
 type list struct {
@@ -272,7 +272,7 @@ func (l *Lists) Failures(listURL string) int {
 
 // Run fetches each list once it is due, as due tells, until ctx is done. A
 // list never tried is fetched at once, and so is one that is due by its
-// last try or copy read from reputation.json.
+// last try or copy read from lists.json.
 func (l *Lists) Run(ctx context.Context) {
 	if len(l.lists) == 0 {
 		return
@@ -292,7 +292,7 @@ func (l *Lists) Run(ctx context.Context) {
 }
 
 // Snapshot returns each list that has been tried, with its copy, if it
-// has one, sorted by URL, as reputation.json lists them.
+// has one, sorted by URL, as lists.json lists them.
 func (l *Lists) Snapshot() []List {
 	l.mu.Lock()
 
@@ -313,7 +313,7 @@ func (l *Lists) Snapshot() []List {
 	return tried
 }
 
-// Load puts lists, read from reputation.json, in place of the last tries
+// Load puts lists, read from lists.json, in place of the last tries
 // and copies held. A list Params does not name is dropped. A copy with a
 // line that parse refuses is an error, and then nothing changes.
 func (l *Lists) Load(lists []List) error {

internal/reputation/reputation_test.go

@@ -260,7 +260,7 @@ func TestKeptCopyIsFetchedAgainOnceRefreshHasPassedSinceItWasFetched(t *testing.
 		lists.SetTransport(servers)
 
 		// drop.txt was fetched an hour ago, and tor.txt a refresh ago, as
-		// reputation.json says at start.
+		// lists.json says at start.
 		err := lists.Load([]reputation.List{
 			{URL: dropURL, Fetched: time.Now().Add(-time.Hour), Lines: []string{"203.0.113.9"}},
 			{URL: torURL, Fetched: time.Now().Add(-refresh), Lines: []string{"198.51.100.9"}},
@@ -297,7 +297,7 @@ func TestRestartWaitsRefreshAfterTheLastTryEvenOneThatFailed(t *testing.T) {
 		time.Sleep(refresh)
 		wantFetches(t, servers, 2)
 
-		// Restarted with what reputation.json keeps, it waits a refresh
+		// Restarted with what lists.json keeps, it waits a refresh
 		// after the failed try, as it does while it runs.
 		restarted := &standIn{lists: map[string]string{dropURL: "198.51.100.2\n"}}
 		again := reputation.New(params(dropURL))
@@ -353,7 +353,7 @@ func TestFetchCutOffAsItStopsIsNoFailureButARestartWaitsForIt(t *testing.T) {
 				waiting(queue))
 		}
 
-		// Restarted an hour later with what reputation.json keeps, it fetches
+		// Restarted an hour later with what lists.json keeps, it fetches
 		// tor.txt, never tried, at once, and drop.txt a refresh after its
 		// cut-off try.
 		time.Sleep(time.Hour)